runtime: fatal error: bad map state

[mikioh]

See http://build.golang.org/log/2b7d4e774fad87aea6381dbfc940a0fae573c6dd.

fatal error: bad map state

goroutine 52 [running]:
runtime.throw(0x283b50, 0xd)
    /tmp/workdir/go/src/runtime/panic.go:527 +0x100 fp=0x132db9a0 sp=0x132db990
runtime.evacuate(0x21fe40, 0x132dbc04, 0x0, 0x132aa380)
    /tmp/workdir/go/src/runtime/hashmap.go:825 +0x3c0 fp=0x132dba08 sp=0x132db9a0
runtime.growWork(0x21fe40, 0x132dbc04, 0x1, 0x12a280e0)
    /tmp/workdir/go/src/runtime/hashmap.go:795 +0xa0 fp=0x132dba20 sp=0x132dba08
runtime.mapassign1(0x21fe40, 0x132dbc04, 0x132b08a8, 0x132dbb90)
    /tmp/workdir/go/src/runtime/hashmap.go:433 +0x1a0 fp=0x132dba80 sp=0x132dba20
internal/trace.postProcessTrace(0x132ce000, 0x3b, 0x40, 0x4b, 0x0, 0x0)
    /tmp/workdir/go/src/internal/trace/parser.go:459 +0x2100 fp=0x132dbe40 sp=0x132dba80
internal/trace.Parse(0xfeec4420, 0x132a2b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3659)
    /tmp/workdir/go/src/internal/trace/parser.go:73 +0x1e0 fp=0x132dbe90 sp=0x132dbe40
runtime/trace_test.parseTrace(0xfeec4420, 0x132a2b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:84 +0x60 fp=0x132dbf00 sp=0x132dbe90
runtime/trace_test.TestTraceStressStartStop(0x132a2ae0, 0xe)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:364 +0x2a0 fp=0x132dbf80 sp=0x132dbf00
testing.tRunner(0x132a2ae0, 0x6e1cbc)
    /tmp/workdir/go/src/testing/testing.go:456 +0xe0 fp=0x132dbfa8 sp=0x132dbf80
runtime.goexit()
    /tmp/workdir/go/src/runtime/asm_amd64p32.s:1127 +0x1 fp=0x132dbfb0 sp=0x132dbfa8
created by testing.RunTests
    /tmp/workdir/go/src/testing/testing.go:561 +0x980

goroutine 1 [chan receive]:
testing.RunTests(0x2bff44, 0x6e1c80, 0x7, 0x7, 0x1, 0x6d01e0)
    /tmp/workdir/go/src/testing/testing.go:562 +0x9c0
testing.(*M).Run(0x10937f68, 0x1092fe78, 0x10, 0x10)
    /tmp/workdir/go/src/testing/testing.go:494 +0xe0
main.main()
    runtime/trace/_test/_testmain.go:68 +0x140

goroutine 57 [runnable, locked to thread]:
runtime.Gosched()
    /tmp/workdir/go/src/runtime/proc.go:166 +0x20
runtime/trace_test.TestTraceStressStartStop.func1.5(0x10939580, 0x6e1cb0)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:276 +0x100
created by runtime/trace_test.TestTraceStressStartStop.func1
    /tmp/workdir/go/src/runtime/trace/trace_test.go:279 +0x320

goroutine 56 [semacquire]:
sync.runtime_Syncsemacquire(0x132a2bb0, 0x1)
    /tmp/workdir/go/src/runtime/sema.go:237 +0x2e0
sync.(*Cond).Wait(0x132a2ba8, 0x132a2ba0)
    /tmp/workdir/go/src/sync/cond.go:62 +0xe0
syscall.(*queue).waitRead(0x132a2ba0, 0x1, 0x0, 0x0, 0x6e7120, 0x0, 0x0, 0x0)
    /tmp/workdir/go/src/syscall/net_nacl.go:263 +0x180
syscall.(*byteq).read(0x132a2ba0, 0x11898b30, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfee00220)
    /tmp/workdir/go/src/syscall/net_nacl.go:328 +0xe0
syscall.(*pipeFile).read(0x1329c998, 0x11898b30, 0x1, 0x1, 0x0, 0x0, 0x0, 0xe1ef)
    /tmp/workdir/go/src/syscall/fd_nacl.go:303 +0xe0
syscall.Read(0x3, 0x11898b30, 0x1, 0x1, 0x28, 0x0, 0x0, 0x0)
    /tmp/workdir/go/src/syscall/fd_nacl.go:162 +0xe0
os.(*File).read(0x1329c9a8, 0x11898b30, 0x1, 0x1, 0x33780, 0x0, 0x0, 0x21b9c0)
    /tmp/workdir/go/src/os/file_unix.go:211 +0x60
os.(*File).Read(0x1329c9a8, 0x11898b30, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
    /tmp/workdir/go/src/os/file.go:95 +0x80
runtime/trace_test.TestTraceStressStartStop.func1.4(0x1329c9a8, 0x10939580, 0x11898b00, 0x0)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:263 +0x80
created by runtime/trace_test.TestTraceStressStartStop.func1
    /tmp/workdir/go/src/runtime/trace/trace_test.go:266 +0x2e0

goroutine 55 [chan receive]:
runtime/trace_test.TestTraceStressStartStop.func1.2(0x10939580, 0x11898b00)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:248 +0x40
created by runtime/trace_test.TestTraceStressStartStop.func1
    /tmp/workdir/go/src/runtime/trace/trace_test.go:250 +0x100

goroutine 53 [runnable]:
runtime/trace_test.TestTraceStressStartStop.func1(0x10939540, 0x132a2ae0)
    /tmp/workdir/go/src/runtime/trace/trace_test.go:284 +0x380
created by runtime/trace_test.TestTraceStressStartStop
    /tmp/workdir/go/src/runtime/trace/trace_test.go:355 +0xe0

goroutine 20 [select (no cases)]:
runtime/trace_test.TestTraceStress.func9()
    /tmp/workdir/go/src/runtime/trace/trace_test.go:212 +0x40
created by runtime/trace_test.TestTraceStress
    /tmp/workdir/go/src/runtime/trace/trace_test.go:213 +0xb60
FAIL    runtime/trace   4.032s

[randall77]

I don't see how this is possible unless there is some memory corruption.
The map is thread-local so there can't be a race causing it.

[ianlancetaylor]

CC @RLH @aclements

[aclements]

This has happened a total of twice on the builders, both times on nacl:

2015-07-05T03:52:56-cc3f031/nacl-arm-cheney
2015-07-23T03:52:20-5659964/nacl-amd64p32

[aclements]

Never mind about 2015-07-05T03:52:56-cc3f031/nacl-arm-cheney. It looks like that dumped the entire contents of some Go binary, which of course contained this error string.

[rsc]

Can reproduce quite easily in nacl/amd64p32. Cannot reproduce on nacl/386, nor on linux/386, nor on linux/amd64. Reproduces even with gcstoptheworld=1,gcshrinkstackoff=1, suggesting that maybe it is a liveness bitmap problem.

[gopherbot]

CL https://golang.org/cl/12971 mentions this issue.