My program will get panic issue from time to time.
I don’t have procedure to reproduce this issue.
So far, the issue is happen on 10/23, 10/25, 10/30, 11/14 and 11/22.
In the last try, I capture packets by wireshark, it seems no packet incoming when panic.
I also try go build -race. There is no useful clues in crash dump.
Any suggestion is welcome.
go version go1.12.7 linux/amd64
go env
GOARCH=“amd64”
GOBIN=""
GOCACHE="/home/ubuntu/.cache/go-build"
GOEXE=""
GOFLAGS=""
GOHOSTARCH=“amd64”
GOHOSTOS=“linux”
GOOS=“linux”
GOPATH="/home/ubuntu/go"
GOPROXY=""
GORACE=""
GOROOT="/usr/local/go"
GOTMPDIR=""
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
GCCGO=“gccgo”
CC=“gcc”
CXX=“g++”
CGO_ENABLED=“1”
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG=“pkg-config”
GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build954623632=/tmp/go-build -gno-record-gcc-switches"
execption dump on 20191122 17:01:21(the program is auto bring up by crontab in 10 seconds. The actually issue is 17:01:11~17:01:21):
[GIN] 2019/11/22 - 08:33:29 | 404 | 486ns | 38.21.245.102 | GET /
[GIN] 2019/11/22 - 08:33:29 | 404 | 864ns | 38.21.245.102 | GET /HNAP1/
[GIN] 2019/11/22 - 09:20:16 | 404 | 1.066µs | 94.242.26.158 | GET /
[GIN] 2019/11/22 - 10:07:25 | 404 | 482ns | 107.189.11.148 | GET /script
[GIN] 2019/11/22 - 10:19:53 | 404 | 918ns | 80.82.70.187 | GET /cache/global/img/gs.gif
[GIN] 2019/11/22 - 11:20:51 | 404 | 774ns | 218.214.116.175 | GET /
[GIN] 2019/11/22 - 11:56:36 | 404 | 485ns | 43.255.216.26 | GET /
[GIN] 2019/11/22 - 12:09:17 | 404 | 450ns | 170.245.173.234 | GET /
[GIN] 2019/11/22 - 12:41:12 | 404 | 728ns | 109.1.110.70 | GET /
[GIN] 2019/11/22 - 13:16:05 | 404 | 698ns | 216.218.206.68 | GET /
[GIN] 2019/11/22 - 13:40:07 | 404 | 753ns | 187.102.57.223 | GET /
[GIN] 2019/11/22 - 15:04:29 | 404 | 554ns | 119.61.6.133 | GET /
[GIN] 2019/11/22 - 15:24:29 | 404 | 822ns | 103.249.181.40 | GET /
runtime: unexpected return pc for net/http.(*Server).Serve called from 0x0
stack: frame={sp:0xc000137d78, fp:0xc000137e40} stack=[0xc000137000,0xc000138000)
000000c000137c78: 000000000000000c 000000c000137cc0
000000c000137c88: 0000000000535948 <net.(*TCPListener).AcceptTCP+72> 000000c0002cc088
000000c000137c98: 000000c00002ca00 000000000040e169 <runtime.mallocgc+745>
000000c000137ca8: 000000c000098780 00000000000000a0
000000c000137cb8: 000000c000137d00 000000c000137d30
000000c000137cc8: 0000000000b24686 <go-template/vendor/github.com/fvbock/endless.(*endlessListener).Accept+70> 000000c0002cc088
000000c000137cd8: 00000000ea73cddc 62f827d6a4dee03e
000000c000137ce8: 000000000044e458 <time.now+56> 000d4614f8da2885
000000c000137cf8: 000000c016ed78e0 16ed78e000137d38
000000c000137d08: 000000005dd78357 000000c000137d38
000000c000137d18: 00000000004b7d56 <time.Now+38> 000000005dd78357
000000c000137d28: 000000000043908f <runtime.newproc+111> 000000c000137d68
000000c000137d38: 0000000000763d3c <net/http.(*onceCloseListener).Accept+60> 000000c000296140
000000c000137d48: 000000c000137d88 0000000000000018
000000c000137d58: 000000c000498780 000000000073fd54 <net/http.(*Server).Serve+756>
000000c000137d68: 000000c000137e30 000000000073fc8d <net/http.(*Server).Serve+557>
000000c000137d78: <000000c000402480 0000000000e13940
000000c000137d88: 000000c000098780 0000000000f46000
000000c000137d98: 000000c000402540 000000c0003f9200
000000c000137da8: 0000000000f46000 000000c000402540
000000c000137db8: 0000000000000000 0000000000f53a20
000000c000137dc8: 000000000042f231 <runtime.deferreturn+145> 0000000000f46000
000000c000137dd8: 000000c000296b00 00000000004735fe <sync.(*RWMutex).Unlock+126>
000000c000137de8: 0000000d00402540 000000c000138b38
000000c000137df8: 00007f7950798330 0000000000000001
000000c000137e08: 0000000000000000 000000c000582000
000000c000137e18: 000000c000000000 0000000000008000
000000c000137e28: 0000000000000000 0000000000000000
000000c000137e38: !0000000000000000 >0000000000000212
000000c000137e48: 0000000000000070 0000000000000002
000000c000137e58: 0000000000000001 0000000000000049
000000c000137e68: 0000000000000025 000000c0000f4070
000000c000137e78: 000000c000138b38 000000c00002ca00
000000c000137e88: 0000000000000067 ffffffffffffffe0
000000c000137e98: 00000000004aba60 <syscall.Syscall+48> 000000c000138ae8
000000c000137ea8: 00000000004aba60 <syscall.Syscall+48> 0000000000000212
000000c000137eb8: 0000000000000033 0000000000000000
000000c000137ec8: 0000000000000000 0000000000000000
000000c000137ed8: 0000000000000000 000000c000137fc0
000000c000137ee8: 0000000000000000 0000000000000000
000000c000137ef8: 0000000000f42fc0 000000000000000c
000000c000137f08: 000000c000304120 000000c0002cc088
000000c000137f18: 000000c000285000 0000000000f1b660
000000c000137f28: 0000000000000000 0000000000f1b630
000000c000137f38: 000000c000137f70
fatal error: unknown caller pc
runtime stack:
runtime.throw(0xde01c3, 0x11)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000498780, 0x0, 0x0, 0x7fffffff, 0x7f794b46cb08, 0x0, 0x0, …)
/usr/local/go/src/runtime/traceback.go:275 +0x1cd1
runtime.scanstack(0xc000498780, 0xc00002dc70)
/usr/local/go/src/runtime/mgcmark.go:711 +0x15f
runtime.scang(0xc000498780, 0xc00002dc70)
/usr/local/go/src/runtime/proc.go:888 +0x1e4
runtime.markroot.func1()
/usr/local/go/src/runtime/mgcmark.go:221 +0x6e
runtime.markroot(0xc00002dc70, 0x7f790000001b)
/usr/local/go/src/runtime/mgcmark.go:202 +0x2e9
runtime.gcDrain(0xc00002dc70, 0x3)
/usr/local/go/src/runtime/mgcmark.go:899 +0x112
runtime.gcBgMarkWorker.func2()
/usr/local/go/src/runtime/mgc.go:1903 +0x80
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1153
goroutine 50 [GC worker (idle), 2 minutes]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc0002d0760 sp=0xc0002d0758 pc=0x45aa40
runtime.gcBgMarkWorker(0xc00002ca00)
/usr/local/go/src/runtime/mgc.go:1890 +0x1be fp=0xc0002d07d8 sp=0xc0002d0760 pc=0x41e4be
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc0002d07e0 sp=0xc0002d07d8 pc=0x45cb11
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1784 +0x77
goroutine 1 [IO wait, 4 minutes]:
internal/poll.runtime_pollWait(0x7f7949bf7ec8, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000331d18, 0x72, 0x0, 0x0, 0xdd741e)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(…)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000331d00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc000331d00, 0x7f7950a013b8, 0x203000, 0x203000)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00055cfb8, 0xc00002a500, 0x40e169, 0xc000098780)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc00055cfb8, 0x83e35326, 0xa0df53898b7b9c8, 0x44e458)
/usr/local/go/src/net/tcpsock.go:247 +0x48
go-template/vendor/github.com/fvbock/endless.(*endlessListener).Accept(0xc0003fb100, 0xc00052bab0, 0x18, 0xc000000180, 0x73fd54)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:490 +0x46
net/http.(*Server).Serve(0xc000383b00, 0xf41bc0, 0xc0003fb100, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2859 +0x22d
go-template/vendor/github.com/fvbock/endless.(*endlessServer).Serve(0xc000383b00, 0x0, 0x0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:195 +0xec
go-template/vendor/github.com/fvbock/endless.(*endlessServer).ListenAndServe(0xc000383b00, 0xc, 0xf30cc0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:229 +0x1bf
go-template/vendor/github.com/fvbock/endless.ListenAndServe(0xc000284f90, 0xc, 0xf30cc0, 0xc0003839e0, 0xf56ce0, 0xc0003839e0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:153 +0x57
go-template/fcs-go/http.Start()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/http.go:40 +0x1f2
go-template/fcs-go/cmd.startServices()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/cmd/root.go:70 +0xe0
go-template/fcs-go/cmd.glob…func1(0x16fb2e0, 0xc000296ea0, 0x0, 0x2)
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/cmd/root.go:85 +0x33
go-template/vendor/github.com/spf13/cobra.(*Command).execute(0x16fb2e0, 0xc000020190, 0x2, 0x2, 0x16fb2e0, 0xc000020190)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/spf13/cobra/command.go:766 +0x2ae
go-template/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x16fb2e0, 0xc0001adf68, 0xbfc7ee, 0x16fb2e0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/spf13/cobra/command.go:852 +0x2ec
go-template/vendor/github.com/spf13/cobra.(*Command).Execute(…)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/spf13/cobra/command.go:800
go-template/fcs-go/cmd.Execute()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/cmd/root.go:92 +0x32
main.main()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/main.go:8 +0x20
goroutine 18 [syscall, 22848 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 20 [select, 14530 minutes]:
go-template/fcs-go/http/v1.(*Hub).run(0xc000294b80)
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/v1/alarm.go:126 +0x240
created by go-template/fcs-go/http/v1.init.0
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/v1/alarm.go:43 +0x41
goroutine 21 [sleep, 2 minutes]:
runtime.goparkunlock(…)
/usr/local/go/src/runtime/proc.go:307
time.Sleep(0x2540be400)
/usr/local/go/src/runtime/time.go:105 +0x159
go-template/fcs-go/http/v1.listen()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/v1/alarm.go:90 +0xf4
created by go-template/fcs-go/http/v1.init.0
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/v1/alarm.go:44 +0x59
goroutine 35 [select, 22848 minutes]:
database/sql.(*DB).connectionOpener(0xc0002fc0c0, 0xf45f40, 0xc0002c8040)
/usr/local/go/src/database/sql/sql.go:1000 +0xe8
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:670 +0x15e
goroutine 36 [select, 1014 minutes]:
database/sql.(*DB).connectionResetter(0xc0002fc0c0, 0xf45f40, 0xc0002c8040)
/usr/local/go/src/database/sql/sql.go:1013 +0xfb
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:671 +0x194
goroutine 10 [chan receive, 2 minutes]:
go-template/fcs-go/models.InitScheduling.func1(0xc00021a570)
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/models/scheduling.go:430 +0x58
created by go-template/fcs-go/models.InitScheduling
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/models/scheduling.go:427 +0xe0
goroutine 11 [IO wait, 99 minutes]:
internal/poll.runtime_pollWait(0x7f7949bf8068, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc0003c2098, 0x72, 0x0, 0x0, 0xdd741e)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(…)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0003c2080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc0003c2080, 0x7f7950a013b8, 0x203000, 0x203000)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0002cc078, 0xc000028000, 0x40e169, 0xc000098780)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc0002cc078, 0x12f91435, 0x3af71f9f9ea876bd, 0x44e458)
/usr/local/go/src/net/tcpsock.go:247 +0x48
go-template/vendor/github.com/fvbock/endless.(*endlessListener).Accept(0xc000296100, 0xc00013bd88, 0x18, 0xc000498600, 0x73fd54)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:490 +0x46
net/http.(*Server).Serve(0xc0003e7c20, 0xf41bc0, 0xc000296100, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2859 +0x22d
go-template/vendor/github.com/fvbock/endless.(*endlessServer).Serve(0xc0003e7c20, 0x0, 0x0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:195 +0xec
go-template/vendor/github.com/fvbock/endless.(*endlessServer).ListenAndServe(0xc0003e7c20, 0xc, 0xf30cc0)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:229 +0x1bf
go-template/vendor/github.com/fvbock/endless.ListenAndServe(0xc000284ff0, 0xc, 0xf30cc0, 0xc0003e7b00, 0xf56ce0, 0xc0003e7b00)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:153 +0x57
go-template/fcs-go/http.StartTPKDataRouter()
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/http.go:101 +0x1c6
created by go-template/fcs-go/http.Start
/home/jessica/0909/go-template/.workspace/src/go-template/fcs-go/http/http.go:20 +0x39
goroutine 12 [IO wait (scan), 140 minutes]:
internal/poll.runtime_pollWait(0x7f7949bf7f98, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000022218, 0x72, 0x0, 0x0, 0xdd741e)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(…)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000022200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc000022200, 0x7f7950a013b8, 0x203000, 0x203000)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0002cc088, 0xc00002ca00, 0x40e169, 0xc000098780)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc0002cc088, 0xea73cddc, 0x62f827d6a4dee03e, 0x44e458)
/usr/local/go/src/net/tcpsock.go:247 +0x48
go-template/vendor/github.com/fvbock/endless.(*endlessListener).Accept(0xc000296140, 0xc000137d88, 0x18, 0xc000498780, 0x73fd54)
/home/jessica/0909/go-template/.workspace/src/go-template/vendor/github.com/fvbock/endless/endless.go:490 +0x46
fatal error: unexpected signal during runtime execution
panic during panic
[signal SIGSEGV: segmentation violation code=0x1 addr=0x119 pc=0x451543]
runtime stack:
runtime.throw(0xdf459e, 0x2a)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:374 +0x4a9
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000498780, 0x0, 0x0, 0x64, 0x0, 0x0, 0x0, …)
/usr/local/go/src/runtime/traceback.go:263 +0x15d3
runtime.traceback1(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000498780, 0x0)
/usr/local/go/src/runtime/traceback.go:746 +0xf1
runtime.traceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000498780)
/usr/local/go/src/runtime/traceback.go:700 +0x52
runtime.tracebackothers(0xc000001200)
/usr/local/go/src/runtime/traceback.go:953 +0x1a8
runtime.dopanic_m(0xc000001200, 0x42fc12, 0x7f794b46c770, 0x1)
/usr/local/go/src/runtime/panic.go:817 +0x2a0
runtime.fatalthrow.func1()
/usr/local/go/src/runtime/panic.go:672 +0x5f
runtime.fatalthrow()
/usr/local/go/src/runtime/panic.go:669 +0x57
runtime.throw(0xde01c3, 0x11)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000498780, 0x0, 0x0, 0x7fffffff, 0x7f794b46cb08, 0x0, 0x0, …)
/usr/local/go/src/runtime/traceback.go:275 +0x1cd1
runtime.scanstack(0xc000498780, 0xc00002dc70)
/usr/local/go/src/runtime/mgcmark.go:711 +0x15f
runtime.scang(0xc000498780, 0xc00002dc70)
/usr/local/go/src/runtime/proc.go:888 +0x1e4
runtime.markroot.func1()
/usr/local/go/src/runtime/mgcmark.go:221 +0x6e
runtime.markroot(0xc00002dc70, 0x7f790000001b)
/usr/local/go/src/runtime/mgcmark.go:202 +0x2e9
runtime.gcDrain(0xc00002dc70, 0x3)
/usr/local/go/src/runtime/mgcmark.go:899 +0x112
runtime.gcBgMarkWorker.func2()
/usr/local/go/src/runtime/mgc.go:1903 +0x80
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1153
[ e[0;32minfoe[0m ] 17:01:22 (checkPointMappingList.go:136) checkPointMpList: