From nobody@FreeBSD.org  Tue Aug  9 08:08:26 2011
Return-Path: <nobody@FreeBSD.org>
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34])
	by hub.freebsd.org (Postfix) with ESMTP id 5CC70106564A
	for <freebsd-gnats-submit@FreeBSD.org>; Tue,  9 Aug 2011 08:08:26 +0000 (UTC)
	(envelope-from nobody@FreeBSD.org)
Received: from red.freebsd.org (red.freebsd.org [IPv6:2001:4f8:fff6::22])
	by mx1.freebsd.org (Postfix) with ESMTP id 423278FC15
	for <freebsd-gnats-submit@FreeBSD.org>; Tue,  9 Aug 2011 08:08:26 +0000 (UTC)
Received: from red.freebsd.org (localhost [127.0.0.1])
	by red.freebsd.org (8.14.4/8.14.4) with ESMTP id p7988QZj083881
	for <freebsd-gnats-submit@FreeBSD.org>; Tue, 9 Aug 2011 08:08:26 GMT
	(envelope-from nobody@red.freebsd.org)
Received: (from nobody@localhost)
	by red.freebsd.org (8.14.4/8.14.4/Submit) id p7988QOo083880;
	Tue, 9 Aug 2011 08:08:26 GMT
	(envelope-from nobody)
Message-Id: <201108090808.p7988QOo083880@red.freebsd.org>
Date: Tue, 9 Aug 2011 08:08:26 GMT
From: Pavel Polyakov <bsd@kobyla.org>
To: freebsd-gnats-submit@FreeBSD.org
Subject: [panic] panic: soabort: so_count
X-Send-Pr-Version: www-3.1
X-GNATS-Notify:

>Number:         159621
>Category:       kern
>Synopsis:       [tcp] [panic] panic: soabort: so_count
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-net
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:  
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Tue Aug 09 08:10:11 UTC 2011
>Closed-Date:    
>Last-Modified:  Fri Jan 27 15:30:07 UTC 2012
>Originator:     Pavel Polyakov
>Release:        FreeBSD 8.2-STABLE 224067
>Organization:
>Environment:
FreeBSD 8.2-STABLE #8 r224067: Sat Jul 16 01:23:48 EEST 2011

>Description:
Panic during normal operations.
kern/145825 may be related.

======

db:0:kdb.enter.panic>  run lockinfo
db:1:lockinfo> show locks
exclusive rw tcpinp (tcpinp) r = 0 (0x8d01df64) locked @ /usr/src/sys/netinet/tcp_input.c:962
exclusive rw tcp (tcp) r = 0 (0x80c3eb0c) locked @ /usr/src/sys/netinet/tcp_usrreq.c:1507


db:1:locks>  show alllocks
..
Process 93609 (sshd) thread 0x88c4f8a0 (100118)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0x89f686fc) locked @ /usr/src/sys/kern/uipc_sockbuf.c:148
Process 2819 (vtund) thread 0x88c878a0 (100125)
exclusive rw tcpinp (tcpinp) r = 0 (0x8d01df64) locked @ /usr/src/sys/netinet/tcp_input.c:962
exclusive rw tcp (tcp) r = 0 (0x80c3eb0c) locked @ /usr/src/sys/netinet/tcp_usrreq.c:1507
Process 2692 (mysqld) thread 0x89475000 (100259)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0x89f96228) locked @ /usr/src/sys/kern/uipc_sockbuf.c:148
Process 2692 (mysqld) thread 0x88c515c0 (100109)
exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0x8d0a7898) locked @ /usr/src/sys/kern/uipc_sockbuf.c:148

db:0:kdb.enter.panic>  show pcpu
cpuid        = 0
dynamic pcpu = 0x567300
curthread    = 0x88c878a0: pid 2819 "vtund"
curpcb       = 0xd6780d80
fpcurthread  = none
idlethread   = 0x868718a0: tid 100003 "idle"
APIC ID      = 0
currentldt   = 0x50
spin locks held:

 2819     1  2819     0  Rs      CPU 0               vtund


db:0:kdb.enter.panic>  bt
Tracing pid 2819 tid 100125 td 0x88c878a0
kdb_enter(80a0e225,80a0e225,80a186dd,d6780840,d6780840,...) at kdb_enter+0x3a
panic(80a186dd,346,0,d67808b8,0,...) at panic+0xd1
soabort(88961670,80a2dc03,346,0,0,...) at soabort+0x1b
syncache_expand(d6780a60,d6780a40,8c78cb5c,d6780a5c,8c78cb00,...) at syncache_expand+0xdb3
tcp_input(8c78cb00,14,86c17800,1,0,...) at tcp_input+0xc49
ip_input(8c78cb00,4,80a0c98c,109,24,...) at ip_input+0x65b
netisr_dispatch_src(1,0,8c78cb00,d6780b34,80766ae1,...) at netisr_dispatch_src+0xd1
netisr_dispatch(1,8c78cb00,0,86c17800,8c78cb3a,...) at netisr_dispatch+0x20
ether_demux(86c17800,8c78cb00,3,0,3,...) at ether_demux+0x1a1
ether_input(86c16400,8c78cb00,6,2,2,...) at ether_input+0x393
tapwrite(88b85d00,d6780c28,0,0,0,...) at tapwrite+0xef
devfs_write_f(89f7c8c0,d6780c28,8686cb80,0,88c878a0,...) at devfs_write_f+0xac
dofilewrite(d6780c28,ffffffff,ffffffff,0,89f7c8c0,...) at dofilewrite+0x95
kern_writev(88c878a0,5,d6780c28,d6780c48,1,...) at kern_writev+0x58
write(88c878a0,d6780cec,d6780d28,80a12db8,0,...) at write+0x4f
syscallenter(88c878a0,d6780ce4,d6780ce4,0,806d4a26,...) at syscallenter+0x246
syscall(d6780d28) at syscall+0x34
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (4, FreeBSD ELF32, write), eip = 0x28324d2b, esp = 0x7fbfeb2c, ebp = 0x7fbfec08 ---


>How-To-Repeat:

>Fix:


>Release-Note:
>Audit-Trail:
Responsible-Changed-From-To: freebsd-bugs->freebsd-net 
Responsible-Changed-By: linimon 
Responsible-Changed-When: Sun Aug 14 10:24:24 UTC 2011 
Responsible-Changed-Why:  
Appears to be tcp-related. 

http://www.freebsd.org/cgi/query-pr.cgi?pr=159621 

From: "Pavel Polyakov" <bsd@kobyla.org>
To: bug-followup@freebsd.org, bsd@kobyla.org
Cc:  
Subject: Re: kern/159621: [tcp] [panic] panic: soabort: so_count
Date: Fri, 27 Jan 2012 16:48:10 +0200

 Dump header from device /dev/ada1p20
    Architecture: amd64
    Architecture Version: 1
    Dump Length: 465408B (0 MB)
    Blocksize: 512
    Dumptime: Wed Jan 25 00:14:21 2012
    Hostname: shared
    Magic: FreeBSD Text Dump
    Version String: FreeBSD 8.2-STABLE #5 r230331: Mon Jan 23 22:05:26 UTC  
 2012
    Panic String: soabort: so_count
    Dump Parity: 1378570526
    Bounds: 2
    Dump Status: good
 
 
 
 ichwd0: timer reloaded
 panic: soabort: so_count
 cpuid = 1
 KDB: enter: panic
 
 0xffffff0008d8a3b0: tag syncer, type VNON
      usecount 1, writecount 0, refcount 2 mountedhere 0
      flags ()
      lock type syncer: EXCL by thread 0xffffff0008657000 (pid 20)
 
 0xffffff02104dcce8: tag ufs, type VDIR
      usecount 1, writecount 0, refcount 3 mountedhere 0
      flags ()
      v_object 0xffffff009f19a798 ref 0 pages 1
      lock type ufs: SHARED (count 1)
          ino 89806906, on dev mirror/shared0home
 
 0xffffff0085b4b760: tag ufs, type VDIR
      usecount 1, writecount 0, refcount 3 mountedhere 0
      flags ()
      v_object 0xffffff013d38c5e8 ref 0 pages 1
      lock type ufs: SHARED (count 1)
          ino 57062185, on dev mirror/shared0home
 
 0xffffff01afaa21d8: tag nfs, type VDIR
      usecount 2, writecount 0, refcount 10934 mountedhere 0
      flags ()
      v_object 0xffffff01a4955000 ref 0 pages 15576
      lock type nfs: SHARED (count 1)
          fileid 43900928 fsid 0x500ff01
 
 
 
 db:0:kdb.enter.panic>  run lockinfo
 db:1:lockinfo> show locks
 exclusive rw tcpinp (tcpinp) r = 0 (0xffffff0042d89670) locked @  
 /usr/src/sys/netinet/tcp_input.c:946
 exclusive rw tcp (tcp) r = 0 (0xffffffff80b35108) locked @  
 /usr/src/sys/netinet/tcp_usrreq.c:1507
 db:1:locks>  show alllocks
 Process 65325 (mysqldump) thread 0xffffff00be85f460 (100712)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff01ae596648) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 63966 (exim-4.77-0) thread 0xffffff01ba777000 (100319)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff00be7170f8) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 62099 (cpsrvd-ssl) thread 0xffffff01d78de460 (103292)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff00255b4b98) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 34226 (pkgacct) thread 0xffffff011936e8c0 (101408)
 shared lockmgr nfs (nfs) r = 0 (0xffffff01afaa2270) locked @  
 /usr/src/sys/kern/vfs_syscalls.c:4100
 Process 33445 (pkgacct) thread 0xffffff01ba0c5460 (101331)
 exclusive lockmgr bufwait (bufwait) r = 0 (0xffffff81ee9040e0) locked @  
 /usr/src/sys/kern/vfs_bio.c:1891
 shared lockmgr ufs (ufs) r = 0 (0xffffff02104dcd80) locked @  
 /usr/src/sys/kern/vfs_syscalls.c:4100
 Process 26396 (pkgacct) thread 0xffffff01baa80460 (100415)
 exclusive lockmgr bufwait (bufwait) r = 0 (0xffffff81ee66b748) locked @  
 /usr/src/sys/kern/vfs_bio.c:1891
 shared lockmgr ufs (ufs) r = 0 (0xffffff0085b4b7f8) locked @  
 /usr/src/sys/kern/vfs_syscalls.c:4100
 Process 20 (syncer) thread 0xffffff0008657000 (100115)
 exclusive sleep mutex struct mount mtx (struct mount mtx) r = 0  
 (0xffffff00087e1bc0) locked @ /usr/src/sys/ufs/ufs/ufs_quota.c:939
 exclusive lockmgr syncer (syncer) r = 0 (0xffffff0008d8a448) locked @  
 /usr/src/sys/kern/vfs_subr.c:1770
 Process 0 (kernel) thread 0xffffff0002b07000 (100076)
 exclusive rw tcpinp (tcpinp) r = 0 (0xffffff0042d89670) locked @  
 /usr/src/sys/netinet/tcp_input.c:946
 exclusive rw tcp (tcp) r = 0 (0xffffffff80b35108) locked @  
 /usr/src/sys/netinet/tcp_usrreq.c:1507
 db:1:alllocks>  show lockedvnods
 Locked vnodes
 db:0:kdb.enter.panic>  show pcpu
 cpuid        = 1
 dynamic pcpu = 0xffffff807f6d5780
 curthread    = 0xffffff0002b07000: pid 0 "em0 rxq"
 curpcb       = 0xffffff824101dd10
 fpcurthread  = none
 idlethread   = 0xffffff00023778c0: tid 100017 "idle: cpu1"
 curpmap      = 0xffffffff8096b670
 tssp         = 0xffffffff80b7a768
 commontssp   = 0xffffffff80b7a768
 rsp0         = 0xffffff824101dd10
 gs32p        = 0xffffffff80b795a0
 ldt          = 0xffffffff80b795e0
 tss          = 0xffffffff80b795d0
 spin locks held:
 db:0:kdb.enter.panic>  bt
 Tracing pid 0 tid 100076 td 0xffffff0002b07000
 kdb_enter() at kdb_enter+0x3b
 panic() at panic+0x17b
 soabort() at soabort+0x9c
 syncache_expand() at syncache_expand+0x2ca
 tcp_input() at tcp_input+0xf05
 ip_input() at ip_input+0xb3
 netisr_dispatch_src() at netisr_dispatch_src+0x9e
 ether_demux() at ether_demux+0x176
 ether_input() at ether_input+0x198
 em_rxeof() at em_rxeof+0x19d
 em_handle_rx() at em_handle_rx+0x1c
 taskqueue_run_locked() at taskqueue_run_locked+0x93
 taskqueue_thread_loop() at taskqueue_thread_loop+0x3f
 fork_exit() at fork_exit+0x12a
 fork_trampoline() at fork_trampoline+0xe
 --- trap 0, rip = 0, rsp = 0xffffff824101dd00, rbp = 0 ---
 
 
      1     0     1     0  SLs     wait     0xffffff00023648e0 [init]
     10     0     0     0  SL      audit_wo 0xffffffff80b3fdd0 [audit]
      0     0     0     0  RLs     (threaded)                  kernel
 100321                   D       -        0xffffff01ba75c500 [aiod_bio  
 taskq]
 100107                   Run     CPU 7                       [dummynet]
 100105                   D       -        0xffffffff8096e5c4 [deadlkres]
 100083                   D       -        0xffffff0002b21380 [em1 txq]
 100081                   D       -        0xffffff0002b21500 [em1 rxq]
 100078                   D       -        0xffffff0002b0f980 [em0 txq]
 100076                   Run     CPU 1                       [em0 rxq]
 100053                   D       -        0xffffff0002466880 [kqueue taskq]
 100049                   D       -        0xffffff00024a2380 [thread taskq]
 100047                   D       -        0xffffff00024a2500 [ffs_trim  
 taskq]
 100046                   D       -        0xffffff0002444280 [acpi_task_2]
 100045                   D       -        0xffffff0002444280 [acpi_task_1]
 100044                   D       -        0xffffff0002444280 [acpi_task_0]
 100040                   D       -        0xffffff0002363880 [firmware  
 taskq]
 100000                   D       sched    0xffffffff8096aca0 [swapper]

From: "Pavel Polyakov" <bsd@kobyla.org>
To: bug-followup@freebsd.org, bsd@kobyla.org
Cc:  
Subject: Re: kern/159621: [tcp] [panic] panic: soabort: so_count
Date: Fri, 27 Jan 2012 16:44:28 +0200

 Dump header from device /dev/ada1p20
    Architecture: amd64
    Architecture Version: 1
    Dump Length: 223744B (0 MB)
    Blocksize: 512
    Dumptime: Tue Jan 24 05:32:13 2012
    Hostname: shared
    Magic: FreeBSD Text Dump
    Version String: FreeBSD 8.2-STABLE #5 r230331: Mon Jan 23 22:05:26 UTC  
 2012
    Panic String: soabort: so_count
    Dump Parity: 1380996126
    Bounds: 1
    Dump Status: good
 
 
 ichwd0: timer reloaded
 panic: soabort: so_count
 cpuid = 9
 KDB: enter: panic
 
 0xffffff0008b293b0: tag syncer, type VNON
      usecount 1, writecount 0, refcount 2 mountedhere 0
      flags ()
      lock type syncer: EXCL by thread 0xffffff0008657000 (pid 20)
 
 0xffffff00bb6a8000: tag ufs, type VDIR
      usecount 1, writecount 0, refcount 758 mountedhere 0
      flags ()
      v_object 0xffffff00af22d948 ref 0 pages 6032
      lock type ufs: EXCL by thread 0xffffff01c252a8c0 (pid 73631)
   with exclusive waiters pending
          ino 70625292, on dev mirror/shared0home
 
 0xffffff00b47c0ce8: tag ufs, type VREG
      usecount 4, writecount 4, refcount 1128 mountedhere 0
      flags ()
      v_object 0xffffff0014183360 ref 0 pages 8984
      lock type ufs: SHARED (count 4)
          ino 6584279, on dev mirror/shared0home
 
 0xffffff00bc8b8938: tag ufs, type VREG
      usecount 4, writecount 0, refcount 5 mountedhere 0
      flags ()
      v_object 0xffffff017f4e9438 ref 2 pages 9
      lock type ufs: SHARED (count 1)
          ino 103787584, on dev mirror/shared0home
 
 
 
 
 db:0:kdb.enter.panic>  run lockinfo
 db:1:lockinfo> show locks
 exclusive rw tcpinp (tcpinp) r = 0 (0xffffff01c2425670) locked @  
 /usr/src/sys/netinet/tcp_input.c:946
 exclusive rw tcp (tcp) r = 0 (0xffffffff80b35108) locked @  
 /usr/src/sys/netinet/tcp_usrreq.c:1507
 db:1:locks>  show alllocks
 Process 75843 (php) thread 0xffffff01c2b9a8c0 (100468)
 exclusive lockmgr bufwait (bufwait) r = 0 (0xffffff81ede351c8) locked @  
 /usr/src/sys/vm/vm_pager.c:311
 shared lockmgr ufs (ufs) r = 0 (0xffffff00bc8b89d0) locked @  
 /usr/src/sys/kern/vfs_subr.c:2170
 Process 75837 (php) thread 0xffffff00b723e000 (100232)
 exclusive sleep mutex pmap (pmap) r = 0 (0xffffff0008f98d70) locked @  
 /usr/src/sys/amd64/amd64/pmap.c:4363
 exclusive sleep mutex vm page queue mutex (vm page queue mutex) r = 0  
 (0xffffffff80b41880) locked @ /usr/src/sys/vm/vm_object.c:1101
 exclusive sleep mutex vm object (standard object) r = 0  
 (0xffffff00b41de0d8) locked @ /usr/src/sys/vm/vm_object.c:1048
 shared sx user map (user map) r = 0 (0xffffff0008f98cb8) locked @  
 /usr/src/sys/vm/vm_map.c:2023
 Process 75836 (php) thread 0xffffff01c2e228c0 (100490)
 exclusive sleep mutex vm object (standard object) r = 0  
 (0xffffff00bb23dbd0) locked @ /usr/src/sys/vm/vm_object.c:1048
 shared sx user map (user map) r = 0 (0xffffff0008a44078) locked @  
 /usr/src/sys/vm/vm_map.c:2023
 Process 75148 (php) thread 0xffffff010568c8c0 (100744)
 shared lockmgr ufs (ufs) r = 0 (0xffffff00b47c0d80) locked @  
 /usr/src/sys/kern/vfs_vnops.c:549
 Process 74926 (exim-4.77-0) thread 0xffffff00b71ea000 (100323)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff01fc50c8f0) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 74366 (exim-4.77-0) thread 0xffffff00b713c000 (100677)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff0060a908f0) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 74205 (php) thread 0xffffff01c2c2f8c0 (100694)
 shared lockmgr ufs (ufs) r = 0 (0xffffff00b47c0d80) locked @  
 /usr/src/sys/kern/vfs_vnops.c:549
 Process 74118 (exim-4.77-0) thread 0xffffff00b7d748c0 (100685)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff01fc990e40) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 74105 (php) thread 0xffffff01c262d460 (100442)
 exclusive lockmgr bufwait (bufwait) r = 0 (0xffffff81ee2c5e90) locked @  
 /usr/src/sys/kern/vfs_bio.c:1891
 shared lockmgr ufs (ufs) r = 0 (0xffffff00b47c0d80) locked @  
 /usr/src/sys/kern/vfs_vnops.c:549
 Process 73987 (exim-4.77-0) thread 0xffffff00b715a460 (100223)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff00600c68f0) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 73728 (php) thread 0xffffff0008f14000 (100195)
 shared lockmgr ufs (ufs) r = 0 (0xffffff00b47c0d80) locked @  
 /usr/src/sys/kern/vfs_vnops.c:549
 Process 73631 (exim-4.77-0) thread 0xffffff01c252a8c0 (100353)
 exclusive lockmgr bufwait (bufwait) r = 0 (0xffffff81ee2d62b8) locked @  
 /usr/src/sys/kern/vfs_bio.c:1891
 exclusive sx dirhash (dirhash) r = 0 (0xffffff0092e50000) locked @  
 /usr/src/sys/ufs/ufs/ufs_dirhash.c:216
 exclusive lockmgr ufs (ufs) r = 0 (0xffffff00bb6a8098) locked @  
 /usr/src/sys/kern/vfs_subr.c:2170
 Process 73552 (exim-4.77-0) thread 0xffffff0008bda000 (100206)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff005c0b80f8) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 73547 (exim-4.77-0) thread 0xffffff01c252e000 (100349)
 exclusive sx so_rcv_sx (so_rcv_sx) r = 0 (0xffffff01c27d6648) locked @  
 /usr/src/sys/kern/uipc_sockbuf.c:148
 Process 2234 (zabbix_agentd) thread 0xffffff0008c018c0 (100272)
 exclusive sleep mutex sigio lock (sigio lock) r = 0 (0xffffffff8096bb60)  
 locked @ /usr/src/sys/kern/kern_descrip.c:1095
 Process 20 (syncer) thread 0xffffff0008657000 (100115)
 exclusive lockmgr syncer (syncer) r = 0 (0xffffff0008b29448) locked @  
 /usr/src/sys/kern/vfs_subr.c:1770
 Process 12 (intr) thread 0xffffff0002b068c0 (100077)
 exclusive sleep mutex em0:tx(0) (em0:tx(0)) r = 0 (0xffffff0002561808)  
 locked @ /usr/src/sys/modules/em/../../dev/e1000/if_em.c:1523
 Process 12 (intr) thread 0xffffff0002b07460 (100075)
 exclusive rw tcpinp (tcpinp) r = 0 (0xffffff01c2425670) locked @  
 /usr/src/sys/netinet/tcp_input.c:946
 exclusive rw tcp (tcp) r = 0 (0xffffffff80b35108) locked @  
 /usr/src/sys/netinet/tcp_usrreq.c:1507
 db:1:alllocks>  show lockedvnods
 Locked vnodes
 db:0:kdb.enter.panic>  show pcpu
 cpuid        = 9
 dynamic pcpu = 0xffffff807f70d780
 curthread    = 0xffffff0002b07460: pid 12 "irq256: em0:rx 0"
 curpcb       = 0xffffff8241018d10
 fpcurthread  = none
 idlethread   = 0xffffff0002369460: tid 100009 "idle: cpu9"
 curpmap      = 0xffffffff8096b670
 tssp         = 0xffffffff80b7aaa8
 commontssp   = 0xffffffff80b7aaa8
 rsp0         = 0xffffff8241018d10
 gs32p        = 0xffffffff80b798e0
 ldt          = 0xffffffff80b79920
 tss          = 0xffffffff80b79910
 spin locks held:
 db:0:kdb.enter.panic>  bt
 Tracing pid 12 tid 100075 td 0xffffff0002b07460
 kdb_enter() at kdb_enter+0x3b
 panic() at panic+0x17b
 soabort() at soabort+0x9c
 syncache_expand() at syncache_expand+0x2ca
 tcp_input() at tcp_input+0xf05
 ip_input() at ip_input+0xb3
 netisr_dispatch_src() at netisr_dispatch_src+0x9e
 ether_demux() at ether_demux+0x176
 ether_input() at ether_input+0x198
 em_rxeof() at em_rxeof+0x19d
 em_msix_rx() at em_msix_rx+0x24
 intr_event_execute_handlers() at intr_event_execute_handlers+0x67
 ithread_loop() at ithread_loop+0xad
 fork_exit() at fork_exit+0x12a
 fork_trampoline() at fork_trampoline+0xe
 --- trap 0, rip = 0, rsp = 0xffffff8241018d00, rbp = 0 ---
 
 
      6     0     0     0  SL      crypto_r 0xffffffff80e03ce0 [crypto  
 returns]
      5     0     0     0  SL      crypto_w 0xffffffff80e03ca0 [crypto]
     13     0     0     0  SL      -        0xffffffff8096e5c4 [yarrow]
      4     0     0     0  SL      -        0xffffffff8096ab68 [g_down]
      3     0     0     0  SL      -        0xffffffff8096ab60 [g_up]
      2     0     0     0  SL      -        0xffffffff8096ab50 [g_event]
     12     0     0     0  RL      (threaded)                  intr
>Unformatted:
 >> kdb_enter (80a0e225,80a0e225,80a186dd,d6780840,d6780840,...) @ kdb_enter=806e1930 + 0x3a == 806e196a
 /usr/src/sys/kern/subr_kdb.c:351
 
 >> panic (80a186dd,346,0,d67808b8,0,...) @ panic=806b23d0 + 0xd1 == 806b24a1
 /usr/src/sys/kern/kern_shutdown.c:597
 
 >> soabort (88961670,80a2dc03,346,0,0,...) @ soabort=8071ba90 + 0x1b == 8071baab
 /usr/src/sys/kern/uipc_socket.c:758
 
 >> syncache_expand (d6780a60,d6780a40,8c78cb5c,d6780a5c,8c78cb00,...) @ syncache_expand=80848750 + 0xdb3 == 80849503
 /usr/src/sys/netinet/tcp_syncache.c:841
 
 >> tcp_input (8c78cb00,14,86c17800,1,0,...) @ tcp_input=8083f5a0 + 0xc49 == 808401e9
 /usr/src/sys/netinet/tcp_input.c:1036
 
 >> ip_input (8c78cb00,4,80a0c98c,109,24,...) @ ip_input=807daa70 + 0x65b == 807db0cb
 /usr/src/sys/netinet/ip_input.c:787
 
 >> netisr_dispatch_src (1,0,8c78cb00,d6780b34,80766ae1,...) @ netisr_dispatch_src=80771420 + 0xd1 == 807714f1
 /usr/src/sys/net/netisr.c:859
 
 >> netisr_dispatch (1,8c78cb00,0,86c17800,8c78cb3a,...) @ netisr_dispatch=80771750 + 0x20 == 80771770
 /usr/src/sys/net/netisr.c:947
 
 >> ether_demux (86c17800,8c78cb00,3,0,3,...) @ ether_demux=80766940 + 0x1a1 == 80766ae1
 /usr/src/sys/net/if_ethersubr.c:894
 
 >> ether_input (86c16400,8c78cb00,6,2,2,...) @ ether_input=80766d40 + 0x393 == 807670d3
 /usr/src/sys/net/if_ethersubr.c:755
 
 
 
 
