[PCA 2.x] Compute Node Kernel Panic on "xenvif_start_xmit" or "netif_receive_skb_internal" (Doc ID 2502163.1)

Last updated on AUGUST 24, 2022

Applies to:

Symptoms

Compute node got kernel panic randomly, you may see call trace like followings:

crash7lates> bt
PID: 0 TASK: ffff880492842a00 CPU: 19 COMMAND: "swapper/19"
#0 [ffff8804a6ac3560] panic at ffffffff816ec727
#1 [ffff8804a6ac35e0] oops_end at ffffffff8101a89a
#2 [ffff8804a6ac3610] no_context at ffffffff8106ea91
#3 [ffff8804a6ac3660] __bad_area_nosemaphore at ffffffff8106ec8d
#4 [ffff8804a6ac36b0] bad_area_nosemaphore at ffffffff8106eda3
#5 [ffff8804a6ac36c0] __do_page_fault at ffffffff8106f328
#6 [ffff8804a6ac3730] do_page_fault at ffffffff8106f637
#7 [ffff8804a6ac3770] page_fault at ffffffff816f7d5f
[exception RIP: xenvif_start_xmit+268]
RIP: ffffffffa0764c3c RSP: ffff8804a6ac3828 RFLAGS: 00010297
RAX: 000000000000103a RBX: ffff88044d9b5840 RCX: dead000000000200
RDX: ffffc9000c762f80 RSI: 00000000000002f8 RDI: 00000000000002f8
RBP: ffff8804a6ac3848 R8: ffff88044046be30 R9: ffff880492a2a800
R10: ffff880495c00780 R11: 0000000000000000 R12: ffff88044ddb3f00
R13: ffffc9000c757de8 R14: 0000000000000001 R15: 00000000000000e6
ORIG_RAX: ffffffffffffffff CS: e030 SS: e02b
#8 [ffff8804a6ac3850] xmit_one at ffffffff81615622
#9 [ffff8804a6ac38a0] dev_hard_start_xmit at ffffffff81615790
#10 [ffff8804a6ac38e0] sch_direct_xmit at ffffffff8163ac72
#11 [ffff8804a6ac3940] __dev_queue_xmit at ffffffff8161620f
#12 [ffff8804a6ac39a0] dev_queue_xmit_sk at ffffffff81616633
#13 [ffff8804a6ac39b0] br_dev_queue_push_xmit at ffffffffa05d6f6a [bridge]
#14 [ffff8804a6ac39d0] br_forward_finish at ffffffffa05d74ef [bridge]
#15 [ffff8804a6ac3a20] __br_forward at ffffffffa05d703d [bridge]
#16 [ffff8804a6ac3a80] br_forward at ffffffffa05d7166 [bridge]
#17 [ffff8804a6ac3ab0] br_handle_frame_finish at ffffffffa05d884e [bridge]
#18 [ffff8804a6ac3b10] br_handle_frame at ffffffffa05d8c3f [bridge]
#19 [ffff8804a6ac3b70] __netif_receive_skb_core at ffffffff81617a57
#20 [ffff8804a6ac3be0] __netif_receive_skb at ffffffff816180f2
#21 [ffff8804a6ac3c00] netif_receive_skb_internal at ffffffff81618313
#22 [ffff8804a6ac3c40] netif_receive_skb_sk at ffffffff81619024
#23 [ffff8804a6ac3c60] xsvnic_send_skb at ffffffffa0549ce6 [xsvnic]
#24 [ffff8804a6ac3d30] xsvnic_process_rx_skb at ffffffffa054a0e2 [xsvnic]
#25 [ffff8804a6ac3d60] xsvnic_poll at ffffffffa054c48c [xsvnic]
#26 [ffff8804a6ac3e40] napi_poll at ffffffff81618a1b
#27 [ffff8804a6ac3e90] net_rx_action at ffffffff81618bc0
#28 [ffff8804a6ac3ef0] __do_softirq at ffffffff8108cbba
#29 [ffff8804a6ac3f70] irq_exit at ffffffff8108cf75
#30 [ffff8804a6ac3f90] xen_evtchn_do_upcall at ffffffff813f9de9
#31 [ffff8804a6ac3fb0] xen_do_hypervisor_callback at ffffffff816f6e7e
--- <IRQ stack> ---
#32 [ffff88049285bd90] xen_do_hypervisor_callback at ffffffff816f6e7e
[exception RIP: xen_hypercall_sched_op+10]
RIP: ffffffff810013aa RSP: 0000000000000001 RFLAGS: 00000000
RAX: 0000000000000000 RBX: ffff880492858008 RCX: 0000000000000246
RDX: 0000000000000001 RSI: 00000001b65b8981 RDI: 0000000000000000
RBP: ffffffff816f6e45 R8: ffff88049285be80 R9: 0000000000000013
R10: ffff88049285800a R11: ffff88049285800a R12: ffffffff816f6db1
R13: ffffffff816f6db8 R14: ffffffff816f6dbf R15: ffffffff816f6dc6
ORIG_RAX: 0000000000000000 CS: ffff880492858000 SS: ffffffffffffffff
bt: WARNING: possibly bogus exception frame
#33 [ffff88049285be78] xen_safe_halt at ffffffff8100bc50
#34 [ffff88049285be88] default_idle at ffffffff816f101e
#35 [ffff88049285bea8] arch_cpu_idle at ffffffff810225cf
#36 [ffff88049285beb8] cpuidle_idle_call at ffffffff810cca25
#37 [ffff88049285bee8] cpu_idle_loop at ffffffff810ccb6c
#38 [ffff88049285bf28] cpu_startup_entry at ffffffff810ccc2f
#39 [ffff88049285bf48] cpu_bringup_and_idle at ffffffff81013ba8
crash7lates>

crash7lates> bt
PID: 0 TASK: ffff880492842a00 CPU: 19 COMMAND: "swapper/19"
#0 [ffff8804a6ac3a50] panic at ffffffff816ec727
#1 [ffff8804a6ac3ad0] oops_end at ffffffff8101a89a
#2 [ffff8804a6ac3b00] no_context at ffffffff8106ea91
#3 [ffff8804a6ac3b50] __bad_area_nosemaphore at ffffffff8106ec8d
#4 [ffff8804a6ac3ba0] bad_area_nosemaphore at ffffffff8106eda3
#5 [ffff8804a6ac3bb0] __do_page_fault at ffffffff8106f328
#6 [ffff8804a6ac3c20] do_page_fault at ffffffff8106f637
#7 [ffff8804a6ac3c60] page_fault at ffffffff816f7d5f
[exception RIP: netif_receive_skb_internal+35]
RIP: ffffffff81618313 RSP: ffff8804a6ac3d18 RFLAGS: 00010296
RAX: 0000000000000000 RBX: ffff880429066400 RCX: 00000000000000ee
RDX: 0000000000000000 RSI: 0000000000000200 RDI: ffffffff81616264
RBP: ffff8804a6ac3d48 R8: 0000000000000040 R9: 00000000ffffff7f
R10: 0000000000000001 R11: 0000000000000000 R12: ffff880429066400
R13: ffffc9000d654130 R14: 00000000000000e5 R15: 0000000000000001
ORIG_RAX: ffffffffffffffff CS: e030 SS: e02b
#8 [ffff8804a6ac3d20] skb_checksum_setup_ip at ffffffff81604c10
#9 [ffff8804a6ac3d50] netif_receive_skb_sk at ffffffff81619024
#10 [ffff8804a6ac3d70] xenvif_tx_submit at ffffffffa076569a [xen_netback]
#11 [ffff8804a6ac3de0] xenvif_tx_action at ffffffffa0766f3c [xen_netback]
#12 [ffff8804a6ac3e10] xenvif_poll at ffffffffa076973c [xen_netback]
#13 [ffff8804a6ac3e40] napi_poll at ffffffff81618a1b
#14 [ffff8804a6ac3e90] net_rx_action at ffffffff81618bc0
#15 [ffff8804a6ac3ef0] __do_softirq at ffffffff8108cbba
#16 [ffff8804a6ac3f70] irq_exit at ffffffff8108cf75
#17 [ffff8804a6ac3f90] xen_evtchn_do_upcall at ffffffff813f9de9
#18 [ffff8804a6ac3fb0] xen_do_hypervisor_callback at ffffffff816f6e7e
--- <IRQ stack> ---
#19 [ffff88049285bd90] xen_do_hypervisor_callback at ffffffff816f6e7e
[exception RIP: xen_hypercall_sched_op+10]
RIP: ffffffff810013aa RSP: 0000000000000001 RFLAGS: 00000000
RAX: 0000000000000000 RBX: ffff880492858008 RCX: 0000000000000246
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffffff816f6e45 R8: ffff88049285be80 R9: 0000000000000013
R10: ffff88049285800a R11: ffff88049285800a R12: ffffffff816f6db1
R13: ffffffff816f6db8 R14: ffffffff816f6dbf R15: ffffffff816f6dc6
ORIG_RAX: 0000000000000000 CS: ffff880492858000 SS: ffffffffffffffff
bt: WARNING: possibly bogus exception frame
#20 [ffff88049285be78] xen_safe_halt at ffffffff8100bc50
#21 [ffff88049285be88] default_idle at ffffffff816f101e
#22 [ffff88049285bea8] arch_cpu_idle at ffffffff810225cf
#23 [ffff88049285beb8] cpuidle_idle_call at ffffffff810cca25
#24 [ffff88049285bee8] cpu_idle_loop at ffffffff810ccb6c
#25 [ffff88049285bf28] cpu_startup_entry at ffffffff810ccc2f
#26 [ffff88049285bf48] cpu_bringup_and_idle at ffffffff81013ba8

Cause

My Oracle Support provides customers with access to over a million knowledge articles and a vibrant support community of peers and Oracle experts.