Kernel Panic at RIP: dequeue_hwpoisoned_huge_page (Doc ID 2697301.1)

Last updated on JANUARY 02, 2024

Applies to:

Symptoms

A server crashed unexpectedly. The following call trace was found in a core dump (vmcore from the system):
 

KERNEL: /share/linuxrpm/vmlinux_repo/64/4.1.12-124.30.1.el6uek.x86_64/vmlinux
 DUMPFILE: vmcore [PARTIAL DUMP]
 CPUS: 88
 DATE: Sat Jul 18 15:41:04 2020
 UPTIME: 118 days, 11:51:11
LOAD AVERAGE: 31.39, 18.80, 15.26
 TASKS: 2893
 NODENAME: HOSTNAME
 RELEASE: 4.1.12-124.30.1.el6uek.x86_64
 VERSION: #2 SMP Wed Aug 7 18:44:21 PDT 2019
 MACHINE: x86_64 (2195 Mhz)
 MEMORY: 255.9 GB
 PANIC: "BUG: unable to handle kernel NULL pointer dereference at 0000000000000028"
 PID: 19014
 COMMAND: "mcelog"
 TASK: ffff883feabf4600 [THREAD_INFO: ffff882dba01c000]
 CPU: 29
 STATE: TASK_RUNNING (PANIC)

bt
PID: 19014 TASK: ffff883feabf4600 CPU: 29 COMMAND: "mcelog"
 #0 [ffff882dba01f990] machine_kexec at ffffffff810604fc
 #1 [ffff882dba01f9f0] crash_kexec at ffffffff81115673
 #2 [ffff882dba01fac0] oops_end at ffffffff8101a7a8
 #3 [ffff882dba01faf0] no_context at ffffffff8106eccf
 #4 [ffff882dba01fb60] __bad_area_nosemaphore at ffffffff8106ef9d
 #5 [ffff882dba01fbb0] bad_area_nosemaphore at ffffffff8106f143
 #6 [ffff882dba01fbc0] __do_page_fault at ffffffff8106f80b
 #7 [ffff882dba01fc20] do_page_fault at ffffffff8106fc2f
 #8 [ffff882dba01fc60] page_fault at ffffffff816fb1a7
 [exception RIP: dequeue_hwpoisoned_huge_page+233]
 RIP: ffffffff811e04f9 RSP: ffff882dba01fd18 RFLAGS: 00010246
 RAX: ffffea0004f90020 RBX: 0000000000000000 RCX: dead000000000200
 RDX: ffff88207ffda388 RSI: ffffffff820dd7b0 RDI: ffffea003f228020
 RBP: ffff882dba01fd38 R8: ffffea003f228020 R9: 0000000000001000
 R10: 0000000000000010 R11: 000000000000001d R12: ffffea003f228000
 R13: 0000000000000000 R14: ffffea003f228020 R15: 0000000000000001
 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
 #9 [ffff882dba01fd40] soft_offline_page at ffffffff81205019
#10 [ffff882dba01fd90] store_soft_offline_page at ffffffff81493fad
#11 [ffff882dba01fdc0] dev_attr_store at ffffffff81478ccb
#12 [ffff882dba01fdd0] sysfs_kf_write at ffffffff81291183
#13 [ffff882dba01fde0] kernfs_fop_write at ffffffff8129035a
#14 [ffff882dba01fe30] __vfs_write at ffffffff8120d91b
#15 [ffff882dba01feb0] vfs_write at ffffffff8120e079
#16 [ffff882dba01ff00] sys_write at ffffffff8120ee86
#17 [ffff882dba01ff50] system_call_fastpath at ffffffff816f5436
 RIP: 00007f907414a850 RSP: 00007ffc6089cc08 RFLAGS: 00000246
 RAX: ffffffffffffffda RBX: 0000000000000007 RCX: 00007f907414a850
 RDX: 000000000000000b RSI: 000000000176cb40 RDI: 0000000000000007
 RBP: 0000000000413b1d R8: 00000000ffffffff R9: 0000000000000000
 R10: 000000000000000c R11: 0000000000000246 R12: 00000000ffffffff
 R13: 0000000000000001 R14: 00000000ffffffff R15: 000000005f13180f
 ORIG_RAX: 0000000000000001 CS: 0033 SS: 002b


logs

[10235744.356975] mce: [Hardware Error]: Machine check events logged
[10235744.416429] BUG: unable to handle kernel NULL pointer dereference at 0000000000000028
[10235744.425516] IP: [<ffffffff811e04f9>] dequeue_hwpoisoned_huge_page+0xe9/0x140
[10235744.433713] PGD 2d7886d067 PUD 2d740aa067 PMD 0
[10235744.439204] Oops: 0002 [#1] SMP
[10235744.443134] Modules linked in: tcp_diag inet_diag iptable_filter ip_tables seos(POE) nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 nfs fscache lockd sunrpc grace ipmi_poweroff ipmi_devintf bonding ksplice_ml4n9es4_rds_rdma_new(O) ksplice_ml4n9es4(O) ksplice_bi8a67a1(O) ksplice_e7p7fg0o(O) ksplice_aaopxeie(O) ksplice_6didogye_rds_rdma_new(O) ksplice_6didogye(O) ksplice_hl2oj1k0(O) ksplice_l9dufylv_vmlinux_new(O) ksplice_l9dufylv(O) ksplice_s7aryxkq(O) ksplice_bmje1ftz(O) ksplice_hpv12rnp(O) ksplice_nbpjr43i(O) ksplice_nqvymyta(O) ksplice_5joc5gam(O) ksplice_5v8vc7u7_rds_rdma_new(O) ksplice_5v8vc7u7(O) ksplice_cbhnf9wh_i2c_core_new(O) ksplice_cbhnf9wh(O) ksplice_eot0cppt_vmlinux_new(O) ksplice_eot0cppt(O) ksplice_72k73x2z(O) ksplice_luuif47m(O) ksplice_podyb7ry_vmlinux_new(O) ksplice_podyb7ry(O)
[10235744.522582] ksplice_733segf9_vmlinux_new(O) ksplice_733segf9(O) ksplice_hvilt320_vmlinux_new(O) ksplice_hvilt320(O) ksplice_sewx21fe_sg_new(O) ksplice_sewx21fe(O) ksplice_dapfserc_rds_rdma_new(O) ksplice_dapfserc(O) ksplice_oym6plp3_ext4_new(O) ksplice_oym6plp3(O) ksplice_p15beq9p_vmlinux_new(O) ksplice_p15beq9p(O) ksplice_fsllhqxv(O) ksplice_m9jjla3h_vmlinux_new(O) ksplice_m9jjla3h(O) ksplice_hoctvhe7_vmlinux_new(O) ksplice_hoctvhe7(O) ksplice_dvmmfcdz_vmlinux_new(O) ksplice_dvmmfcdz(O) ksplice_m29bqqxo_vmlinux_new(O) ksplice_m29bqqxo(O) ksplice_150x02dl(O) ksplice_s00w8b2m_vmlinux_new(O) ksplice_s00w8b2m(O) ksplice_85fi46ii(O) ksplice_c6xiosgt_rds_new(O) ksplice_c6xiosgt(O) ksplice_c3m9jqd9_vmlinux_new(O) ksplice_c3m9jqd9(O) ksplice_etwjlupv(O) ksplice_l1xdge71_vmlinux_new(O) ksplice_l1xdge71(O)
[10235744.601194] ksplice_op40465s(O) ksplice_ccg7rycz(O) ksplice_515cfpia_vmlinux_new(O) ksplice_515cfpia(O) ksplice_m8a2u1pc_vmlinux_new(O) ksplice_m8a2u1pc(O) ksplice_6h0hcs5l_vmlinux_new(O) ksplice_6h0hcs5l(O) ksplice_ksa6xnv2_vmlinux_new(O) ksplice_ksa6xnv2(O) ksplice_mz736abu_vmlinux_new(O) ksplice_mz736abu(O) ksplice_gfw7o55c_vmlinux_new(O) ksplice_gfw7o55c(O) rds_rdma rds ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad dm_multipath bnx2i cnic uio cxgb4i libcxgbi cxgb4 ib_iser rdma_cm ib_cm iw_cm iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi fuse iTCO_wdt iTCO_vendor_support ipmi_ssif ipmi_si ipmi_msghandler sb_edac edac_core i2c_i801 i2c_core lpc_ich mfd_core sg ixgbe dca ptp pps_core vxlan udp_tunnel ip6_udp_tunnel mdio wmi mlx4_ib ib_sa ib_mad ib_core ib_addr ipv6 acpi_pad mlx4_core ext4 jbd2
[10235744.681407] mbcache2 sd_mod ahci libahci megaraid_sas dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ksplice_ml4n9es4_rds_rdma_old]
[10235744.694531] CPU: 29 PID: 19014 Comm: mcelog Tainted: P OE 4.1.12-124.30.1.el6uek.x86_64 #2
[10235744.705706] Hardware name: Oracle Corporation ORACLE SERVER X6-2/ASM,MOTHERBOARD,1U, BIOS 38310100 08/30/2019
[10235744.717467] task: ffff883feabf4600 ti: ffff882dba01c000 task.ti: ffff882dba01c000
[10235744.726501] RIP: 0010:[<ffffffff811e04f9>] [<ffffffff811e04f9>] dequeue_hwpoisoned_huge_page+0xe9/0x140
[10235744.737785] RSP: 0018:ffff882dba01fd18 EFLAGS: 00010246
[10235744.744205] RAX: ffffea0004f90020 RBX: 0000000000000000 RCX: dead000000000200
[10235744.752849] RDX: ffff88207ffda388 RSI: ffffffff820dd7b0 RDI: ffffea003f228020
[10235744.761496] RBP: ffff882dba01fd38 R08: ffffea003f228020 R09: 0000000000001000
[10235744.770139] R10: 0000000000000010 R11: 000000000000001d R12: ffffea003f228000
[10235744.778781] R13: 0000000000000000 R14: ffffea003f228020 R15: 0000000000000001
[10235744.787426] FS: 00007f9074617700(0000) GS:ffff883ffebc0000(0000) knlGS:0000000000000000
[10235744.797138] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[10235744.804046] CR2: 0000000000000028 CR3: 0000002db1c04000 CR4: 00000000003606f0
[10235744.812689] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[10235744.821336] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[10235744.829974] Stack:
[10235744.832703] ffffea003f22d080 0000000000000001 0000000000fc8b42 ffffea003f228000
[10235744.841686] ffff882dba01fd88 ffffffff81205019 ffff882dba01fd48 ffff882dba01fd48
[10235744.850670] ffff882dba01fd78 000000000000000b ffff883fefda8a40 ffff8827fb95d618
[10235744.859646] Call Trace:
[10235744.862866] [<ffffffff81205019>] soft_offline_page+0x159/0x560
[10235744.869973] [<ffffffff81493fad>] store_soft_offline_page+0x9d/0xe0
[10235744.877466] [<ffffffff81478ccb>] dev_attr_store+0x1b/0x30
[10235744.884085] [<ffffffff81291183>] sysfs_kf_write+0x43/0x50
[10235744.890709] [<ffffffff8129035a>] kernfs_fop_write+0x12a/0x190
[10235744.897726] [<ffffffff8120d91b>] __vfs_write+0x2b/0x110
[10235744.904155] [<ffffffff81210669>] ? __sb_start_write+0x49/0x100
[10235744.911262] [<ffffffff812b6979>] ? security_file_permission+0x29/0xb0
[10235744.919046] [<ffffffff8120e079>] vfs_write+0xa9/0x1b0
[10235744.925275] [<ffffffff8120ee86>] SyS_write+0x46/0xb0
[10235744.931408] [<ffffffff81027216>] ? syscall_trace_leave+0xd6/0x180
[10235744.938811] [<ffffffff816f5436>] system_call_fastpath+0x18/0xee
[10235744.946013] Code: 69 4c 89 e0 8b 40 1c 85 c0 75 47 4d 8d 74 24 20 4c 89 f7 e8 6a 77 16 00 4d 89 74 24 20 4d 89 74 24 28 41 c7 44 24 1c 01 00 00 00 <48> 83 6b 28 01 42 83 ac ab 58 50 00 00 01 31 db 48 c7 c7 08 ef
[10235744.968743] RIP [<ffffffff811e04f9>] dequeue_hwpoisoned_huge_page+0xe9/0x140
[10235744.977404] RSP <ffff882dba01fd18>
[10235744.981773] CR2: 0000000000000028

Changes

Cause

My Oracle Support provides customers with access to over a million knowledge articles and a vibrant support community of peers and Oracle experts.