luxadm command hangs with ddi_dma_buf_bind_handle() failed due to depleted DVMA space (Doc ID 1923572.1)

Last updated on SEPTEMBER 07, 2014

Applies to:

Solaris SPARC Operating System - Version 10 8/11 U10 and later
Information in this document applies to any platform.

Symptoms

luxadm command hang due to depletion of DVMA space in Solaris 10U10. From the dump we can usually see the following stack trace caused by luxadm command.

CAT(vmcore.2/10U)> proc 21086
    addr       PID    PPID   RUID/UID     size      RSS     swresv  lwpcnt command
============= ====== ====== ========== ========== ======== ======== ====== =====
0x6019c767968  21086      1          0    3850240  2269184  1024000      1 luxadm -e dump_map /devices/pci@1,700000/pci@0/pci@9/pci@0/pci@1/fibre-channel@
        thread: 0x3003e17d580  state: slp   wchan: 0x3000db9801e  sobj: condition var (from genunix:vmem_xalloc+0x714)

CAT(vmcore.2/10U)> thread 0x3003e17d580
==== user (LWP_SYS) thread: 0x3003e17d580  PID: 21086 ====
cmd: luxadm -e dump_map /devices/pci@1,700000/pci@0/pci@9/pci@0/pci@1/fibre-channel@
t_wchan: 0x3000db9801e  sobj: condition var (from genunix:vmem_xalloc+0x714)
t_procp: 0x6019c767968
   p_as: 0x601c23e81d8  size: 3850240  RSS: 2269184
      a_hat: 0x30040dd8900
      cnum: CPU0:40259/41
      cpusran: 0,1,2,3,4,5,6,7,48,49,50,51,52,53,54,55,88,89,90,91,92,93,94,95
   p_zone: 0x19a0ab8 (global)
t_stk: 0x2a1040e5ae0  sp: 0x2a1040e4291  t_stkbase: 0x2a1040e0000
t_pri: 59 (TS)  pctcpu: 0.000479
t_transience: 10 (TRANSIENT)  t_wkld_flags: 0
t_lwp: 0x60196560be0  t_tid: 1
   machpcb: 0x2a1040e5ae0
   lwp_ap:   0x2a1040e5bd0
   t_mstate: LMS_SLEEP  ms_prev: LMS_SYSTEM
   ms_state_start: 274 days 4 hours 22 minutes 32.839113307 seconds earlier
   ms_start: 274 days 15 hours 4 minutes 30.168044877 seconds earlier
t_cpupart: 0x191f870(0)  last CPU: 88
idle: 604 ticks (6.04s)
start: Wed Jul  9 19:30:02 2014
age: 3851730 seconds (44 days 13 hours 55 minutes 30 seconds)
t_state:     TS_SLEEP
t_flag:      0x1000 (T_LWPREUSE)
t_proc_flag: 0x104 (TP_TWAIT|TP_MSACCT)
t_schedflag: 3 (TS_LOAD|TS_DONT_SWAP)
p_flag:      0x4a004900 (SKILLED|SEXTKILLED|SEXECED|SMSACCT|SAUTOLPG|SMSFORK)

pc:      genunix:cv_wait+0x38:   call   unix:swtch

void genunix:cv_wait+0x38((kcondvar_t *)0x3000db9801e, (kmutex_t *)0x3000db98020)
void *genunix:vmem_xalloc+0x714((vmem_t *)0x3000db98000, (size_t)0x100000, (size_t)0x100000, (size_t)0, (size_t)0, (void *)0, (void *)0, (int))
void *genunix:vmem_alloc+0x1d4((vmem_t *)0x3000db98000, (size_t)0x100000, (int))
kmem_slab_t *genunix:kmem_slab_create+0x44((kmem_cache_t *)0x3000fa14008, (int)0)
void *genunix:kmem_slab_alloc+0x30((kmem_cache_t *)0x3000fa14008, (int)0)
void *genunix:kmem_cache_alloc+0x2dc((kmem_cache_t *)0x3000fa14008, (int)0)
int px:px_dvma_map+0xbc((ddi_dma_impl_t *)0x6014ce57b00, (ddi_dma_req_t *)0x2a1040e51d8, (px_mmu_t *)0x300028b21e8)
int px:px_dma_bindhdl+0xcc((dev_info_t *), (dev_info_t *), (ddi_dma_handle_t), (ddi_dma_req_t *)0x2a1040e51d8, (ddi_dma_cookie_t *)0x2a1040e5300, (uint_t *)0x601a2b63b98)
int genunix:ddi_dma_addr_bind_handle+0x70((ddi_dma_handle_t)0x6014ce57b00, (struct as *)0, (caddr_t)0x300220b3c80, (size_t), (uint_t)0x11, (int (*)())1, (caddr_t)0, (ddi_dma_cookie_t *)0x2a1040e5300, (uint_t *))
fp_cmd_t *fp:fp_alloc_pkt+0x2a0((fc_port_t *)0x600fc0f3000, (int)0x14, (int)0x260, (int)0, (fc_port_device_t *)0)
int fp:fp_ns_query+0xb8((fc_port_t *)0x600fc0f3000, (fctl_ns_req_t *)0x601bbfc9f68, (job_request_t *)0x601acaa36d0, (int)1, (int)0)
int fp:fp_ns_get_devcount+0x158((fc_port_t *)0x600fc0f3000, (job_request_t *)0x601acaa36d0, (int)0, (int)0)
int fp:fp_fciocmd+0x1380((fc_port_t *)0x600fc0f3000, (intptr_t)0xffbfeec8, (int)0x100005, (fcio_t *)0x2a1040e5868)
int fp:fp_ioctl+0x1b8((dev_t), (int), (intptr_t)0xffbfeec8, (int), (cred_t *)0x601853bd2f8, (int *)0x2a1040e5adc)
specfs:spec_ioctl((struct vnode *)0x6019b3f9d80, (int)0x47ce, (intptr_t)0xffbfeec8, (int)0x100005, (struct cred *), (int *)0x2a1040e5adc) - frame recycled
int genunix:fop_ioctl+0x2c((vnode_t *)0x6019b3f9d80, (int)0x47ce, (intptr_t)0xffbfeec8, (int)0x100005, (cred_t *), (int *)0x2a1040e5adc)
int genunix:ioctl+0x184((int), (int), (intptr_t))
unix:syscall_trap32+0xcc()
-- switch to user thread's user stack --

From mdb command ::kmastat, the following vmem and slab cache memory allocation failures in the px arena can be observed.

vmem                         memory     memory    memory     alloc alloc
name                         in use      total    import   succeed  fail
------------------------- --------- ---------- --------- --------- -----


px0_dvma_vmem_1m          176160768 1072693248         0 110325337     0
   px0_dvma_vmem_16       15859712   16777216  16777216 4985490465     0
px0_dvma_vmem_8k                  0  134217728         0         0     0
ip_minor_arena_sa               384     262140         0        20     0
ip_minor_arena_la              1344 4294705152         0        28     0
px1_dvma_vmem_1m          1072693248 1072693248         0  26695053 4461032567
   px1_dvma_vmem_16       31981568   32505856  32505856 1167060905 3765091160
px1_dvma_vmem_8k           37322752  134217728         0 4460518783     0


cache                        buf    buf    buf    memory     alloc alloc
name                        size in use  total    in use   succeed  fail
------------------------- ------ ------ ------ --------- --------- -----
px1_dvma_cache1             8192  51405  51456 421527552 2053726082 60660098
px1_dvma_cache2            16384     40    192   3145728 475257554 92101
px1_dvma_cache8            65536   7070   7072 463470592 256663511 345841116

This is actually described by bug 17532406 - ddi_dma_buf_bind_handle() failed due to depleted DVMA space

 

 

Cause

Sign In with your My Oracle Support account

Don't have a My Oracle Support account? Click to get started

My Oracle Support provides customers with access to over a
Million Knowledge Articles and hundreds of Community platforms