Extract is hung/slow with lots of small lob related files generated in dirtmp directory (Doc ID 1958670.1)

Last updated on JULY 17, 2017

Applies to:

Oracle GoldenGate - Version 12.1.2.0.3 and later
Information in this document applies to any platform.

Symptoms

Extract hangs in 12.1.2.0.0, and 12.1.2.0.3, with lots of small files generated in dirtmp directory.

1. ggsci showch and status outputs

GGSCI (apollo26) 1> info EHVD01, showch
EXTRACT EHVD01 Last Started 2014-12-28 21:46 Status RUNNING
Checkpoint Lag 10:36:03 (updated 14:24:55 ago)
Process ID 16411
Log Read Checkpoint Oracle Redo Logs
2014-12-28 11:13:54 Seqno 125198, RBA 107681204
SCN 404.1871009412 (1737037796996)

Current Checkpoint Detail:

Read Checkpoint #1
Oracle Redo Log
....
Recovery Checkpoint (position of oldest unprocessed transaction in the data
source):
Thread #: 1
Sequence #: 125195
RBA: 66682384
Timestamp: 2014-12-28 11:12:35.000000
SCN: 404.1871008100 (1737037795684)
Redo File: /u01/app/oracle/admin/finprod/arch/arch_125195_1_673238363.arc

Current Checkpoint (position of last record read in the data source):
Thread #: 1
Sequence #: 125198
RBA: 107681204
Timestamp: 2014-12-28 11:13:54.000000
SCN: 404.1871009412 (1737037796996)
Redo File: /u01/app/oracle/admin/finprod/arch/arch_125198_1_673238363.arc
.....

GGSCI (apollo26) 20> STATS EHVD01
Sending STATS request to EXTRACT EHVD01 ...
ERROR: sending message to EXTRACT EHVD01 (Timeout waiting for message).

GGSCI (apollo26) 2> send EHVD01, cachemgr cachestat
Sending CACHEMGR request to EXTRACT EHVD01 ...
ERROR: sending message to EXTRACT EHVD01 (Timeout waiting for message).

2. lots of small lob related files under dirtmp

(1)(xxx)(/u01/app/oracle/product/12.1.2goldengate/dirtmp) ls |wc -l
14961
(2)(xxx)(/u01/app/oracle/product/12.1.2goldengate/dirtmp) ls -lt |head -20
total 1916928
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108ceff38_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108cefca0_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108cefa08_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108cef770_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108cef4d8_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108cef240_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108ceefa8_00000.cm
-rw-r----- 1 oracle dba 65536 Dec 29 13:12 ehvd01_p2704_ORA-LOB-MEMPOOL_108ceed10_00000.cm
......
(3)(apollo26)(/u01/app/oracle/product/12.1.2goldengate/dirtmp) du -sk .
959472 .

 

3. pstack output:

----------------- lwp# 1 / thread# 1 --------------------
ffffffff738dbd58 nanosleep (ffffffff7fffa300, 0)
0000000100319160 __1cMggDelayMsecs6Fl_v_ (fffffffffff07fc0, ffffffffffffc200, 1, ffffffffffffffe1, 20, 0) + 50
000000010033d63c __1cHcm_mmap6FpnMcache_object_LLpnCmf_Lipippc_i_ (105cd42b8, 1, 200000, 108123c90, 1024c74d0, 0) + 49c
0000000100348acc __1cWcm_mf_cursor_mmap_mbuf6FpnMcache_object_pi_pnEmbuf__ (105cd42b8, ffffffff7fffa8ac, 108cf4710, 100800, 102000, 100000) + 554
000000010033c3d0 $XDsBl8K79IbUkLt.__1cQcm_item_get_next6FpnMcache_object_pipC23_i_ (105cd42b8, ffffffff7fffab94, ffffffff7fffab80, 0, ffffffff7fffab7a, 105cd42b8) + c04
000000010033b6b8 __1cVFM_tran_get_next_item6FpnKobjectpool_pnMcache_object_pipC45_h_ (102a71d50, 105cd42b8, ffffffff7fffab94, ffffffff7fffab80, ffffffff7fffab7c, ffffffff7fffab7a) + 20
00000001004f3e8c __1cbGdelete_onegroup_marked_lob_trans6FpnJora_xid_t__v_ (104f91010, 1051ea530, 100d6c, ffffffff7fffabb0, 1, 104f91010) + 158
00000001004f3c60 __1cbFREDO_delete_all_marked_lob_tran6F_v_ (0, 0, 100d6cd88, 100d6cd60, 100d6cd80, 0) + 9c
00000001004eac2c __1cJREDO_read6Fpcpipl0phiradaUc_h_ (1, 8, 1, 102a71a50, 10206a523, 105130010) + d6c
000000010011e594 __1cTREAD_EXTRACT_RECORD6FpnPchkpt_context_t_hpcLpnN__std_rec_hdr_pi5plhh2ph276radaUc_h_ (104293010, 3, 102a7e010, 100e42ff0, 10206b538, 0) + 324
0000000100217450 __1cUprocess_extract_loop6F_h_ (100d3d, 100d3f000, 10206b538, 100d3d068, 0, 100d48b38) + 748
0000000100243340 __1cMextract_main6Fippc_i_ (100d3f, 102622de0, 0, 100d3f, 100d3d068, 100c00) + 634
00000001003dc4e0 __1cDggsFgglibOMultiThreadingKMainThreadIExecMain6M_I_ (ffffffff7ffff0e0, 100c00, 8, 100d5d, 8f, 100d5d1c8) + 84
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (0, 100b41f08, 0, 100800, 1003dc780, ffffffff7ffff0e0) + 170
00000001003dc18c __1cDggsFgglibOMultiThreadingKMainThreadDRun6Mippc_I_ (ffffffff7ffff0e0, 1024a96e0, 100b42, 1003dc000, 100800, 100800) + a8
00000001002444b4 main (8, ffffffff7ffff248, ffffffff7ffff290, ffffffff7ffff0e0, 100242000, 100242) + 28
000000010010cfdc _start (0, 0, 0, 0, 0, 0) + 17c
----------------- lwp# 2 / thread# 2 --------------------
ffffffff738d8b9c lwp_park (0, 0, 0)
ffffffff738d2388 cond_wait_queue (10250df30, 10250df18, 0, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (10250df30, 10250df18, 0, 10250df1c, 16bab4, 0) + 2d4
ffffffff738d290c cond_wait (10250df30, 10250df18, 2, 0, 0, ffffffff73a4a300) + 10
ffffffff738d2948 pthread_cond_wait (10250df30, 10250df18, 54a199960012da8a, 54a199960012da8a, ffffffff783dfe50, 2e8) + 8
ffffffff780cf458 __1cDggsFgglibFgglogLLoggingTimeKTimeThread6Fpv_I_ (0, 0, 0, 10250df10, 1, 10250df18) + 390
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (100d5d, 100c00, 1003da, ffffffff7fffe528, ffffffff780cf0c8, 0) + 170
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 3 / thread# 3 --------------------
ffffffff738d8b9c lwp_park (0, 0, 0)
ffffffff738d2388 cond_wait_queue (10250de70, 10250de58, 0, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (10250de70, 10250de58, 0, 10250de5c, 16bab4, 0) + 2d4
ffffffff738d290c cond_wait (10250de70, 10250de58, 0, 1cc4, 1c00, 2) + 10
ffffffff738d2948 pthread_cond_wait (10250de70, 10250de58, 2, 0, 0, ffffffff73a4a300) + 8
00000001003dbb64 __1cDggsFgglibOMultiThreadingFEventEWait6MI_b_ (10250de50, 0, 0, 0, 10250de58, 10250de70) + 254
ffffffff781e8cdc __1cDggsFgglibFgglogNLogBufferImplPPublisherThread6Fpv_I_ (ffffffff783dfe38, 20, 0, ffffffff783d2800, 1e9b64, ffffffff72701500) + 44
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (100d5d, 100c00, 1003da, ffffffff7fffe5d8, ffffffff781e8c98, 0) + 170
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 4 / thread# 4 --------------------
ffffffff738d8b9c lwp_park (0, ffffffff71afb8d0, 0)
ffffffff738d2388 cond_wait_queue (10265f2b0, 10265f298, ffffffff71afb8d0, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (10265f2b0, 10265f298, ffffffff71afb8d0, 10265f29c, 16, 0) + 2d4
ffffffff738d299c _cond_timedwait (10265f2b0, 10265f298, ffffffff71afbaf0, 3, 110374, 0) + 34
ffffffff738d2a38 cond_timedwait (10265f2b0, 10265f298, ffffffff71afbaf0, 1cc4, 1c00, 0) + 14
ffffffff738d2a78 pthread_cond_timedwait (10265f2b0, 10265f298, ffffffff71afbaf0, 3c55dca, 773594, 3cd16dc) + c
00000001003dba0c __1cDggsFgglibOMultiThreadingFEventEWait6MI_b_ (10265f290, 1dcd6500, 0, 0, 10265f298, 54a2ff2f) + fc
ffffffff781d0464 __1cDggsFgglibFgglogTDOMConfiguratorImplLwatchThread6Fpv_I_ (102a6d970, 0, 400, 17400, 8f, ffffffff783dfc48) + 164
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (0, 100b41f08, 0, 100800, ffffffff781d0300, 102a6d970) + 170
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 5 / thread# 5 --------------------
ffffffff738d8b9c lwp_park (0, ffffffff716fbb40, 0)
ffffffff738d2388 cond_wait_queue (102625b30, 102625b18, ffffffff716fbb40, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (102625b30, 102625b18, ffffffff716fbb40, 102625b1c, 16, 0) + 2d4
ffffffff738d299c _cond_timedwait (102625b30, 102625b18, ffffffff716fbd60, 3, 3e, 0) + 34
ffffffff738d2a38 cond_timedwait (102625b30, 102625b18, ffffffff716fbd60, 1cc4, 1c00, ff) + 14
ffffffff738d2a78 pthread_cond_timedwait (102625b30, 102625b18, ffffffff716fbd60, 42fcae8, 0, 4385df0) + c
00000001003dba0c __1cDggsFgglibOMultiThreadingFEventEWait6MI_b_ (102625b10, 0, 0, 0, 102625b18, 21c2ef8) + fc
ffffffff7818abc8 __1cTCMessageFactoryImplQRepetitionThread6Fpv_I_ (1026258d0, 927c0, 92400, 7400, 8f, ffffffff716fbf10) + 24
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (0, 100b41f08, 0, 100800, ffffffff7818aba4, 1026258d0) + 170
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 6 / thread# 6 --------------------
ffffffff738d8b9c lwp_park (0, ffffffff711fbe60, 0)
ffffffff738d2388 cond_wait_queue (ffffffff7af36c38, ffffffff7af36c58, ffffffff711fbe60, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (ffffffff7af36c38, ffffffff7af36c58, ffffffff711fbe60, ffffffff7af36c5c, 16, 0) + 2d4
ffffffff738d299c _cond_timedwait (ffffffff7af36c38, ffffffff7af36c58, ffffffff711fbf28, 3, ffffffff7af24000, ffffffff711fbf38) + 34
ffffffff7ae11ff8 umem_update_thread (54a2ff33, 0, 0, ffffffff7af24000, ffffffff7af28730, ffffffff7af37ce0) + 218
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 7 / thread# 7 --------------------
ffffffff738d8b9c lwp_park (0, 0, 0)
ffffffff738d2388 cond_wait_queue (104613630, 104613618, 0, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (104613630, 104613618, 0, 10461361c, 16bab4, 0) + 2d4
ffffffff738d290c cond_wait (104613630, 104613618, 2, 0, 0, ffffffff73a4a300) + 10
ffffffff738d2948 pthread_cond_wait (104613630, 104613618, 104ecc442, 100d54, ffffffff72f09240, 0) + 8
0000000100344ec4 __1cVcm_filecaching_thread6Fpv_0_ (104613590, 100800, 102000, 100800, 0, 104613618) + 114
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)
----------------- lwp# 8 / thread# 8 --------------------
ffffffff738d8b9c lwp_park (0, ffffffff70cfaac0, 0)
ffffffff738d2388 cond_wait_queue (1050eaaf0, 1050eaad8, ffffffff70cfaac0, 0, 0, 0) + 4c
ffffffff738d2810 cond_wait_common (1050eaaf0, 1050eaad8, ffffffff70cfaac0, 1050eaadc, 16, 0) + 2d4
ffffffff738d299c _cond_timedwait (1050eaaf0, 1050eaad8, ffffffff70cfacf0, 3, 3e, 0) + 34
ffffffff738d2a38 cond_timedwait (1050eaaf0, 1050eaad8, ffffffff70cfacf0, 1cc4, 1c00, fffc00) + 14
ffffffff738d2a78 pthread_cond_timedwait (1050eaaf0, 1050eaad8, ffffffff70cfacf0, 44b82c00, 0, ffffffff70cface0) + c
00000001003ee024 __1cUTHREADUTIL_cond_wait6FpnRthreadutil_cond_s_il_i_ (1050eaad8, 0, 0, 0, 0, 5f5e100) + cc
00000001003b5c34 __1cYCQdDUEUE_read_wait_timeout6FpnIcqueue_t_pipLppvl_b_ (0, ffffffff70cfbe34, ffffffff70cfbe28, ffffffff70cfbe38, 5f5e100, 1050eaa50) + fc
00000001003a832c __1cDggsFgglibLAsyncReaderLAsyncReaderMReaderThread6Fpv_I_ (104fb4ad0, 1050ea910, ffffffff70cfbe28, 100d56ad8, 100b3856e, 100b385cf) + 158
00000001003da928 __1cDggsFgglibOMultiThreadingGThreadJRunThread6Fpn0DKThreadArgs__I_ (0, 100b41f08, 0, 100800, 1003a81d4, 104fb4ad0) + 170
ffffffff738d8afc _lwp_start (0, 0, 0, 0, 0, 0)

 

 

Cause

Sign In with your My Oracle Support account

Don't have a My Oracle Support account? Click to get started

My Oracle Support provides customers with access to over a
Million Knowledge Articles and hundreds of Community platforms