OPSS - Servers operations very slow and the thread dumps showing multiple threads on the Auditing
(Doc ID 2488750.1)
Last updated on OCTOBER 17, 2019
Applies to:
Oracle Platform Security for Java - Version 12.2.1.3.0 and laterInformation in this document applies to any platform.
Symptoms
On : 12.2.1.3.0 version, Java Platform Security
All the servers in the domain are very slow and the thread dumps, observing multiple threads on the OPSS Auditing
"[ACTIVE] ExecuteThread: '<EXECUTE_THREAD>' for queue: 'weblogic.kernel.Default (self-tuning)'" daemon prio=5 os_prio=0 tid=<TID> nid=<NID> runnable [<ENCRYPTED_FILE>]
java.lang.Thread.State: RUNNABLE
at java.io.UnixFileSystem.list(Native Method)
at java.io.File.list(File.java:1122)
at oracle.security.audit.ajl.FileLogWriter.getSegmentFiles(FileLogWriter.java:439)
at oracle.security.audit.ajl.FileLogWriter.rotateAndDelete(FileLogWriter.java:323)
at oracle.security.audit.ajl.BusStopLogWriter.unsecureWrite(BusStopLogWriter.java:306)
at oracle.security.audit.ajl.BusStopLogWriter.write(BusStopLogWriter.java:237)
at oracle.security.audit.spi.impl.OracleChannelImpl.log(OracleChannelImpl.java:93)
at oracle.security.audit.Auditor.log(Auditor.java:547)
at oracle.security.jps.internal.audit.DelegateAuditor.log(DelegateAuditor.java:104)
at oracle.security.jps.internal.common.audit.util.AuditDelegateImpl.log(AuditDelegateImpl.java:103)
at oracle.security.jps.util.AuthZAuditUtil.log(AuthZAuditUtil.java:136)
at oracle.security.jps.util.AuthZAuditUtil.auditCheckPerm(AuthZAuditUtil.java:245)
at oracle.security.jps.util.JpsAuth.checkPermission(JpsAuth.java:643)
at oracle.security.jps.util.JpsAuth.checkPermission(JpsAuth.java:606)
at oracle.security.opss.ee.runtime.LocalUserSecurityContext.setAttributeKey(LocalUserSecurityContext.java:103)
at oracle.security.jps.ee.runtime.AppLocalSecurityContext.setAttributeKey(AppLocalSecurityContext.java:54)
at oracle.security.jps.ee.util.AttributeKeyAction.run(AttributeKeyAction.java:45)
at oracle.security.jps.ee.util.AttributeKeyAction.run(AttributeKeyAction.java:33)
at java.security.AccessController.doPrivileged(Native Method)
at oracle.security.jps.internal.jaas.AccActionExecutor.setupIDCKey(AccActionExecutor.java:94)
at oracle.security.jps.internal.jaas.AccActionExecutor.execute(AccActionExecutor.java:68)
at oracle.security.jps.internal.jaas.AbstractSubjectSecurity$ActionExecutorWrapper.execute(AbstractSubjectSecurity.java:266)
at oracle.security.jps.internal.jaas.CascadeActionExecutor$SubjectPrivilegedExceptionAction.run(CascadeActionExecutor.java:82)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:368)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:163)
at weblogic.security.Security.runAs(Security.java:62)
at oracle.security.jps.wls.jaas.WlsActionExecutor.execute(WlsActionExecutor.java:50)
at oracle.security.jps.internal.jaas.CascadeActionExecutor.execute(CascadeActionExecutor.java:55)
at oracle.security.jps.internal.jaas.AbstractSubjectSecurity$ActionExecutorWrapper.execute(AbstractSubjectSecurity.java:266)
at oracle.wsm.policymanager.bean.DocumentManagerBean.retrieveDocuments(DocumentManagerBean.java:838)
at oracle.wsm.policymanager.bean.ejb.impl.DocumentManagerEJB.retrieveDocuments(DocumentManagerEJB.java:224)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.bea.core.repackaged.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at oracle.wsm.policymanager.bean.ejb.impl.EjbUtil.removeCurrentADFContext(EjbUtil.java:89)
at oracle.wsm.policymanager.bean.ejb.impl.DocumentManagerEJB.removeCurrentADFContext(DocumentManagerEJB.java:203)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:101)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73)
at org.jboss.weld.ejb.SessionBeanInterceptor.aroundInvoke(SessionBeanInterceptor.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor$1.run(JpsAbsInterceptor.java:131)
at java.security.AccessController.doPrivileged(Native Method)
at oracle.security.jps.util.JpsSubject.doAsPrivileged(JpsSubject.java:315)
at oracle.security.jps.ee.util.JpsPlatformUtil.runJaasMode(JpsPlatformUtil.java:650)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor.runJaasMode(JpsAbsInterceptor.java:118)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor.intercept(JpsAbsInterceptor.java:197)
at oracle.security.jps.ee.ejb.JpsInterceptor.intercept(JpsInterceptor.java:112)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.bea.core.repackaged.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133)
at com.bea.core.repackaged.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.bea.core.repackaged.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy257.retrieveDocuments(Unknown Source)
java.lang.Thread.State: RUNNABLE
at java.io.UnixFileSystem.list(Native Method)
at java.io.File.list(File.java:1122)
at oracle.security.audit.ajl.FileLogWriter.getSegmentFiles(FileLogWriter.java:439)
at oracle.security.audit.ajl.FileLogWriter.rotateAndDelete(FileLogWriter.java:323)
at oracle.security.audit.ajl.BusStopLogWriter.unsecureWrite(BusStopLogWriter.java:306)
at oracle.security.audit.ajl.BusStopLogWriter.write(BusStopLogWriter.java:237)
at oracle.security.audit.spi.impl.OracleChannelImpl.log(OracleChannelImpl.java:93)
at oracle.security.audit.Auditor.log(Auditor.java:547)
at oracle.security.jps.internal.audit.DelegateAuditor.log(DelegateAuditor.java:104)
at oracle.security.jps.internal.common.audit.util.AuditDelegateImpl.log(AuditDelegateImpl.java:103)
at oracle.security.jps.util.AuthZAuditUtil.log(AuthZAuditUtil.java:136)
at oracle.security.jps.util.AuthZAuditUtil.auditCheckPerm(AuthZAuditUtil.java:245)
at oracle.security.jps.util.JpsAuth.checkPermission(JpsAuth.java:643)
at oracle.security.jps.util.JpsAuth.checkPermission(JpsAuth.java:606)
at oracle.security.opss.ee.runtime.LocalUserSecurityContext.setAttributeKey(LocalUserSecurityContext.java:103)
at oracle.security.jps.ee.runtime.AppLocalSecurityContext.setAttributeKey(AppLocalSecurityContext.java:54)
at oracle.security.jps.ee.util.AttributeKeyAction.run(AttributeKeyAction.java:45)
at oracle.security.jps.ee.util.AttributeKeyAction.run(AttributeKeyAction.java:33)
at java.security.AccessController.doPrivileged(Native Method)
at oracle.security.jps.internal.jaas.AccActionExecutor.setupIDCKey(AccActionExecutor.java:94)
at oracle.security.jps.internal.jaas.AccActionExecutor.execute(AccActionExecutor.java:68)
at oracle.security.jps.internal.jaas.AbstractSubjectSecurity$ActionExecutorWrapper.execute(AbstractSubjectSecurity.java:266)
at oracle.security.jps.internal.jaas.CascadeActionExecutor$SubjectPrivilegedExceptionAction.run(CascadeActionExecutor.java:82)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:368)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:163)
at weblogic.security.Security.runAs(Security.java:62)
at oracle.security.jps.wls.jaas.WlsActionExecutor.execute(WlsActionExecutor.java:50)
at oracle.security.jps.internal.jaas.CascadeActionExecutor.execute(CascadeActionExecutor.java:55)
at oracle.security.jps.internal.jaas.AbstractSubjectSecurity$ActionExecutorWrapper.execute(AbstractSubjectSecurity.java:266)
at oracle.wsm.policymanager.bean.DocumentManagerBean.retrieveDocuments(DocumentManagerBean.java:838)
at oracle.wsm.policymanager.bean.ejb.impl.DocumentManagerEJB.retrieveDocuments(DocumentManagerEJB.java:224)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.bea.core.repackaged.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at oracle.wsm.policymanager.bean.ejb.impl.EjbUtil.removeCurrentADFContext(EjbUtil.java:89)
at oracle.wsm.policymanager.bean.ejb.impl.DocumentManagerEJB.removeCurrentADFContext(DocumentManagerEJB.java:203)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:101)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73)
at org.jboss.weld.ejb.SessionBeanInterceptor.aroundInvoke(SessionBeanInterceptor.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.oracle.pitchfork.intercept.MethodInvocationInvocationContext.proceed(MethodInvocationInvocationContext.java:101)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor$1.run(JpsAbsInterceptor.java:131)
at java.security.AccessController.doPrivileged(Native Method)
at oracle.security.jps.util.JpsSubject.doAsPrivileged(JpsSubject.java:315)
at oracle.security.jps.ee.util.JpsPlatformUtil.runJaasMode(JpsPlatformUtil.java:650)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor.runJaasMode(JpsAbsInterceptor.java:118)
at oracle.security.jps.ee.ejb.JpsAbsInterceptor.intercept(JpsAbsInterceptor.java:197)
at oracle.security.jps.ee.ejb.JpsInterceptor.intercept(JpsInterceptor.java:112)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.oracle.pitchfork.intercept.JeeInterceptorInterceptor.invoke(JeeInterceptorInterceptor.java:94)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.bea.core.repackaged.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133)
at com.bea.core.repackaged.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121)
at com.bea.core.repackaged.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at com.bea.core.repackaged.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy257.retrieveDocuments(Unknown Source)
Cause
To view full details, sign in with your My Oracle Support account. |
|
Don't have a My Oracle Support account? Click to get started! |
In this Document
| Symptoms |
| Cause |
| Solution |