New OAM 11.1.2.3 HA Installation Oam_policy_mgr Down On Both Nodes Of The Cluster (Doc ID 2124034.1)

Last updated on MARCH 08, 2017

Applies to:

Oracle Access Manager - Version 11.1.2.3.0 and later
Information in this document applies to any platform.

Symptoms

1. Post OAM 11.1.2.3 installation. deployment of oam_policy_mgr.ear file fails on WLS_PolicyManager1 and WLS_PolicyManager2.

2. OAM server has multiple ip addresses.

####<Feb 22, 2016 11:17:41 AM EST> <Error> <Deployer> <ORS-21ORA-13> <wls_oama1> <[ACTIVE] ExecuteThread: '0' for queue: 'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <> <0000LC9lJ1s1barN4IYBUE1MmnHt000003> <1456157861232> <BEA-149231> <Unable to set the activation state to true for the application 'oam_policy_mgr [Version=11.1.2.0.0]'.
weblogic.application.ModuleException:
at weblogic.servlet.internal.WebAppModule.startContexts(WebAppModule.java:1529)
at weblogic.servlet.internal.WebAppModule.start(WebAppModule.java:486)
at weblogic.application.internal.flow.ModuleStateDriver$3.next(ModuleStateDriver.java:425)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.flow.ModuleStateDriver.start(ModuleStateDriver.java:119)
at weblogic.application.internal.flow.ScopedModuleDriver.start(ScopedModuleDriver.java:200)
at weblogic.application.internal.flow.ModuleListenerInvoker.start(ModuleListenerInvoker.java:247)
at weblogic.application.internal.flow.ModuleStateDriver$3.next(ModuleStateDriver.java:425)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.flow.ModuleStateDriver.start(ModuleStateDriver.java:119)
at weblogic.application.internal.flow.StartModulesFlow.activate(StartModulesFlow.java:27)
at weblogic.application.internal.BaseDeployment$2.next(BaseDeployment.java:671)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.BaseDeployment.activate(BaseDeployment.java:212)
at weblogic.application.internal.EarDeployment.activate(EarDeployment.java:59)
at weblogic.application.internal.DeploymentStateChecker.activate(DeploymentStateChecker.java:161)
at weblogic.deploy.internal.targetserver.AppContainerInvoker.activate(AppContainerInvoker.java:80)
at weblogic.deploy.internal.targetserver.BasicDeployment.activate(BasicDeployment.java:187)
at weblogic.deploy.internal.targetserver.BasicDeployment.activateFromServerLifecycle(BasicDeployment.java:379)
at weblogic.management.deploy.internal.DeploymentAdapter$1.doActivate(DeploymentAdapter.java:51)
at weblogic.management.deploy.internal.DeploymentAdapter.activate(DeploymentAdapter.java:200)
at weblogic.management.deploy.internal.AppTransition$2.transitionApp(AppTransition.java:30)
at weblogic.management.deploy.internal.ConfiguredDeployments.transitionApps(ConfiguredDeployments.java:240)
at weblogic.management.deploy.internal.ConfiguredDeployments.activate(ConfiguredDeployments.java:169)
at weblogic.management.deploy.internal.ConfiguredDeployments.deploy(ConfiguredDeployments.java:123)
at weblogic.management.deploy.internal.DeploymentServerService.resume(DeploymentServerService.java:180)
at weblogic.management.deploy.internal.DeploymentServerService.start(DeploymentServerService.java:96)
at weblogic.t3.srvr.SubsystemRequest.run(SubsystemRequest.java:64)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:263)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:221)
Caused By: java.lang.RuntimeException: Failed to start Service "Cluster" (ServiceState=SERVICE_STOPPED, STATE_JOINING)
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service.start(Service.CDB:38)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.start(Grid.CDB:6)
at com.tangosol.coherence.component.net.Cluster.onStart(Cluster.CDB:56)
at com.tangosol.coherence.component.net.Cluster.start(Cluster.CDB:11)
at com.tangosol.coherence.component.util.SafeCluster.startCluster(SafeCluster.CDB:3)
at com.tangosol.coherence.component.util.SafeCluster.restartCluster(SafeCluster.CDB:10)
at com.tangosol.coherence.component.util.SafeCluster.ensureRunningCluster(SafeCluster.CDB:26)
at com.tangosol.coherence.component.util.SafeCluster.start(SafeCluster.CDB:2)
at com.tangosol.net.CacheFactory.ensureCluster(CacheFactory.java:427)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureServiceInternal(DefaultConfigurableCacheFactory.java:973)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureService(DefaultConfigurableCacheFactory.java:942)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:924)
at com.tangosol.net.DefaultConfigurableCacheFactory.configureCache(DefaultConfigurableCacheFactory.java:1301)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:302)
at com.tangosol.net.CacheFactory.getCache(CacheFactory.java:204)
at com.tangosol.net.CacheFactory.getCache(CacheFactory.java:181)
at oracle.security.am.foundation.mapimpl.coherence.CoherenceMapImpl.init(CoherenceMapImpl.java:238)
at oracle.security.am.foundation.mapimpl.coherence.CoherenceMapImpl.<init>(CoherenceMapImpl.java:226)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:339)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:323)
at oracle.security.am.admin.config.util.GenericFactory.readObject(GenericFactory.java:233)
at oracle.security.am.admin.config.util.GenericFactory.validateAndGetObjectForMap(GenericFactory.java:120)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:104)
at oracle.security.am.admin.config.util.NotificationLockHelper.<init>(NotificationLockHelper.java:142)
at oracle.security.am.admin.config.util.NotificationLockHelper.<init>(NotificationLockHelper.java:114)
at oracle.security.am.lifecycle.config.DistributionListenerImpl.<init>(DistributionListenerImpl.java:59)
at oracle.security.am.lifecycle.config.ConfigLifecycleImpl.<clinit>(ConfigLifecycleImpl.java:75)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:335)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:323)
at oracle.security.am.admin.config.util.GenericFactory.readObject(GenericFactory.java:233)
at oracle.security.am.admin.config.util.GenericFactory.validateAndGetObjectForMap(GenericFactory.java:120)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:104)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:191)
at oracle.security.am.lifecycle.ApplicationLifecycle.initComponentBootstrap(ApplicationLifecycle.java:212)
at oracle.security.am.lifecycle.ApplicationLifecycle.initComponentBootstrap(ApplicationLifecycle.java:198)
at oracle.security.am.lifecycle.ApplicationLifecycle.contextInitialized(ApplicationLifecycle.java:109)
at weblogic.servlet.internal.EventsManager$FireContextListenerAction.run(EventsManager.java:481)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.internal.EventsManager.notifyContextCreatedEvent(EventsManager.java:181)
at weblogic.servlet.internal.WebAppServletContext.preloadResources(WebAppServletContext.java:1871)
at weblogic.servlet.internal.WebAppServletContext.start(WebAppServletContext.java:3173)
at weblogic.servlet.internal.WebAppModule.startContexts(WebAppModule.java:1527)
at weblogic.servlet.internal.WebAppModule.start(WebAppModule.java:486)
at weblogic.application.internal.flow.ModuleStateDriver$3.next(ModuleStateDriver.java:425)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.flow.ModuleStateDriver.start(ModuleStateDriver.java:119)
at weblogic.application.internal.flow.ScopedModuleDriver.start(ScopedModuleDriver.java:200)
at weblogic.application.internal.flow.ModuleListenerInvoker.start(ModuleListenerInvoker.java:247)
at weblogic.application.internal.flow.ModuleStateDriver$3.next(ModuleStateDriver.java:425)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.flow.ModuleStateDriver.start(ModuleStateDriver.java:119)
at weblogic.application.internal.flow.StartModulesFlow.activate(StartModulesFlow.java:27)
at weblogic.application.internal.BaseDeployment$2.next(BaseDeployment.java:671)
at weblogic.application.utils.StateMachineDriver.nextState(StateMachineDriver.java:52)
at weblogic.application.internal.BaseDeployment.activate(BaseDeployment.java:212)
at weblogic.application.internal.EarDeployment.activate(EarDeployment.java:59)
at weblogic.application.internal.DeploymentStateChecker.activate(DeploymentStateChecker.java:161)
at weblogic.deploy.internal.targetserver.AppContainerInvoker.activate(AppContainerInvoker.java:80)
at weblogic.deploy.internal.targetserver.BasicDeployment.activate(BasicDeployment.java:187)
at weblogic.deploy.internal.targetserver.BasicDeployment.activateFromServerLifecycle(BasicDeployment.java:379)
at weblogic.management.deploy.internal.DeploymentAdapter$1.doActivate(DeploymentAdapter.java:51)
at weblogic.management.deploy.internal.DeploymentAdapter.activate(DeploymentAdapter.java:200)
at weblogic.management.deploy.internal.AppTransition$2.transitionApp(AppTransition.java:30)
at weblogic.management.deploy.internal.ConfiguredDeployments.transitionApps(ConfiguredDeployments.java:240)
at weblogic.management.deploy.internal.ConfiguredDeployments.activate(ConfiguredDeployments.java:169)
at weblogic.management.deploy.internal.ConfiguredDeployments.deploy(ConfiguredDeployments.java:123)
at weblogic.management.deploy.internal.DeploymentServerService.resume(DeploymentServerService.java:180)
at weblogic.management.deploy.internal.DeploymentServerService.start(DeploymentServerService.java:96)
at weblogic.t3.srvr.SubsystemRequest.run(SubsystemRequest.java:64)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:263)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:221)

 

[2016-01-19T05:03:18.382-05:00] [WLS_AMA2] [ERROR] [] [Coherence] [tid: Logger@1396430337 3.7.1.13] [userId: <anonymous>] [ecid: 0000L97ICwY1barN4IV4EE1MaMe_000004,1:26299] [APP: oam_policy_mgr#11.1.2.0.0] 2016-01-19 05:03:18.382/302924.047 Oracle Coherence GE 3.7.1.13 <Error> (thread=Cluster, member=n/a): This member is not authorized to join the cluster. Rejected by Member(Id=1, Timestamp=2016-01-14 21:06:49.903, Address=137.187.219.47:9095, MachineId=36788, Location=site:,machine:ors-21ora-13,process:9604, Role=WeblogicServer).
[2016-01-19T05:03:18.389-05:00] [WLS_AMA2] [ERROR] [] [Coherence] [tid: Logger@1396430337 3.7.1.13] [userId: <anonymous>] [ecid: 0000L97ICwY1barN4IV4EE1MaMe_000004,1:26299] [APP: oam_policy_mgr#11.1.2.0.0] 2016-01-19 05:03:18.389/302924.054 Oracle Coherence GE 3.7.1.13 <Error> (thread=Configuration Store Observer, member=n/a): Error while starting cluster: java.lang.RuntimeException: Failed to start Service "Cluster" (ServiceState=SERVICE_STOPPED, STATE_JOINING)[[
at com.tangosol.coherence.component.util.daemon.queueProcessor.Service.start(Service.CDB:38)
at com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.start(Grid.CDB:6)
at com.tangosol.coherence.component.net.Cluster.onStart(Cluster.CDB:56)
at com.tangosol.coherence.component.net.Cluster.start(Cluster.CDB:11)
at com.tangosol.coherence.component.util.SafeCluster.startCluster(SafeCluster.CDB:3)
at com.tangosol.coherence.component.util.SafeCluster.restartCluster(SafeCluster.CDB:10)
at com.tangosol.coherence.component.util.SafeCluster.ensureRunningCluster(SafeCluster.CDB:26)
at com.tangosol.coherence.component.util.SafeCluster.start(SafeCluster.CDB:2)
at com.tangosol.net.CacheFactory.ensureCluster(CacheFactory.java:427)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureServiceInternal(DefaultConfigurableCacheFactory.java:973)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureService(DefaultConfigurableCacheFactory.java:942)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:924)
at com.tangosol.net.DefaultConfigurableCacheFactory.configureCache(DefaultConfigurableCacheFactory.java:1301)
at com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:302)
at com.tangosol.net.CacheFactory.getCache(CacheFactory.java:204)
at com.tangosol.net.CacheFactory.getCache(CacheFactory.java:181)
at oracle.security.am.foundation.mapimpl.coherence.CoherenceMapImpl.init(CoherenceMapImpl.java:238)
at oracle.security.am.foundation.mapimpl.coherence.CoherenceMapImpl.<init>(CoherenceMapImpl.java:226)
at sun.reflect.GeneratedConstructorAccessor291.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:339)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:323)
at oracle.security.am.admin.config.util.GenericFactory.readObject(GenericFactory.java:233)
at oracle.security.am.admin.config.util.GenericFactory.validateAndGetObjectForMap(GenericFactory.java:120)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:104)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:191)
at oracle.security.am.foundation.distributor.coherence.DistributorImpl.init(DistributorImpl.java:92)
at oracle.security.am.foundation.distributor.coherence.DistributorImpl.<init>(DistributorImpl.java:71)
at sun.reflect.GeneratedConstructorAccessor351.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:339)
at oracle.security.am.admin.config.util.GenericFactory.getManager(GenericFactory.java:323)
at oracle.security.am.admin.config.util.GenericFactory.readObject(GenericFactory.java:233)
at oracle.security.am.admin.config.util.GenericFactory.validateAndGetObjectForMap(GenericFactory.java:120)
at oracle.security.am.admin.config.util.GenericFactory.getInstance(GenericFactory.java:163)
at oracle.security.am.admin.config.util.ConfigUtil.getConfigDistributor(ConfigUtil.java:145)
at oracle.security.am.admin.config.util.ConfigUtil.getConfigDistributor(ConfigUtil.java:125)
at oracle.security.am.admin.config.util.observable.CohStore.getDistributor(CohStore.java:81)
at oracle.security.am.admin.config.util.observable.CohStore.storeChanged(CohStore.java:118)
at oracle.security.am.admin.config.util.observable.ObservableConfigStore$StoreWatcher.run(ObservableConfigStore.java:227)
at java.util.TimerThread.mainLoop(Timer.java:555)
at java.util.TimerThread.run(Timer.java:505)

]]

Cause

Sign In with your My Oracle Support account

Don't have a My Oracle Support account? Click to get started

My Oracle Support provides customers with access to over a
Million Knowledge Articles and hundreds of Community platforms