java - RabbitMQ impacting JDBC connection pool -
i have web servlet has been running well. decided replace tcp connection backend server rabbitmq. while doing load testing noticed started fail timeouts waiting idle jdbc connection.
error [http-bio-8080-exec-12] jdbcexceptionreporter.logexceptions(234) | cannot connection, pool error timeout waiting idle object debug [http-bio-8080-exec-12] jdbcexceptionreporter.logexceptions(225) | cannot open connection [???] org.apache.commons.dbcp.sqlnestedexception: cannot connection, pool error timeout waiting idle object @ org.apache.commons.dbcp.poolingdatasource.getconnection(poolingdatasource.java:114) @ org.apache.commons.dbcp.basicdatasource.getconnection(basicdatasource.java:1044) @ org.springframework.orm.hibernate3.localdatasourceconnectionprovider.getconnection(localdatasourceconnectionprovider.java:85) @ org.hibernate.jdbc.connectionmanager.openconnection(connectionmanager.java:446) @ org.hibernate.jdbc.connectionmanager.getconnection(connectionmanager.java:167) @ org.hibernate.jdbc.jdbccontext.connection(jdbccontext.java:160) @ org.hibernate.transaction.jdbctransaction.begin(jdbctransaction.java:81) @ org.hibernate.impl.sessionimpl.begintransaction(sessionimpl.java:1473) @ org.springframework.orm.hibernate3.hibernatetransactionmanager.dobegin(hibernatetransactionmanager.java:560) @ org.springframework.transaction.support.abstractplatformtransactionmanager.gettransaction(abstractplatformtransactionmanager.java:373) @ org.springframework.transaction.interceptor.transactionaspectsupport.createtransactionifnecessary(transactionaspectsupport.java:438) @ org.springframework.transaction.interceptor.transactionaspectsupport.invokewithintransaction(transactionaspectsupport.java:261) @ org.springframework.transaction.interceptor.transactioninterceptor.invoke(transactioninterceptor.java:95) @ org.springframework.aop.framework.reflectivemethodinvocation.proceed(reflectivemethodinvocation.java:179) @ org.springframework.aop.interceptor.exposeinvocationinterceptor.invoke(exposeinvocationinterceptor.java:92) @ org.springframework.aop.framework.reflectivemethodinvocation.proceed(reflectivemethodinvocation.java:179) @ org.springframework.aop.framework.cglibaopproxy$dynamicadvisedinterceptor.intercept(cglibaopproxy.java:653) @ net.hedtech.degreeworks.security.service.shppassportservice$$enhancerbyspringcglib$$9c441f05.getandcheckpassportforrequest(<generated>) @ net.hedtech.degreeworks.security.spring.shppassportfilter.dofilter(shppassportfilter.java:124) @ net.hedtech.degreeworks.security.spring.shppassportfilter$$fastclassbyspringcglib$$8067e598.invoke(<generated>) @ org.springframework.cglib.proxy.methodproxy.invoke(methodproxy.java:204) @ org.springframework.aop.framework.cglibaopproxy$cglibmethodinvocation.invokejoinpoint(cglibaopproxy.java:717) @ org.springframework.aop.framework.reflectivemethodinvocation.proceed(reflectivemethodinvocation.java:157) @ org.springframework.aop.aspectj.methodinvocationproceedingjoinpoint.proceed(methodinvocationproceedingjoinpoint.java:85) @ com.sungardhe.degreeworks.util.springbeansprofiler.aroundadvice(springbeansprofiler.java:92) @ sun.reflect.generatedmethodaccessor167.invoke(unknown source) @ sun.reflect.delegatingmethodaccessorimpl.invoke(delegatingmethodaccessorimpl.java:43) @ java.lang.reflect.method.invoke(method.java:606) @ org.springframework.aop.aspectj.abstractaspectjadvice.invokeadvicemethodwithgivenargs(abstractaspectjadvice.java:621) @ org.springframework.aop.aspectj.abstractaspectjadvice.invokeadvicemethod(abstractaspectjadvice.java:610) @ org.springframework.aop.aspectj.aspectjaroundadvice.invoke(aspectjaroundadvice.java:68) @ org.springframework.aop.framework.reflectivemethodinvocation.proceed(reflectivemethodinvocation.java:179) @ org.springframework.aop.interceptor.exposeinvocationinterceptor.invoke(exposeinvocationinterceptor.java:92) @ org.springframework.aop.framework.reflectivemethodinvocation.proceed(reflectivemethodinvocation.java:179) @ org.springframework.aop.framework.cglibaopproxy$dynamicadvisedinterceptor.intercept(cglibaopproxy.java:653) @ net.hedtech.degreeworks.security.spring.shppassportfilter$$enhancerbyspringcglib$$c08501d.dofilter(<generated>) @ org.springframework.security.web.filterchainproxy$virtualfilterchain.dofilter(filterchainproxy.java:342) @ org.springframework.security.web.authentication.logout.logoutfilter.dofilter(logoutfilter.java:110) @ org.springframework.security.web.filterchainproxy$virtualfilterchain.dofilter(filterchainproxy.java:342) @ org.springframework.security.web.context.request.async.webasyncmanagerintegrationfilter.dofilterinternal(webasyncmanagerintegrationfilter.java:50) @ org.springframework.web.filter.onceperrequestfilter.dofilter(onceperrequestfilter.java:107) @ org.springframework.security.web.filterchainproxy$virtualfilterchain.dofilter(filterchainproxy.java:342) @ org.springframework.security.web.context.securitycontextpersistencefilter.dofilter(securitycontextpersistencefilter.java:87) @ org.springframework.security.web.filterchainproxy$virtualfilterchain.dofilter(filterchainproxy.java:342) @ org.springframework.security.web.filterchainproxy.dofilterinternal(filterchainproxy.java:192) @ org.springframework.security.web.filterchainproxy.dofilter(filterchainproxy.java:160) @ org.springframework.web.filter.delegatingfilterproxy.invokedelegate(delegatingfilterproxy.java:344) @ org.springframework.web.filter.delegatingfilterproxy.dofilter(delegatingfilterproxy.java:261) @ org.apache.catalina.core.applicationfilterchain.internaldofilter(applicationfilterchain.java:241) @ org.apache.catalina.core.applicationfilterchain.dofilter(applicationfilterchain.java:208) @ org.springframework.web.filter.characterencodingfilter.dofilterinternal(characterencodingfilter.java:88) @ org.springframework.web.filter.onceperrequestfilter.dofilter(onceperrequestfilter.java:107) @ org.apache.catalina.core.applicationfilterchain.internaldofilter(applicationfilterchain.java:241) @ org.apache.catalina.core.applicationfilterchain.dofilter(applicationfilterchain.java:208) @ org.apache.catalina.core.standardwrappervalve.invoke(standardwrappervalve.java:220) @ org.apache.catalina.core.standardcontextvalve.invoke(standardcontextvalve.java:122) @ org.apache.catalina.authenticator.authenticatorbase.invoke(authenticatorbase.java:504) @ org.apache.catalina.core.standardhostvalve.invoke(standardhostvalve.java:170) @ org.apache.catalina.valves.errorreportvalve.invoke(errorreportvalve.java:103) @ org.apache.catalina.core.standardenginevalve.invoke(standardenginevalve.java:116) @ org.apache.catalina.connector.coyoteadapter.service(coyoteadapter.java:421) @ org.apache.coyote.http11.abstracthttp11processor.process(abstracthttp11processor.java:1074) @ org.apache.coyote.abstractprotocol$abstractconnectionhandler.process(abstractprotocol.java:611) @ org.apache.tomcat.util.net.jioendpoint$socketprocessor.run(jioendpoint.java:316) @ java.util.concurrent.threadpoolexecutor.runworker(threadpoolexecutor.java:1145) @ java.util.concurrent.threadpoolexecutor$worker.run(threadpoolexecutor.java:615) @ org.apache.tomcat.util.threads.taskthread$wrappingrunnable.run(taskthread.java:61) @ java.lang.thread.run(thread.java:745) caused by: java.util.nosuchelementexception: timeout waiting idle object @ org.apache.commons.pool.impl.genericobjectpool.borrowobject(genericobjectpool.java:1174) @ org.apache.commons.dbcp.poolingdatasource.getconnection(poolingdatasource.java:106) i have not implemented pooling myself @ point. rabbitmq connection singleton shared between session threads. channel opened , closed each request.
my research has not turned connection between rabbitmq , jdbc connection pools, have admit have passing knowledge of either.
here code using:
string requestqueuename = "dw-dev-prototype"; queueingconsumer consumer; channel channel = null; string correlationid = uuid.randomuuid().tostring(); string thisresponse = null; string totalresponse = ""; stringbuffer sbresponse; // create new reply queue each request string replyqueuename = ""; try { // send replyqueuename request server knows how send // response channel = _connection.createchannel(); replyqueuename = channel.queuedeclare().getqueue(); consumer = new queueingconsumer(channel); channel.basicconsume(replyqueuename, auto_acknowledge, consumer); basicproperties props = new basicproperties.builder().correlationid(correlationid).replyto(replyqueuename) .build(); channel.basicpublish(no_exchange, requestqueuename, props, snamevaluepairs.getbytes("utf-8")); } catch (ioexception e) { log.error("new connection ioexception: " + e.getmessage(), e); abortconnection(_connection, channel); throw new runtimeexception(e); } while (true) { queueingconsumer.delivery delivery; try { // next response delivery = consumer.nextdelivery(); thisresponse = new string(delivery.getbody(), "utf-8"); // ignore response if not our correlation-id else if (delivery.getproperties().getcorrelationid().equals(correlationid)) { if (thisresponse.equals(this._servicefinishedmsg)) { // no more text response - done break; } else { totalresponse += thisresponse; } } } catch (shutdownsignalexception | consumercancelledexception | interruptedexception e) { log.error("consumer exception: " + e.getmessage(), e); throw new runtimeexception(e); } catch (unsupportedencodingexception e) { log.error("unsupportedencoding exception: " + e.getmessage(), e); throw new runtimeexception(e); } } // close done try { if (channel != null && channel.isopen()) { channel.close(); } } catch (ioexception | timeoutexception e) { log.error("error closing channel or connection", e); } i running in tomcat container. using spring, not spring amqp yet.
the jdbc calls in security filter runs before request gets method.
any hints how might troubleshoot welcome.
thanks in advance.
a call initiated invoked aop -> spring service -> hibernate. hibernate invoking commons dbcp , asking connection pool. you're running out of connections pool or db has become unreachable. i'd vote former.
integration java melody app , deploy again. monitor connection pool , check if run out. if do...
- consider increasing connection pool limit
- if increasing connection pool limit not help, might have dbcp pool leak. investigate if returning connections pool correctly.
Comments
Post a Comment