Hi,
Our APP use JDBC to query data from Dremio. I just found an issue which is very strange.
If the time cost of a query is more than 1 minutes. I got an error like this:
And the stacktrace is like:
org.springframework.jdbc.UncategorizedSQLException: StatementCallback; uncategorized SQLException for SQL [SELECT “event_name”, EXTRACT(DOW FROM “slogtime_date”) AS day_period, COUNT(DISTINCT “session_id”) AS sessions FROM Datahub.kipawa.client_events WHERE “slogtime_datemonth” BETWEEN ‘2022-09-01’ AND ‘2022-11-01’ GROUP BY 1, 2 ORDER BY 1, 2]; SQL state [null]; error code [0]; Connection null <–> null (user client) closed unexpectedly. SabotNode down?; nested exception is java.sql.SQLException: Connection null <–> null (user client) closed unexpectedly. SabotNode down?
at org.springframework.jdbc.core.JdbcTemplate.translateException(JdbcTemplate.java:1542)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:393)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:406)
at com.telenav.datahub.model.driver.TNJdbcTemplate.query(TNJdbcTemplate.java:82)
at com.telenav.datahub.dao.v2.impl.NavDaoImpl.getNavSessionsByDay(NavDaoImpl.java:291)
at com.telenav.datahub.dao.v2.impl.NavDaoImpl$$FastClassBySpringCGLIB$$e9408fec.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:89)
at com.telenav.datahub.common.profiler.ProfilerAspect.logExecutionTime(ProfilerAspect.java:48)
at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:634)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:624)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:72)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:708)
at com.telenav.datahub.dao.v2.impl.NavDaoImpl$$EnhancerBySpringCGLIB$$84e10e94.getNavSessionsByDay()
at com.telenav.datahub.service.NavService.getNavSessionsByDay(NavService.java:137)
at com.telenav.datahub.service.NavService$$FastClassBySpringCGLIB$$9435a830.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:793)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:89)
at com.telenav.datahub.common.profiler.ProfilerAspect.logExecutionTime(ProfilerAspect.java:48)
at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:634)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:624)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:72)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.cache.interceptor.CacheInterceptor.lambda$invoke$0(CacheInterceptor.java:54)
at org.springframework.cache.interceptor.CacheAspectSupport.invokeOperation(CacheAspectSupport.java:366)
at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:421)
at org.springframework.cache.interceptor.CacheAspectSupport.execute(CacheAspectSupport.java:345)
at org.springframework.cache.interceptor.CacheInterceptor.invoke(CacheInterceptor.java:64)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:763)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:708)
at com.telenav.datahub.service.NavService$$EnhancerBySpringCGLIB$$f9c3f1b1.getNavSessionsByDay()
at com.telenav.datahub.service.ProductService.lambda$getProductDistribution$32(ProductService.java:477)
at com.telenav.datahub.common.measure.MeasureRunnable.run(MeasureRunnable.java:14)
at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1736)
at java.base/java.util.concurrent.CompletableFuture$AsyncRun.exec(CompletableFuture.java:1728)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:177)
Caused by: java.sql.SQLException: Connection null <–> null (user client) closed unexpectedly. SabotNode down?
at com.dremio.jdbc.impl.DremioCursor.nextRowInternally(DremioCursor.java:542)
at com.dremio.jdbc.impl.DremioCursor.loadInitialSchema(DremioCursor.java:628)
at com.dremio.jdbc.impl.DremioResultSetImpl.execute(DremioResultSetImpl.java:1883)
at com.dremio.jdbc.impl.DremioResultSetImpl.execute(DremioResultSetImpl.java:59)
at cdjd.org.apache.calcite.avatica.AvaticaConnection$1.execute(AvaticaConnection.java:667)
at com.dremio.jdbc.impl.DremioMetaImpl.prepareAndExecute(DremioMetaImpl.java:184)
at cdjd.org.apache.calcite.avatica.AvaticaConnection.prepareAndExecuteInternal(AvaticaConnection.java:675)
at com.dremio.jdbc.impl.DremioConnectionImpl.prepareAndExecuteInternal(DremioConnectionImpl.java:128)
at cdjd.org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:156)
at cdjd.org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:227)
at com.dremio.jdbc.impl.DremioStatementImpl.executeQuery(DremioStatementImpl.java:108)
at com.zaxxer.hikari.pool.ProxyStatement.executeQuery(ProxyStatement.java:110)
at com.zaxxer.hikari.pool.HikariProxyStatement.executeQuery(HikariProxyStatement.java)
at com.telenav.datahub.model.driver.TNJdbcTemplate$1QueryStatementCallback.doInStatement(TNJdbcTemplate.java:69)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:381)
… 60 common frames omitted
Caused by: cdjd.com.dremio.common.exceptions.UserException: Connection null <–> null (user client) closed unexpectedly. SabotNode down?
at cdjd.com.dremio.common.exceptions.UserException$Builder.build(UserException.java:804)
at cdjd.com.dremio.sabot.rpc.user.QueryResultHandler$ChannelClosedHandler$1.operationComplete(QueryResultHandler.java:374)
at cdjd.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577)
at cdjd.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570)
at cdjd.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549)
at cdjd.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490)
at cdjd.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615)
at cdjd.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:604)
at cdjd.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)
at cdjd.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
at cdjd.io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1158)
at cdjd.io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:760)
at cdjd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:736)
at cdjd.io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:607)
at cdjd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.closeOnRead(AbstractNioByteChannel.java:105)
at cdjd.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at cdjd.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
at cdjd.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)
at cdjd.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)
at cdjd.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
at cdjd.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
at cdjd.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.lang.Thread.run(Thread.java:834)
Is there any internal timeout setting?
Driver version is dremio-jdbc-driver:4.9.1-202010230218060541-2e764ed0
693ef986-5c2b-422e-bb75-424d6f2891eb.zip (608.8 KB)