I have Dremio Installed on azure Kubernate service its was running fime from last 15 days but suddenly today it was Getting error in GUI,
Here is some of the log Captured from container
seableThreadPool@4abbe01c[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 1746391]
com.dremio.common.exceptions.UserException: RejectedExecutionException: Task com.dremio.exec.work.foreman.AttemptManager@2a5835bf rejected from com.dremio.exec.rpc.CloseableThreadPool@4abbe01c[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 1746391]
at com.dremio.common.exceptions.UserException$Builder.build(UserException.java:746) ~[dremio-common-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$ResponseSenderImpl.sendFailure(RpcBus.java:287) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$ResponseSenderImpl.access$700(RpcBus.java:220) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$RequestEvent.run(RpcBus.java:467) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.common.SerializedExecutor$RunnableProcessor.run(SerializedExecutor.java:87) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$SameExecutor.execute(RpcBus.java:328) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.common.SerializedExecutor.execute(SerializedExecutor.java:121) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:362) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$InboundHandler.decode(RpcBus.java:333) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88) [netty-codec-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.22.Final.jar:4.1.22.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1414) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:945) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:146) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) [netty-transport-4.1.22.Final.jar:4.1.22.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:886) [netty-common-4.1.22.Final.jar:4.1.22.Final]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_181]
Caused by: com.dremio.exec.rpc.UserRpcException: java.util.concurrent.RejectedExecutionException: Task com.dremio.exec.work.foreman.AttemptManager@2a5835bf rejected from com.dremio.exec.rpc.CloseableThreadPool@4abbe01c[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 1746391]
at com.dremio.exec.rpc.RpcBus$RequestEvent.run(RpcBus.java:466) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
… 33 common frames omitted
Caused by: java.util.concurrent.RejectedExecutionException: Task com.dremio.exec.work.foreman.AttemptManager@2a5835bf rejected from com.dremio.exec.rpc.CloseableThreadPool@4abbe01c[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 1746391]
at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) ~[na:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[na:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[na:1.8.0_181]
at com.dremio.exec.work.protector.Foreman.newAttempt(Foreman.java:164) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.work.protector.Foreman.start(Foreman.java:132) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.work.protector.ForemenWorkManager.submit(ForemenWorkManager.java:185) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.work.protector.ForemenWorkManager$UserWorkerImpl.submitWork(ForemenWorkManager.java:482) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.sabot.rpc.user.UserRPCServer.handle(UserRPCServer.java:226) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.sabot.rpc.user.UserRPCServer.handle(UserRPCServer.java:92) ~[dremio-sabot-kernel-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
at com.dremio.exec.rpc.RpcBus$RequestEvent.run(RpcBus.java:462) [dremio-services-base-rpc-3.1.1-201901281837360699-30c9d74.jar:3.1.1-201901281837360699-30c9d74]
… 33 common frames omitted