HTTP ERROR 500 Problem accessing /jobs. Reason:

HTTP ERROR 500
Problem accessing /jobs. Reason:

Server Error

Caused by:
com.dremio.datastore.DatastoreException: Failed to get from store id: sys.options, config: name: sys.options, key serializer:com.dremio.datastore.StringSerializer, value serializer: com.dremio.datastore.PassThroughSerializer, version extractor: null, document converter: null
at com.dremio.datastore.RemoteKVStore.get(RemoteKVStore.java:91)
at com.dremio.exec.store.sys.store.KVPersistentStore.get(KVPersistentStore.java:81)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:195)
at com.dremio.exec.server.options.BaseOptionManager.getOptionSafe(BaseOptionManager.java:34)
at com.dremio.exec.server.options.BaseOptionManager.getOption(BaseOptionManager.java:54)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:53)
at com.dremio.dac.server.IndexServlet.service(IndexServlet.java:107)
at com.dremio.dac.server.DremioServlet.service(DremioServlet.java:74)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)
at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:365)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:258)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.dremio.exec.rpc.RpcException: java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.dremio.services.fabric.simple.EndpointCreator$RequestSender.send(EndpointCreator.java:84)
at com.dremio.datastore.DatastoreRpcClient.get(DatastoreRpcClient.java:91)
at com.dremio.datastore.RemoteKVStore.get(RemoteKVStore.java:84)
… 27 more
Caused by: java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:269)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96)
at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:69)
at com.google.common.util.concurrent.AbstractCheckedFuture.checkedGet(AbstractCheckedFuture.java:107)
at com.dremio.services.fabric.simple.EndpointCreator$RequestSender.send(EndpointCreator.java:82)
… 29 more
Caused by:
com.dremio.exec.rpc.RpcException: java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.dremio.services.fabric.simple.EndpointCreator$RequestSender.send(EndpointCreator.java:84)
at com.dremio.datastore.DatastoreRpcClient.get(DatastoreRpcClient.java:91)
at com.dremio.datastore.RemoteKVStore.get(RemoteKVStore.java:84)
at com.dremio.exec.store.sys.store.KVPersistentStore.get(KVPersistentStore.java:81)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:195)
at com.dremio.exec.server.options.BaseOptionManager.getOptionSafe(BaseOptionManager.java:34)
at com.dremio.exec.server.options.BaseOptionManager.getOption(BaseOptionManager.java:54)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:53)
at com.dremio.dac.server.IndexServlet.service(IndexServlet.java:107)
at com.dremio.dac.server.DremioServlet.service(DremioServlet.java:74)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)
at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:365)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:258)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:269)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96)
at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:69)
at com.google.common.util.concurrent.AbstractCheckedFuture.checkedGet(AbstractCheckedFuture.java:107)
at com.dremio.services.fabric.simple.EndpointCreator$RequestSender.send(EndpointCreator.java:82)
… 29 more
Caused by:
java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:269)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96)
at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:69)
at com.google.common.util.concurrent.AbstractCheckedFuture.checkedGet(AbstractCheckedFuture.java:107)
at com.dremio.services.fabric.simple.EndpointCreator$RequestSender.send(EndpointCreator.java:82)
at com.dremio.datastore.DatastoreRpcClient.get(DatastoreRpcClient.java:91)
at com.dremio.datastore.RemoteKVStore.get(RemoteKVStore.java:84)
at com.dremio.exec.store.sys.store.KVPersistentStore.get(KVPersistentStore.java:81)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:195)
at com.dremio.exec.server.options.BaseOptionManager.getOptionSafe(BaseOptionManager.java:34)
at com.dremio.exec.server.options.BaseOptionManager.getOption(BaseOptionManager.java:54)
at com.dremio.exec.server.options.SystemOptionManager.getOption(SystemOptionManager.java:53)
at com.dremio.dac.server.IndexServlet.service(IndexServlet.java:107)
at com.dremio.dac.server.DremioServlet.service(DremioServlet.java:74)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1669)
at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83)
at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:365)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.handler.RequestLogHandler.handle(RequestLogHandler.java:95)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:258)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:745)

Hi @JoyJava

Can we please make sure other Dremio functionality is working. Like can you submit new queries? What happens when you refresh the jobs page?

Thanks,
@balaji.ramaswamy