even though I have used the dremio-admin upgrade command - which completed successfully, dremio will not start and I’m seeing the following error in the server.out file:
Dremio is exiting. Failure while starting services.
java.lang.IllegalStateException: KVStore has an older version (3.1.11-201904261857420193-c674472) than the server (3.2.0-201905102005330382-0598733), please run the upgrade tool first
at com.google.common.base.Preconditions.checkState(Preconditions.java:721)
at com.dremio.dac.daemon.DremioDaemon$AutoUpgrade.ensureUpgradeSupported(DremioDaemon.java:67)
at com.dremio.dac.cmd.upgrade.Upgrade.run(Upgrade.java:178)
at com.dremio.dac.cmd.upgrade.Upgrade.run(Upgrade.java:160)
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:97)
and the following in server.log:
019-05-15 14:52:24,119 [main] ERROR ROOT - Dremio is exiting. Failure while starting services.
java.lang.IllegalStateException: KVStore has an older version (3.1.11-201904261857420193-c674472) than the server (3.2.0-201905102005330382-0598733), please run the upgrade tool first
at com.google.common.base.Preconditions.checkState(Preconditions.java:721) ~[guava-20.0.jar:na]
at com.dremio.dac.daemon.DremioDaemon$AutoUpgrade.ensureUpgradeSupported(DremioDaemon.java:67) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.cmd.upgrade.Upgrade.run(Upgrade.java:178) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.cmd.upgrade.Upgrade.run(Upgrade.java:160) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:97) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
I reran ./dremio-admin upgrade as the dremio user account (instead of root) and now I’m getting a new error in server.out:
Dremio is exiting. Failure while starting services.
java.lang.NoSuchMethodError: org.eclipse.jetty.http.pathmap.PathMappings.put(Lorg/eclipse/jetty/http/pathmap/PathSpec;Ljava/lang/Object;)Z
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1421)
at org.eclipse.jetty.servlet.ServletHandler.doStart(ServletHandler.java:165)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.server.handler.ScopedHandler.doStart(ScopedHandler.java:123)
at org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:913)
at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:365)
at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:852)
at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:278)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.server.Server.start(Server.java:415)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.server.Server.doStart(Server.java:382)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at com.dremio.dac.server.WebServer.start(WebServer.java:257)
at com.dremio.service.SingletonRegistry$AbstractServiceReference.start(SingletonRegistry.java:137)
at com.dremio.service.ServiceRegistry.start(ServiceRegistry.java:74)
at com.dremio.service.SingletonRegistry.start(SingletonRegistry.java:33)
at com.dremio.dac.daemon.DACDaemon.startServices(DACDaemon.java:183)
at com.dremio.dac.daemon.DACDaemon.init(DACDaemon.java:189)
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:102)
2019-05-15 15:11:17,755 [main] ERROR ROOT - Dremio is exiting. Failure while starting services.
java.lang.NoSuchMethodError: org.eclipse.jetty.http.pathmap.PathMappings.put(Lorg/eclipse/jetty/http/pathmap/PathSpec;Ljava/lang/Object;)Z
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1421) ~[jetty-servlet-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.servlet.ServletHandler.doStart(ServletHandler.java:165) ~[jetty-servlet-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.ScopedHandler.doStart(ScopedHandler.java:123) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:913) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:365) ~[jetty-servlet-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:852) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:278) ~[jetty-servlet-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.Server.start(Server.java:415) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.server.Server.doStart(Server.java:382) ~[jetty-server-9.4.15.v20190215.jar:9.4.15.v20190215]
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) ~[jetty-util-9.4.15.v20190215.jar:9.4.15.v20190215]
at com.dremio.dac.server.WebServer.start(WebServer.java:257) ~[dremio-dac-backend-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.service.SingletonRegistry$AbstractServiceReference.start(SingletonRegistry.java:137) ~[dremio-common-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.service.ServiceRegistry.start(ServiceRegistry.java:74) ~[dremio-common-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.service.SingletonRegistry.start(SingletonRegistry.java:33) ~[dremio-common-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.daemon.DACDaemon.startServices(DACDaemon.java:183) ~[dremio-dac-backend-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.daemon.DACDaemon.init(DACDaemon.java:189) ~[dremio-dac-backend-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:102) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
I see the below. Can you please do a df -h from the mount rocksDB is on?
2019-05-15 19:29:21,974 [main] ERROR ROOT - Dremio is exiting. Failure while starting services.
java.lang.ExceptionInInitializerError: null
at org.rocksdb.Options.(Options.java:25) ~[rocksdbjni-5.14.2.jar:na]
at com.dremio.datastore.ByteStoreManager.start(ByteStoreManager.java:201) ~[dremio-services-datastore-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.datastore.CoreStoreProviderImpl.start(CoreStoreProviderImpl.java:195) ~[dremio-services-datastore-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.datastore.LocalKVStoreProvider.start(LocalKVStoreProvider.java:157) ~[dremio-services-datastore-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.cmd.upgrade.Upgrade.run(Upgrade.java:158) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:97) ~[dremio-dac-daemon-3.2.0-201905102005330382-0598733.jar:3.2.0-201905102005330382-0598733]
Caused by: java.lang.RuntimeException: Unable to load the RocksDB shared libraryjava.io.IOException: No space left on device
at org.rocksdb.RocksDB.loadLibrary(RocksDB.java:67) ~[rocksdbjni-5.14.2.jar:na]
at org.rocksdb.RocksDB.(RocksDB.java:35) ~[rocksdbjni-5.14.2.jar:na]
… 6 common frames omitted
How is the location of the rocksdb configured? I see our /tmp directory is 100% full and it looks like the rocksdb is saved there, but I’m not sure why it would be saved there. It looks like I need to change the location for rocksdb to something other than /tmp.
I think I have fixed the issue with /tmp running out of space, however now I’m now getting a new error message. I’m growing concerned because I have a proof of concept demo on Monday. The new error message I’m seeing is:
Dremio is exiting. Failure while starting services.
java.lang.NoSuchMethodError: org.eclipse.jetty.http.pathmap.PathMappings.put(Lorg/eclipse/jetty/http/pathmap/PathSpec;Ljava/lang/Object;)Z
at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1421)
at org.eclipse.jetty.servlet.ServletHandler.doStart(ServletHandler.java:165)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.server.handler.ScopedHandler.doStart(ScopedHandler.java:123)
at org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:913)
at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:365)
at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:852)
at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:278)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:138)
at org.eclipse.jetty.server.Server.start(Server.java:415)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:108)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:113)
at org.eclipse.jetty.server.Server.doStart(Server.java:382)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at com.dremio.dac.server.WebServer.start(WebServer.java:257)
at com.dremio.service.SingletonRegistry$AbstractServiceReference.start(SingletonRegistry.java:137)
at com.dremio.service.ServiceRegistry.start(ServiceRegistry.java:74)
at com.dremio.service.SingletonRegistry.start(SingletonRegistry.java:33)
at com.dremio.dac.daemon.DACDaemon.startServices(DACDaemon.java:183)
at com.dremio.dac.daemon.DACDaemon.init(DACDaemon.java:189)
at com.dremio.dac.daemon.DremioDaemon.main(DremioDaemon.java:102)
This suggests Dremio is trying to use the wrong jars. Can you do a recursive ls of your Dremio’s jars folder and post it here ? Thanks!
Also how did you install Dremio, rpm, tar ?