Error Message:
cdjd.com.dremio.exec.vector.accessor.InvalidAccessException: Requesting value of type Timestamp for an object of type TIME:OPTIONAL is not allowed.
at cdjd.com.dremio.exec.vector.accessor.AbstractSqlAccessor.newInvalidAccessException(AbstractSqlAccessor.java:114)
at cdjd.com.dremio.exec.vector.accessor.AbstractSqlAccessor.getTimestamp(AbstractSqlAccessor.java:108)
at cdjd.com.dremio.exec.vector.accessor.BoundCheckingAccessor.getTimestamp(BoundCheckingAccessor.java:133)
at com.dremio.jdbc.impl.TypeConvertingSqlAccessor.getTimestamp(TypeConvertingSqlAccessor.java:722)
at com.dremio.jdbc.impl.SqlAccessorWrapper.getTimestamp(SqlAccessorWrapper.java:222)
at cdjd.org.apache.calcite.avatica.AvaticaResultSet.getTimestamp(AvaticaResultSet.java:288)
at com.dremio.jdbc.impl.DremioResultSetImpl.getTimestamp(DremioResultSetImpl.java:262)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$org$apache$spark$sql$execution$datasources$jdbc$JdbcUtils$$makeGetter$11.apply(JdbcUtils.scala:448)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$org$apache$spark$sql$execution$datasources$jdbc$JdbcUtils$$makeGetter$11.apply(JdbcUtils.scala:447)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:356)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:338)
at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636)
at org.apache.spark.sql.execution.UnsafeExternalRowSorter.sort(UnsafeExternalRowSorter.java:216)
at org.apache.spark.sql.execution.SortExec$$anonfun$1.apply(SortExec.scala:108)
at org.apache.spark.sql.execution.SortExec$$anonfun$1.apply(SortExec.scala:101)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:836)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:836)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
More specifically:
Requesting value of type Timestamp for an object of type TIME:OPTIONAL is not allowed
I try to connect to dremio using jdbc with pyspark using following code
df = glueContext.read.format("jdbc").option("dbtable", args["SOURCE_CONNECTION_DBTABLE"]).option("url", f"jdbc:dremio:direct={host}:{port}").option("user", uid).option("password", pwd).option("driver", f"{driver}").option("SSL", "true").option("disableHostVerification", "true").option("DisableCertificateVerification", "true").load()