I have an index in elasticsearch that have the mapping
{
"mappings": {
"_doc": {
"timestamp": {
"type": "date",
"format": "epoch_second"
}
}
}
}
Yet somehow dremio failed to parse value in field timestamp. Here is the log
2020-09-22 06:57:38,606 [20965f9f-11fa-7260-7599-ed4dbfa9e900/0:foreman-planning] INFO c.d.p.elastic.execution.WriteHolders - User Error Occurred [ErrorId: 53dc6bbc-50c0-439d-9970-e2ce1558cd22]
com.dremio.common.exceptions.UserException: Failed to parse date time value 1.558344897E9 in field timestamp.
at com.dremio.common.exceptions.UserException$Builder.build(UserException.java:802)
at com.dremio.plugins.elastic.execution.WriteHolders.getMillis(WriteHolders.java:370)
at com.dremio.plugins.elastic.execution.WriteHolders.access$100(WriteHolders.java:45)
at com.dremio.plugins.elastic.execution.WriteHolders$TimestampWriteHolder.write(WriteHolders.java:356)
at com.dremio.plugins.elastic.execution.WriteHolders$TimestampWriteHolder.writeMap(WriteHolders.java:347)
at com.dremio.plugins.elastic.execution.FieldReadDefinition.writeMap(FieldReadDefinition.java:178)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.writeDeclaredMap(ElasticsearchJsonReader.java:326)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.writeToVector(ElasticsearchJsonReader.java:213)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.write(ElasticsearchJsonReader.java:181)
at com.dremio.plugins.elastic.execution.ElasticsearchRecordReader.next(ElasticsearchRecordReader.java:279)
at com.dremio.plugins.elastic.ElasticDatasetMetadata.getSampledSchema(ElasticDatasetMetadata.java:162)
at com.dremio.plugins.elastic.ElasticDatasetMetadata.build(ElasticDatasetMetadata.java:97)
at com.dremio.plugins.elastic.ElasticsearchStoragePlugin.getDatasetMetadata(ElasticsearchStoragePlugin.java:402)
at com.dremio.exec.catalog.DatasetSaver.save(DatasetSaver.java:98)
at com.dremio.exec.catalog.DatasetSaver.save(DatasetSaver.java:142)
at com.dremio.exec.catalog.DatasetManager.getTableFromPlugin(DatasetManager.java:356)
at com.dremio.exec.catalog.DatasetManager.getTable(DatasetManager.java:207)
at com.dremio.exec.catalog.CatalogImpl.getTable(CatalogImpl.java:185)
at com.dremio.exec.catalog.SourceAccessChecker.lambda$getTable$3(SourceAccessChecker.java:126)
at com.dremio.exec.catalog.SourceAccessChecker.getIfVisible(SourceAccessChecker.java:90)
at com.dremio.exec.catalog.SourceAccessChecker.getTable(SourceAccessChecker.java:126)
at com.dremio.exec.catalog.DelegatingCatalog.getTable(DelegatingCatalog.java:88)
at com.dremio.exec.catalog.CachingCatalog.getTable(CachingCatalog.java:93)
at com.dremio.exec.catalog.DremioCatalogReader.getTable(DremioCatalogReader.java:94)
at com.dremio.exec.catalog.DremioCatalogReader.getTable(DremioCatalogReader.java:71)
at org.apache.calcite.sql.validate.EmptyScope.getTableNamespace(EmptyScope.java:76)
at org.apache.calcite.sql.validate.DelegatingScope.getTableNamespace(DelegatingScope.java:197)
at org.apache.calcite.sql.validate.IdentifierNamespace.resolveImpl(IdentifierNamespace.java:102)
at org.apache.calcite.sql.validate.IdentifierNamespace.validateImpl(IdentifierNamespace.java:120)
at org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:943)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:924)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:2971)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:2956)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3197)
at org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
at org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:943)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:924)
at org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:226)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:899)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:609)
at com.dremio.exec.planner.sql.SqlConverter.validate(SqlConverter.java:226)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateNode(PrelTransformer.java:196)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateAndConvert(PrelTransformer.java:181)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateAndConvert(PrelTransformer.java:177)
at com.dremio.exec.planner.sql.handlers.query.NormalHandler.getPlan(NormalHandler.java:47)
at com.dremio.exec.planner.sql.handlers.commands.HandlerToExec.plan(HandlerToExec.java:59)
at com.dremio.exec.work.foreman.AttemptManager.plan(AttemptManager.java:428)
at com.dremio.exec.work.foreman.AttemptManager.lambda$run$1(AttemptManager.java:337)
at com.dremio.service.commandpool.CommandWrapper.run(CommandWrapper.java:62)
at com.dremio.context.RequestContext.run(RequestContext.java:95)
at com.dremio.common.concurrent.ContextMigratingExecutorService.lambda$decorate$3(ContextMigratingExecutorService.java:199)
at com.dremio.common.concurrent.ContextMigratingExecutorService$ComparableRunnable.run(ContextMigratingExecutorService.java:180)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-09-22 06:57:38,622 [20965f9f-11fa-7260-7599-ed4dbfa9e900/0:foreman-planning] ERROR c.d.s.commandpool.CommandWrapper - command 20965f9f-11fa-7260-7599-ed4dbfa9e900/0:foreman-planning failed
com.dremio.common.exceptions.UserException: Failed to parse date time value 1.558344897E9 in field timestamp.
at com.dremio.common.exceptions.UserException$Builder.build(UserException.java:802)
at com.dremio.plugins.elastic.execution.WriteHolders.getMillis(WriteHolders.java:370)
at com.dremio.plugins.elastic.execution.WriteHolders.access$100(WriteHolders.java:45)
at com.dremio.plugins.elastic.execution.WriteHolders$TimestampWriteHolder.write(WriteHolders.java:356)
at com.dremio.plugins.elastic.execution.WriteHolders$TimestampWriteHolder.writeMap(WriteHolders.java:347)
at com.dremio.plugins.elastic.execution.FieldReadDefinition.writeMap(FieldReadDefinition.java:178)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.writeDeclaredMap(ElasticsearchJsonReader.java:326)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.writeToVector(ElasticsearchJsonReader.java:213)
at com.dremio.plugins.elastic.execution.ElasticsearchJsonReader.write(ElasticsearchJsonReader.java:181)
at com.dremio.plugins.elastic.execution.ElasticsearchRecordReader.next(ElasticsearchRecordReader.java:279)
at com.dremio.plugins.elastic.ElasticDatasetMetadata.getSampledSchema(ElasticDatasetMetadata.java:162)
at com.dremio.plugins.elastic.ElasticDatasetMetadata.build(ElasticDatasetMetadata.java:97)
at com.dremio.plugins.elastic.ElasticsearchStoragePlugin.getDatasetMetadata(ElasticsearchStoragePlugin.java:402)
at com.dremio.exec.catalog.DatasetSaver.save(DatasetSaver.java:98)
at com.dremio.exec.catalog.DatasetSaver.save(DatasetSaver.java:142)
at com.dremio.exec.catalog.DatasetManager.getTableFromPlugin(DatasetManager.java:356)
at com.dremio.exec.catalog.DatasetManager.getTable(DatasetManager.java:207)
at com.dremio.exec.catalog.CatalogImpl.getTable(CatalogImpl.java:185)
at com.dremio.exec.catalog.SourceAccessChecker.lambda$getTable$3(SourceAccessChecker.java:126)
at com.dremio.exec.catalog.SourceAccessChecker.getIfVisible(SourceAccessChecker.java:90)
at com.dremio.exec.catalog.SourceAccessChecker.getTable(SourceAccessChecker.java:126)
at com.dremio.exec.catalog.DelegatingCatalog.getTable(DelegatingCatalog.java:88)
at com.dremio.exec.catalog.CachingCatalog.getTable(CachingCatalog.java:93)
at com.dremio.exec.catalog.DremioCatalogReader.getTable(DremioCatalogReader.java:94)
at com.dremio.exec.catalog.DremioCatalogReader.getTable(DremioCatalogReader.java:71)
at org.apache.calcite.sql.validate.EmptyScope.getTableNamespace(EmptyScope.java:76)
at org.apache.calcite.sql.validate.DelegatingScope.getTableNamespace(DelegatingScope.java:197)
at org.apache.calcite.sql.validate.IdentifierNamespace.resolveImpl(IdentifierNamespace.java:102)
at org.apache.calcite.sql.validate.IdentifierNamespace.validateImpl(IdentifierNamespace.java:120)
at org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:943)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:924)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:2971)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateFrom(SqlValidatorImpl.java:2956)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3197)
at org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:60)
at org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:84)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:943)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:924)
at org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:226)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:899)
at org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:609)
at com.dremio.exec.planner.sql.SqlConverter.validate(SqlConverter.java:226)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateNode(PrelTransformer.java:196)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateAndConvert(PrelTransformer.java:181)
at com.dremio.exec.planner.sql.handlers.PrelTransformer.validateAndConvert(PrelTransformer.java:177)
at com.dremio.exec.planner.sql.handlers.query.NormalHandler.getPlan(NormalHandler.java:47)
at com.dremio.exec.planner.sql.handlers.commands.HandlerToExec.plan(HandlerToExec.java:59)
at com.dremio.exec.work.foreman.AttemptManager.plan(AttemptManager.java:428)
at com.dremio.exec.work.foreman.AttemptManager.lambda$run$1(AttemptManager.java:337)
at com.dremio.service.commandpool.CommandWrapper.run(CommandWrapper.java:62)
at com.dremio.context.RequestContext.run(RequestContext.java:95)
at com.dremio.common.concurrent.ContextMigratingExecutorService.lambda$decorate$3(ContextMigratingExecutorService.java:199)
at com.dremio.common.concurrent.ContextMigratingExecutorService$ComparableRunnable.run(ContextMigratingExecutorService.java:180)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2020-09-22 06:57:38,632 [out-of-band-observer] INFO query.logger - {"queryId":"20965f9f-11fa-7260-7599-ed4dbfa9e900","schema":"[datasource_sna]","queryText":"SELECT education FROM \"datasource_sna\".\"sna-0\".\"_doc\"","start":1600757856578,"finish":1600757858623,"outcome":"FAILED","outcomeReason":"Failed to parse date time value 1.558344897E9 in field timestamp.","username":"admin"}
I’ve checked every topic that discussed this bug but it seemed that there was no solution to this. Or maybe I miss something?