From 299defb7410ec7d7ed851200c41fafdb0db12741 Mon Sep 17 00:00:00 2001 From: DOU <15730031613@163.com> Date: Fri, 16 Sep 2022 10:56:26 +0800 Subject: [PATCH 1/3] Parameter omnidata.client.task.timeout can be configured --- .../io/prestosql/plugin/hive/HiveConfig.java | 14 ++++++++++ .../plugin/hive/orc/OrcPageSourceFactory.java | 5 ++++ .../parquet/ParquetPageSourceFactory.java | 5 ++++ .../omnidata/transfer/OmniDataProperty.java | 5 ++++ .../boostkit/omnidata/util/ConfigUtils.java | 27 +++++++++++++++++++ 5 files changed, 56 insertions(+) create mode 100644 omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HiveConfig.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HiveConfig.java index b5cfceb7d..0b0894590 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HiveConfig.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HiveConfig.java @@ -238,6 +238,8 @@ public class HiveConfig private boolean workerMetaStoreCacheEnabled; + private int omniDataClientTaskTimeout = 300; // OmniData client task timeout, 300 seconds + @Config("hive.max-initial-splits") public HiveConfig setMaxInitialSplits(int maxInitialSplits) { @@ -1987,4 +1989,16 @@ public class HiveConfig { return minOffloadRowNumber; } + + @Config("omnidata.client.task.timeout") + public HiveConfig setOmniDataClientTaskTimeout(int omniDataClientTaskTimeout) + { + this.omniDataClientTaskTimeout = omniDataClientTaskTimeout; + return this; + } + + public int getOmniDataClientTaskTimeout() + { + return this.omniDataClientTaskTimeout; + } } diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java index b106d365b..165b603b9 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java @@ -98,6 +98,7 @@ import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Maps.uniqueIndex; import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TARGET_LIST; +import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TASK_TIMEOUT; import static io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.prestosql.orc.OrcReader.INITIAL_BATCH_SIZE; import static io.prestosql.orc.OrcReader.handleCacheLoadException; @@ -154,6 +155,7 @@ public class OrcPageSourceFactory private final OrcCacheStore orcCacheStore; private final int domainCompactionThreshold; private final DateTimeZone legacyTimeZone; + private final int omniDataClientTaskTimeout; private String omniDataServerTarget; @Inject @@ -167,6 +169,8 @@ public class OrcPageSourceFactory this.orcCacheStore = orcCacheStore; this.domainCompactionThreshold = config.getDomainCompactionThreshold(); this.legacyTimeZone = requireNonNull(config, "hiveConfig is null").getOrcLegacyDateTimeZone(); + this.omniDataClientTaskTimeout = + requireNonNull(config, "hiveConfig is null").getOmniDataClientTaskTimeout(); this.omniDataServerTarget = null; } @@ -544,6 +548,7 @@ public class OrcPageSourceFactory AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); Properties transProperties = new Properties(); transProperties.put(OMNIDATA_CLIENT_TARGET_LIST, omniDataServerTarget); + transProperties.put(OMNIDATA_CLIENT_TASK_TIMEOUT, this.omniDataClientTaskTimeout); DataSource orcPushDownDataSource = new com.huawei.boostkit.omnidata.model.datasource.hdfs.HdfsOrcDataSource( path.toString(), diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java index dbccc9416..4bd1204a9 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -78,6 +78,7 @@ import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.nullToEmpty; import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TARGET_LIST; +import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TASK_TIMEOUT; import static io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.prestosql.parquet.ParquetTypeUtils.getColumnIO; import static io.prestosql.parquet.ParquetTypeUtils.getDescriptors; @@ -114,6 +115,7 @@ public class ParquetPageSourceFactory private final FileFormatDataSourceStats stats; private final DateTimeZone timeZone; + private final int omniDataClientTaskTimeout; private String omniDataServerTarget; @Inject @@ -123,6 +125,8 @@ public class ParquetPageSourceFactory this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.stats = requireNonNull(stats, "stats is null"); this.timeZone = requireNonNull(hiveConfig, "hiveConfig is null").getParquetDateTimeZone(); + this.omniDataClientTaskTimeout = + requireNonNull(hiveConfig, "hiveConfig is null").getOmniDataClientTaskTimeout(); } @Override @@ -346,6 +350,7 @@ public class ParquetPageSourceFactory AggregatedMemoryContext systemMemoryUsage = newSimpleAggregatedMemoryContext(); Properties transProperties = new Properties(); transProperties.put(OMNIDATA_CLIENT_TARGET_LIST, omniDataServerTarget); + transProperties.put(OMNIDATA_CLIENT_TASK_TIMEOUT, this.omniDataClientTaskTimeout); DataSource parquetPushDownDataSource = new com.huawei.boostkit.omnidata.model.datasource.hdfs.HdfsParquetDataSource(path.toString(), start, length, false); diff --git a/omnidata/omnidata-openlookeng-connector/stub/client/src/main/java/com/huawei/boostkit/omnidata/transfer/OmniDataProperty.java b/omnidata/omnidata-openlookeng-connector/stub/client/src/main/java/com/huawei/boostkit/omnidata/transfer/OmniDataProperty.java index 328cf8abe..c2326f65c 100644 --- a/omnidata/omnidata-openlookeng-connector/stub/client/src/main/java/com/huawei/boostkit/omnidata/transfer/OmniDataProperty.java +++ b/omnidata/omnidata-openlookeng-connector/stub/client/src/main/java/com/huawei/boostkit/omnidata/transfer/OmniDataProperty.java @@ -27,5 +27,10 @@ public class OmniDataProperty { * constant string for "omnidata.client.target" */ public static final String OMNIDATA_CLIENT_TARGET = "omnidata.client.target"; + + /** + * constant string for "omnidata.client.task.timeout" + */ + public static final String OMNIDATA_CLIENT_TASK_TIMEOUT = "omnidata.client.task.timeout"; } diff --git a/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java b/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java new file mode 100644 index 000000000..cb6cd7826 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.huawei.boostkit.omnidata.util; + +public class ConfigUtils { + public static void load(String path) + { + } + + public static T loadConfig(Class classConfig) + { + return null; + } +} -- Gitee From 2644e0a714fdde12b036c59a283d8870f1b9e787 Mon Sep 17 00:00:00 2001 From: DOU <15730031613@163.com> Date: Sat, 17 Sep 2022 16:09:21 +0800 Subject: [PATCH 2/3] Fit the timeout --- .../prestosql/plugin/hive/HivePageSourceProvider.java | 11 +++++++++-- .../io/prestosql/plugin/hive/TestHiveFileFormats.java | 8 ++++++-- .../plugin/hive/TestOrcPageSourceMemoryTracking.java | 6 +++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java index 9af74c750..ae5d0af4e 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java @@ -90,6 +90,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.Maps.uniqueIndex; import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.HOSTADDRESS_DELIMITER; import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TARGET_LIST; +import static com.huawei.boostkit.omnidata.transfer.OmniDataProperty.OMNIDATA_CLIENT_TASK_TIMEOUT; import static io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.DUMMY_OFFLOADED; import static io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; import static io.prestosql.plugin.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX; @@ -117,6 +118,7 @@ public class HivePageSourceProvider private final IndexCache indexCache; private final Set selectivePageSourceFactories; private final OmniDataNodeManager omniDataNodeManager; + private final int omniDataClientTaskTimeout; @Inject public HivePageSourceProvider( @@ -138,6 +140,8 @@ public class HivePageSourceProvider this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.indexCache = indexCache; this.selectivePageSourceFactories = selectivePageSourceFactories; + this.omniDataClientTaskTimeout = + requireNonNull(hiveConfig, "hiveConfig is null").getOmniDataClientTaskTimeout(); } public HivePageSourceProvider( @@ -158,6 +162,8 @@ public class HivePageSourceProvider this.indexCache = indexCache; this.selectivePageSourceFactories = selectivePageSourceFactories; this.omniDataNodeManager = null; + this.omniDataClientTaskTimeout = + requireNonNull(hiveConfig, "hiveConfig is null").getOmniDataClientTaskTimeout(); } @Override @@ -516,7 +522,7 @@ public class HivePageSourceProvider throw new IllegalStateException("Could not find a file reader for split " + split); } - public static Optional createHivePageSource( + public Optional createHivePageSource( Set cursorProviders, Set pageSourceFactories, Configuration configuration, @@ -674,7 +680,7 @@ public class HivePageSourceProvider return Optional.empty(); } - private static ConnectorPageSource createPushDownPageSource( + private ConnectorPageSource createPushDownPageSource( Path path, long start, long length, @@ -686,6 +692,7 @@ public class HivePageSourceProvider AggregatedMemoryContext systemMemoryUsage = AggregatedMemoryContext.newSimpleAggregatedMemoryContext(); Properties transProperties = new Properties(); transProperties.put(OMNIDATA_CLIENT_TARGET_LIST, omniDataServerTarget); + transProperties.put(OMNIDATA_CLIENT_TASK_TIMEOUT, this.omniDataClientTaskTimeout); DataSource pushDownDataSource = new HdfsRecordDataSource(path.toString(), start, length, fileSize, schema); diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java index 1074c515a..25af4c844 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestHiveFileFormats.java @@ -76,6 +76,7 @@ import static io.prestosql.plugin.hive.HiveStorageFormat.TEXTFILE; import static io.prestosql.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER; import static io.prestosql.plugin.hive.HiveTestUtils.createGenericHiveRecordCursorProvider; +import static io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; @@ -94,6 +95,7 @@ public class TestHiveFileFormats private static final FileFormatDataSourceStats STATS = new FileFormatDataSourceStats(); private static TestingConnectorSession parquetPageSourceSession = new TestingConnectorSession(new HiveSessionProperties(createParquetHiveConfig(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); private static TestingConnectorSession parquetPageSourceSessionUseName = new TestingConnectorSession(new HiveSessionProperties(createParquetHiveConfig(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); + private static HivePageSourceProvider hivePageSourceProvider; @DataProvider(name = "rowCount") public static Object[][] rowCountProvider() @@ -104,6 +106,8 @@ public class TestHiveFileFormats @BeforeClass(alwaysRun = true) public void setUp() { + HiveConfig config = new HiveConfig(); + hivePageSourceProvider = new HivePageSourceProvider(config, HiveTestUtils.createTestHdfsEnvironment(config), HiveTestUtils.getDefaultHiveRecordCursorProvider(config), HiveTestUtils.getDefaultHiveDataStreamFactories(config), HiveTestUtils.TYPE_MANAGER, HiveTestUtils.getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config)); // ensure the expected timezone is configured for this VM assertEquals(TimeZone.getDefault().getID(), "America/Bahia_Banderas", @@ -582,7 +586,7 @@ public class TestHiveFileFormats Configuration configuration = new Configuration(); configuration.set("io.compression.codecs", LzoCodec.class.getName() + "," + LzopCodec.class.getName()); - Optional pageSource = HivePageSourceProvider.createHivePageSource( + Optional pageSource = hivePageSourceProvider.createHivePageSource( ImmutableSet.of(cursorProvider), ImmutableSet.of(), configuration, @@ -638,7 +642,7 @@ public class TestHiveFileFormats List columnHandles = getColumnHandles(testColumns); - Optional pageSource = HivePageSourceProvider.createHivePageSource( + Optional pageSource = hivePageSourceProvider.createHivePageSource( ImmutableSet.of(), ImmutableSet.of(sourceFactory), new Configuration(), diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestOrcPageSourceMemoryTracking.java b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestOrcPageSourceMemoryTracking.java index 5cb88ec92..4d8ff7917 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestOrcPageSourceMemoryTracking.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/TestOrcPageSourceMemoryTracking.java @@ -113,6 +113,7 @@ import static io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR; import static io.prestosql.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.prestosql.plugin.hive.HiveTestUtils.SESSION; import static io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER; +import static io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories; import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; import static io.prestosql.sql.relational.Expressions.field; import static io.prestosql.testing.TestingHandles.TEST_TABLE_HANDLE; @@ -153,6 +154,7 @@ public class TestOrcPageSourceMemoryTracking private File tempFile; private TestPreparer testPreparer; + private static HivePageSourceProvider hivePageSourceProvider; @DataProvider(name = "rowCount") public static Object[][] rowCount() @@ -164,6 +166,8 @@ public class TestOrcPageSourceMemoryTracking public void setUp() throws Exception { + HiveConfig config = new HiveConfig(); + hivePageSourceProvider = new HivePageSourceProvider(config, HiveTestUtils.createTestHdfsEnvironment(config), HiveTestUtils.getDefaultHiveRecordCursorProvider(config), HiveTestUtils.getDefaultHiveDataStreamFactories(config), HiveTestUtils.TYPE_MANAGER, HiveTestUtils.getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config)); tempFile = File.createTempFile("presto_test_orc_page_source_memory_tracking", "orc"); tempFile.delete(); testPreparer = new TestPreparer(tempFile.getCanonicalPath()); @@ -492,7 +496,7 @@ public class TestOrcPageSourceMemoryTracking Duration.ofMillis(new HiveConfig().getOrcBloomFiltersCacheTtl().toMillis()), new HiveConfig().getOrcRowDataCacheMaximumWeight(), Duration.ofMillis(new HiveConfig().getOrcRowDataCacheTtl().toMillis()), new HiveConfig().isOrcCacheStatsMetricCollectionEnabled())); - return HivePageSourceProvider.createHivePageSource( + return hivePageSourceProvider.createHivePageSource( ImmutableSet.of(), ImmutableSet.of(orcPageSourceFactory), new Configuration(), -- Gitee From ca93e649ec79b7d9faf6840c81f67a5d4392ccfa Mon Sep 17 00:00:00 2001 From: DOU <15730031613@163.com> Date: Sat, 17 Sep 2022 16:33:41 +0800 Subject: [PATCH 3/3] delete ConfigUtils.java file --- .../boostkit/omnidata/util/ConfigUtils.java | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java diff --git a/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java b/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java deleted file mode 100644 index cb6cd7826..000000000 --- a/omnidata/omnidata-openlookeng-connector/stub/common/src/main/java/com/huawei/boostkit/omnidata/util/ConfigUtils.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.huawei.boostkit.omnidata.util; - -public class ConfigUtils { - public static void load(String path) - { - } - - public static T loadConfig(Class classConfig) - { - return null; - } -} -- Gitee