From 34b0c0975c242cca418fee2c2554b2395dd53afa Mon Sep 17 00:00:00 2001 From: hyy_cyan Date: Mon, 30 Sep 2024 14:22:18 +0800 Subject: [PATCH 1/2] remove filesystem --- .../omniop-native-reader/cpp/src/orcfile/OrcFileOverride.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/omnioperator/omniop-native-reader/cpp/src/orcfile/OrcFileOverride.cc b/omnioperator/omniop-native-reader/cpp/src/orcfile/OrcFileOverride.cc index 763bbcfe6..c5e771912 100644 --- a/omnioperator/omniop-native-reader/cpp/src/orcfile/OrcFileOverride.cc +++ b/omnioperator/omniop-native-reader/cpp/src/orcfile/OrcFileOverride.cc @@ -17,7 +17,6 @@ */ #include "OrcFileOverride.hh" -#include #define O_BINARY 0 @@ -35,11 +34,6 @@ namespace orc { return orc::createHdfsFileOutputStream(uri); } else { - std::filesystem::path path(std::string(uri.Path())); - auto res = std::filesystem::create_directories(path.parent_path()); - if (!res) { - throw std::runtime_error("Create local directories fail"); - } return orc::writeLocalFile(std::string(uri.Path())); } } -- Gitee From 355921b29e0211c23e6f6000822c2a32020e6382 Mon Sep 17 00:00:00 2001 From: hyy_cyan Date: Mon, 30 Sep 2024 14:37:15 +0800 Subject: [PATCH 2/2] remove filesystem dependency ut --- .../spark/TableWriteBasicFunctionSuite.scala | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/TableWriteBasicFunctionSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/TableWriteBasicFunctionSuite.scala index 0fb1c6055..d2c06b15b 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/TableWriteBasicFunctionSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/TableWriteBasicFunctionSuite.scala @@ -48,39 +48,6 @@ class TableWriteBasicFunctionSuite extends QueryTest with SharedSparkSession { super.beforeAll() } - test("Insert basic data (Non-partitioned table)") { - val drop = spark.sql("drop table if exists employees_for_table_write_ut_test") - drop.collect() - val employees = Seq[(String, String, Int, Int)]( - ("Lisa", "Sales", 10000, 35), - ).toDF("name", "dept", "salary", "age") - employees.write.format("orc").saveAsTable("employees_for_table_write_ut_test") - - val insert = spark.sql("insert into " + - "employees_for_table_write_ut_test values('Maggie', 'Sales', 1, 2)") - insert.collect() - val select = spark.sql("select * from employees_for_table_write_ut_test") - val runRows = select.collect() - val expectedRows = Seq(Row("Lisa", "Sales", 10000, 35), Row("Maggie", "Sales", 1, 2)) - assert(QueryTest.sameRows(runRows, expectedRows).isEmpty, "the run value is error") - } - - test("Insert Basic data (Partitioned table)") { - val drop = spark.sql("drop table if exists employees_for_table_write_ut_partition_test") - drop.collect() - val employees = Seq(("Lisa", "Sales", 10000, 35)).toDF("name", "dept", "salary", "age") - employees.write.format("orc").partitionBy("age") - .saveAsTable("employees_for_table_write_ut_partition_test") - val insert = spark.sql("insert into employees_for_table_write_ut_partition_test " + - "values('Maggie','Sales',200,30),('Bob','Sales',2000,30),('Tom','Sales',5000,20)") - insert.collect() - val select = spark.sql("select * from employees_for_table_write_ut_partition_test") - val runRows = select.collect() - val expectedRows = Seq(Row("Lisa", "Sales", 10000, 35), Row("Maggie", "Sales", 200, 30), - Row("Bob", "Sales", 2000, 30), Row("Tom", "Sales", 5000, 20)) - assert(QueryTest.sameRows(runRows, expectedRows).isEmpty, "the run value is error") - } - test("Unsupported Scenarios") { val data = Seq[(Int, Int)]( (10000, 35), -- Gitee