diff --git a/storage/rocksdb/rocksdb/options/customizable_test.cc b/storage/rocksdb/rocksdb/options/customizable_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..100ed6787d719163f3b73e9eaa341ef3058c1842 --- /dev/null +++ b/storage/rocksdb/rocksdb/options/customizable_test.cc @@ -0,0 +1,625 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "rocksdb/customizable.h" + +#include +#include +#include +#include + +#include "options/configurable_helper.h" +#include "options/customizable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "rocksdb/convenience.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "table/mock_table.h" +#include "test_util/testharness.h" +#include "test_util/testutil.h" + +#ifndef GFLAGS +bool FLAGS_enable_print = false; +#else +#include "util/gflags_compat.h" +using GFLAGS_NAMESPACE::ParseCommandLineFlags; +DEFINE_bool(enable_print, false, "Print options generated to console."); +#endif // GFLAGS + +namespace ROCKSDB_NAMESPACE { +class StringLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[1000]; + vsnprintf(buffer, sizeof(buffer), format, ap); + string_.append(buffer); + } + const std::string& str() const { return string_; } + void clear() { string_.clear(); } + + private: + std::string string_; +}; + +class TestCustomizable : public Customizable { + public: + TestCustomizable(const std::string& name) : name_(name) {} + // Method to allow CheckedCast to work for this class + static const char* kClassName() { + return "TestCustomizable"; + ; + } + + const char* Name() const override { return name_.c_str(); } + static const char* Type() { return "test.custom"; } + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::unique_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + std::shared_ptr* result); + static Status CreateFromString(const ConfigOptions& opts, + const std::string& value, + TestCustomizable** result); + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return Customizable::IsInstanceOf(name); + } + } + + protected: + const std::string name_; +}; + +struct AOptions { + int i = 0; + bool b = false; +}; + +static std::unordered_map a_option_info = { +#ifndef ROCKSDB_LITE + {"int", + {offsetof(struct AOptions, i), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bool", + {offsetof(struct AOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; +class ACustomizable : public TestCustomizable { + public: + ACustomizable(const std::string& id) : TestCustomizable("A"), id_(id) { + ConfigurableHelper::RegisterOptions(*this, "A", &opts_, &a_option_info); + } + std::string GetId() const override { return id_; } + static const char* kClassName() { return "A"; } + + private: + AOptions opts_; + const std::string id_; +}; + +#ifndef ROCKSDB_LITE +static int A_count = 0; +const FactoryFunc& a_func = + ObjectLibrary::Default()->Register( + "A.*", + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new ACustomizable(name)); + A_count++; + return guard->get(); + }); +#endif // ROCKSDB_LITE + +struct BOptions { + std::string s; + bool b = false; +}; + +static std::unordered_map b_option_info = { +#ifndef ROCKSDB_LITE + {"string", + {offsetof(struct BOptions, s), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"bool", + {offsetof(struct BOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +#endif // ROCKSDB_LITE +}; + +class BCustomizable : public TestCustomizable { + private: + public: + BCustomizable(const std::string& name) : TestCustomizable(name) { + ConfigurableHelper::RegisterOptions(*this, name, &opts_, &b_option_info); + } + static const char* kClassName() { return "B"; } + + private: + BOptions opts_; +}; + +static bool LoadSharedB(const std::string& id, + std::shared_ptr* result) { + if (id == "B") { + result->reset(new BCustomizable(id)); + return true; + } else if (id.empty()) { + result->reset(); + return true; + } else { + return false; + } +} +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + return LoadSharedObject(config_options, value, LoadSharedB, + result); +} + +Status TestCustomizable::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::unique_ptr* result) { + return LoadUniqueObject( + config_options, value, + [](const std::string& id, std::unique_ptr* u) { + if (id == "B") { + u->reset(new BCustomizable(id)); + return true; + } else if (id.empty()) { + u->reset(); + return true; + } else { + return false; + } + }, + result); +} + +Status TestCustomizable::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + TestCustomizable** result) { + return LoadStaticObject( + config_options, value, + [](const std::string& id, TestCustomizable** ptr) { + if (id == "B") { + *ptr = new BCustomizable(id); + return true; + } else if (id.empty()) { + *ptr = nullptr; + return true; + } else { + return false; + } + }, + result); +} + +#ifndef ROCKSDB_LITE +const FactoryFunc& s_func = + ObjectLibrary::Default()->Register( + "S", [](const std::string& name, + std::unique_ptr* /* guard */, + std::string* /* msg */) { return new BCustomizable(name); }); +#endif // ROCKSDB_LITE + +struct SimpleOptions { + bool b = true; + std::unique_ptr cu; + std::shared_ptr cs; + TestCustomizable* cp = nullptr; +}; + +static std::unordered_map simple_option_info = { +#ifndef ROCKSDB_LITE + {"bool", + {offsetof(struct SimpleOptions, b), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"unique", OptionTypeInfo::AsCustomUniquePtr( + offsetof(struct SimpleOptions, cu), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"shared", OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct SimpleOptions, cs), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"pointer", OptionTypeInfo::AsCustomRawPtr( + offsetof(struct SimpleOptions, cp), + OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, +#endif // ROCKSDB_LITE +}; + +class SimpleConfigurable : public Configurable { + private: + SimpleOptions simple_; + + public: + SimpleConfigurable() { + ConfigurableHelper::RegisterOptions(*this, "simple", &simple_, + &simple_option_info); + } + + SimpleConfigurable( + const std::unordered_map* map) { + ConfigurableHelper::RegisterOptions(*this, "simple", &simple_, map); + } +}; + +class CustomizableTest : public testing::Test { + public: + ConfigOptions config_options_; +}; + +#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE +// Tests that a Customizable can be created by: +// - a simple name +// - a XXX.id option +// - a property with a name +TEST_F(CustomizableTest, CreateByNameTest) { + ObjectLibrary::Default()->Register( + "TEST.*", + [](const std::string& name, std::unique_ptr* guard, + std::string* /* msg */) { + guard->reset(new TestCustomizable(name)); + return guard->get(); + }); + std::unique_ptr configurable(new SimpleConfigurable()); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique={id=TEST_1}")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_1"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique.id=TEST_2")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_2"); + ASSERT_OK( + configurable->ConfigureFromString(config_options_, "unique=TEST_3")); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "TEST_3"); +} + +TEST_F(CustomizableTest, ToStringTest) { + std::unique_ptr custom(new TestCustomizable("test")); + ASSERT_EQ(custom->ToString(config_options_), "test"); +} + +TEST_F(CustomizableTest, SimpleConfigureTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=B;string=s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +static void GetMapFromProperties( + const std::string& props, + std::unordered_map* map) { + std::istringstream iss(props); + std::unordered_map copy_map; + std::string line; + map->clear(); + for (int line_num = 0; std::getline(iss, line); line_num++) { + std::string name; + std::string value; + ASSERT_OK( + RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num)); + (*map)[name] = value; + } +} + +TEST_F(CustomizableTest, ConfigureFromPropsTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); + std::string opt_str; + std::string mismatch; + config_options_.delimiter = "\n"; + std::unordered_map props; + ASSERT_OK(configurable->GetOptionString(config_options_, &opt_str)); + GetMapFromProperties(opt_str, &props); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromMap(config_options_, props)); + ASSERT_TRUE( + configurable->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CustomizableTest, ConfigureFromShortTest) { + std::unordered_map opt_map = { + {"unique.id", "A"}, {"unique.A.int", "1"}, {"unique.A.bool", "true"}, + {"shared.id", "B"}, {"shared.B.string", "s"}, + }; + std::unique_ptr configurable(new SimpleConfigurable()); + ASSERT_OK(configurable->ConfigureFromMap(config_options_, opt_map)); + SimpleOptions* simple = configurable->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), "A"); +} + +TEST_F(CustomizableTest, AreEquivalentOptionsTest) { + std::unordered_map opt_map = { + {"unique", "id=A;int=1;bool=true"}, + {"shared", "id=A;int=1;bool=true"}, + }; + std::string mismatch; + ConfigOptions config_options = config_options_; + config_options.invoke_prepare_options = false; + std::unique_ptr c1(new SimpleConfigurable()); + std::unique_ptr c2(new SimpleConfigurable()); + ASSERT_OK(c1->ConfigureFromMap(config_options, opt_map)); + ASSERT_OK(c2->ConfigureFromMap(config_options, opt_map)); + ASSERT_TRUE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + SimpleOptions* simple = c1->GetOptions("simple"); + ASSERT_TRUE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_OK(simple->cu->ConfigureOption(config_options, "int", "2")); + ASSERT_FALSE( + simple->cu->AreEquivalent(config_options, simple->cs.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ConfigOptions loosely = config_options; + loosely.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible; + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_TRUE(simple->cu->AreEquivalent(loosely, simple->cs.get(), &mismatch)); + + ASSERT_OK(c1->ConfigureOption(config_options, "shared", "id=B;string=3")); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); + ASSERT_FALSE(simple->cs->AreEquivalent(loosely, simple->cu.get(), &mismatch)); + simple->cs.reset(); + ASSERT_TRUE(c1->AreEquivalent(loosely, c2.get(), &mismatch)); + ASSERT_FALSE(c1->AreEquivalent(config_options, c2.get(), &mismatch)); +} + +// Tests that we can initialize a customizable from its options +TEST_F(CustomizableTest, ConfigureStandaloneCustomTest) { + std::unique_ptr base, copy; + auto registry = ObjectRegistry::NewInstance(); + ASSERT_OK(registry->NewUniqueObject("A", &base)); + ASSERT_OK(registry->NewUniqueObject("A", ©)); + ASSERT_OK(base->ConfigureFromString(config_options_, "int=33;bool=true")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +// Tests that we fail appropriately if the pattern is not registered +TEST_F(CustomizableTest, BadNameTest) { + config_options_.ignore_unsupported_options = false; + std::unique_ptr c1(new SimpleConfigurable()); + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); + config_options_.ignore_unsupported_options = true; + ASSERT_OK( + c1->ConfigureFromString(config_options_, "unique.shared.id=bad name")); +} + +// Tests that we fail appropriately if a bad option is passed to the underlying +// configurable +TEST_F(CustomizableTest, BadOptionTest) { + std::unique_ptr c1(new SimpleConfigurable()); + ConfigOptions ignore = config_options_; + ignore.ignore_unknown_options = true; + + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "shared={id=B;int=1}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared={id=A;string=s}")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "B.int=11")); + ASSERT_OK(c1->ConfigureFromString(ignore, "B.int=11")); + ASSERT_NOK(c1->ConfigureFromString(config_options_, "A.string=s")); + ASSERT_OK(c1->ConfigureFromString(ignore, "A.string=s")); + // Test as detached + ASSERT_NOK( + c1->ConfigureFromString(config_options_, "shared.id=A;A.string=b}")); + ASSERT_OK(c1->ConfigureFromString(ignore, "shared.id=A;A.string=s}")); +} + +// Tests that different IDs lead to different objects +TEST_F(CustomizableTest, UniqueIdTest) { + std::unique_ptr base(new SimpleConfigurable()); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(simple->cu->GetId(), std::string("A_1")); + std::string opt_str; + std::string mismatch; + ASSERT_OK(base->GetOptionString(config_options_, &opt_str)); + std::unique_ptr copy(new SimpleConfigurable()); + ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str)); + ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=true}")); + ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_EQ(simple->cu->GetId(), std::string("A_2")); +} + +TEST_F(CustomizableTest, IsInstanceOfTest) { + std::shared_ptr tc = std::make_shared("A"); + + ASSERT_TRUE(tc->IsInstanceOf("A")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("B")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); + + tc.reset(new BCustomizable("B")); + ASSERT_TRUE(tc->IsInstanceOf("B")); + ASSERT_TRUE(tc->IsInstanceOf("TestCustomizable")); + ASSERT_FALSE(tc->IsInstanceOf("A")); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), tc.get()); + ASSERT_EQ(tc->CheckedCast(), nullptr); +} + +static std::unordered_map inner_option_info = { +#ifndef ROCKSDB_LITE + {"inner", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kNormal, OptionTypeFlags::kStringNameOnly)} +#endif // ROCKSDB_LITE +}; + +class ShallowCustomizable : public Customizable { + public: + ShallowCustomizable() { + inner_ = std::make_shared("a"); + ConfigurableHelper::RegisterOptions(*this, "inner", &inner_, + &inner_option_info); + }; + static const char* kClassName() { return "shallow"; } + const char* Name() const override { return kClassName(); } + + private: + std::shared_ptr inner_; +}; + +TEST_F(CustomizableTest, TestStringDepth) { + ConfigOptions shallow = config_options_; + std::unique_ptr c(new ShallowCustomizable()); + std::string opt_str; + shallow.depth = ConfigOptions::Depth::kDepthShallow; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_EQ(opt_str, "inner=a;"); + shallow.depth = ConfigOptions::Depth::kDepthDetailed; + ASSERT_OK(c->GetOptionString(shallow, &opt_str)); + ASSERT_NE(opt_str, "inner=a;"); +} + +// Tests that we only get a new customizable when it changes +TEST_F(CustomizableTest, NewCustomizableTest) { + std::unique_ptr base(new SimpleConfigurable()); + A_count = 0; + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=true}")); + SimpleOptions* simple = base->GetOptions("simple"); + ASSERT_NE(simple, nullptr); + ASSERT_NE(simple->cu, nullptr); + ASSERT_EQ(A_count, 1); // Created one A + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_1;int=1;bool=false}")); + ASSERT_EQ(A_count, 2); // Create another A_1 + ASSERT_OK(base->ConfigureFromString(config_options_, + "unique={id=A_2;int=1;bool=false}")); + ASSERT_EQ(A_count, 3); // Created another A + ASSERT_OK(base->ConfigureFromString(config_options_, "unique=")); + ASSERT_EQ(simple->cu, nullptr); + ASSERT_EQ(A_count, 3); +} + +TEST_F(CustomizableTest, IgnoreUnknownObjects) { + ConfigOptions ignore = config_options_; + std::shared_ptr shared; + std::unique_ptr unique; + TestCustomizable* pointer = nullptr; + ignore.ignore_unsupported_options = false; + ASSERT_NOK( + LoadSharedObject(ignore, "Unknown", nullptr, &shared)); + ASSERT_NOK( + LoadUniqueObject(ignore, "Unknown", nullptr, &unique)); + ASSERT_NOK( + LoadStaticObject(ignore, "Unknown", nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ignore.ignore_unsupported_options = true; + ASSERT_OK( + LoadSharedObject(ignore, "Unknown", nullptr, &shared)); + ASSERT_OK( + LoadUniqueObject(ignore, "Unknown", nullptr, &unique)); + ASSERT_OK( + LoadStaticObject(ignore, "Unknown", nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown", nullptr, + &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown", nullptr, + &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown", nullptr, + &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_OK(LoadSharedObject(ignore, "id=Unknown;option=bad", + nullptr, &shared)); + ASSERT_OK(LoadUniqueObject(ignore, "id=Unknown;option=bad", + nullptr, &unique)); + ASSERT_OK(LoadStaticObject(ignore, "id=Unknown;option=bad", + nullptr, &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); +} + +TEST_F(CustomizableTest, FactoryFunctionTest) { + std::shared_ptr shared; + std::unique_ptr unique; + TestCustomizable* pointer = nullptr; + ConfigOptions ignore = config_options_; + ignore.ignore_unsupported_options = false; + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &shared)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &unique)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "B", &pointer)); + ASSERT_NE(shared.get(), nullptr); + ASSERT_NE(unique.get(), nullptr); + ASSERT_NE(pointer, nullptr); + delete pointer; + pointer = nullptr; + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &shared)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &unique)); + ASSERT_OK(TestCustomizable::CreateFromString(ignore, "", &pointer)); + ASSERT_EQ(shared.get(), nullptr); + ASSERT_EQ(unique.get(), nullptr); + ASSERT_EQ(pointer, nullptr); + ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &shared)); + ASSERT_NOK(TestCustomizable::CreateFromString(ignore, "option=bad", &unique)); + ASSERT_NOK( + TestCustomizable::CreateFromString(ignore, "option=bad", &pointer)); + ASSERT_EQ(pointer, nullptr); +} + +#endif // !ROCKSDB_LITE + +} // namespace ROCKSDB_NAMESPACE +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); +#ifdef GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif // GFLAGS + return RUN_ALL_TESTS(); +} diff --git a/storage/rocksdb/rocksdb/options/db_options.cc b/storage/rocksdb/rocksdb/options/db_options.cc new file mode 100644 index 0000000000000000000000000000000000000000..3733d448c7ac25b8ea47a3acaa29d57a800a3ef0 --- /dev/null +++ b/storage/rocksdb/rocksdb/options/db_options.cc @@ -0,0 +1,835 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "options/db_options.h" + +#include + +#include "logging/logging.h" +#include "options/configurable_helper.h" +#include "options/options_helper.h" +#include "options/options_parser.h" +#include "port/port.h" +#include "rocksdb/configurable.h" +#include "rocksdb/env.h" +#include "rocksdb/file_system.h" +#include "rocksdb/rate_limiter.h" +#include "rocksdb/sst_file_manager.h" +#include "rocksdb/utilities/options_type.h" +#include "rocksdb/wal_filter.h" +#include "util/string_util.h" + +namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE +static std::unordered_map + wal_recovery_mode_string_map = { + {"kTolerateCorruptedTailRecords", + WALRecoveryMode::kTolerateCorruptedTailRecords}, + {"kAbsoluteConsistency", WALRecoveryMode::kAbsoluteConsistency}, + {"kPointInTimeRecovery", WALRecoveryMode::kPointInTimeRecovery}, + {"kSkipAnyCorruptedRecords", + WALRecoveryMode::kSkipAnyCorruptedRecords}}; + +static std::unordered_map + access_hint_string_map = {{"NONE", DBOptions::AccessHint::NONE}, + {"NORMAL", DBOptions::AccessHint::NORMAL}, + {"SEQUENTIAL", DBOptions::AccessHint::SEQUENTIAL}, + {"WILLNEED", DBOptions::AccessHint::WILLNEED}}; + +static std::unordered_map info_log_level_string_map = + {{"DEBUG_LEVEL", InfoLogLevel::DEBUG_LEVEL}, + {"INFO_LEVEL", InfoLogLevel::INFO_LEVEL}, + {"WARN_LEVEL", InfoLogLevel::WARN_LEVEL}, + {"ERROR_LEVEL", InfoLogLevel::ERROR_LEVEL}, + {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, + {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; + +static std::unordered_map + db_mutable_options_type_info = { + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable}}, + {"max_background_jobs", + {offsetof(struct MutableDBOptions, max_background_jobs), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_compactions", + {offsetof(struct MutableDBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"base_background_compactions", + {offsetof(struct MutableDBOptions, base_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_subcompactions", + {offsetof(struct MutableDBOptions, max_subcompactions), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"avoid_flush_during_shutdown", + {offsetof(struct MutableDBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"writable_file_max_buffer_size", + {offsetof(struct MutableDBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delayed_write_rate", + {offsetof(struct MutableDBOptions, delayed_write_rate), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_total_wal_size", + {offsetof(struct MutableDBOptions, max_total_wal_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_dump_period_sec", + {offsetof(struct MutableDBOptions, stats_dump_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_persist_period_sec", + {offsetof(struct MutableDBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"stats_history_buffer_size", + {offsetof(struct MutableDBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_open_files", + {offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"bytes_per_sync", + {offsetof(struct MutableDBOptions, bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"wal_bytes_per_sync", + {offsetof(struct MutableDBOptions, wal_bytes_per_sync), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"strict_bytes_per_sync", + {offsetof(struct MutableDBOptions, strict_bytes_per_sync), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"compaction_readahead_size", + {offsetof(struct MutableDBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"max_background_flushes", + {offsetof(struct MutableDBOptions, max_background_flushes), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, +}; + +static std::unordered_map + db_immutable_options_type_info = { + /* + // not yet supported + std::shared_ptr row_cache; + std::shared_ptr delete_scheduler; + std::shared_ptr info_log; + std::shared_ptr rate_limiter; + std::shared_ptr statistics; + std::vector db_paths; + std::vector> listeners; + */ + {"advise_random_on_open", + {offsetof(struct ImmutableDBOptions, advise_random_on_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_mmap_reads", + {offsetof(struct ImmutableDBOptions, allow_mmap_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_fallocate", + {offsetof(struct ImmutableDBOptions, allow_fallocate), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_mmap_writes", + {offsetof(struct ImmutableDBOptions, allow_mmap_writes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_direct_reads", + {offsetof(struct ImmutableDBOptions, use_direct_reads), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_direct_writes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"use_direct_io_for_flush_and_compaction", + {offsetof(struct ImmutableDBOptions, + use_direct_io_for_flush_and_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_2pc", + {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"create_if_missing", + {offsetof(struct ImmutableDBOptions, create_if_missing), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"create_missing_column_families", + {offsetof(struct ImmutableDBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"disableDataSync", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"disable_data_sync", // for compatibility + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"enable_thread_tracking", + {offsetof(struct ImmutableDBOptions, enable_thread_tracking), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"error_if_exists", + {offsetof(struct ImmutableDBOptions, error_if_exists), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"is_fd_close_on_exec", + {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"paranoid_checks", + {offsetof(struct ImmutableDBOptions, paranoid_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"track_and_verify_wals_in_manifest", + {offsetof(struct ImmutableDBOptions, + track_and_verify_wals_in_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"skip_log_error_on_recovery", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"skip_stats_update_on_db_open", + {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"skip_checking_sst_file_sizes_on_db_open", + {offsetof(struct ImmutableDBOptions, + skip_checking_sst_file_sizes_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"new_table_reader_for_compaction_inputs", + {offsetof(struct ImmutableDBOptions, + new_table_reader_for_compaction_inputs), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"random_access_max_buffer_size", + {offsetof(struct ImmutableDBOptions, random_access_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_adaptive_mutex", + {offsetof(struct ImmutableDBOptions, use_adaptive_mutex), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"use_fsync", + {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"max_file_opening_threads", + {offsetof(struct ImmutableDBOptions, max_file_opening_threads), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"table_cache_numshardbits", + {offsetof(struct ImmutableDBOptions, table_cache_numshardbits), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_write_buffer_size", + {offsetof(struct ImmutableDBOptions, db_write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"keep_log_file_num", + {offsetof(struct ImmutableDBOptions, keep_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"recycle_log_file_num", + {offsetof(struct ImmutableDBOptions, recycle_log_file_num), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_file_time_to_roll", + {offsetof(struct ImmutableDBOptions, log_file_time_to_roll), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"manifest_preallocation_size", + {offsetof(struct ImmutableDBOptions, manifest_preallocation_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_log_file_size", + {offsetof(struct ImmutableDBOptions, max_log_file_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_log_dir", + {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"wal_dir", + {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"WAL_size_limit_MB", + {offsetof(struct ImmutableDBOptions, wal_size_limit_mb), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"WAL_ttl_seconds", + {offsetof(struct ImmutableDBOptions, wal_ttl_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_manifest_file_size", + {offsetof(struct ImmutableDBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"persist_stats_to_disk", + {offsetof(struct ImmutableDBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"fail_if_options_file_error", + {offsetof(struct ImmutableDBOptions, fail_if_options_file_error), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"enable_pipelined_write", + {offsetof(struct ImmutableDBOptions, enable_pipelined_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"unordered_write", + {offsetof(struct ImmutableDBOptions, unordered_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_concurrent_memtable_write", + {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"wal_recovery_mode", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, wal_recovery_mode), + &wal_recovery_mode_string_map)}, + {"enable_write_thread_adaptive_yield", + {offsetof(struct ImmutableDBOptions, + enable_write_thread_adaptive_yield), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_thread_slow_yield_usec", + {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_write_batch_group_size_bytes", + {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_thread_max_yield_usec", + {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"access_hint_on_compaction_start", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, + access_hint_on_compaction_start), + &access_hint_string_map)}, + {"info_log_level", + OptionTypeInfo::Enum( + offsetof(struct ImmutableDBOptions, info_log_level), + &info_log_level_string_map)}, + {"dump_malloc_stats", + {offsetof(struct ImmutableDBOptions, dump_malloc_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"avoid_flush_during_recovery", + {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"allow_ingest_behind", + {offsetof(struct ImmutableDBOptions, allow_ingest_behind), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"preserve_deletes", + {offsetof(struct ImmutableDBOptions, preserve_deletes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"concurrent_prepare", // Deprecated by two_write_queues + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"two_write_queues", + {offsetof(struct ImmutableDBOptions, two_write_queues), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"manual_wal_flush", + {offsetof(struct ImmutableDBOptions, manual_wal_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"seq_per_batch", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone}}, + {"atomic_flush", + {offsetof(struct ImmutableDBOptions, atomic_flush), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"avoid_unnecessary_blocking_io", + {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"write_dbid_to_manifest", + {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"log_readahead_size", + {offsetof(struct ImmutableDBOptions, log_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"best_efforts_recovery", + {offsetof(struct ImmutableDBOptions, best_efforts_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"max_bgerror_resume_count", + {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"bgerror_resume_retry_interval", + {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, + {"db_host_id", + {offsetof(struct ImmutableDBOptions, db_host_id), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}}, + // The following properties were handled as special cases in ParseOption + // This means that the properties could be read from the options file + // but never written to the file or compared to each other. + {"rate_limiter_bytes_per_sec", + {offsetof(struct ImmutableDBOptions, rate_limiter), + OptionType::kUnknown, OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), + // Parse the input value as a RateLimiter + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto limiter = + reinterpret_cast*>(addr); + limiter->reset(NewGenericRateLimiter( + static_cast(ParseUint64(value)))); + return Status::OK(); + }}}, + {"env", + {offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown, + OptionVerificationType::kNormal, + (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), + // Parse the input value as an Env + [](const ConfigOptions& /*opts*/, const std::string& /*name*/, + const std::string& value, char* addr) { + auto old_env = reinterpret_cast(addr); // Get the old value + Env* new_env = *old_env; // Set new to old + Status s = Env::LoadEnv(value, &new_env); // Update new value + if (s.ok()) { // It worked + *old_env = new_env; // Update the old one + } + return s; + }}}, + {"allow_data_in_errors", + {offsetof(struct ImmutableDBOptions, allow_data_in_errors), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, +}; + +const std::string OptionsHelper::kDBOptionsName = "DBOptions"; + +class MutableDBConfigurable : public Configurable { + public: + MutableDBConfigurable(const MutableDBOptions& mdb) { + mutable_ = mdb; + ConfigurableHelper::RegisterOptions(*this, &mutable_, + &db_mutable_options_type_info); + } + + protected: + MutableDBOptions mutable_; +}; + +class DBOptionsConfigurable : public MutableDBConfigurable { + public: + DBOptionsConfigurable(const DBOptions& opts) + : MutableDBConfigurable(MutableDBOptions(opts)), db_options_(opts) { + // The ImmutableDBOptions currently requires the env to be non-null. Make + // sure it is + if (opts.env != nullptr) { + immutable_ = ImmutableDBOptions(opts); + } else { + DBOptions copy = opts; + copy.env = Env::Default(); + immutable_ = ImmutableDBOptions(copy); + } + ConfigurableHelper::RegisterOptions(*this, &immutable_, + &db_immutable_options_type_info); + } + + protected: + Status ConfigureOptions( + const ConfigOptions& config_options, + const std::unordered_map& opts_map, + std::unordered_map* unused) override { + Status s = ConfigurableHelper::ConfigureOptions(config_options, *this, + opts_map, unused); + if (s.ok()) { + db_options_ = BuildDBOptions(immutable_, mutable_); + s = PrepareOptions(config_options); + } + return s; + } + + const void* GetOptionsPtr(const std::string& name) const override { + if (name == OptionsHelper::kDBOptionsName) { + return &db_options_; + } else { + return MutableDBConfigurable::GetOptionsPtr(name); + } + } + + private: + ImmutableDBOptions immutable_; + DBOptions db_options_; +}; + +std::unique_ptr DBOptionsAsConfigurable( + const MutableDBOptions& opts) { + std::unique_ptr ptr(new MutableDBConfigurable(opts)); + return ptr; +} +std::unique_ptr DBOptionsAsConfigurable(const DBOptions& opts) { + std::unique_ptr ptr(new DBOptionsConfigurable(opts)); + return ptr; +} +#endif // ROCKSDB_LITE + +ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} + +ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) + : create_if_missing(options.create_if_missing), + create_missing_column_families(options.create_missing_column_families), + error_if_exists(options.error_if_exists), + paranoid_checks(options.paranoid_checks), + track_and_verify_wals_in_manifest( + options.track_and_verify_wals_in_manifest), + env(options.env), + fs(options.env->GetFileSystem()), + rate_limiter(options.rate_limiter), + sst_file_manager(options.sst_file_manager), + info_log(options.info_log), + info_log_level(options.info_log_level), + max_file_opening_threads(options.max_file_opening_threads), + statistics(options.statistics), + use_fsync(options.use_fsync), + db_paths(options.db_paths), + db_log_dir(options.db_log_dir), + wal_dir(options.wal_dir), + max_log_file_size(options.max_log_file_size), + log_file_time_to_roll(options.log_file_time_to_roll), + keep_log_file_num(options.keep_log_file_num), + recycle_log_file_num(options.recycle_log_file_num), + max_manifest_file_size(options.max_manifest_file_size), + table_cache_numshardbits(options.table_cache_numshardbits), + wal_ttl_seconds(options.WAL_ttl_seconds), + wal_size_limit_mb(options.WAL_size_limit_MB), + max_write_batch_group_size_bytes( + options.max_write_batch_group_size_bytes), + manifest_preallocation_size(options.manifest_preallocation_size), + allow_mmap_reads(options.allow_mmap_reads), + allow_mmap_writes(options.allow_mmap_writes), + use_direct_reads(options.use_direct_reads), + use_direct_io_for_flush_and_compaction( + options.use_direct_io_for_flush_and_compaction), + allow_fallocate(options.allow_fallocate), + is_fd_close_on_exec(options.is_fd_close_on_exec), + advise_random_on_open(options.advise_random_on_open), + db_write_buffer_size(options.db_write_buffer_size), + write_buffer_manager(options.write_buffer_manager), + access_hint_on_compaction_start(options.access_hint_on_compaction_start), + new_table_reader_for_compaction_inputs( + options.new_table_reader_for_compaction_inputs), + random_access_max_buffer_size(options.random_access_max_buffer_size), + use_adaptive_mutex(options.use_adaptive_mutex), + listeners(options.listeners), + enable_thread_tracking(options.enable_thread_tracking), + enable_pipelined_write(options.enable_pipelined_write), + unordered_write(options.unordered_write), + allow_concurrent_memtable_write(options.allow_concurrent_memtable_write), + enable_write_thread_adaptive_yield( + options.enable_write_thread_adaptive_yield), + write_thread_max_yield_usec(options.write_thread_max_yield_usec), + write_thread_slow_yield_usec(options.write_thread_slow_yield_usec), + skip_stats_update_on_db_open(options.skip_stats_update_on_db_open), + skip_checking_sst_file_sizes_on_db_open( + options.skip_checking_sst_file_sizes_on_db_open), + wal_recovery_mode(options.wal_recovery_mode), + allow_2pc(options.allow_2pc), + row_cache(options.row_cache), +#ifndef ROCKSDB_LITE + wal_filter(options.wal_filter), +#endif // ROCKSDB_LITE + fail_if_options_file_error(options.fail_if_options_file_error), + dump_malloc_stats(options.dump_malloc_stats), + avoid_flush_during_recovery(options.avoid_flush_during_recovery), + allow_ingest_behind(options.allow_ingest_behind), + preserve_deletes(options.preserve_deletes), + two_write_queues(options.two_write_queues), + manual_wal_flush(options.manual_wal_flush), + atomic_flush(options.atomic_flush), + avoid_unnecessary_blocking_io(options.avoid_unnecessary_blocking_io), + persist_stats_to_disk(options.persist_stats_to_disk), + write_dbid_to_manifest(options.write_dbid_to_manifest), + log_readahead_size(options.log_readahead_size), + file_checksum_gen_factory(options.file_checksum_gen_factory), + best_efforts_recovery(options.best_efforts_recovery), + max_bgerror_resume_count(options.max_bgerror_resume_count), + bgerror_resume_retry_interval(options.bgerror_resume_retry_interval), + allow_data_in_errors(options.allow_data_in_errors), + db_host_id(options.db_host_id) { +} + +void ImmutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.error_if_exists: %d", + error_if_exists); + ROCKS_LOG_HEADER(log, " Options.create_if_missing: %d", + create_if_missing); + ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d", + paranoid_checks); + ROCKS_LOG_HEADER(log, + " " + "Options.track_and_verify_wals_in_manifest: %d", + track_and_verify_wals_in_manifest); + ROCKS_LOG_HEADER(log, " Options.env: %p", + env); + ROCKS_LOG_HEADER(log, " Options.fs: %s", + fs->Name()); + ROCKS_LOG_HEADER(log, " Options.info_log: %p", + info_log.get()); + ROCKS_LOG_HEADER(log, " Options.max_file_opening_threads: %d", + max_file_opening_threads); + ROCKS_LOG_HEADER(log, " Options.statistics: %p", + statistics.get()); + ROCKS_LOG_HEADER(log, " Options.use_fsync: %d", + use_fsync); + ROCKS_LOG_HEADER( + log, " Options.max_log_file_size: %" ROCKSDB_PRIszt, + max_log_file_size); + ROCKS_LOG_HEADER(log, + " Options.max_manifest_file_size: %" PRIu64, + max_manifest_file_size); + ROCKS_LOG_HEADER( + log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt, + log_file_time_to_roll); + ROCKS_LOG_HEADER( + log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt, + keep_log_file_num); + ROCKS_LOG_HEADER( + log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt, + recycle_log_file_num); + ROCKS_LOG_HEADER(log, " Options.allow_fallocate: %d", + allow_fallocate); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_reads: %d", + allow_mmap_reads); + ROCKS_LOG_HEADER(log, " Options.allow_mmap_writes: %d", + allow_mmap_writes); + ROCKS_LOG_HEADER(log, " Options.use_direct_reads: %d", + use_direct_reads); + ROCKS_LOG_HEADER(log, + " " + "Options.use_direct_io_for_flush_and_compaction: %d", + use_direct_io_for_flush_and_compaction); + ROCKS_LOG_HEADER(log, " Options.create_missing_column_families: %d", + create_missing_column_families); + ROCKS_LOG_HEADER(log, " Options.db_log_dir: %s", + db_log_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.wal_dir: %s", + wal_dir.c_str()); + ROCKS_LOG_HEADER(log, " Options.table_cache_numshardbits: %d", + table_cache_numshardbits); + ROCKS_LOG_HEADER(log, + " Options.WAL_ttl_seconds: %" PRIu64, + wal_ttl_seconds); + ROCKS_LOG_HEADER(log, + " Options.WAL_size_limit_MB: %" PRIu64, + wal_size_limit_mb); + ROCKS_LOG_HEADER(log, + " " + "Options.max_write_batch_group_size_bytes: %" PRIu64, + max_write_batch_group_size_bytes); + ROCKS_LOG_HEADER( + log, " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt, + manifest_preallocation_size); + ROCKS_LOG_HEADER(log, " Options.is_fd_close_on_exec: %d", + is_fd_close_on_exec); + ROCKS_LOG_HEADER(log, " Options.advise_random_on_open: %d", + advise_random_on_open); + ROCKS_LOG_HEADER( + log, " Options.db_write_buffer_size: %" ROCKSDB_PRIszt, + db_write_buffer_size); + ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p", + write_buffer_manager.get()); + ROCKS_LOG_HEADER(log, " Options.access_hint_on_compaction_start: %d", + static_cast(access_hint_on_compaction_start)); + ROCKS_LOG_HEADER(log, " Options.new_table_reader_for_compaction_inputs: %d", + new_table_reader_for_compaction_inputs); + ROCKS_LOG_HEADER( + log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt, + random_access_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d", + use_adaptive_mutex); + ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p", + rate_limiter.get()); + Header( + log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64, + sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0); + ROCKS_LOG_HEADER(log, " Options.wal_recovery_mode: %d", + static_cast(wal_recovery_mode)); + ROCKS_LOG_HEADER(log, " Options.enable_thread_tracking: %d", + enable_thread_tracking); + ROCKS_LOG_HEADER(log, " Options.enable_pipelined_write: %d", + enable_pipelined_write); + ROCKS_LOG_HEADER(log, " Options.unordered_write: %d", + unordered_write); + ROCKS_LOG_HEADER(log, " Options.allow_concurrent_memtable_write: %d", + allow_concurrent_memtable_write); + ROCKS_LOG_HEADER(log, " Options.enable_write_thread_adaptive_yield: %d", + enable_write_thread_adaptive_yield); + ROCKS_LOG_HEADER(log, + " Options.write_thread_max_yield_usec: %" PRIu64, + write_thread_max_yield_usec); + ROCKS_LOG_HEADER(log, + " Options.write_thread_slow_yield_usec: %" PRIu64, + write_thread_slow_yield_usec); + if (row_cache) { + ROCKS_LOG_HEADER( + log, + " Options.row_cache: %" ROCKSDB_PRIszt, + row_cache->GetCapacity()); + } else { + ROCKS_LOG_HEADER(log, + " Options.row_cache: None"); + } +#ifndef ROCKSDB_LITE + ROCKS_LOG_HEADER(log, " Options.wal_filter: %s", + wal_filter ? wal_filter->Name() : "None"); +#endif // ROCKDB_LITE + + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_recovery: %d", + avoid_flush_during_recovery); + ROCKS_LOG_HEADER(log, " Options.allow_ingest_behind: %d", + allow_ingest_behind); + ROCKS_LOG_HEADER(log, " Options.preserve_deletes: %d", + preserve_deletes); + ROCKS_LOG_HEADER(log, " Options.two_write_queues: %d", + two_write_queues); + ROCKS_LOG_HEADER(log, " Options.manual_wal_flush: %d", + manual_wal_flush); + ROCKS_LOG_HEADER(log, " Options.atomic_flush: %d", atomic_flush); + ROCKS_LOG_HEADER(log, + " Options.avoid_unnecessary_blocking_io: %d", + avoid_unnecessary_blocking_io); + ROCKS_LOG_HEADER(log, " Options.persist_stats_to_disk: %u", + persist_stats_to_disk); + ROCKS_LOG_HEADER(log, " Options.write_dbid_to_manifest: %d", + write_dbid_to_manifest); + ROCKS_LOG_HEADER( + log, " Options.log_readahead_size: %" ROCKSDB_PRIszt, + log_readahead_size); + ROCKS_LOG_HEADER(log, " Options.file_checksum_gen_factory: %s", + file_checksum_gen_factory ? file_checksum_gen_factory->Name() + : kUnknownFileChecksumFuncName); + ROCKS_LOG_HEADER(log, " Options.best_efforts_recovery: %d", + static_cast(best_efforts_recovery)); + ROCKS_LOG_HEADER(log, " Options.max_bgerror_resume_count: %d", + max_bgerror_resume_count); + ROCKS_LOG_HEADER(log, + " Options.bgerror_resume_retry_interval: %" PRIu64, + bgerror_resume_retry_interval); + ROCKS_LOG_HEADER(log, " Options.allow_data_in_errors: %d", + allow_data_in_errors); + ROCKS_LOG_HEADER(log, " Options.db_host_id: %s", + db_host_id.c_str()); +} + +MutableDBOptions::MutableDBOptions() + : max_background_jobs(2), + base_background_compactions(-1), + max_background_compactions(-1), + max_subcompactions(0), + avoid_flush_during_shutdown(false), + writable_file_max_buffer_size(1024 * 1024), + delayed_write_rate(2 * 1024U * 1024U), + max_total_wal_size(0), + delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000), + stats_dump_period_sec(600), + stats_persist_period_sec(600), + stats_history_buffer_size(1024 * 1024), + max_open_files(-1), + bytes_per_sync(0), + wal_bytes_per_sync(0), + strict_bytes_per_sync(false), + compaction_readahead_size(0), + max_background_flushes(-1) {} + +MutableDBOptions::MutableDBOptions(const DBOptions& options) + : max_background_jobs(options.max_background_jobs), + base_background_compactions(options.base_background_compactions), + max_background_compactions(options.max_background_compactions), + max_subcompactions(options.max_subcompactions), + avoid_flush_during_shutdown(options.avoid_flush_during_shutdown), + writable_file_max_buffer_size(options.writable_file_max_buffer_size), + delayed_write_rate(options.delayed_write_rate), + max_total_wal_size(options.max_total_wal_size), + delete_obsolete_files_period_micros( + options.delete_obsolete_files_period_micros), + stats_dump_period_sec(options.stats_dump_period_sec), + stats_persist_period_sec(options.stats_persist_period_sec), + stats_history_buffer_size(options.stats_history_buffer_size), + max_open_files(options.max_open_files), + bytes_per_sync(options.bytes_per_sync), + wal_bytes_per_sync(options.wal_bytes_per_sync), + strict_bytes_per_sync(options.strict_bytes_per_sync), + compaction_readahead_size(options.compaction_readahead_size), + max_background_flushes(options.max_background_flushes) {} + +void MutableDBOptions::Dump(Logger* log) const { + ROCKS_LOG_HEADER(log, " Options.max_background_jobs: %d", + max_background_jobs); + ROCKS_LOG_HEADER(log, " Options.max_background_compactions: %d", + max_background_compactions); + ROCKS_LOG_HEADER(log, " Options.max_subcompactions: %" PRIu32, + max_subcompactions); + ROCKS_LOG_HEADER(log, " Options.avoid_flush_during_shutdown: %d", + avoid_flush_during_shutdown); + ROCKS_LOG_HEADER( + log, " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt, + writable_file_max_buffer_size); + ROCKS_LOG_HEADER(log, " Options.delayed_write_rate : %" PRIu64, + delayed_write_rate); + ROCKS_LOG_HEADER(log, " Options.max_total_wal_size: %" PRIu64, + max_total_wal_size); + ROCKS_LOG_HEADER( + log, " Options.delete_obsolete_files_period_micros: %" PRIu64, + delete_obsolete_files_period_micros); + ROCKS_LOG_HEADER(log, " Options.stats_dump_period_sec: %u", + stats_dump_period_sec); + ROCKS_LOG_HEADER(log, " Options.stats_persist_period_sec: %d", + stats_persist_period_sec); + ROCKS_LOG_HEADER( + log, + " Options.stats_history_buffer_size: %" ROCKSDB_PRIszt, + stats_history_buffer_size); + ROCKS_LOG_HEADER(log, " Options.max_open_files: %d", + max_open_files); + ROCKS_LOG_HEADER(log, + " Options.bytes_per_sync: %" PRIu64, + bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.wal_bytes_per_sync: %" PRIu64, + wal_bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.strict_bytes_per_sync: %d", + strict_bytes_per_sync); + ROCKS_LOG_HEADER(log, + " Options.compaction_readahead_size: %" ROCKSDB_PRIszt, + compaction_readahead_size); + ROCKS_LOG_HEADER(log, " Options.max_background_flushes: %d", + max_background_flushes); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/storage/rocksdb/rocksdb/options/db_options.h b/storage/rocksdb/rocksdb/options/db_options.h new file mode 100644 index 0000000000000000000000000000000000000000..42a58e2567b9a2373f83ce00f378c005cbb61c34 --- /dev/null +++ b/storage/rocksdb/rocksdb/options/db_options.h @@ -0,0 +1,126 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include + +#include "rocksdb/options.h" + +namespace ROCKSDB_NAMESPACE { + +struct ImmutableDBOptions { + static const char* kName() { return "ImmutableDBOptions"; } + ImmutableDBOptions(); + explicit ImmutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + bool create_if_missing; + bool create_missing_column_families; + bool error_if_exists; + bool paranoid_checks; + bool track_and_verify_wals_in_manifest; + Env* env; + std::shared_ptr fs; + std::shared_ptr rate_limiter; + std::shared_ptr sst_file_manager; + std::shared_ptr info_log; + InfoLogLevel info_log_level; + int max_file_opening_threads; + std::shared_ptr statistics; + bool use_fsync; + std::vector db_paths; + std::string db_log_dir; + std::string wal_dir; + size_t max_log_file_size; + size_t log_file_time_to_roll; + size_t keep_log_file_num; + size_t recycle_log_file_num; + uint64_t max_manifest_file_size; + int table_cache_numshardbits; + uint64_t wal_ttl_seconds; + uint64_t wal_size_limit_mb; + uint64_t max_write_batch_group_size_bytes; + size_t manifest_preallocation_size; + bool allow_mmap_reads; + bool allow_mmap_writes; + bool use_direct_reads; + bool use_direct_io_for_flush_and_compaction; + bool allow_fallocate; + bool is_fd_close_on_exec; + bool advise_random_on_open; + size_t db_write_buffer_size; + std::shared_ptr write_buffer_manager; + DBOptions::AccessHint access_hint_on_compaction_start; + bool new_table_reader_for_compaction_inputs; + size_t random_access_max_buffer_size; + bool use_adaptive_mutex; + std::vector> listeners; + bool enable_thread_tracking; + bool enable_pipelined_write; + bool unordered_write; + bool allow_concurrent_memtable_write; + bool enable_write_thread_adaptive_yield; + uint64_t write_thread_max_yield_usec; + uint64_t write_thread_slow_yield_usec; + bool skip_stats_update_on_db_open; + bool skip_checking_sst_file_sizes_on_db_open; + WALRecoveryMode wal_recovery_mode; + bool allow_2pc; + std::shared_ptr row_cache; +#ifndef ROCKSDB_LITE + WalFilter* wal_filter; +#endif // ROCKSDB_LITE + bool fail_if_options_file_error; + bool dump_malloc_stats; + bool avoid_flush_during_recovery; + bool allow_ingest_behind; + bool preserve_deletes; + bool two_write_queues; + bool manual_wal_flush; + bool atomic_flush; + bool avoid_unnecessary_blocking_io; + bool persist_stats_to_disk; + bool write_dbid_to_manifest; + size_t log_readahead_size; + std::shared_ptr file_checksum_gen_factory; + bool best_efforts_recovery; + int max_bgerror_resume_count; + uint64_t bgerror_resume_retry_interval; + bool allow_data_in_errors; + std::string db_host_id; +}; + +struct MutableDBOptions { + static const char* kName() { return "MutableDBOptions"; } + MutableDBOptions(); + explicit MutableDBOptions(const MutableDBOptions& options) = default; + explicit MutableDBOptions(const DBOptions& options); + + void Dump(Logger* log) const; + + int max_background_jobs; + int base_background_compactions; + int max_background_compactions; + uint32_t max_subcompactions; + bool avoid_flush_during_shutdown; + size_t writable_file_max_buffer_size; + uint64_t delayed_write_rate; + uint64_t max_total_wal_size; + uint64_t delete_obsolete_files_period_micros; + unsigned int stats_dump_period_sec; + unsigned int stats_persist_period_sec; + size_t stats_history_buffer_size; + int max_open_files; + uint64_t bytes_per_sync; + uint64_t wal_bytes_per_sync; + bool strict_bytes_per_sync; + size_t compaction_readahead_size; + int max_background_flushes; +}; + +} // namespace ROCKSDB_NAMESPACE