From cd366641f9895ec86223ad1c6e62ae3a4eea4621 Mon Sep 17 00:00:00 2001 From: htt1997 Date: Sat, 29 Mar 2025 16:31:33 +0800 Subject: [PATCH] update Signed-off-by: htt1997 --- data_object/CMakeLists.txt | 2 +- .../include/cloud_db_proxy.h | 70 + .../collaboration_edit/include/db_error.h | 3 + .../collaboration_edit/include/db_store.h | 9 +- .../include/db_store_manager.h | 11 + .../include/db_thread_pool.h | 48 + .../include/grd_api_manager.h | 27 +- .../include/grd_type_export.h | 103 + .../collaboration_edit/include/rd_adapter.h | 10 +- .../collaboration_edit/include/rd_type.h | 42 + .../collaboration_edit/include/rd_utils.h | 24 +- .../collaboration_edit/src/cloud_db_proxy.cpp | 402 + .../collaboration_edit/src/db_store.cpp | 106 +- .../src/db_store_manager.cpp | 84 +- .../collaboration_edit/src/db_thread_pool.cpp | 29 +- .../src/grd_api_manager.cpp | 12 +- .../collaboration_edit/src/rd_adapter.cpp | 57 +- .../collaboration_edit/src/rd_utils.cpp | 121 +- .../include/adaptor/asset_change_timer.h | 4 - .../include/adaptor/client_adaptor.h | 1 - .../include/adaptor/distributed_object_impl.h | 2 - .../adaptor/distributed_objectstore_impl.h | 5 +- .../adaptor/flat_object_storage_engine.h | 6 - .../include/adaptor/flat_object_store.h | 4 - .../include/adaptor/object_service.h | 5 - .../include/adaptor/object_storage_engine.h | 5 - .../innerkitsimpl/include/adaptor/watcher.h | 3 - .../innerkitsimpl/include/common/bytes.h | 11 +- .../include/common/bytes_utils.h | 3 - .../innerkitsimpl/include/common/logger.h | 1 - .../include/common/object_radar_reporter.h | 14 +- .../include/common/object_utils.h | 2 - .../include/common/string_utils.h | 5 - .../app_device_status_change_listener.h | 4 +- .../include/communicator/app_pipe_handler.h | 7 - .../include/communicator/app_pipe_mgr.h | 6 - .../include/communicator/app_types.h | 10 +- .../communicator/ark_communication_provider.h | 2 - .../communicator/communication_provider.h | 2 - .../communication_provider_impl.h | 2 - .../include/communicator/dev_manager.h | 2 - .../include/communicator/softbus_adapter.h | 8 - ...uteddata_object_store_ipc_interface_code.h | 4 +- .../innerkitsimpl/include/iobject_service.h | 3 - .../innerkitsimpl/include/object_callback.h | 2 - .../include/object_callback_stub.h | 1 - .../innerkitsimpl/include/object_service.h | 4 - .../include/object_service_proxy.h | 2 - .../src/adaptor/asset_change_timer.cpp | 2 + .../src/adaptor/distributed_object_impl.cpp | 2 - .../adaptor/distributed_object_store_impl.cpp | 19 +- .../adaptor/flat_object_storage_engine.cpp | 3 - .../src/adaptor/flat_object_store.cpp | 5 - .../src/communicator/app_pipe_handler.cpp | 4 - .../src/communicator/app_pipe_mgr.cpp | 2 +- .../ark_communication_provider.cpp | 2 - .../communicator/communication_provider.cpp | 2 - .../communication_provider_impl.cpp | 2 - .../src/communicator/dev_manager.cpp | 6 +- .../communicator/softbus_adapter_standard.cpp | 5 +- .../src/object_callback_stub.cpp | 1 - .../src/object_service_proxy.cpp | 10 +- .../innerkitsimpl/test/unittest/BUILD.gn | 2 +- .../unittest/src/asset_change_timer_test.cpp | 10 +- .../test/unittest/src/object_store_test.cpp | 5 +- .../jskitsimpl/collaboration_edit/BUILD.gn | 13 +- .../collaboration_edit/include/js_utils.h | 28 + .../include/napi_async_call.h | 83 + .../include/napi_cloud_db.h | 87 + .../include/napi_collaboration_edit_object.h | 19 +- .../include/napi_const_properties.h | 26 + .../include/napi_edit_unit.h | 2 + .../collaboration_edit/include/napi_errno.h | 1 + .../collaboration_edit/include/napi_node.h | 1 + .../collaboration_edit/include/napi_parser.h | 16 + .../include/napi_sync_service.h | 43 + .../collaboration_edit/include/napi_utils.h | 32 +- .../collaboration_edit/src/entry_point.cpp | 2 + .../collaboration_edit/src/js_utils.cpp | 28 + .../src/napi_async_call.cpp | 169 + .../collaboration_edit/src/napi_cloud_db.cpp | 651 + .../src/napi_collaboration_edit_object.cpp | 398 +- .../src/napi_const_properties.cpp | 90 + .../collaboration_edit/src/napi_edit_unit.cpp | 88 +- .../collaboration_edit/src/napi_node.cpp | 29 + .../collaboration_edit/src/napi_parser.cpp | 197 +- .../src/napi_sync_service.cpp | 58 + .../collaboration_edit/src/napi_text.cpp | 1 + .../src/napi_undo_manager.cpp | 5 +- .../collaboration_edit/src/napi_utils.cpp | 327 +- .../collaboration_edit/test/BUILD.gn | 2 +- .../AppScope/app.json | 21 + .../resources/base/element/string.json | 8 + .../collaboration_edit_js_test}/BUILD.gn | 60 +- .../collaboration_edit_js_test/Test.json | 26 + .../src/main/ets/Application/AbilityStage.ts | 23 + .../src/main/ets/MainAbility/MainAbility.ts | 50 + .../ets/MainAbility/pages/index/index.ets | 55 + .../src/main/ets/TestAbility/TestAbility.ts | 43 + .../src/main/ets/TestAbility/pages/index.ets | 48 + .../ets/TestRunner/OpenHarmonyTestRunner.ts | 72 + .../entry/src/main/ets/test/CloudDbMock.ets | 75 + .../entry/src/main/ets/test/EditUnit.ets} | 166 +- .../src/main/ets/test/GetEditObject.ets} | 73 +- .../entry/src/main/ets/test/List.test.ets | 30 + .../entry/src/main/ets/test/Node.ets} | 311 +- .../entry/src/main/ets/test/SetCloudDb.ets | 441 + .../entry/src/main/ets/test/TestUtils.ets | 20 + .../entry/src/main/ets/test/Text.ets} | 95 +- .../entry/src/main/ets/test/UndoRedo.ets} | 189 +- .../entry/src/main/module.json | 52 + .../main/resources/base/element/string.json | 52 + .../src/main/resources/base/media/icon.png | Bin 0 -> 20093 bytes .../resources/base/profile/main_pages.json | 5 + .../base/profile/shortcuts_config.json | 5 + .../signature}/openharmony_sx.p7b | Bin 3437 -> 3468 bytes .../test/unittest/config.json | 63 - .../jskitsimpl/include/adaptor/js_common.h | 1 - .../include/adaptor/js_distributedobject.h | 3 - .../adaptor/js_distributedobjectstore.h | 5 - .../include/adaptor/js_object_wrapper.h | 4 - .../jskitsimpl/include/adaptor/js_watcher.h | 7 +- .../include/adaptor/notifier_impl.h | 3 - .../jskitsimpl/include/common/js_ability.h | 5 - .../jskitsimpl/include/common/js_util.h | 14 +- .../jskitsimpl/include/common/napi_queue.h | 2 - .../jskitsimpl/include/common/object_error.h | 2 +- .../jskitsimpl/include/common/uv_queue.h | 3 - .../src/adaptor/js_distributedobject.cpp | 2 - .../src/adaptor/js_distributedobjectstore.cpp | 28 +- .../jskitsimpl/src/adaptor/js_module_init.cpp | 1 - .../src/adaptor/js_object_wrapper.cpp | 1 + .../jskitsimpl/src/adaptor/notifier_impl.cpp | 7 +- .../jskitsimpl/src/common/js_ability.cpp | 1 - .../jskitsimpl/src/common/js_util.cpp | 2 - .../jskitsimpl/src/common/object_error.cpp | 8 +- .../jskitsimpl/src/common/uv_queue.cpp | 1 + .../jskitsimpl/test/unittest/src/BUILD.gn | 2 +- .../unittest/src/ObjectStoreJsunit.test.js | 40 +- .../jskitsimpl/test/unittest/src/config.json | 2 +- .../interfaces/innerkits/distributed_object.h | 24 +- .../innerkits/distributed_objectstore.h | 21 +- .../interfaces/innerkits/object_types.h | 16 +- .../interfaces/innerkits/objectstore_errors.h | 2 - data_share/CMakeLists.txt | 2 +- .../js/napi/common/src/datashare_js_utils.cpp | 7 +- .../frameworks/js/napi/dataShare/BUILD.gn | 1 + .../dataShare/src/napi_datashare_helper.cpp | 36 +- .../js/napi/observer/include/napi_observer.h | 2 +- .../include/napi_subscriber_manager.h | 5 +- .../js/napi/observer/src/napi_observer.cpp | 8 +- .../observer/src/napi_subscriber_manager.cpp | 45 +- .../native/common/include/callbacks_manager.h | 1 - .../native/common/include/datashare_log.h | 1 - .../common/include/datashare_radar_reporter.h | 5 - ...ibuteddata_data_share_ipc_interface_code.h | 27 +- .../common/src/datashare_itypes_utils.cpp | 8 +- .../common/src/ishared_result_set_stub.cpp | 19 +- .../consumer/include/datashare_helper_impl.h | 5 +- .../native/consumer/src/datashare_helper.cpp | 44 +- .../consumer/src/datashare_helper_impl.cpp | 96 +- .../provider/include/datashare_uv_queue.h | 1 - .../include/js_datashare_ext_ability.h | 158 +- .../provider/src/datashare_stub_impl.cpp | 177 +- .../provider/src/js_datashare_ext_ability.cpp | 163 +- .../proxy/include/data_share_service_proxy.h | 10 + .../proxy/include/rdb_subscriber_manager.h | 3 +- .../proxy/src/data_share_service_proxy.cpp | 72 +- .../src/published_data_subscriber_manager.cpp | 3 +- .../proxy/src/rdb_subscriber_manager.cpp | 31 +- .../common/include/datashare_errno.h | 5 + .../consumer/include/datashare_helper.h | 13 +- .../test/js/data_share/unittest/config.json | 2 +- .../test/js/data_share/unittest/src/BUILD.gn | 2 +- datamgr_service/OAT.xml | 7 + datamgr_service/bundle.json | 14 +- datamgr_service/conf/config.json | 3 +- .../distributeddataservice/adapter/BUILD.gn | 83 - .../adapter/CMakeLists.txt | 9 +- .../adapter/account/BUILD.gn | 5 +- .../src/account_delegate_normal_impl.cpp | 28 +- .../src/account_delegate_normal_impl.h | 3 +- .../adapter/account/test/BUILD.gn | 3 +- .../broadcaster/src/broadcast_sender_impl.cpp | 63 - .../adapter/communicator/BUILD.gn | 11 +- .../communicator/src/app_pipe_handler.cpp | 2 +- .../communicator/src/app_pipe_handler.h | 2 +- .../adapter/communicator/src/app_pipe_mgr.cpp | 2 +- .../src/process_communicator_impl.cpp | 33 +- .../src/softbus_adapter_standard.cpp | 86 +- .../adapter/communicator/test/BUILD.gn | 229 +- .../fuzztest/softbusadapter_fuzzer/BUILD.gn | 9 +- .../unittest/app_pipe_mgr_service_test.cpp | 290 + .../unittest/communicator_context_test.cpp | 163 + .../test/unittest/data_buffer_test.cpp | 56 + .../unittest/device_manager_adapter_test.cpp | 204 + .../process_communicator_impl_test.cpp | 506 + .../softbus_adapter_standard_test.cpp | 442 +- .../test/unittest/softbus_client_test.cpp | 189 + .../adapter/dfx/BUILD.gn | 68 +- .../src/behaviour/behaviour_reporter_impl.h | 2 +- .../dfx/src/fault/cloud_sync_fault_impl.h | 4 +- .../dfx/src/fault/communication_fault_impl.h | 4 +- .../dfx/src/fault/database_fault_impl.h | 2 +- .../adapter/dfx/src/fault/fault_reporter.cpp | 2 +- .../dfx/src/fault/runtime_fault_impl.h | 2 +- .../dfx/src/fault/service_fault_impl.h | 2 +- .../adapter/dfx/src/hiview_adapter.cpp | 4 +- .../adapter/dfx/src/hiview_adapter.h | 2 +- .../adapter/dfx/src/radar_reporter.cpp | 2 +- .../src/{reporter.cpp => reporter_impl.cpp} | 63 +- .../api_performance_statistic_impl.h | 4 +- .../src/statistic/database_statistic_impl.h | 2 +- .../dfx/src/statistic/statistic_reporter.cpp | 2 +- .../src/statistic/traffic_statistic_impl.h | 2 +- .../dfx/src/statistic/visit_statistic_impl.h | 2 +- .../adapter/dfx/test/BUILD.gn | 60 +- .../unittest/distributeddata_dfx_mst_test.cpp | 11 +- .../unittest/distributeddata_dfx_ut_test.cpp | 97 +- .../test/unittest/hiview_adapter_dfx_test.cpp | 249 + .../communicator/process_communicator_impl.h | 6 +- .../include/communicator/route_head_handler.h | 6 +- .../adapter/include/dfx/reporter_impl.h | 54 + .../include/schema_helper/get_schema_helper.h | 68 + .../{service => adapter}/network/BUILD.gn | 34 +- .../src/network_delegate_default_impl.cpp | 50 + .../src/network_delegate_default_impl.h} | 37 +- .../src/network_delegate_normal_impl.cpp} | 64 +- .../src/network_delegate_normal_impl.h} | 45 +- .../{permission => network}/test/BUILD.gn | 82 +- .../network_delegate_normal_impl_test.cpp | 217 + .../network/test/network_delegate_test.cpp} | 69 +- .../{broadcaster => schema_helper}/BUILD.gn | 33 +- .../schema_helper/src/get_schema_helper.cpp | 121 + .../adapter/screenlock/BUILD.gn | 5 +- .../adapter/screenlock/src/screen_lock.cpp | 2 +- .../adapter/screenlock/test/BUILD.gn | 2 +- .../adapter/test/BUILD.gn | 5 + .../adapter/utils/BUILD.gn | 20 +- .../distributeddataservice/app/BUILD.gn | 8 +- .../distributeddataservice/app/CMakeLists.txt | 2 +- .../app/src/checker/BUILD.gn | 8 +- .../app/src/clone/clone_backup_info.cpp | 33 +- .../app/src/clone/secret_key_backup_data.cpp | 2 + .../app/src/clone/secret_key_backup_data.h | 1 + .../kvstore_flowctrl_manager.cpp | 74 - .../kvstore_flowctrl_manager.h | 61 - .../app/src/installer/BUILD.gn | 7 +- .../app/src/installer/installer_impl.cpp | 25 +- .../app/src/installer/installer_impl.h | 1 + .../app/src/kvstore_data_service.cpp | 314 +- .../app/src/kvstore_data_service.h | 29 +- .../app/src/kvstore_data_service_stub.cpp | 4 +- .../app/src/kvstore_meta_manager.cpp | 127 +- .../app/src/kvstore_meta_manager.h | 6 + .../route_head_handler_impl.cpp | 80 +- .../session_manager/route_head_handler_impl.h | 5 +- .../src/session_manager/session_manager.cpp | 73 +- .../app/src/session_manager/session_manager.h | 8 +- .../distributeddataservice/app/test/BUILD.gn | 68 +- .../fuzztest/dataservicestub_fuzzer/BUILD.gn | 5 - .../kvstore_data_service_clear_test.cpp | 79 +- .../unittest/kvstore_data_service_test.cpp | 11 +- .../kvstore_flowctrl_manager_test.cpp | 280 - .../test/unittest/session_manager_test.cpp | 5 +- .../distributeddataservice/framework/BUILD.gn | 20 +- .../framework/CMakeLists.txt | 4 +- .../framework/cloud/cloud_db.cpp | 2 +- .../framework/cloud/cloud_info.cpp | 12 + .../framework/cloud/cloud_last_sync_info.cpp | 54 + .../framework/cloud/cloud_mark.cpp | 47 + .../framework/cloud/schema_meta.cpp | 10 + .../dfx/reporter.cpp} | 30 +- .../framework/directory/directory_manager.cpp | 23 + .../framework/feature/static_acts.cpp | 14 + .../include/account/account_delegate.h | 1 + .../framework/include/cloud/asset_loader.h | 2 +- .../framework/include/cloud/cloud_db.h | 2 +- .../framework/include/cloud/cloud_event.h | 1 + .../include/cloud/cloud_extra_data.h | 2 +- .../framework/include/cloud/cloud_info.h | 2 + .../include/cloud/cloud_last_sync_info.h | 41 + .../framework/include/cloud/cloud_mark.h | 42 + .../framework/include/cloud/schema_meta.h | 1 + .../framework/include/cloud/sync_event.h | 2 +- .../include/dfx/behaviour_reporter.h | 6 +- .../include/dfx/db_meta_callback_delegate.h | 6 +- .../include/dfx/dfx_types.h | 8 +- .../include/dfx/fault_reporter.h | 10 +- .../include/dfx/radar_reporter.h | 8 +- .../include/dfx/reporter.h | 56 +- .../include/dfx/statistic_reporter.h | 4 +- .../include/directory/directory_manager.h | 3 + .../framework/include/error/general_error.h | 2 +- .../include/feature/feature_system.h | 2 +- .../framework/include/feature/static_acts.h | 9 + .../include/metadata/device_meta_data.h | 34 + .../include/metadata/meta_data_manager.h | 4 +- .../include/metadata/secret_key_meta_data.h | 1 + .../include/metadata/store_meta_data.h | 2 + .../include/metadata/store_meta_data_local.h | 12 + .../include/metadata/strategy_meta_data.h | 1 + .../include/network/network_delegate.h} | 39 +- .../framework/include/store/auto_cache.h | 13 +- .../framework/include/store/general_store.h | 12 + .../include/utils/time_utils.h | 10 +- .../framework/metadata/device_meta_data.cpp | 47 + .../framework/metadata/meta_data_manager.cpp | 8 +- .../metadata/secret_key_meta_data.cpp | 2 + .../framework/metadata/store_meta_data.cpp | 10 +- .../metadata/store_meta_data_local.cpp | 19 + .../framework/network/network_delegate.cpp | 40 + .../framework/store/auto_cache.cpp | 70 +- .../framework/test/BUILD.gn | 17 +- .../framework/test/cloud_test.cpp | 7 +- .../framework/test/meta_data_manager_test.cpp | 84 +- .../framework/test/meta_data_test.cpp | 462 +- .../framework/test/store_test.cpp | 93 +- .../framework/test/utils_test.cpp | 7 +- .../framework/utils/corrupt_reporter.cpp | 3 + .../src => framework/utils}/time_utils.cpp | 6 +- .../rust/connect_adapter/BUILD.gn | 10 +- .../rust/extension/BUILD.gn | 6 +- .../distributeddataservice/service/BUILD.gn | 10 + .../service/CMakeLists.txt | 2 +- .../service/backup/BUILD.gn | 3 + .../service/backup/src/backup_manager.cpp | 12 +- .../service/bootstrap/BUILD.gn | 2 + .../service/cloud/BUILD.gn | 11 +- .../service/cloud/cloud_service_impl.cpp | 207 +- .../service/cloud/cloud_service_impl.h | 15 +- .../service/cloud/cloud_types_util.cpp | 4 +- .../service/cloud/sync_manager.cpp | 205 +- .../service/cloud/sync_manager.h | 19 +- .../sync_strategies/network_sync_strategy.cpp | 19 +- .../service/common/BUILD.gn | 2 + .../service/config/BUILD.gn | 2 + .../service/config/include/config_factory.h | 2 +- .../config/include/model/global_config.h | 2 +- .../config/src/model/directory_config.cpp | 2 + .../service/crypto/BUILD.gn | 2 + .../service/crypto/include/crypto_manager.h | 66 +- .../service/crypto/src/crypto_manager.cpp | 258 +- .../service/data_share/BUILD.gn | 3 + .../data_share/common/bundle_mgr_proxy.cpp | 15 +- .../service/data_share/common/db_delegate.h | 5 +- .../service/data_share/common/kv_delegate.cpp | 34 +- .../service/data_share/common/kv_delegate.h | 7 +- .../data_share/common/rdb_delegate.cpp | 11 +- .../service/data_share/common/uri_utils.cpp | 1 - .../data_share/data/published_data.cpp | 10 +- .../service/data_share/data/template_data.cpp | 18 +- .../data_share/data_provider_config.cpp | 1 + .../service/data_share/data_provider_config.h | 1 + .../data_share/data_share_db_config.cpp | 4 + .../data_share/data_share_profile_config.cpp | 4 + .../data_share/data_share_service_impl.cpp | 161 +- .../data_share/data_share_service_impl.h | 4 + .../data_share/data_share_service_stub.cpp | 57 +- .../data_share/data_share_service_stub.h | 2 + .../service/data_share/dfx/hiview_adapter.cpp | 4 +- .../data_share/dfx/hiview_fault_adapter.cpp | 54 +- .../data_share/dfx/hiview_fault_adapter.h | 54 +- .../service/data_share/idata_share_service.h | 26 +- ...g_from_data_share_bundle_info_strategy.cpp | 1 + .../strategies/publish_strategy.cpp | 2 +- .../service/dumper/BUILD.gn | 2 + .../service/kvdb/BUILD.gn | 4 +- .../service/kvdb/kvdb_exporter.cpp | 4 +- .../service/kvdb/kvdb_general_store.cpp | 44 +- .../service/kvdb/kvdb_service_impl.cpp | 201 +- .../service/kvdb/kvdb_service_impl.h | 50 +- .../service/kvdb/kvdb_service_stub.cpp | 94 +- .../service/kvdb/upgrade.cpp | 30 +- .../service/kvdb/upgrade.h | 1 - .../service/matrix/BUILD.gn | 3 + .../service/object/BUILD.gn | 3 + .../service/object/src/object_manager.cpp | 4 +- .../service/permission/BUILD.gn | 2 + .../service/rdb/BUILD.gn | 5 +- .../service/rdb/rdb_asset_loader.cpp | 4 +- .../service/rdb/rdb_general_store.cpp | 173 +- .../service/rdb/rdb_general_store.h | 2 + .../service/rdb/rdb_hiview_adapter.cpp | 106 + .../service/rdb/rdb_hiview_adapter.h | 40 + .../service/rdb/rdb_query.cpp | 2 +- .../service/rdb/rdb_query.h | 2 +- .../service/rdb/rdb_result_set_impl.h | 11 + .../service/rdb/rdb_result_set_stub.h | 4 + .../service/rdb/rdb_schema_config.cpp | 10 +- .../service/rdb/rdb_service_impl.cpp | 209 +- .../service/rdb/rdb_service_impl.h | 24 +- .../service/rdb/rdb_service_stub.cpp | 47 +- .../service/rdb/rdb_service_stub.h | 13 +- .../service/test/BUILD.gn | 575 +- .../service/test/auth_delegate_mock_test.cpp | 177 + .../test/backup_manager_service_test.cpp | 328 + .../test/bootstrap_mock_test.cpp} | 74 +- .../service/test/cloud_data_test.cpp | 671 +- .../service/test/cloud_service_impl_test.cpp | 335 +- .../service/test/crypto_manager_test.cpp | 156 +- .../service/test/data_share_common_test.cpp | 92 + .../test/data_share_service_impl_test.cpp | 43 +- .../test/data_share_service_stub_test.cpp | 32 - .../test/data_share_types_util_test.cpp | 174 + .../service/test/device_matrix_test.cpp | 678 +- .../fuzztest/cloudservicestub_fuzzer/BUILD.gn | 6 +- .../datashareservicestub_fuzzer/BUILD.gn | 1 + .../test/fuzztest/dumphelper_fuzzer/BUILD.gn | 1 + .../fuzztest/kvdbservicestub_fuzzer/BUILD.gn | 1 - .../objectservicestub_fuzzer/BUILD.gn | 1 + .../fuzztest/rdbresultsetstub_fuzzer/BUILD.gn | 1 + .../fuzztest/rdbservicestub_fuzzer/BUILD.gn | 3 +- .../test/fuzztest/udmfservice_fuzzer/BUILD.gn | 5 + .../udmfservice_fuzzer/udmfservice_fuzzer.cpp | 250 +- .../service/test/kv_dalegate_test.cpp | 115 + .../service/test/kvdb_general_store_test.cpp | 295 +- .../service/test/kvdb_service_impl_test.cpp | 812 +- .../test/kvdb_service_stub_unittest.cpp | 354 + .../service/test/kvdb_service_test.cpp | 142 +- .../service/test/mock/BUILD.gn | 5 + .../service/test/mock/access_token_mock.cpp | 36 + .../service/test/mock/access_token_mock.h | 45 + .../test/mock/account_delegate_mock.cpp | 36 + .../service/test/mock/account_delegate_mock.h | 51 + .../test/mock/config_factory_mock.cpp} | 83 +- .../test/mock/device_manager_adapter_mock.cpp | 80 + .../test/mock/device_manager_adapter_mock.h | 25 +- .../test/mock/meta_data_manager_mock.cpp | 34 + .../test/mock/meta_data_manager_mock.h | 39 + .../test/mock/network_delegate_mock.cpp | 34 + .../service/test/mock/network_delegate_mock.h | 32 + .../service/test/mock/screen_lock_mock.cpp | 49 + .../service/test/mock/screen_lock_mock.h | 35 + .../service/test/mock/udmf_types_util.h | 42 + .../service/test/mock/user_delegate_mock.cpp | 103 + .../service/test/mock/user_delegate_mock.h | 41 + .../service/test/object_asset_loader_test.cpp | 50 + .../test/object_asset_machine_test.cpp | 33 +- .../service/test/object_manager_test.cpp | 176 +- .../service/test/object_service_impl_test.cpp | 154 + .../service/test/object_service_stub_test.cpp | 108 + .../test/permission_validator_mock_test.cpp | 93 + .../test/permit_delegate_mock_test.cpp | 208 + .../service/test/query_helper_test.cpp | 590 + .../service/test/rdb_asset_loader_test.cpp | 27 + .../service/test/rdb_cloud_test.cpp | 2 + .../service/test/rdb_general_store_test.cpp | 2 + .../service/test/udmf_run_time_store_test.cpp | 12 +- .../service/test/udmf_service_impl_test.cpp | 43 + .../test/udmf_service_stub_mock_test.cpp | 258 + .../service/test/udmf_service_stub_test.cpp | 243 + .../service/test/upgrade_mock_test.cpp | 132 + .../service/test/user_delegate_mock_test.cpp | 151 + .../service/test/value_proxy_test.cpp | 99 + .../service/udmf/BUILD.gn | 24 +- .../udmf/lifecycle/lifecycle_manager.cpp | 7 +- .../udmf/lifecycle/lifecycle_manager.h | 8 - .../udmf/lifecycle/lifecycle_policy.cpp | 16 +- .../service/udmf/lifecycle/lifecycle_policy.h | 5 - .../udmf/permission/checker_manager.cpp | 2 +- .../service/udmf/permission/checker_manager.h | 2 - .../permission/uri_permission_manager.cpp | 5 +- .../udmf/permission/uri_permission_manager.h | 5 - .../service/udmf/preprocess/data_handler.cpp | 23 +- .../service/udmf/preprocess/data_handler.h | 26 +- .../udmf/preprocess/preprocess_utils.cpp | 180 +- .../udmf/preprocess/preprocess_utils.h | 14 +- .../service/udmf/preprocess/udmf_dialog.cpp | 99 - .../service/udmf/preprocess/udmf_dialog.h | 52 - .../service/udmf/store/runtime_store.cpp | 128 +- .../service/udmf/store/runtime_store.h | 8 +- .../service/udmf/store/store.h | 7 +- .../udmf/store/store_account_observer.cpp | 2 - .../service/udmf/store/store_cache.cpp | 9 +- .../service/udmf/store/store_cache.h | 5 +- .../service/udmf/udmf_service_impl.cpp | 225 +- .../service/udmf/udmf_service_impl.h | 15 +- .../service/udmf/udmf_service_stub.cpp | 46 +- .../service/udmf/udmf_service_stub.h | 8 +- googletest/BUILD.gn | 7 + googletest/bundle.json | 6 + .../api/@ohos.data.relationalStore.d.ts | 10 + kv_store/BUILD.gn | 2 +- kv_store/databaseutils/CMakeLists.txt | 2 +- kv_store/databaseutils/test/BUILD.gn | 2 +- kv_store/frameworks/CMakeLists.txt | 4 +- kv_store/frameworks/common/BUILD.gn | 2 +- kv_store/frameworks/common/test/BUILD.gn | 4 +- .../src/distributed_kv_data_manager.cpp | 26 +- .../kvdb/include/auto_sync_timer.h | 1 + .../kvdb/include/backup_manager.h | 13 +- .../kvdb/include/kv_hiview_reporter.h | 9 +- .../innerkitsimpl/kvdb/include/kvdb_service.h | 39 +- .../kvdb/include/kvdb_service_client.h | 40 +- .../kvdb/include/observer_bridge.h | 4 +- .../kvdb/include/single_store_impl.h | 4 +- .../kvdb/include/store_factory.h | 15 +- .../kvdb/include/store_manager.h | 11 +- .../kvdb/src/auto_sync_timer.cpp | 2 +- .../kvdb/src/auto_sync_timer_mock.cpp | 2 +- .../innerkitsimpl/kvdb/src/backup_manager.cpp | 28 +- .../kvdb/src/kv_hiview_reporter.cpp | 27 +- .../kvdb/src/kv_hiview_reporter_mock.cpp | 3 +- .../kvdb/src/kvdb_service_client.cpp | 74 +- .../kvdb/src/observer_bridge.cpp | 17 +- .../kvdb/src/single_store_impl.cpp | 53 +- .../innerkitsimpl/kvdb/src/store_factory.cpp | 46 +- .../innerkitsimpl/kvdb/src/store_manager.cpp | 64 +- .../innerkitsimpl/kvdb/test/BUILD.gn | 89 +- .../kvdb/test/backup_manager_test.cpp | 63 + .../mock/src/kvdb_service_client_mock.cpp | 39 +- .../test/mock/src/observer_bridge_mock.cpp | 5 +- .../kvdb/test/single_store_impl_test.cpp | 20 +- .../distributeddata/include/uv_queue.h | 7 - .../distributeddata/src/js_kv_store.cpp | 3 +- .../distributeddata/src/napi_queue.cpp | 12 +- .../distributeddata/src/uv_queue.cpp | 99 +- .../include/js_field_node.h | 6 +- .../include/js_kv_manager.h | 2 +- .../distributedkvstore/include/uv_queue.h | 7 - .../distributedkvstore/src/js_field_node.cpp | 31 +- .../distributedkvstore/src/js_kv_manager.cpp | 2 +- .../src/js_single_kv_store.cpp | 1 + .../distributedkvstore/src/napi_queue.cpp | 12 +- .../distributedkvstore/src/uv_queue.cpp | 99 +- kv_store/frameworks/libs/CMakeLists.txt | 4 +- .../frameworks/libs/distributeddb/BUILD.gn | 1 + .../common/include/cloud/cloud_db_constant.h | 2 - .../distributeddb/common/include/db_common.h | 3 + .../common/include/db_constant.h | 4 +- .../distributeddb/common/include/db_errno.h | 2 +- .../relational/relational_schema_object.h | 1 + .../common/include/relational/table_info.h | 4 + .../common/include/runtime_context.h | 2 + .../common/include/schema_negotiate.h | 8 +- .../src/cloud/assets_download_manager.cpp | 1 + .../common/src/concurrent_adapter.cpp | 5 +- .../distributeddb/common/src/db_common.cpp | 21 +- .../common/src/evloop/src/event_impl.cpp | 1 + .../common/src/query_expression.cpp | 30 +- .../relational/relational_schema_object.cpp | 45 +- .../common/src/relational/table_info.cpp | 24 + .../common/src/runtime_context_impl.cpp | 10 + .../common/src/runtime_context_impl.h | 2 + .../common/src/schema_negotiate.cpp | 57 +- .../common/src/task_pool_impl.cpp | 7 +- .../distributeddb/common/src/task_pool_impl.h | 1 + .../include/communicator_aggregator.h | 12 +- .../communicator/include/iadapter.h | 7 +- .../include/icommunicator_aggregator.h | 2 + .../src/communicator_aggregator.cpp | 63 +- .../communicator/src/communicator_linker.cpp | 9 + .../communicator/src/communicator_linker.h | 2 + .../communicator/src/network_adapter.cpp | 31 +- .../gaussdb_rd/src/common/src/json_common.cpp | 3 + .../gaussdb_rd/test/unittest/BUILD.gn | 2 +- .../distributeddb/include/query_expression.h | 5 + .../include/cloud/cloud_store_types.h | 10 + .../include/iprocess_communicator.h | 16 +- .../interfaces/include/kv_store_nb_delegate.h | 5 + .../relational/relational_store_client.h | 7 + .../relational/relational_store_delegate.h | 20 + .../interfaces/include/store_types.h | 7 +- .../src/kv_store_nb_delegate_impl.cpp | 99 +- .../src/kv_store_nb_delegate_impl.h | 6 + .../relational_store_delegate_impl.cpp | 64 + .../relational_store_delegate_impl.h | 8 + .../relational/relational_store_manager.cpp | 8 +- .../relational_store_sqlite_ext.cpp | 86 +- .../relational/relational_sync_able_storage.h | 8 + .../interfaces/src/runtime_config.cpp | 15 +- .../sqlite_adapter/include/tokenizer_sqlite.h | 4 +- .../sqlite_adapter/src/tokenizer_api.h | 26 +- .../src/tokenizer_export_type.h | 4 + .../sqlite_adapter/src/tokenizer_sqlite.cpp | 23 +- .../include/cloud/cloud_storage_utils.h | 11 +- .../include/icloud_sync_storage_interface.h | 10 + .../storage/include/ikvdb_connection.h | 2 + .../include/relational_store_connection.h | 5 + .../storage/include/storage_proxy.h | 4 +- .../storage/src/cloud/cloud_storage_utils.cpp | 136 +- .../storage/src/cloud/schema_mgr.cpp | 5 +- .../storage/src/data_transformer.h | 1 + .../storage/src/gaussdb_rd/rd_utils.cpp | 21 +- .../storage/src/kv/generic_kvdb.cpp | 3 + .../src/kv/generic_kvdb_connection.cpp | 5 + .../storage/src/kv/generic_kvdb_connection.h | 2 + .../src/kv/sync_able_kvdb_connection.cpp | 2 +- .../storage/src/operation/database_oper.cpp | 5 +- .../storage/src/operation/database_oper.h | 5 +- .../relational_sync_able_storage.cpp | 100 +- .../relational_sync_able_storage_extend.cpp | 50 +- .../relational_sync_data_inserter.cpp | 215 +- .../relational_sync_data_inserter.h | 25 +- .../single_ver_natural_store_connection.cpp | 2 +- .../storage/src/sqlite/query_object.cpp | 11 +- .../storage/src/sqlite/query_object.h | 7 +- .../storage/src/sqlite/query_sync_object.cpp | 8 + .../cloud_sync_log_table_manager.cpp | 6 +- .../collaboration_log_table_manager.cpp | 3 +- .../relational/sqlite_relational_store.cpp | 214 +- .../relational/sqlite_relational_store.h | 10 + .../sqlite_relational_store_connection.cpp | 62 +- .../sqlite_relational_store_connection.h | 6 + .../relational/sqlite_relational_utils.cpp | 204 +- .../relational/sqlite_relational_utils.h | 8 + ...qlite_single_relational_storage_engine.cpp | 36 +- .../sqlite_single_relational_storage_engine.h | 5 +- ...e_single_ver_relational_continue_token.cpp | 6 +- ...single_ver_relational_storage_executor.cpp | 274 +- ...e_single_ver_relational_storage_executor.h | 34 +- ...ver_relational_storage_executor_extend.cpp | 105 +- ...ver_relational_storage_extend_executor.cpp | 64 +- .../sqlite/sqlite_cloud_kv_executor_utils.cpp | 16 +- .../src/sqlite/sqlite_cloud_kv_store.cpp | 69 +- .../src/sqlite/sqlite_cloud_kv_store.h | 5 + .../sqlite/sqlite_local_storage_executor.cpp | 12 +- .../src/sqlite/sqlite_log_table_manager.cpp | 30 + .../src/sqlite/sqlite_log_table_manager.h | 3 + .../src/sqlite/sqlite_meta_executor.cpp | 7 +- .../sqlite/sqlite_multi_ver_transaction.cpp | 87 +- .../src/sqlite/sqlite_query_helper.cpp | 27 +- .../sqlite/sqlite_single_ver_natural_store.h | 2 + ...te_single_ver_natural_store_connection.cpp | 10 + ...lite_single_ver_natural_store_connection.h | 1 + ...sqlite_single_ver_natural_store_extend.cpp | 10 + .../sqlite_single_ver_storage_engine.cpp | 8 + .../sqlite_single_ver_storage_executor.cpp | 128 +- ...lite_single_ver_storage_executor_cache.cpp | 41 +- ...ite_single_ver_storage_executor_extend.cpp | 63 +- ..._single_ver_storage_executor_subscribe.cpp | 22 +- .../storage/src/sqlite/sqlite_utils.cpp | 62 +- .../storage/src/sqlite/sqlite_utils.h | 7 + .../src/sqlite/sqlite_utils_extend.cpp | 81 +- .../storage/src/storage_proxy.cpp | 36 +- .../syncer/src/cloud/cloud_merge_strategy.cpp | 5 +- .../syncer/src/cloud/cloud_sync_utils.cpp | 47 +- .../syncer/src/cloud/cloud_sync_utils.h | 3 + .../syncer/src/cloud/cloud_syncer.cpp | 92 +- .../syncer/src/cloud/cloud_syncer.h | 10 +- .../syncer/src/cloud/cloud_syncer_extend.cpp | 98 +- .../syncer/src/device/ability_sync.cpp | 6 +- .../syncer/src/device/generic_syncer.cpp | 8 +- .../syncer/src/device/meta_data.cpp | 15 +- .../syncer/src/device/meta_data.h | 2 +- .../device/singlever/single_ver_data_sync.cpp | 25 +- .../device/singlever/single_ver_data_sync.h | 2 +- .../singlever/single_ver_data_sync_extend.cpp | 8 +- .../singlever/single_ver_data_sync_utils.cpp | 12 +- .../single_ver_relational_syncer.cpp | 86 +- .../singlever/single_ver_relational_syncer.h | 4 +- .../single_ver_serialize_manager.cpp | 6 +- .../single_ver_sync_state_machine.cpp | 7 +- .../single_ver_sync_task_context.cpp | 4 + .../syncer/src/device/sync_engine.cpp | 3 + .../distributeddb/syncer/src/time_helper.cpp | 7 - .../distributeddb/syncer/src/time_helper.h | 2 - .../libs/distributeddb/test/BUILD.gn | 37 +- .../test/fuzztest/json_fuzzer/json_fuzzer.cpp | 6 +- .../jsoninner_fuzzer/jsoninner_fuzzer.cpp | 6 +- .../fuzztest/parseckeck_fuzzer/project.xml | 2 +- .../test/fuzztest/query_fuzzer/BUILD.gn | 1 + .../fuzztest/query_fuzzer/query_fuzzer.cpp | 19 + .../storage_fuzzer/storage_fuzzer.cpp | 3 +- .../common/common/basic_unit_test.cpp | 94 + .../unittest/common/common/basic_unit_test.h | 51 + ...ibuteddb_relational_schema_object_test.cpp | 57 +- .../common/distributeddb_tools_unit_test.cpp | 68 +- .../common/distributeddb_tools_unit_test.h | 5 + .../common/common/evloop_timer_unit_test.cpp | 6 +- .../unittest/common/common/kv_general_ut.cpp | 209 + .../unittest/common/common/kv_general_ut.h | 50 + .../common/process_communicator_test_stub.h | 13 + .../common/common/rdb_data_generator.cpp | 47 +- .../common/common/rdb_data_generator.h | 20 +- .../unittest/common/common/rdb_general_ut.cpp | 377 + .../unittest/common/common/rdb_general_ut.h | 74 + .../common/communicator/adapter_stub.cpp | 34 +- .../common/communicator/adapter_stub.h | 5 +- .../distributeddb_communicator_deep_test.cpp | 22 +- .../communicator/mock_process_communicator.h | 4 +- ...b_cloud_interfaces_relational_ext_test.cpp | 12 +- ...ces_relational_remove_device_data_test.cpp | 46 +- ...tributeddb_interfaces_auto_launch_test.cpp | 4 + ...b_interfaces_import_and_export_rd_test.cpp | 1 + ...eddb_interfaces_import_and_export_test.cpp | 120 + ...ddb_interfaces_nb_delegate_extend_test.cpp | 2 +- ...buteddb_interfaces_nb_delegate_rd_test.cpp | 49 +- ...uteddb_interfaces_relational_sync_test.cpp | 71 +- ...stributeddb_interfaces_relational_test.cpp | 84 +- ...terfaces_relational_tracker_table_test.cpp | 177 +- .../process_system_api_adapter_impl.cpp | 37 +- .../process_system_api_adapter_impl.h | 3 + ...teddb_cloud_assets_operation_sync_test.cpp | 73 + .../distributeddb_cloud_check_sync_test.cpp | 168 +- ..._cloud_interfaces_relational_sync_test.cpp | 7 +- .../distributeddb_cloud_meta_data_test.cpp | 96 + ...stributeddb_cloud_save_cloud_data_test.cpp | 5 +- .../distributeddb_rdb_collaboration_test.cpp | 2214 +- ...relational_cloud_syncable_storage_test.cpp | 14 +- ...distributeddb_relational_get_data_test.cpp | 2 + .../kv/distributeddb_basic_kv_test.cpp | 60 + .../kv/distributeddb_kv_data_status_test.cpp | 318 + .../rdb/distributeddb_basic_rdb_test.cpp | 94 + .../distributeddb_rdb_data_status_test.cpp | 209 + .../syncer/cloud/cloud_db_sync_utils_test.cpp | 7 +- ...teddb_cloud_async_download_assets_test.cpp | 254 +- .../distributeddb_cloud_kv_syncer_test.cpp | 98 + .../distributeddb_cloud_kvstore_test.cpp | 144 + .../distributeddb_cloud_strategy_test.cpp | 2 +- ...cloud_syncer_download_assets_only_test.cpp | 125 +- ...eddb_cloud_syncer_download_assets_test.cpp | 75 +- .../distributeddb_cloud_syncer_lock_test.cpp | 102 +- ...ddb_cloud_syncer_progress_manager_test.cpp | 2 +- .../syncer/cloud/virtual_asset_loader.cpp | 19 +- .../syncer/cloud/virtual_asset_loader.h | 13 +- .../syncer/distributeddb_meta_data_test.cpp | 20 + ...stributeddb_relational_multi_user_test.cpp | 12 + ...ributeddb_relational_ver_p2p_sync_test.cpp | 16 - ...buteddb_single_ver_multi_sub_user_test.cpp | 55 +- ...stributeddb_single_ver_multi_user_test.cpp | 4 +- ...buteddb_single_ver_p2p_query_sync_test.cpp | 2 +- ...uteddb_single_ver_p2p_simple_sync_test.cpp | 58 +- ...ddb_single_ver_p2p_subscribe_sync_test.cpp | 44 +- ...buteddb_single_ver_p2p_sync_check_test.cpp | 97 +- .../syncer/distributeddb_time_sync_test.cpp | 2 +- .../syncer/relational_virtual_device.cpp | 5 + .../common/syncer/relational_virtual_device.h | 1 + .../virtual_communicator_aggregator.cpp | 12 + .../syncer/virtual_communicator_aggregator.h | 4 + ...rtual_relational_ver_sync_db_interface.cpp | 8 + ...virtual_relational_ver_sync_db_interface.h | 3 + .../innerkits/distributeddata/BUILD.gn | 11 +- .../include/distributed_kv_data_manager.h | 16 +- .../distributeddata/include/kv_utils.h | 10 + .../distributeddata/include/single_kvstore.h | 9 + .../innerkits/distributeddatamgr/BUILD.gn | 2 +- .../jskits/distributeddata/BUILD.gn | 2 +- .../jskits/distributedkvstore/BUILD.gn | 2 +- kv_store/kvstoremock/distributeddb/BUILD.gn | 2 +- .../include/distributed_kv_data_manager.h | 11 +- .../src/distributed_kv_data_manager.cpp | 11 +- .../kvdb/include/single_store_impl.h | 1 + .../kvdb/include/store_factory.h | 6 +- .../kvdb/include/store_manager.h | 8 +- .../kvdb/src/single_store_impl.cpp | 4 + .../innerkitsimpl/kvdb/src/store_factory.cpp | 9 +- .../innerkitsimpl/kvdb/src/store_manager.cpp | 10 +- .../innerkits/distributeddata/BUILD.gn | 2 + .../jskits/distributeddata/BUILD.gn | 3 + .../jskits/distributedkvstore/BUILD.gn | 3 + .../single_kvstore_client/BUILD.gn | 4 + kv_store/test/fuzztest/blob_fuzzer/BUILD.gn | 1 + .../distributedkvdatamanager_fuzzer/BUILD.gn | 1 + .../distributedkvdataservice_fuzzer/BUILD.gn | 1 + .../fuzztest/singlekvstore_fuzzer/BUILD.gn | 1 + .../singlekvstorestub_fuzzer/BUILD.gn | 1 + .../test/fuzztest/typesutil_fuzzer/BUILD.gn | 1 + .../test/unittest/distributedKVStore/BUILD.gn | 2 +- .../SingleKvStoreKVPromiseJsTest.js | 25 + .../unittest/distributedKVStore/config.json | 2 +- .../test/unittest/distributeddata/BUILD.gn | 2 +- .../test/unittest/distributeddata/config.json | 2 +- mock/CMakeLists.txt | 12 +- .../include/dataobs_mgr_client.h | 153 +- .../libaccesstoken_sdk/include/tokenid_kit.h | 1 + .../file_api/interfaces/kits/security_label.h | 89 +- .../huks/libhukssdk/include/hks_type.h | 640 +- .../huks/libhukssdk/include/hks_type_enum.h | 697 + .../global_resmgr/include/resource_manager.h | 254 +- mock/sqlite/BUILD.gn | 273 +- mock/sqlite/OAT.xml | 111 + mock/sqlite/README.OpenSource | 7 +- mock/sqlite/README.md | 4 +- mock/sqlite/bundle.json | 54 + mock/sqlite/ext/misc/cksumvfs.c | 916 + mock/sqlite/include/sqlite3.h | 1506 +- mock/sqlite/include/sqlite3ext.h | 59 +- mock/sqlite/include/sqlite3icu.h | 50 + mock/sqlite/include/sqlite3sym.h | 8 + mock/sqlite/include/sqlite3tokenizer.h | 145 + .../patch/0001-BaselineWithHistoryOH.patch | 3639 ++ mock/sqlite/patch/0002-busy-debug.patch | 813 + .../patch/0003-suport-meta-recovery.patch | 1346 + .../patch/0004-Enable-and-optimize-ICU.patch | 2119 + ...ort-corruption-when-runtime-decteted.patch | 1446 + ...dd-extention-cksumvfs-and-check-page.patch | 1079 + .../patch/0007-BugFix-CurrVersion.patch | 72 + mock/sqlite/patch/BUILD.gn | 41 + mock/sqlite/patch/apply_patch.sh | 67 + .../src/BUILD.gn => mock/sqlite/sqlite.gni | 17 +- mock/sqlite/src/shell.c | 29352 +++++---- mock/sqlite/src/sqlite3.c | 50986 +++++++++++----- mock/sqlite/src/sqlite3icu.c | 925 + mock/src/mock_aa_fwk.cpp | 4 + mock/src/mock_account.cpp | 36 +- .../src/mock_begetutil.cpp | 14 +- mock/src/mock_dms.cpp | 1 + mock/src/mock_huks.cpp | 58 +- mock/src/mock_resouce_manager.cpp | 224 +- preferences/CMakeLists.txt | 4 +- .../frameworks/common/include/log_print.h | 8 - .../js/napi/common/include/uv_queue.h | 11 - .../js/napi/common/src/napi_async_call.cpp | 3 +- .../js/napi/common/src/uv_queue.cpp | 97 +- .../napi/preferences/src/napi_preferences.cpp | 90 +- .../src/napi_preferences_helper.cpp | 22 +- .../src/napi_preferences.cpp | 19 +- .../src/napi_preferences_helper.cpp | 6 +- .../js/napi/storage/src/napi_storage.cpp | 4 +- .../native/include/concurrent_map.h | 2 +- .../native/include/preferences_base.h | 5 +- .../native/include/preferences_enhance_impl.h | 6 +- .../native/include/preferences_impl.h | 26 +- .../native/include/preferences_xml_utils.h | 8 +- .../frameworks/native/include/visibility.h | 3 + .../platform/include/preferences_db_adapter.h | 14 +- .../include/preferences_dfx_adapter.h | 2 +- .../include/preferences_file_operation.h | 54 +- .../platform/src/preferences_db_adapter.cpp | 37 +- .../platform/src/preferences_dfx_adapter.cpp | 11 +- .../platform/src/preferences_file_lock.cpp | 2 +- .../frameworks/native/src/base64_helper.cpp | 15 +- .../native/src/preferences_base.cpp | 18 +- .../native/src/preferences_enhance_impl.cpp | 36 +- .../native/src/preferences_helper.cpp | 28 +- .../native/src/preferences_impl.cpp | 486 +- .../native/src/preferences_xml_utils.cpp | 724 +- .../ndk/include/oh_preferences_impl.h | 2 +- .../frameworks/ndk/src/oh_convertor.cpp | 4 +- .../frameworks/ndk/src/oh_preferences.cpp | 4 +- .../ndk/src/oh_preferences_option.cpp | 4 +- .../inner_api/include/preferences.h | 16 +- .../ndk/include/oh_preferences_option.h | 6 +- .../test/js/unittest/preferences/src/BUILD.gn | 2 +- .../src/PreferencesSyncJsunit.test.js | 72 +- .../stage_unittest/preferences/src/BUILD.gn | 4 +- .../ets/test/StagePreferencesSynctest.ets | 32 +- .../test/js/unittest/storage/src/BUILD.gn | 2 +- .../js/unittest/system_storage/src/BUILD.gn | 2 +- preferences/test/native/BUILD.gn | 2 +- .../preferences_fuzzer/preferences_fuzzer.cpp | 3 + .../native/unittest/base64_helper_test.cpp | 1 - .../native/unittest/preferences_file_test.cpp | 131 +- .../unittest/preferences_helper_test.cpp | 1 - .../preferences_storage_type_test.cpp | 613 + .../test/native/unittest/preferences_test.cpp | 22 +- .../unittest/preferences_xml_utils_test.cpp | 208 +- preferences/test/ndk/BUILD.gn | 2 +- .../preferences_ndk_storage_type_test.cpp | 62 +- relational_store/CMakeLists.txt | 2 +- relational_store/bundle.json | 20 +- .../cj/src/relational_store_ffi.cpp | 5 +- .../cj/src/relational_store_impl_rdbstore.cpp | 9 +- .../relational_store_impl_resultsetproxy.cpp | 8 +- .../js/napi/cloud_data/src/js_cloud_utils.cpp | 1 + .../cloud_data/src/js_const_properties.cpp | 11 + .../frameworks/js/napi/graphstore/BUILD.gn | 9 +- .../napi/graphstore/include/napi_async_call.h | 2 +- .../graphstore/include/napi_gdb_transaction.h | 2 +- .../js/napi/graphstore/src/napi_gdb_error.cpp | 2 +- .../include/napi_rdb_js_utils.h | 1 + .../relationalstore/include/napi_rdb_store.h | 10 +- .../mock/include/napi_rdb_store.h | 4 +- .../src/napi_rdb_const_properties.cpp | 24 +- .../relationalstore/src/napi_rdb_error.cpp | 1 + .../relationalstore/src/napi_rdb_js_utils.cpp | 7 +- .../relationalstore/src/napi_rdb_store.cpp | 68 +- .../src/napi_rdb_store_helper.cpp | 16 +- .../relationalstore/src/napi_result_set.cpp | 64 +- .../cloud_data/src/cloud_types_util.cpp | 4 +- .../native/dfx/include/rdb_dfx_errno.h | 74 + .../dfx/include/rdb_fault_hiview_reporter.h | 42 +- .../native/dfx/include/rdb_stat_reporter.h | 62 + .../dfx/src/rdb_fault_hiview_reporter.cpp | 124 +- .../native/dfx/src/rdb_stat_reporter.cpp | 85 + .../native/gdb/adapter/include/grd_adapter.h | 3 +- .../native/gdb/adapter/include/grd_error.h | 164 - .../native/gdb/adapter/src/grd_adapter.cpp | 8 +- .../native/gdb/include/db_store_impl.h | 2 +- .../frameworks/native/gdb/include/gdb_utils.h | 7 +- .../native/gdb/include/graph_connection.h | 1 + .../native/gdb/include/transaction_impl.h | 8 +- .../frameworks/native/gdb/src/connection.cpp | 2 +- .../native/gdb/src/connection_pool.cpp | 2 +- .../frameworks/native/gdb/src/db_helper.cpp | 2 +- .../native/gdb/src/db_store_impl.cpp | 6 +- .../native/gdb/src/db_store_manager.cpp | 2 +- .../frameworks/native/gdb/src/edge.cpp | 2 +- .../frameworks/native/gdb/src/full_result.cpp | 4 +- .../frameworks/native/gdb/src/gdb_utils.cpp | 2 +- .../native/gdb/src/graph_connection.cpp | 17 +- .../native/gdb/src/graph_statement.cpp | 21 +- .../frameworks/native/gdb/src/path.cpp | 2 +- .../native/gdb/src/path_segment.cpp | 2 +- .../native/gdb/src/store_config.cpp | 12 +- .../frameworks/native/gdb/src/trans_db.cpp | 2 +- .../frameworks/native/gdb/src/transaction.cpp | 7 +- .../native/gdb/src/transaction_impl.cpp | 2 +- .../frameworks/native/gdb/src/vertex.cpp | 2 +- .../frameworks/native/icu/src/icu_collect.cpp | 10 +- .../native/rdb/include/connection.h | 15 +- .../native/rdb/include/connection_pool.h | 18 +- .../frameworks/native/rdb/include/grd_error.h | 68 +- .../native/rdb/include/rd_connection.h | 8 +- .../native/rdb/include/rdb_service_proxy.h | 12 +- .../native/rdb/include/rdb_store_impl.h | 28 +- .../native/rdb/include/rdb_store_manager.h | 12 +- .../native/rdb/include/rdb_types_util.h | 12 +- .../include/shared_block_serializer_info.h | 9 + .../native/rdb/include/sqlite_connection.h | 37 +- .../native/rdb/include/sqlite_errno.h | 4 + .../native/rdb/include/sqlite_global_config.h | 8 + .../native/rdb/include/sqlite_statement.h | 4 + .../native/rdb/include/sqlite_utils.h | 19 +- .../native/rdb/include/string_utils.h | 1 + .../native/rdb/include/transaction_impl.h | 2 +- .../native/rdb/mock/include/rdb_store_impl.h | 18 +- .../mock/src/rdb_fault_hiview_reporter.cpp | 8 +- .../frameworks/native/rdb/src/connection.cpp | 10 + .../native/rdb/src/connection_pool.cpp | 255 +- .../native/rdb/src/rd_connection.cpp | 9 +- .../native/rdb/src/rd_statement.cpp | 2 +- .../frameworks/native/rdb/src/rd_utils.cpp | 97 +- .../frameworks/native/rdb/src/rdb_helper.cpp | 33 +- .../native/rdb/src/rdb_manager_impl.cpp | 2 + .../native/rdb/src/rdb_security_manager.cpp | 31 +- .../native/rdb/src/rdb_service_proxy.cpp | 58 +- .../native/rdb/src/rdb_sql_utils.cpp | 4 + .../native/rdb/src/rdb_store_config.cpp | 88 +- .../native/rdb/src/rdb_store_impl.cpp | 532 +- .../native/rdb/src/rdb_store_manager.cpp | 136 +- .../native/rdb/src/rdb_time_utils.cpp | 8 +- .../native/rdb/src/rdb_types_util.cpp | 27 +- .../frameworks/native/rdb/src/share_block.cpp | 8 +- .../rdb/src/shared_block_serializer_info.cpp | 21 +- .../native/rdb/src/sqlite_connection.cpp | 273 +- .../native/rdb/src/sqlite_global_config.cpp | 51 +- .../rdb/src/sqlite_shared_result_set.cpp | 11 +- .../native/rdb/src/sqlite_sql_builder.cpp | 7 +- .../native/rdb/src/sqlite_statement.cpp | 90 +- .../native/rdb/src/sqlite_utils.cpp | 166 +- .../native/rdb/src/step_result_set.cpp | 11 +- .../native/rdb/src/string_utils.cpp | 6 + .../frameworks/native/rdb/src/trans_db.cpp | 22 +- .../native/rdb/src/transaction_impl.cpp | 50 +- .../rdb_data_share_adapter/src/rdb_utils.cpp | 165 +- .../cloud_data/include/cloud_types.h | 19 +- .../interfaces/inner_api/gdb/BUILD.gn | 13 +- .../inner_api}/gdb/include/edge.h | 21 +- .../inner_api/gdb/include/gdb_errors.h} | 6 +- .../inner_api/gdb/include/gdb_helper.h | 2 +- .../inner_api/gdb/include/gdb_store.h | 2 +- .../inner_api/gdb/include/gdb_store_config.h | 16 +- .../{transaction.h => gdb_transaction.h} | 14 +- .../inner_api}/gdb/include/path.h | 27 +- .../inner_api}/gdb/include/path_segment.h | 23 +- .../interfaces/inner_api/gdb/include/result.h | 2 +- .../inner_api}/gdb/include/vertex.h | 27 +- .../interfaces/inner_api/rdb/BUILD.gn | 2 +- .../inner_api/rdb/include/cache_result_set.h | 10 + ...data_relational_store_ipc_interface_code.h | 2 + .../inner_api/rdb/include/rdb_errno.h | 10 + .../inner_api/rdb/include/rdb_service.h | 4 + .../inner_api/rdb/include/rdb_store_config.h | 96 +- .../inner_api/rdb/include/rdb_types.h | 68 +- .../inner_api/rdb/include/remote_result_set.h | 29 +- .../inner_api/rdb/include/transaction.h | 1 + .../rdb/mock/include/rdb_store_config.h | 85 +- .../inner_api/rdb_data_share_adapter/BUILD.gn | 2 + .../include/rdb_utils.h | 4 + .../interfaces/ndk/include/oh_rdb_types.h | 1 - .../interfaces/ndk/include/relational_store.h | 13 + .../ndk/include/relational_store_error_code.h | 2 +- .../interfaces/ndk/src/oh_rdb_transaction.cpp | 4 +- .../interfaces/ndk/src/relational_store.cpp | 32 +- .../rdbmock/frameworks/native/rdb/file_ex.h | 15 - .../rdbmock/frameworks/native/rdb/hks_api.h | 11 + .../native/rdb/relational_store_client.h | 6 + .../rdbmock/frameworks/native/rdb/sys/file.h | 25 +- .../test/js/clouddata/unittest/src/BUILD.gn | 2 +- .../test/js/dataability/unittest/src/BUILD.gn | 2 +- .../test/js/gdb/performance/src/BUILD.gn | 2 +- .../test/js/gdb/unittest/src/BUILD.gn | 2 +- .../test/js/rdb/performance/src/BUILD.gn | 2 +- .../test/js/rdb/unittest/src/BUILD.gn | 2 +- .../src/RdbstorePredicatesJsunit.test.js | 1 - .../relationalstore/performance/src/BUILD.gn | 2 +- .../performance/src/RdbStorePromisePerf.js | 18 + .../src/SceneGetValuesBucketPerf.js | 2 +- .../js/relationalstore/unittest/src/BUILD.gn | 2 +- .../unittest/src/RdbStoreAttachJsunit.test.js | 92 + .../unittest/src/RdbStoreCloud.test.js | 91 +- .../src/RdbStoreRdExecuteJsunit.test.js | 4 +- .../src/RdbStoreResultSetGetRow.test.js | 48 - .../src/RdbStoreResultSetGetValue.test.js | 58 - .../src/RdbStoreResultSetJsunit.test.js | 628 +- .../src/RdbStoreResultSetSyncJsunit.test.js | 600 +- .../unittest/src/RdbStoreTransaction.test.js | 10 +- .../unittest/src/RdbStoreVectorJsunit.test.js | 230 + .../RdbstoreConcurrentQueryWithCrud.test.js | 704 + .../src/RdbstoreMemoryDbJsunit.test.js | 1617 + .../src/RdbstoreRdbstoreJsunit.test.js | 152 +- .../test/native/appdatafwk/BUILD.gn | 2 +- .../test/native/clouddata/BUILD.gn | 2 +- .../test/native/dataability/BUILD.gn | 2 +- .../unittest/data_ability_predicates_test.cpp | 16 - relational_store/test/native/gdb/BUILD.gn | 138 +- .../gdb/fuzztest/gdbstore_fuzzer/BUILD.gn | 2 +- .../test/native/gdb/mock/grd_adapter.cpp | 407 + .../test/native/gdb/mock/grd_adapter.h | 73 + .../native/gdb/unittest/gdb_encrypt_test.cpp | 4 +- .../native/gdb/unittest/gdb_execute_test.cpp | 27 +- .../native/gdb/unittest/gdb_function_test.cpp | 2 +- .../gdb/unittest/gdb_grd_adapter_test.cpp | 455 + .../native/gdb/unittest/gdb_grdapi_test.cpp | 2 +- .../gdb/unittest/gdb_multi_thread_test.cpp | 2 +- .../native/gdb/unittest/gdb_query_test.cpp | 2 +- .../gdb/unittest/gdb_store_config_test.cpp | 41 + .../gdb/unittest/gdb_transaction_test.cpp | 203 +- relational_store/test/native/rdb/BUILD.gn | 4 +- .../rdb_store_impl_test/BUILD.gn | 2 +- .../test/native/rdb/unittest/common.cpp | 21 - .../test/native/rdb/unittest/common.h | 3 - .../rdb_store_multiprocess_createDB_test.cpp | 22 +- .../native/rdb/unittest/rdb_attach_test.cpp | 152 + .../native/rdb/unittest/rdb_delete_test.cpp | 58 +- .../rdb/unittest/rdb_double_write_test.cpp | 13 +- .../native/rdb/unittest/rdb_execute_test.cpp | 157 +- .../native/rdb/unittest/rdb_helper_test.cpp | 173 +- .../native/rdb/unittest/rdb_insert_test.cpp | 125 +- .../rdb/unittest/rdb_memory_db_test.cpp | 522 + .../rdb/unittest/rdb_open_callback_test.cpp | 87 + .../rdb/unittest/rdb_predicates_test.cpp | 98 +- .../rdb/unittest/rdb_rd_data_aging_test.cpp | 218 + .../rdb/unittest/rdb_sql_utils_test.cpp | 24 + .../rdb_sqlite_shared_result_set_test.cpp | 117 +- .../unittest/rdb_step_result_get_row_test.cpp | 4 - .../rdb/unittest/rdb_step_result_set_test.cpp | 195 +- .../rdb_store_backup_restore_test.cpp | 2 +- .../unittest/rdb_store_concurrent_test.cpp | 55 +- .../rdb/unittest/rdb_store_config_test.cpp | 197 +- .../rdb/unittest/rdb_store_impl_test.cpp | 179 +- .../rdb/unittest/rdb_store_subscribe_test.cpp | 156 + .../native/rdb/unittest/rdb_update_test.cpp | 181 +- .../rdb/unittest/rdb_wal_limit_test.cpp | 4 - .../native/rdb/unittest/transaction_test.cpp | 82 +- .../native/rdb_data_ability_adapter/BUILD.gn | 3 +- .../native/rdb_data_share_adapter/BUILD.gn | 2 +- relational_store/test/ndk/BUILD.gn | 2 +- .../ndk/unittest/rdb_store_configv2_test.cpp | 10 + .../test/ndk/unittest/rdb_store_test.cpp | 63 + test/CMakeLists.txt | 10 +- udmf/BUILD.gn | 1 + udmf/CMakeLists.txt | 2 +- udmf/adapter/BUILD.gn | 16 +- .../innerkitsimpl/client/udmf_client.h | 28 +- .../innerkitsimpl/client/utd_client.h | 16 +- .../innerkitsimpl/common/unified_meta.cpp | 35 +- udmf/bundle.json | 17 +- .../framework/common/custom_utd_json_parser.h | 3 +- udmf/framework/common/custom_utd_store.h | 5 +- udmf/framework/common/graph.h | 13 +- udmf/framework/common/tlv_tag.h | 14 + udmf/framework/common/tlv_util.cpp | 302 +- udmf/framework/common/tlv_util.h | 18 + udmf/framework/common/udmf_copy_file.cpp | 56 +- udmf/framework/common/udmf_copy_file.h | 5 +- udmf/framework/common/udmf_types_util.cpp | 4 +- udmf/framework/common/udmf_utils.cpp | 3 +- udmf/framework/common/udmf_utils.h | 1 + udmf/framework/common/unittest/BUILD.gn | 10 +- .../common/unittest/tlv_util_test.cpp | 152 + .../udmf_types_util_abnormal_test.cpp | 1 + .../common/unittest/udmf_types_util_test.cpp | 1 - udmf/framework/common/utd_cfgs_checker.h | 2 +- udmf/framework/common/utd_graph.cpp | 3 +- udmf/framework/common/utd_graph.h | 2 +- .../client/udmf_async_client.cpp | 33 +- .../innerkitsimpl/client/udmf_client.cpp | 10 + .../innerkitsimpl/common/progress_queue.cpp | 6 + .../innerkitsimpl/common/unified_key.cpp | 26 +- .../innerkitsimpl/common/unified_meta.cpp | 39 +- .../innerkitsimpl/convert/udmf_conversion.cpp | 2 + .../data/application_defined_record.cpp | 2 +- udmf/framework/innerkitsimpl/data/audio.cpp | 2 + udmf/framework/innerkitsimpl/data/file.cpp | 18 +- udmf/framework/innerkitsimpl/data/folder.cpp | 2 + udmf/framework/innerkitsimpl/data/html.cpp | 4 +- udmf/framework/innerkitsimpl/data/image.cpp | 2 + udmf/framework/innerkitsimpl/data/link.cpp | 4 +- .../innerkitsimpl/data/plain_text.cpp | 4 +- .../data/preset_type_descriptors.h | 2 +- .../data/system_defined_appitem.cpp | 6 +- .../data/system_defined_form.cpp | 5 +- .../data/system_defined_pixelmap.cpp | 3 +- .../data/system_defined_record.cpp | 2 +- udmf/framework/innerkitsimpl/data/text.cpp | 2 +- .../innerkitsimpl/data/unified_data.cpp | 95 +- .../data/unified_data_helper.cpp | 46 +- .../data/unified_html_record_process.cpp | 217 + .../innerkitsimpl/data/unified_record.cpp | 77 +- udmf/framework/innerkitsimpl/data/video.cpp | 2 + .../distributeddata_udmf_ipc_interface_code.h | 1 - .../innerkitsimpl/service/udmf_service.h | 1 - .../service/udmf_service_client.cpp | 5 - .../service/udmf_service_client.h | 1 - .../service/udmf_service_proxy.cpp | 11 - .../service/udmf_service_proxy.h | 1 - .../udmfclient_fuzzer/udmf_client_fuzzer.cpp | 5 +- .../innerkitsimpl/test/unittest/BUILD.gn | 182 +- .../test/unittest/udmf_async_client_test.cpp | 570 + .../udmf_client_hap_permission_test.cpp | 355 + .../unittest/udmf_client_sa_invoke_test.cpp | 430 + .../test/unittest/udmf_client_test.cpp | 984 +- .../test/unittest/unified_data_test.cpp | 166 +- .../test/unittest/unified_record_test.cpp | 167 +- .../jskitsimpl/common/napi_data_utils.cpp | 107 +- .../jskitsimpl/common/napi_error_utils.cpp | 2 - .../data/unified_data_channel_napi.cpp | 9 +- .../jskitsimpl/data/unified_data_napi.cpp | 3 +- .../jskitsimpl/data/unified_record_napi.cpp | 48 +- .../intelligence/aip_napi_error.cpp | 97 +- .../intelligence/aip_napi_utils.cpp | 1082 +- .../intelligence/i_aip_core_manager_impl.cpp | 11 + .../intelligence/image_embedding_napi.cpp | 46 +- .../jskitsimpl/intelligence/js_ability.cpp | 181 + .../native_module_intelligence.cpp | 1 + .../intelligence/retrieval_napi.cpp | 370 + .../intelligence/text_embedding_napi.cpp | 65 +- .../jskitsimpl/unittest/AipRdbJsTest.js | 981 + .../jskitsimpl/unittest/UdmfCallbackJsTest.js | 4 +- .../jskitsimpl/unittest/UdmfJsTest.js | 1028 +- .../jskitsimpl/unittest/UdmfPromiseJsTest.js | 10 +- udmf/framework/ndkimpl/data/udmf.cpp | 48 +- udmf/framework/ndkimpl/unittest/BUILD.gn | 18 +- .../interfaces/components}/BUILD.gn | 48 +- udmf/interfaces/components/UdmfComponents.js | 935 + .../components/source/UdmfComponents.ets | 660 + udmf/interfaces/components/udmfcomponents.cpp | 45 + udmf/interfaces/innerkits/BUILD.gn | 5 +- .../innerkits/client/getter_system.h | 8 +- .../innerkits/client/udmf_async_client.h | 4 +- .../interfaces/innerkits/client/udmf_client.h | 28 +- udmf/interfaces/innerkits/client/utd_client.h | 20 +- .../innerkits/common/async_task_params.h | 12 +- .../innerkits/common/progress_queue.h | 10 +- .../interfaces/innerkits/common/unified_key.h | 1 + .../innerkits/common/unified_meta.h | 20 +- .../innerkits/common/unified_types.h | 7 + .../innerkits/convert/ndk_data_conversion.h | 6 +- .../data/application_defined_record.h | 20 +- udmf/interfaces/innerkits/data/audio.h | 8 +- udmf/interfaces/innerkits/data/file.h | 24 +- .../interfaces/innerkits/data/flexible_type.h | 3 +- udmf/interfaces/innerkits/data/folder.h | 8 +- udmf/interfaces/innerkits/data/html.h | 18 +- udmf/interfaces/innerkits/data/image.h | 8 +- udmf/interfaces/innerkits/data/link.h | 20 +- udmf/interfaces/innerkits/data/plain_text.h | 18 +- .../innerkits/data/system_defined_appitem.h | 36 +- .../innerkits/data/system_defined_form.h | 33 +- .../innerkits/data/system_defined_pixelmap.h | 14 +- .../innerkits/data/system_defined_record.h | 16 +- udmf/interfaces/innerkits/data/text.h | 14 +- .../innerkits/data/type_descriptor.h | 32 +- udmf/interfaces/innerkits/data/unified_data.h | 50 +- .../innerkits/data/unified_data_helper.h | 1 + .../innerkits/data/unified_data_properties.h | 3 +- .../data/unified_html_record_process.h | 41 + .../innerkits/data/unified_record.h | 74 +- udmf/interfaces/jskits/BUILD.gn | 6 + .../jskits/common/napi_data_utils.h | 3 + .../jskits/data/unified_record_napi.h | 1 + .../jskits/intelligence/aip_napi_error.h | 57 +- .../jskits/intelligence/aip_napi_utils.h | 219 +- .../jskits/intelligence/i_aip_core_manager.h | 225 + .../intelligence/i_aip_core_manager_impl.h | 3 + .../jskits/intelligence/js_ability.h | 73 + .../intelligence/native_module_intelligence.h | 1 + .../jskits/intelligence/retrieval_napi.h | 72 + udmf/interfaces/ndk/BUILD.gn | 1 + utils_native/CMakeLists.txt | 2 +- 1184 files changed, 123551 insertions(+), 36926 deletions(-) create mode 100644 data_object/frameworks/innerkitsimpl/collaboration_edit/include/cloud_db_proxy.h create mode 100644 data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_thread_pool.h create mode 100644 data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_type.h create mode 100644 data_object/frameworks/innerkitsimpl/collaboration_edit/src/cloud_db_proxy.cpp rename datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.h => data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_thread_pool.cpp (52%) create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/include/js_utils.h create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_async_call.h create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_cloud_db.h create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_const_properties.h create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_sync_service.h create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/src/js_utils.cpp create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_async_call.cpp create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_cloud_db.cpp create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_const_properties.cpp create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_sync_service.cpp create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/app.json create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/resources/base/element/string.json rename {datamgr_service/services/distributeddataservice/adapter/permission => data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test}/BUILD.gn (35%) create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/Test.json create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/Application/AbilityStage.ts create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/MainAbility.ts create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/pages/index/index.ets create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/TestAbility.ts create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/pages/index.ets create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestRunner/OpenHarmonyTestRunner.ts create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/CloudDbMock.ets rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{src/CollaborationEditUnit.test.js => collaboration_edit_js_test/entry/src/main/ets/test/EditUnit.ets} (48%) rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{src/CollaborationGetEditObject.test.js => collaboration_edit_js_test/entry/src/main/ets/test/GetEditObject.ets} (66%) create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/List.test.ets rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{src/CollaborationNode.test.js => collaboration_edit_js_test/entry/src/main/ets/test/Node.ets} (52%) create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/SetCloudDb.ets create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/TestUtils.ets rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{src/CollaborationText.test.js => collaboration_edit_js_test/entry/src/main/ets/test/Text.ets} (75%) rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{src/CollaborationUndoRedo.test.js => collaboration_edit_js_test/entry/src/main/ets/test/UndoRedo.ets} (64%) create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/module.json create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/element/string.json create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/media/icon.png create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/main_pages.json create mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/shortcuts_config.json rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/{ => collaboration_edit_js_test/signature}/openharmony_sx.p7b (50%) delete mode 100644 data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/config.json delete mode 100644 datamgr_service/services/distributeddataservice/adapter/BUILD.gn delete mode 100644 datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/app_pipe_mgr_service_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/communicator_context_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/data_buffer_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/process_communicator_impl_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_client_test.cpp rename datamgr_service/services/distributeddataservice/adapter/dfx/src/{reporter.cpp => reporter_impl.cpp} (64%) create mode 100644 datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/hiview_adapter_dfx_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter_impl.h create mode 100644 datamgr_service/services/distributeddataservice/adapter/include/schema_helper/get_schema_helper.h rename datamgr_service/services/distributeddataservice/{service => adapter}/network/BUILD.gn (64%) create mode 100644 datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.cpp rename datamgr_service/services/distributeddataservice/adapter/{include/broadcaster/broadcast_sender.h => network/src/network_delegate_default_impl.h} (46%) rename datamgr_service/services/distributeddataservice/{service/network/network_adapter.cpp => adapter/network/src/network_delegate_normal_impl.cpp} (71%) rename datamgr_service/services/distributeddataservice/{service/network/network_adapter.h => adapter/network/src/network_delegate_normal_impl.h} (62%) rename datamgr_service/services/distributeddataservice/adapter/{permission => network}/test/BUILD.gn (31%) create mode 100644 datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_normal_impl_test.cpp rename datamgr_service/services/distributeddataservice/{service/test/network_adapter_test.cpp => adapter/network/test/network_delegate_test.cpp} (36%) rename datamgr_service/services/distributeddataservice/adapter/{broadcaster => schema_helper}/BUILD.gn (65%) create mode 100644 datamgr_service/services/distributeddataservice/adapter/schema_helper/src/get_schema_helper.cpp delete mode 100644 datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.cpp delete mode 100644 datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.h delete mode 100644 datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_flowctrl_manager_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/framework/cloud/cloud_last_sync_info.cpp create mode 100644 datamgr_service/services/distributeddataservice/framework/cloud/cloud_mark.cpp rename datamgr_service/services/distributeddataservice/{adapter/broadcaster/src/broadcast_sender.cpp => framework/dfx/reporter.cpp} (52%) create mode 100644 datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_last_sync_info.h create mode 100644 datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_mark.h rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/behaviour_reporter.h (81%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/db_meta_callback_delegate.h (82%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/dfx_types.h (97%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/fault_reporter.h (71%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/radar_reporter.h (91%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/reporter.h (45%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/dfx/statistic_reporter.h (89%) create mode 100644 datamgr_service/services/distributeddataservice/framework/include/metadata/device_meta_data.h rename datamgr_service/services/distributeddataservice/{adapter/include/permission/permission_validator.h => framework/include/network/network_delegate.h} (46%) rename datamgr_service/services/distributeddataservice/{adapter => framework}/include/utils/time_utils.h (79%) create mode 100644 datamgr_service/services/distributeddataservice/framework/metadata/device_meta_data.cpp create mode 100644 datamgr_service/services/distributeddataservice/framework/network/network_delegate.cpp rename datamgr_service/services/distributeddataservice/{adapter/utils/src => framework/utils}/time_utils.cpp (94%) create mode 100644 datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/auth_delegate_mock_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/backup_manager_service_test.cpp rename datamgr_service/services/distributeddataservice/{adapter/permission/test/unittest/permission_validator_test.cpp => service/test/bootstrap_mock_test.cpp} (32%) create mode 100644 datamgr_service/services/distributeddataservice/service/test/data_share_common_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/data_share_types_util_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/kv_dalegate_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/kvdb_service_stub_unittest.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.h rename datamgr_service/services/distributeddataservice/{adapter/permission/src/permission_validator.cpp => service/test/mock/config_factory_mock.cpp} (34%) create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/udmf_types_util.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.h create mode 100644 datamgr_service/services/distributeddataservice/service/test/object_service_impl_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/object_service_stub_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/permission_validator_mock_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/permit_delegate_mock_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/query_helper_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_mock_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/upgrade_mock_test.cpp create mode 100644 datamgr_service/services/distributeddataservice/service/test/user_delegate_mock_test.cpp delete mode 100644 datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.cpp delete mode 100644 datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.h create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.h create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.h create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.h create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_basic_kv_test.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_kv_data_status_test.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_basic_rdb_test.cpp create mode 100644 kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_data_status_test.cpp create mode 100644 mock/innerkits/huks/libhukssdk/include/hks_type_enum.h create mode 100644 mock/sqlite/OAT.xml create mode 100644 mock/sqlite/bundle.json create mode 100644 mock/sqlite/ext/misc/cksumvfs.c create mode 100644 mock/sqlite/include/sqlite3icu.h create mode 100644 mock/sqlite/include/sqlite3tokenizer.h create mode 100644 mock/sqlite/patch/0001-BaselineWithHistoryOH.patch create mode 100644 mock/sqlite/patch/0002-busy-debug.patch create mode 100644 mock/sqlite/patch/0003-suport-meta-recovery.patch create mode 100644 mock/sqlite/patch/0004-Enable-and-optimize-ICU.patch create mode 100644 mock/sqlite/patch/0005-Report-corruption-when-runtime-decteted.patch create mode 100644 mock/sqlite/patch/0006-Add-extention-cksumvfs-and-check-page.patch create mode 100644 mock/sqlite/patch/0007-BugFix-CurrVersion.patch create mode 100644 mock/sqlite/patch/BUILD.gn create mode 100644 mock/sqlite/patch/apply_patch.sh rename data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/BUILD.gn => mock/sqlite/sqlite.gni (64%) create mode 100644 mock/sqlite/src/sqlite3icu.c rename datamgr_service/services/distributeddataservice/framework/utils/time_statistic.cpp => mock/src/mock_begetutil.cpp (79%) create mode 100644 preferences/test/native/unittest/preferences_storage_type_test.cpp create mode 100644 relational_store/frameworks/native/dfx/include/rdb_dfx_errno.h create mode 100644 relational_store/frameworks/native/dfx/include/rdb_stat_reporter.h create mode 100644 relational_store/frameworks/native/dfx/src/rdb_stat_reporter.cpp delete mode 100644 relational_store/frameworks/native/gdb/adapter/include/grd_error.h rename relational_store/{frameworks/native => interfaces/inner_api}/gdb/include/edge.h (65%) rename relational_store/{frameworks/native/gdb/include/aip_errors.h => interfaces/inner_api/gdb/include/gdb_errors.h} (95%) rename relational_store/interfaces/inner_api/gdb/include/{transaction.h => gdb_transaction.h} (76%) rename relational_store/{frameworks/native => interfaces/inner_api}/gdb/include/path.h (63%) rename relational_store/{frameworks/native => interfaces/inner_api}/gdb/include/path_segment.h (63%) rename relational_store/{frameworks/native => interfaces/inner_api}/gdb/include/vertex.h (61%) rename datamgr_service/services/distributeddataservice/framework/include/utils/time_statistic.h => relational_store/rdbmock/frameworks/native/rdb/sys/file.h (68%) create mode 100644 relational_store/test/js/relationalstore/unittest/src/RdbStoreVectorJsunit.test.js create mode 100644 relational_store/test/js/relationalstore/unittest/src/RdbstoreConcurrentQueryWithCrud.test.js create mode 100644 relational_store/test/js/relationalstore/unittest/src/RdbstoreMemoryDbJsunit.test.js create mode 100644 relational_store/test/native/gdb/mock/grd_adapter.cpp create mode 100644 relational_store/test/native/gdb/mock/grd_adapter.h create mode 100644 relational_store/test/native/gdb/unittest/gdb_grd_adapter_test.cpp create mode 100644 relational_store/test/native/gdb/unittest/gdb_store_config_test.cpp create mode 100644 relational_store/test/native/rdb/unittest/rdb_memory_db_test.cpp create mode 100644 relational_store/test/native/rdb/unittest/rdb_rd_data_aging_test.cpp create mode 100644 udmf/framework/innerkitsimpl/data/unified_html_record_process.cpp create mode 100644 udmf/framework/innerkitsimpl/test/unittest/udmf_async_client_test.cpp create mode 100644 udmf/framework/innerkitsimpl/test/unittest/udmf_client_hap_permission_test.cpp create mode 100644 udmf/framework/innerkitsimpl/test/unittest/udmf_client_sa_invoke_test.cpp create mode 100644 udmf/framework/jskitsimpl/intelligence/js_ability.cpp create mode 100644 udmf/framework/jskitsimpl/intelligence/retrieval_napi.cpp create mode 100644 udmf/framework/jskitsimpl/unittest/AipRdbJsTest.js rename {datamgr_service/services/distributeddataservice/app/src/flowctrl_manager => udmf/interfaces/components}/BUILD.gn (47%) create mode 100644 udmf/interfaces/components/UdmfComponents.js create mode 100644 udmf/interfaces/components/source/UdmfComponents.ets create mode 100644 udmf/interfaces/components/udmfcomponents.cpp create mode 100644 udmf/interfaces/innerkits/data/unified_html_record_process.h create mode 100644 udmf/interfaces/jskits/intelligence/js_ability.h create mode 100644 udmf/interfaces/jskits/intelligence/retrieval_napi.h diff --git a/data_object/CMakeLists.txt b/data_object/CMakeLists.txt index 117034f6..5e618eaa 100644 --- a/data_object/CMakeLists.txt +++ b/data_object/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.2) project(data_object) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "-std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/cloud_db_proxy.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/cloud_db_proxy.h new file mode 100644 index 00000000..20a378e9 --- /dev/null +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/cloud_db_proxy.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_CLOUD_DB_PROXY_H +#define COLLABORATION_EDIT_CLOUD_DB_PROXY_H + +#include +#include "stdint.h" +#include "grd_type_export.h" +#include "napi_cloud_db.h" + +namespace OHOS::CollaborationEdit { +typedef struct ExtendRecordField { + GRD_CloudFieldTypeE type; + std::string fieldName; + std::string fieldValue_str; + const void *valuePtr; + uint32_t valueLen; +} ExtendRecordFieldT; + +typedef void (*ExtendFieldParser)(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField); + +class CloudDbProxy { +public: + static int32_t BatchInsert(void *cloudDB, GRD_CloudParamsT *cloudParams); + static int32_t Query(void *cloudDB, GRD_CloudParamsT *cloudParams); + static int32_t DownloadAsset(void *cloudDB, const char *equipId, char *path); + static int32_t UploadAsset(void *cloudDB, char *path); + static int32_t DeleteAsset(void *cloudDB, char *path); + static int32_t DeleteLocalAsset(void *cloudDB, char *path); + + static int32_t SendAwarenessData(void *cloudDB, const uint8_t *data, uint32_t dataSize); + static int32_t Lock(void *cloudDB, uint32_t *lockTimeMs); + static int32_t UnLock(void *cloudDB); + static int32_t HeartBeat(void *cloudDB); + static int32_t Close(void *cloudDB); + + static void CursorParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField); + static void EquipIdParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField); + static void TimestampParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField); + static void SyncLogParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField); + + void SetNapiCloudDb(NapiCloudDb *napiCloudDb); +private: + static int32_t GetCloudRecord(GRD_CloudRecordT &record, CloudParamsAdapterT ¶msAdapter); + static int32_t GetCloudParamsVector(GRD_CloudParamsT *cloudParams, + std::vector &cloudParamsVector); + static int32_t GetQueryParams(GRD_CloudParamsT *cloudParams, std::vector &queryConditions); + static int32_t CreateSingleField(GRD_CloudFieldT &field, ExtendRecordFieldT &srcField); + static int32_t ParseExtendVector(GRD_CloudParamsT *cloudParams, std::vector &extends); + static int32_t ParseExtendField(CloudParamsAdapterT &extend, uint32_t index, GRD_CloudRecordT *records, + uint32_t maxRecordSize); + static void FreeExtendFields(GRD_CloudFieldT *fields, uint8_t fieldSize); + + NapiCloudDb *napiCloudDb_; +}; +} // namespace OHOS::CollaborationEdit +#endif // COLLABORATION_EDIT_CLOUD_DB_PROXY_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_error.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_error.h index 280acdcd..f72f1cf8 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_error.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_error.h @@ -18,5 +18,8 @@ namespace OHOS::CollaborationEdit { constexpr const int E_OK = 0; +constexpr const int E_INVALID_ARGS = 1; +constexpr const int E_OUT_OF_MEMORY = 2; +constexpr const int E_MEMORY_OPERATION_ERROR = 3; } // namespace OHOS::CollaborationEdit #endif // COLLABORATION_EDIT_DB_ERROR_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store.h index e7bc62bf..0c1e798d 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store.h @@ -24,12 +24,19 @@ class DBStore { public: DBStore(GRD_DB *db, std::string name); ~DBStore(); - static const char *GetEquipId(void); + std::string GetLocalId(); GRD_DB *GetDB(); + int32_t Sync(GRD_SyncModeE mode, uint64_t syncId, GRD_SyncTaskCallbackFuncT callbackFunc); + int32_t SetThreadPool(); + int ApplyUpdate(std::string &applyInfo); + int WriteUpdate(const char *equipId, const uint8_t *data, uint32_t size, const std::string &watermark); + int GetRelativePos(const char *tableName, const char *nodeSize, uint32_t pos, std::string &relPos); + int GetAbsolutePos(const char *tableName, const char *relPos, const char *nodeSize, uint32_t *pos); private: GRD_DB *db_; std::string name_; + std::shared_ptr localId_; }; } // namespace OHOS::CollaborationEdit #endif // COLLABORATION_EDIT_DB_STORE_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store_manager.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store_manager.h index ea79dfe5..b14f829d 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store_manager.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_store_manager.h @@ -22,6 +22,8 @@ #include "db_error.h" #include "db_store.h" #include "db_store_config.h" +#include "db_thread_pool.h" +#include "napi_cloud_db.h" namespace OHOS::CollaborationEdit { class DBStoreManager { @@ -31,12 +33,21 @@ public: ~DBStoreManager(); std::shared_ptr GetDBStore(const DBStoreConfig &config); int DeleteDBStore(const DBStoreConfig &config); + int32_t SetCloudDb(std::shared_ptr dbStore, NapiCloudDb *napiCloudDb); + void InitThreadPool(); + static GRD_ThreadPoolT threadPool_; private: std::shared_ptr OpenDBStore(const DBStoreConfig &config); int RemoveDir(const char *dir); + static void Schedule(void *func, void *param); + std::mutex mutex_; std::map> storeCache_; + + std::mutex threadPoolMutex_; + static std::shared_ptr executorPool_; + static std::shared_ptr executors_; }; } // namespace OHOS::CollaborationEdit #endif // COLLABORATION_EDIT_DB_STORE_MANAGER_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_thread_pool.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_thread_pool.h new file mode 100644 index 00000000..8be34178 --- /dev/null +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/db_thread_pool.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_DB_THREAD_POOL_H +#define COLLABORATION_EDIT_DB_THREAD_POOL_H + +#include + +#include "executor_pool.h" + +namespace OHOS::CollaborationEdit { +using Task = std::function; +using TaskId = uint64_t; +using Duration = std::chrono::steady_clock::duration; + +class DBStoreMgrThreadPool { +public: + DBStoreMgrThreadPool() = default; + virtual ~DBStoreMgrThreadPool() = default; + + virtual TaskId Execute(const Task &task) = 0; +}; + +class DbThreadPool : public CollaborationEdit::DBStoreMgrThreadPool { +public: + DbThreadPool() = default; + explicit DbThreadPool(std::shared_ptr executors); + ~DbThreadPool() override; + TaskId Execute(const Task &task) override; + +private: + std::shared_ptr executors_ = nullptr; +}; + +} // namespace OHOS::CollaborationEdit +#endif // COLLABORATION_EDIT_DB_THREAD_POOL_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_api_manager.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_api_manager.h index e84b5a77..5b615281 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_api_manager.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_api_manager.h @@ -22,7 +22,14 @@ namespace OHOS::CollaborationEdit { // 1. Database open/close library interface encapsulation typedef int32_t (*DBOpen)(const char *dbPath, const char *configStr, uint32_t flags, GRD_DB **db); typedef int32_t (*DBClose)(GRD_DB *db, uint32_t flags); -typedef int32_t (*RegisterEquipId)(GRD_DB *db, GrdEquipIdGetFuncT func); +typedef int32_t (*SetLocalId)(GRD_DB *db, const char *equipId); +typedef int32_t (*GetLocalId)(GRD_DB *db, char **localId); +typedef int32_t (*ApplyUpdate)(GRD_DB *db, char *equipId, char **applyInfo); +typedef int32_t (*WriteUpdate)( + GRD_DB *db, const char *equipId, const uint8_t *data, uint32_t size, const char *watermark); +typedef int32_t (*GetRelativePos)(GRD_DB *db, const char *tableName, const char *nodeSize, uint32_t pos, char **relPos); +typedef int32_t (*GetAbsolutePos)( + GRD_DB *db, const char *tableName, const char *relPos, const char *nodeSize, uint32_t *pos); // 2. Node operation interface encapsulation typedef int32_t (*InsertElements)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, uint32_t index, GRD_DocNodeInfoT *nodeInfo, GRD_ElementIdT **outElementId); @@ -49,18 +56,29 @@ typedef int32_t (*TextReadInDeltaMode)(GRD_DB *db, GRD_XmlOpPositionT *opPos, co const char *snapshotPrev, char **delta); // 5. Undo/Redo operation interface encapsulation typedef int32_t (*DocUndoManager)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, GRD_UndoParamT *param); +typedef int32_t (*DocCloseUndoManager)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr); typedef int32_t (*DocUndo)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, char **modify); typedef int32_t (*DocRedo)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, char **modify); typedef int32_t (*DocStopCapturing)(GRD_DB *db, GRD_XmlOpPositionT *elementAddr); +// 6. Sync operation interface encapsulation +typedef int32_t (*Sync)(GRD_DB *db, GRD_SyncConfigT *config); +typedef int32_t (*RegistryThreadPool)(GRD_DB *db, GRD_ThreadPoolT *ThreadPool); // Last. Memory free and others typedef void (*FreeElementId)(GRD_ElementIdT *outElementId); typedef int32_t (*FreeValue)(char *value); +typedef int32_t (*SetCloudDb)(GRD_DB *db, GRD_ICloudDBT *iCloud); + struct GRD_APIInfo { // 1. Database open/close library interface encapsulation DBOpen DBOpenApi = nullptr; DBClose DBCloseApi = nullptr; - RegisterEquipId RegisterEquipIdApi = nullptr; + SetLocalId SetLocalIdApi = nullptr; + GetLocalId GetLocalIdApi = nullptr; + ApplyUpdate ApplyUpdateApi = nullptr; + WriteUpdate WriteUpdateApi = nullptr; + GetRelativePos GetRelativePosApi = nullptr; + GetAbsolutePos GetAbsolutePosApi = nullptr; // 2. Node operation inter InsertElements InsertElementsApi = nullptr; DeleteElements DeleteElementsApi = nullptr; @@ -79,12 +97,17 @@ struct GRD_APIInfo { TextReadInDeltaMode TextReadInDeltaModeApi = nullptr; // 5. Undo/Redo operation interface encapsulation DocUndoManager DocUndoManagerApi = nullptr; + DocCloseUndoManager DocCloseUndoManagerApi = nullptr; DocUndo DocUndoApi = nullptr; DocRedo DocRedoApi = nullptr; DocStopCapturing DocStopCapturingApi = nullptr; + // 6. Sync operation interface encapsulation + Sync SyncApi = nullptr; + RegistryThreadPool RegistryThreadPoolApi = nullptr; // Last. Memory free and others FreeElementId FreeElementIdApi = nullptr; FreeValue FreeValueApi = nullptr; + SetCloudDb SetCloudDbApi = nullptr; }; GRD_APIInfo GetApiInfoInstance(); diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_type_export.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_type_export.h index 49f9e56a..2f7be688 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_type_export.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/grd_type_export.h @@ -110,8 +110,111 @@ typedef struct GRD_DocNodeInfo { const char *content; } GRD_DocNodeInfoT; +typedef enum GRD_SyncMode { + GRD_SYNC_MODE_UPLOAD = 0, + GRD_SYNC_MODE_DOWNLOAD, + GRD_SYNC_MODE_UP_DOWN, + GRD_SYNC_MODE_DOWNLOAD_LOG, + GRD_SYNC_MODE_UP_DOWN_LOG, + GRD_SYNC_MODE_INVALID, +} GRD_SyncModeE; + +typedef enum GRD_SyncProcessStatus { + GRD_SYNC_PROCESS_PREPARED = 0, + GRD_SYNC_PROCESS_PROCESSING, + GRD_SYNC_PROCESS_FINISHED, +} GRD_SyncProcessStatusE; + +typedef struct GRD_SyncProcess { + GRD_SyncProcessStatusE status; + int32_t errCode; + GRD_SyncModeE mode; + void *cloudDB; + uint64_t syncId; +} GRD_SyncProcessT; + +typedef void (*GRD_SyncTaskCallbackFuncT)(GRD_SyncProcessT *syncProcess); + +typedef struct GRD_SyncConfig { + GRD_SyncModeE mode; + char **equipIds; + uint8_t size; // equipId size + GRD_SyncTaskCallbackFuncT callbackFunc; + uint64_t timeout; + uint64_t syncId; +} GRD_SyncConfigT; + +typedef void (*GRD_ScheduleFunc)(void *func, void *param); + +typedef struct GRD_ThreadPool { + GRD_ScheduleFunc schedule; +} GRD_ThreadPoolT; + typedef const char *(*GrdEquipIdGetFuncT)(void); +typedef enum GRD_QueryConditionType { + GRD_QUERY_CONDITION_TYPE_NOT_USE = 0, + GRD_QUERY_CONDITION_TYPE_EQUAL_TO, + GRD_QUERY_CONDITION_TYPE_NOT_EQUAL_TO, + GRD_QUERY_CONDITION_TYPE_GREATER_THAN, + GRD_QUERY_CONDITION_TYPE_LESS_THAN, + GRD_QUERY_CONDITION_TYPE_GREATER_THAN_OR_EQUAL_TO, + GRD_QUERY_CONDITION_TYPE_LESS_THAN_OR_EQUAL_TO, +} GRD_QueryConditionTypeE; + +typedef enum GRD_CloudFieldType { + GRD_CLOUD_FIELD_TYPE_INT = 0, + GRD_CLOUD_FIELD_TYPE_DOUBLE, + GRD_CLOUD_FIELD_TYPE_STRING, + GRD_CLOUD_FIELD_TYPE_BOOL, + GRD_CLOUD_FIELD_TYPE_BYTES, +} GRD_CloudFieldTypeE; + +typedef enum CloudErrorCode { + E_CLOUD_OK = 0, + E_CLOUD_ERROR, + E_QUERY_END, +} CloudErrorCodeE; + +typedef struct GRD_CloudField { + GRD_CloudFieldTypeE type; // field type + char *key; // field name + uint32_t valueLen; // value length + void *value; // value + GRD_QueryConditionTypeE condition; // query condition +} GRD_CloudFieldT; + +typedef struct GRD_CloudRecord { + GRD_CloudFieldT *fields; // fields list + uint8_t fieldSize; +} GRD_CloudRecordT; + +typedef struct GRD_CloudParams { + const char *tableName; + GRD_CloudRecordT *records; + uint32_t recordSize; + GRD_CloudRecordT **extends; + uint32_t *extendSize; +} GRD_CloudParamsT; + +typedef struct AssetLoader { + int32_t (*downloadAsset)(void *cloudDB, const char *equipId, char *path); + int32_t (*uploadAsset)(void *cloudDB, char *path); + int32_t (*deleteAsset)(void *cloudDB, char *path); + int32_t (*deleteLocalAsset)(void *cloudDB, char *path); +} AssetLoaderT; + +typedef struct GRD_ICloudDB { + void *cloudDB; + AssetLoaderT *assetLoader; + int32_t (*batchInsert)(void *cloudDB, GRD_CloudParamsT *cloudParams); + int32_t (*query)(void *cloudDB, GRD_CloudParamsT *cloudParams); + int32_t (*sendAwarenessData)(void *cloudDB, const uint8_t *data, uint32_t dataSize); + int32_t (*lock)(void *cloudDB, uint32_t *lockTimeMs); + int32_t (*unLock)(void *cloudDB); + int32_t (*heartBeat)(void *cloudDB); + int32_t (*close)(void *cloudDB); +} GRD_ICloudDBT; #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_adapter.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_adapter.h index b9b86d36..c34ff0e6 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_adapter.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_adapter.h @@ -16,20 +16,13 @@ #ifndef COLLABORATION_EDIT_RD_ADAPTER_H #define COLLABORATION_EDIT_RD_ADAPTER_H -#include - #include "db_store.h" #include "grd_type_export.h" -#include "napi_errno.h" #include "rd_utils.h" namespace OHOS::CollaborationEdit { static constexpr const uint8_t NUMBER_OF_CHARS_IN_LABEL_PREFIX = 2; const uint8_t LABEL_FRAGMENT = 5; // See kernel struct DmElementContentType -static std::map g_errMap = { - {GRD_OK, Status::SUCCESS}, - {GRD_ARRAY_INDEX_NOT_FOUND, Status::INDEX_OUT_OF_RANGE} -}; struct ID { ID(const std::string deviceId, uint64_t clock) @@ -69,11 +62,10 @@ public: std::pair ReadStringText(); int32_t CreateUndoManager(uint64_t captureTimeout); + int32_t CloseUndoManager(); int32_t Undo(); int32_t Redo(); - int32_t TransferToNapiErrNo(int32_t originNo); - private: std::shared_ptr dbStore_; std::string tableName_; diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_type.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_type.h new file mode 100644 index 00000000..e5dd622f --- /dev/null +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_type.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_RD_TYPE_H +#define COLLABORATION_EDIT_RD_TYPE_H + +namespace OHOS::CollaborationEdit { +enum SyncMode { + PUSH, + PULL, + PULL_PUSH, +}; + +enum Predicate { + EQUAL_TO, + NOT_EQUAL_TO, + GREATER_THAN, + LESS_THAN, + GREATER_THAN_OR_EQUAL_TO, + LESS_THAN_OR_EQUAL_TO, +}; + +enum ProgressCode { + CLOUD_SYNC_SUCCESS, + CLOUD_NOT_SET, + SYNC_INTERNAL_ERROR, + SYNC_EXTERNAL_ERROR, +}; +} // namespace OHOS::CollaborationEdit +#endif // COLLABORATION_EDIT_RD_TYPE_H diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_utils.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_utils.h index d0b93e99..c1bd8d44 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_utils.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/include/rd_utils.h @@ -16,16 +16,31 @@ #ifndef COLLABORATION_EDIT_RD_UTILS_H #define COLLABORATION_EDIT_RD_UTILS_H +#include + +#include #include "grd_error.h" #include "grd_type_export.h" +#include "napi_errno.h" namespace OHOS::CollaborationEdit { +static std::map g_errMap = { + {GRD_OK, Status::SUCCESS}, + {GRD_ARRAY_INDEX_NOT_FOUND, Status::INDEX_OUT_OF_RANGE} +}; class RdUtils { public: // 1. Database open/close library interface encapsulation static int RdDbOpen(const char *dbFile, const char *configStr, uint32_t flags, GRD_DB **db); static int RdDbClose(GRD_DB *db, uint32_t flags); - static int RdRegisterEquipId(GRD_DB *db, GrdEquipIdGetFuncT func); + static int RdSetLocalId(GRD_DB *db, const char *equipId); + static int RdGetLocalId(GRD_DB *db, char **equipId); + static int RdApplyUpdate(GRD_DB *db, char **applyInfo); + static int RdWriteUpdate( + GRD_DB *db, const char *equipId, const uint8_t *data, uint32_t size, const std::string &watermark); + static int RdGetRelativePos(GRD_DB *db, const char *tableName, const char *nodeSize, uint32_t pos, char **relPos); + static int RdGetAbsolutePos( + GRD_DB *db, const char *tableName, const char *relPos, const char *nodeSize, uint32_t *pos); // 2. Node operation inter static int RdInsertElements(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, uint32_t index, GRD_DocNodeInfoT *nodeInfo, GRD_ElementIdT **outElementId); @@ -52,12 +67,19 @@ public: const char *snapshotPerv, char **delta); // 5. Undo/Redo operation interface encapsulation static int RdDocUndoManager(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, GRD_UndoParamT *param); + static int RdDocCloseUndoManager(GRD_DB *db, GRD_XmlOpPositionT *elementAddr); static int RdDocUndo(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, char **modify); static int RdDocRedo(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, char **modify); static int RdDocStopCapturing(GRD_DB *db, GRD_XmlOpPositionT *elementAddr); + // 6. Sync operation interface encapsulation + static int RdSync(GRD_DB *db, GRD_SyncConfig *config); + static int RdRegistryThreadPool(GRD_DB *db, GRD_ThreadPoolT *threadPool); // Last. Memory free and others static void RdFreeElementId(GRD_ElementIdT *outElementId); static int RdFreeValue(char *value); + static int RdSetCloudDb(GRD_DB *db, GRD_ICloudDBT *iCloud); + + static int32_t TransferToNapiErrNo(int32_t originNo); }; } // namepace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/cloud_db_proxy.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/cloud_db_proxy.cpp new file mode 100644 index 00000000..0da7cca9 --- /dev/null +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/cloud_db_proxy.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "CloudDbProxy" + +#include "cloud_db_proxy.h" + +#include +#include +#include + +#include "db_error.h" +#include "log_print.h" + +namespace OHOS::CollaborationEdit { +// batchInsert log field index +constexpr int32_t LOG_EQUIP_ID_FIELD_IDX = 0; +constexpr int32_t LOG_TIMESTAMP_FIELD_IDX = 1; +constexpr int32_t LOG_SYNC_LOG_FIELD_IDX = 2; +constexpr int32_t LOG_MAX_FIELD_SIZE = 3; + +// query result field index +constexpr uint8_t QRY_RSP_EQUIP_ID_IDX = 0u; +constexpr uint8_t QRY_RSP_TIMESTAMP_IDX = 1u; +constexpr uint8_t QRY_RSP_SYNC_LOG_IDX = 2u; +constexpr uint8_t QRY_RSP_CURSOR_IDX = 3u; +constexpr uint8_t QRY_MAX_FIELD_SIZE = 4u; + +const std::string EQUIP_ID = "equipId"; +const std::string CURSOR = "cursor"; +const std::string SYNC_LOG_EVENT = "syncLogEvent"; +const std::string TIMESTAMP = "timestamp"; + +static const std::map PREDICATE_MAP = { + {GRD_QUERY_CONDITION_TYPE_EQUAL_TO, EQUAL_TO}, + {GRD_QUERY_CONDITION_TYPE_NOT_EQUAL_TO, NOT_EQUAL_TO}, + {GRD_QUERY_CONDITION_TYPE_GREATER_THAN, GREATER_THAN}, + {GRD_QUERY_CONDITION_TYPE_LESS_THAN, LESS_THAN}, + {GRD_QUERY_CONDITION_TYPE_GREATER_THAN_OR_EQUAL_TO, GREATER_THAN_OR_EQUAL_TO}, + {GRD_QUERY_CONDITION_TYPE_LESS_THAN_OR_EQUAL_TO, LESS_THAN_OR_EQUAL_TO} +}; + +static const std::map QRY_RSP_PARSER_MAP = { + {QRY_RSP_EQUIP_ID_IDX, CloudDbProxy::EquipIdParseFunc}, + {QRY_RSP_TIMESTAMP_IDX, CloudDbProxy::TimestampParseFunc}, + {QRY_RSP_SYNC_LOG_IDX, CloudDbProxy::SyncLogParseFunc}, + {QRY_RSP_CURSOR_IDX, CloudDbProxy::CursorParseFunc} +}; + +void CloudDbProxy::SetNapiCloudDb(NapiCloudDb *napiCloudDb) +{ + napiCloudDb_ = napiCloudDb; +} + +int32_t CloudDbProxy::GetCloudRecord(GRD_CloudRecordT &record, CloudParamsAdapterT ¶msAdapter) +{ + if (record.fieldSize != LOG_MAX_FIELD_SIZE) { + LOG_ERROR("[GetCloudRecord] field size go wrong, size = %{public}d", record.fieldSize); + return E_INVALID_ARGS; + } + GRD_CloudFieldT equipId = record.fields[LOG_EQUIP_ID_FIELD_IDX]; + paramsAdapter.id.assign(static_cast(equipId.value)); + + GRD_CloudFieldT syncLog = record.fields[LOG_SYNC_LOG_FIELD_IDX]; + paramsAdapter.record.assign(static_cast(syncLog.value), + static_cast(syncLog.value) + syncLog.valueLen); + + GRD_CloudFieldT timestamp = record.fields[LOG_TIMESTAMP_FIELD_IDX]; + uint64_t *timestampPtr = static_cast(timestamp.value); + paramsAdapter.timestamp = *timestampPtr; + return E_OK; +} + +void CloudDbProxy::FreeExtendFields(GRD_CloudFieldT *fields, uint8_t fieldSize) +{ + for (uint8_t i = 0; i < fieldSize; i++) { + if (fields[i].key != nullptr) { + free(fields[i].key); + fields[i].key = nullptr; + } + + if (fields[i].value != nullptr) { + free(fields[i].value); + fields[i].value = nullptr; + } + } +} + +int32_t CloudDbProxy::CreateSingleField(GRD_CloudFieldT &field, ExtendRecordFieldT &srcField) +{ + void *fieldVal = malloc(srcField.valueLen); + if (fieldVal == nullptr) { + LOG_ERROR("alloc field value go wrong"); + return E_OUT_OF_MEMORY; + } + (void)memset_s(fieldVal, srcField.valueLen, 0, srcField.valueLen); + errno_t err = memcpy_s(fieldVal, srcField.valueLen, srcField.valuePtr, srcField.valueLen); + if (err != EOK) { + LOG_ERROR("copy field value go wrong, err: %{public}d", err); + free(fieldVal); + return E_MEMORY_OPERATION_ERROR; + } + + char *fieldKey = static_cast(malloc(srcField.fieldName.length() + 1)); + if (fieldKey == nullptr) { + LOG_ERROR("alloc field key go wrong"); + free(fieldVal); + return E_OUT_OF_MEMORY; + } + err = strcpy_s(fieldKey, srcField.fieldName.length() + 1, srcField.fieldName.c_str()); + if (err != EOK) { + LOG_ERROR("copy field key go wrong, err: %{public}d", err); + free(fieldVal); + free(fieldKey); + return E_MEMORY_OPERATION_ERROR; + } + field.key = fieldKey; + field.value = fieldVal; + field.type = srcField.type; + field.valueLen = srcField.valueLen; + return E_OK; +} + +int32_t CloudDbProxy::ParseExtendField(CloudParamsAdapterT &extend, uint32_t index, GRD_CloudRecordT *records, + uint32_t maxRecordSize) +{ + if (index >= maxRecordSize) { + LOG_ERROR("index is wrong, index:%{public}u, max:%{public}u", index, maxRecordSize); + return E_INVALID_ARGS; + } + size_t fieldSize = sizeof(GRD_CloudFieldT) * QRY_MAX_FIELD_SIZE; + records[index].fields = static_cast(malloc(fieldSize)); + if (records[index].fields == nullptr) { + LOG_ERROR("alloc for extend field go wrong"); + return E_OUT_OF_MEMORY; + } + (void)memset_s(records[index].fields, fieldSize, 0, fieldSize); + + int32_t ret = E_OK; + for (uint8_t i = 0; i < QRY_MAX_FIELD_SIZE; i++) { + ExtendRecordFieldT field = {}; + auto it = QRY_RSP_PARSER_MAP.find(i); + if (it == QRY_RSP_PARSER_MAP.end()) { + LOG_ERROR("field content index is wrong, index=%{public}u", i); + FreeExtendFields(records[index].fields, records[index].fieldSize); + free(records[index].fields); + records[index].fields = nullptr; + return E_INVALID_ARGS; + } + it->second(extend, field); + + ret = CreateSingleField(records[index].fields[i], field); + if (ret != E_OK) { + LOG_ERROR("create single field go wrong, index=%{public}u, ret=%{public}d", i, ret); + FreeExtendFields(records[index].fields, records[index].fieldSize); + free(records[index].fields); + records[index].fields = nullptr; + return ret; + } + records[index].fieldSize++; + } + return E_OK; +} + +int32_t CloudDbProxy::ParseExtendVector(GRD_CloudParamsT *cloudParams, std::vector &extends) +{ + if (cloudParams == nullptr) { + LOG_ERROR("cloud params is nullptr"); + return E_INVALID_ARGS; + } + if (cloudParams->extends == nullptr || *cloudParams->extends != nullptr || cloudParams->extendSize == nullptr) { + LOG_ERROR("extend field is nullptr"); + return E_INVALID_ARGS; + } + size_t totalSize = sizeof(GRD_CloudRecordT) * extends.size(); + *cloudParams->extends = (GRD_CloudRecordT *)malloc(totalSize); + if (*cloudParams->extends == nullptr) { + LOG_ERROR("alloc extends go wrong"); + return E_INVALID_ARGS; + } + (void)memset_s(*cloudParams->extends, totalSize, 0, totalSize); + + int32_t ret = E_OK; + for (CloudParamsAdapterT extend : extends) { + ret = ParseExtendField(extend, *cloudParams->extendSize, *cloudParams->extends, extends.size()); + if (ret != E_OK) { + LOG_ERROR("parse extend field go wrong, ret=%{public}d", ret); + free(*cloudParams->extends); + *(cloudParams->extendSize) = 0; + return ret; + } + (*cloudParams->extendSize)++; + } + return E_OK; +} + +int32_t CloudDbProxy::GetCloudParamsVector(GRD_CloudParamsT *cloudParams, + std::vector &cloudParamsVector) +{ + if (cloudParams == nullptr) { + LOG_ERROR("[GetCloudParamsVector] cloud params is nullptr"); + return E_INVALID_ARGS; + } + for (uint32_t i = 0; i < cloudParams->recordSize; i++) { + CloudParamsAdapterT paramsAdapter; + int32_t ret = GetCloudRecord(cloudParams->records[i], paramsAdapter); + if (ret != 0) { + LOG_ERROR("[GetCloudParamsVector] get cloud record go wrong, ret = %{public}d", ret); + return ret; + } + cloudParamsVector.push_back(paramsAdapter); + } + return E_OK; +} + +int32_t CloudDbProxy::GetQueryParams(GRD_CloudParamsT *cloudParams, std::vector &queryConditions) +{ + if (cloudParams == nullptr) { + LOG_ERROR("cloud params is nullptr"); + return E_INVALID_ARGS; + } + if (cloudParams->records == nullptr || cloudParams->recordSize == 0) { + LOG_ERROR("record field is wrong"); + return E_INVALID_ARGS; + } + + GRD_CloudRecordT record = cloudParams->records[0]; + for (uint8_t i = 0; i < record.fieldSize; i++) { + GRD_CloudFieldT field = record.fields[i]; + QueryConditionT condition; + condition.fieldName = std::string(field.key); + auto it = PREDICATE_MAP.find(field.condition); + if (it == PREDICATE_MAP.end()) { + LOG_ERROR("predicate is wrong, field name: %{public}s, predicate = %{public}d", field.key, field.condition); + queryConditions.clear(); + return E_INVALID_ARGS; + } + condition.predicate = it->second; + if (condition.fieldName == CURSOR) { + if (field.valueLen <= 1) { + condition.fieldValue_num = 0; + queryConditions.push_back(condition); + continue; + } + std::string tmpString; + tmpString.resize(field.valueLen); + errno_t errNo = memcpy_s(&tmpString[0], field.valueLen, field.value, field.valueLen); + if (errNo != EOK) { + LOG_ERROR("copy cursor value wrong"); + return E_MEMORY_OPERATION_ERROR; + } + condition.fieldValue_num = std::stoi(tmpString); + } else if (condition.fieldName == EQUIP_ID) { + condition.fieldValue_str = std::string(static_cast(field.value)); + } else { + LOG_ERROR("condition field name is wrong, field name: %{public}s", field.key); + queryConditions.clear(); + return E_INVALID_ARGS; + } + queryConditions.push_back(condition); + } + return E_OK; +} + +void CloudDbProxy::CursorParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField) +{ + destField.type = GRD_CLOUD_FIELD_TYPE_STRING; + destField.fieldName = CURSOR; + destField.valueLen = std::to_string(extend.cursor).length() + 1; + destField.fieldValue_str = std::to_string(extend.cursor); + destField.valuePtr = reinterpret_cast(destField.fieldValue_str.c_str()); +} + +void CloudDbProxy::EquipIdParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField) +{ + destField.type = GRD_CLOUD_FIELD_TYPE_STRING; + destField.fieldName = EQUIP_ID; + destField.valueLen = extend.id.length() + 1; + destField.valuePtr = reinterpret_cast(extend.id.c_str()); +} + +void CloudDbProxy::TimestampParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField) +{ + destField.type = GRD_CLOUD_FIELD_TYPE_INT; + destField.fieldName = TIMESTAMP; + destField.valueLen = sizeof(uint64_t); + destField.valuePtr = &extend.timestamp; +} + +void CloudDbProxy::SyncLogParseFunc(CloudParamsAdapterT &extend, ExtendRecordFieldT &destField) +{ + destField.type = GRD_CLOUD_FIELD_TYPE_BYTES; + destField.fieldName = SYNC_LOG_EVENT; + destField.valueLen = extend.record.size() * sizeof(uint8_t); + destField.valuePtr = reinterpret_cast(extend.record.data()); +} + +int32_t CloudDbProxy::BatchInsert(void *cloudDB, GRD_CloudParamsT *cloudParams) +{ + std::vector cloudParamsVector; + int32_t ret = GetCloudParamsVector(cloudParams, cloudParamsVector); + if (ret != E_OK) { + LOG_ERROR("[BatchInsert] get cloud params go wrong, ret = %{public}d", ret); + return ret; + } + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + return cloudDbProxy->napiCloudDb_->BatchInsert(cloudParamsVector); +} + +int32_t CloudDbProxy::Query(void *cloudDB, GRD_CloudParamsT *cloudParams) +{ + std::vector queryConditions; + int32_t ret = GetQueryParams(cloudParams, queryConditions); + if (ret != E_OK) { + LOG_ERROR("get cloud params go wrong, ret = %{public}d", ret); + return CloudErrorCodeE::E_CLOUD_ERROR; + } + std::vector extends; + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + ret = cloudDbProxy->napiCloudDb_->Query(queryConditions, extends); + if (ret == CloudErrorCodeE::E_CLOUD_ERROR) { + LOG_ERROR("cloudDb query go wrong, ret = %{public}d", ret); + return ret; + } + if (ret == CloudErrorCodeE::E_QUERY_END) { + return ret; + } + // set query result to cloudParams + ret = ParseExtendVector(cloudParams, extends); + if (ret != E_OK) { + LOG_ERROR("cloudDb query parse result go wrong, ret = %{public}d", ret); + return CloudErrorCodeE::E_CLOUD_ERROR; + } + return CloudErrorCodeE::E_CLOUD_OK; +} + +int32_t CloudDbProxy::DownloadAsset(void *cloudDB, const char *equipId, char *path) +{ + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + std::string equipIdStr(equipId); + std::string pathStr(path); + return cloudDbProxy->napiCloudDb_->DownloadAsset(equipIdStr, pathStr); +} + +int32_t CloudDbProxy::UploadAsset(void *cloudDB, char *path) +{ + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + std::string pathStr(path); + return cloudDbProxy->napiCloudDb_->UploadAsset(pathStr); +} + +int32_t CloudDbProxy::DeleteAsset(void *cloudDB, char *path) +{ + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + std::string pathStr(path); + return cloudDbProxy->napiCloudDb_->DeleteAsset(pathStr); +} + +int32_t CloudDbProxy::DeleteLocalAsset(void *cloudDB, char *path) +{ + CloudDbProxy *cloudDbProxy = static_cast(cloudDB); + std::string pathStr(path); + return cloudDbProxy->napiCloudDb_->DeleteLocalAsset(pathStr); +} + +int32_t CloudDbProxy::SendAwarenessData(void *cloudDB, const uint8_t *data, uint32_t dataSize) +{ + return E_OK; +} + +int32_t CloudDbProxy::Lock(void *cloudDB, uint32_t *lockTimeMs) +{ + return E_OK; +} + +int32_t CloudDbProxy::UnLock(void *cloudDB) +{ + return E_OK; +} + +int32_t CloudDbProxy::HeartBeat(void *cloudDB) +{ + return E_OK; +} + +int32_t CloudDbProxy::Close(void *cloudDB) +{ + return E_OK; +} +} // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store.cpp index 295ae672..7e9dde5c 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store.cpp +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store.cpp @@ -14,34 +14,128 @@ */ #define LOG_TAG "DBStore" +#include #include "db_store.h" +#include "db_store_manager.h" +#include "grd_type_export.h" #include "log_print.h" #include "rd_utils.h" namespace OHOS::CollaborationEdit { DBStore::DBStore(GRD_DB *db, std::string name) : db_(db), name_(name) { - int ret = RdUtils::RdRegisterEquipId(db, reinterpret_cast(DBStore::GetEquipId)); + GetLocalId(); + int32_t ret = SetThreadPool(); if (ret != GRD_OK) { - LOG_ERROR("register equip id go wrong. err: %{public}d", ret); + LOG_ERROR("registry threadpool go wrong, ret: %{public}d", ret); } } DBStore::~DBStore() +{} + +std::string DBStore::GetLocalId() { - RdUtils::RdDbClose(db_, GRD_DB_CLOSE); + if (localId_) { + return *localId_; + } + char *retStr = nullptr; + int ret = RdUtils::RdGetLocalId(db_, &retStr); + if (ret != GRD_OK || retStr == nullptr) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, std::numeric_limits::max()); + uint64_t random_num = dis(gen); + localId_ = std::make_shared(std::to_string(random_num)); + ret = RdUtils::RdSetLocalId(db_, (*localId_).c_str()); + if (ret != GRD_OK) { + LOG_ERROR("Set local id go wrong. err: %{public}d", ret); + return ""; + } + } else { + localId_ = std::make_shared(retStr); + (void)RdUtils::RdFreeValue(retStr); + } + return *localId_; } -const char *DBStore::GetEquipId(void) +int32_t DBStore::SetThreadPool() { - std::string *str = new std::string("A"); - return (*str).c_str(); + DBStoreManager::GetInstance().InitThreadPool(); + int32_t ret = RdUtils::RdRegistryThreadPool(db_, &DBStoreManager::threadPool_); + return ret; +} + +int32_t DBStore::Sync(GRD_SyncModeE mode, uint64_t syncId, GRD_SyncTaskCallbackFuncT callbackFunc) +{ + LOG_INFO("Sync START, mode=%{public}d", static_cast(mode)); + if (mode == GRD_SYNC_MODE_INVALID) { + LOG_ERROR("sync mode is wrong"); + return GRD_INVALID_ARGS; + } + char equipId[] = "cloud"; + char *g_equipIds[] = {equipId}; + GRD_SyncConfig config = { + .mode = mode, + .equipIds = g_equipIds, + .size = 1, + .callbackFunc = callbackFunc, + .timeout = 1u, + .syncId = syncId, + }; + int32_t ret = RdUtils::RdSync(db_, &config); + LOG_INFO("Sync END, ret=%{public}d", ret); + return ret; } GRD_DB *DBStore::GetDB() { return db_; } + +int DBStore::ApplyUpdate(std::string &applyInfo) +{ + char *retStr = nullptr; + int ret = RdUtils::RdApplyUpdate(db_, &retStr); + if (ret != GRD_OK || retStr == nullptr) { + LOG_ERROR("ApplyUpdate go wrong. err: %{public}d", ret); + return RdUtils::TransferToNapiErrNo(ret); + } + applyInfo = std::string(retStr); + (void)RdUtils::RdFreeValue(retStr); + return RdUtils::TransferToNapiErrNo(ret); +} + +int DBStore::WriteUpdate(const char *equipId, const uint8_t *data, uint32_t size, const std::string &watermark) +{ + int ret = RdUtils::RdWriteUpdate(db_, equipId, data, size, watermark); + if (ret != GRD_OK) { + LOG_ERROR("WriteUpdate go wrong. err: %{public}d", ret); + } + return RdUtils::TransferToNapiErrNo(ret); +} + +int DBStore::GetRelativePos(const char *tableName, const char *nodeSize, uint32_t pos, std::string &relPos) +{ + char *retStr = nullptr; + int ret = RdUtils::RdGetRelativePos(db_, tableName, nodeSize, pos, &retStr); + if (ret != GRD_OK || retStr == nullptr) { + LOG_ERROR("GetRelativePos go wrong. err: %{public}d", ret); + return RdUtils::TransferToNapiErrNo(ret); + } + relPos = std::string(retStr); + (void)RdUtils::RdFreeValue(retStr); + return RdUtils::TransferToNapiErrNo(ret); +} + +int DBStore::GetAbsolutePos(const char *tableName, const char *relPos, const char *nodeSize, uint32_t *pos) +{ + int ret = RdUtils::RdGetAbsolutePos(db_, tableName, relPos, nodeSize, pos); + if (ret != GRD_OK) { + LOG_ERROR("GetAbsolutePos go wrong. err: %{public}d", ret); + } + return RdUtils::TransferToNapiErrNo(ret); +} } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store_manager.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store_manager.cpp index 104a8ea5..3cd0e16b 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store_manager.cpp +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_store_manager.cpp @@ -24,8 +24,15 @@ #include "db_common.h" #include "log_print.h" #include "rd_utils.h" +#include "cloud_db_proxy.h" namespace OHOS::CollaborationEdit { +GRD_ThreadPoolT DBStoreManager::threadPool_ = {reinterpret_cast(DBStoreManager::Schedule)}; +std::shared_ptr DBStoreManager::executorPool_ = nullptr; +std::shared_ptr DBStoreManager::executors_ = nullptr; +constexpr size_t MAX_THREADS_SIZE = 12; +constexpr size_t MIN_THREADS_SIZE = 5; + DBStoreManager &DBStoreManager::GetInstance() { static DBStoreManager manager; @@ -36,10 +43,7 @@ DBStoreManager::DBStoreManager() {} DBStoreManager::~DBStoreManager() -{ - std::lock_guard lock(mutex_); - storeCache_.clear(); -} +{} std::shared_ptr DBStoreManager::GetDBStore(const DBStoreConfig &config) { @@ -154,4 +158,76 @@ int DBStoreManager::RemoveDir(const char *dir) } return 0; } + +int DBStoreManager::SetCloudDb(std::shared_ptr dbStore, NapiCloudDb *napiCloudDb) +{ + AssetLoaderT *assetLoader = (AssetLoaderT *)malloc(sizeof(AssetLoaderT)); + if (assetLoader == nullptr) { + LOG_ERROR("[SetCloudDb] malloc AssetLoaderT go wrong"); + return -1; + } + assetLoader->downloadAsset = CloudDbProxy::DownloadAsset; + assetLoader->uploadAsset = CloudDbProxy::UploadAsset; + assetLoader->deleteAsset = CloudDbProxy::DeleteAsset; + assetLoader->deleteLocalAsset = CloudDbProxy::DeleteLocalAsset; + + CloudDbProxy *cloudDbProxy = new (std::nothrow) CloudDbProxy(); + if (cloudDbProxy == nullptr) { + LOG_ERROR("[SetCloudDb] create cloudDbProxy go wrong"); + free(assetLoader); + return -1; + } + cloudDbProxy->SetNapiCloudDb(napiCloudDb); + + GRD_ICloudDBT *cloudDB = (GRD_ICloudDBT *)malloc(sizeof(GRD_ICloudDBT)); + if (cloudDB == nullptr) { + LOG_ERROR("[SetCloudDb] malloc ICloudDBT go wrong"); + free(assetLoader); + delete cloudDbProxy; + return -1; + } + cloudDB->assetLoader = assetLoader; + cloudDB->cloudDB = reinterpret_cast(cloudDbProxy); + cloudDB->batchInsert = CloudDbProxy::BatchInsert; + cloudDB->query = CloudDbProxy::Query; + cloudDB->sendAwarenessData = CloudDbProxy::SendAwarenessData; + cloudDB->heartBeat = CloudDbProxy::HeartBeat; + cloudDB->lock = CloudDbProxy::Lock; + cloudDB->unLock = CloudDbProxy::UnLock; + cloudDB->close = CloudDbProxy::Close; + + int32_t errCode = RdUtils::RdSetCloudDb(dbStore->GetDB(), cloudDB); + if (errCode != GRD_OK) { + LOG_ERROR("[SetCloudDb] RdSetCloudDb go wrong, errCode = %{public}d", errCode); + free(assetLoader); + free(cloudDB); + delete cloudDbProxy; + return -1; + } + return 0; +} + +void DBStoreManager::Schedule(void *func, void *param) +{ + if (executors_ == nullptr) { + LOG_ERROR("executors_ is nullptr."); + return; + } + executors_->Execute([func, param]() { + void (*funcPtr)(void *) = reinterpret_cast(func); + funcPtr(param); + }); +} + +void DBStoreManager::InitThreadPool() +{ + std::lock_guard lock(threadPoolMutex_); + if (executorPool_ == nullptr) { + executorPool_ = std::make_shared(MAX_THREADS_SIZE, MIN_THREADS_SIZE); + } + if (executors_ == nullptr) { + executors_ = std::make_shared(DBStoreManager::executorPool_); + LOG_INFO("init thread pool success"); + } +} } // namespace OHOS::CollaborationEdit diff --git a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.h b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_thread_pool.cpp similarity index 52% rename from datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.h rename to data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_thread_pool.cpp index 5365396f..1a967915 100644 --- a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.h +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/db_thread_pool.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,18 +13,19 @@ * limitations under the License. */ -#ifndef BROADCAST_SENDER_IMPL_H -#define BROADCAST_SENDER_IMPL_H +#include "db_thread_pool.h" -#include "broadcast_sender.h" +namespace OHOS::CollaborationEdit { +DbThreadPool::DbThreadPool(std::shared_ptr executors) : executors_(executors) +{} -namespace OHOS::DistributedKv { -class BroadcastSenderImpl : public BroadcastSender { -public: - void SendEvent(const EventParams ¶ms) override; -private: - static const inline std::string ACTION_NAME = "DistributedDataMgrStarter"; - static const inline std::string PKG_NAME = "pkgName"; -}; -} // namespace OHOS::DistributedKv -#endif // BROADCAST_SENDER_IMPL_H +DbThreadPool::~DbThreadPool() +{ + executors_ = nullptr; +} + +TaskId DbThreadPool::Execute(const Task &task) +{ + return executors_->Execute(task); +} +} // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/grd_api_manager.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/grd_api_manager.cpp index 1e1a7446..199ac173 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/grd_api_manager.cpp +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/grd_api_manager.cpp @@ -30,7 +30,12 @@ void GRD_DBApiInitEnhance(GRD_APIInfo &GRD_DBApiInfo) // 1. Database open/close library interface encapsulation GRD_DBApiInfo.DBOpenApi = (DBOpen)dlsym(g_library, "GRD_DBOpen"); GRD_DBApiInfo.DBCloseApi = (DBClose)dlsym(g_library, "GRD_DBClose"); - GRD_DBApiInfo.RegisterEquipIdApi = (RegisterEquipId)dlsym(g_library, "GRD_RegisterEquipId"); + GRD_DBApiInfo.SetLocalIdApi = (SetLocalId)dlsym(g_library, "GRD_SetEquipId"); + GRD_DBApiInfo.GetLocalIdApi = (GetLocalId)dlsym(g_library, "GRD_GetEquipId"); + GRD_DBApiInfo.ApplyUpdateApi = (ApplyUpdate)dlsym(g_library, "GRD_OplogRelayApply"); + GRD_DBApiInfo.WriteUpdateApi = (WriteUpdate)dlsym(g_library, "GRD_WriteLogWithEquipId"); + GRD_DBApiInfo.GetRelativePosApi = (GetRelativePos)dlsym(g_library, "GRD_AbsolutePositionToRelativePosition"); + GRD_DBApiInfo.GetAbsolutePosApi = (GetAbsolutePos)dlsym(g_library, "GRD_RelativePositionToAbsolutePosition"); // 2. Node operation interface encapsulation GRD_DBApiInfo.InsertElementsApi = (InsertElements)dlsym(g_library, "GRD_XmlFragmentInsert"); GRD_DBApiInfo.DeleteElementsApi = (DeleteElements)dlsym(g_library, "GRD_XmlFragmentDelete"); @@ -49,12 +54,17 @@ void GRD_DBApiInitEnhance(GRD_APIInfo &GRD_DBApiInfo) GRD_DBApiInfo.TextReadInDeltaModeApi = (TextReadInDeltaMode)dlsym(g_library, "GRD_TextReadInDeltaMode"); // 5. Undo/Redo operation interface encapsulation GRD_DBApiInfo.DocUndoManagerApi = (DocUndoManager)dlsym(g_library, "GRD_DocUndoManager"); + GRD_DBApiInfo.DocCloseUndoManagerApi = (DocCloseUndoManager)dlsym(g_library, "GRD_DocUndoManagerClose"); GRD_DBApiInfo.DocUndoApi = (DocUndo)dlsym(g_library, "GRD_DocUndo"); GRD_DBApiInfo.DocRedoApi = (DocRedo)dlsym(g_library, "GRD_DocRedo"); GRD_DBApiInfo.DocStopCapturingApi = (DocStopCapturing)dlsym(g_library, "GRD_DocStopCapturing"); + // 6. Sync operation interface encapsulation + GRD_DBApiInfo.SyncApi = (Sync)dlsym(g_library, "GRD_Sync"); + GRD_DBApiInfo.RegistryThreadPoolApi = (RegistryThreadPool)dlsym(g_library, "GRD_RegistryThreadPool"); // Last. Memory free and others GRD_DBApiInfo.FreeElementIdApi = (FreeElementId)dlsym(g_library, "GRD_XmlFreeElementId"); GRD_DBApiInfo.FreeValueApi = (FreeValue)dlsym(g_library, "GRD_DocFree"); + GRD_DBApiInfo.SetCloudDbApi = (SetCloudDb)dlsym(g_library, "GRD_RegistryCloudDB"); #endif } diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_adapter.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_adapter.cpp index b53431cc..a2084d81 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_adapter.cpp +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_adapter.cpp @@ -78,7 +78,7 @@ std::pair> RdAdapter::InsertNode(uint32_t index, std: int32_t errCode = RdUtils::RdInsertElements(this->dbStore_->GetDB(), &position, index, &nodeInfo, &outElementId); if (errCode != GRD_OK || outElementId == nullptr) { LOG_ERROR("InsertElements go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), std::nullopt); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), std::nullopt); } ID id(std::string(outElementId->equipId), outElementId->incrClock); RdUtils::RdFreeElementId(outElementId); @@ -107,7 +107,7 @@ std::pair> RdAdapter::InsertText(uint32_t index) int32_t errCode = RdUtils::RdInsertElements(this->dbStore_->GetDB(), &position, index, &nodeInfo, &outElementId); if (errCode != GRD_OK || outElementId == nullptr) { LOG_ERROR("InsertElements go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), std::nullopt); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), std::nullopt); } ID id(std::string(outElementId->equipId), outElementId->incrClock); RdUtils::RdFreeElementId(outElementId); @@ -129,7 +129,7 @@ int32_t RdAdapter::DeleteChildren(uint32_t index, uint32_t length) if (errCode != GRD_OK) { LOG_ERROR("DeleteElements go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } std::pair RdAdapter::GetChildren(uint32_t index, uint32_t length) @@ -150,7 +150,7 @@ std::pair RdAdapter::GetChildren(uint32_t index, uint32_t int32_t errCode = RdUtils::RdGetElements(this->dbStore_->GetDB(), &position, index, length, &respXml); if (errCode != GRD_OK) { LOG_ERROR("RdGetElements go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), ""); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), ""); } std::string retString(respXml == nullptr ? "" : respXml); (void)RdUtils::RdFreeValue(respXml); @@ -172,7 +172,7 @@ std::pair RdAdapter::GetJsonString() &replyJson); if (errCode != GRD_OK) { LOG_ERROR("RdFragmentToString go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), ""); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), ""); } std::string retString(replyJson == nullptr ? "" : replyJson); (void)RdUtils::RdFreeValue(replyJson); @@ -197,7 +197,7 @@ int32_t RdAdapter::SetAttribute(const std::string &attributeName, const std::str if (errCode != GRD_OK) { LOG_ERROR("RdElementSetAttribute go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } int32_t RdAdapter::RemoveAttrribute(const std::string &attributeName) @@ -217,7 +217,7 @@ int32_t RdAdapter::RemoveAttrribute(const std::string &attributeName) if (errCode != GRD_OK) { LOG_ERROR("RdElementRemoveAttribute go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } std::pair RdAdapter::GetAttributes() @@ -238,7 +238,7 @@ std::pair RdAdapter::GetAttributes() int32_t errCode = RdUtils::RdElementGetAttributes(this->dbStore_->GetDB(), &position, &fullAttributes); if (errCode != GRD_OK) { LOG_ERROR("RdElementGetAttributes go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), ""); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), ""); } std::string attrsString(fullAttributes == nullptr ? "" : fullAttributes); (void)RdUtils::RdFreeValue(fullAttributes); @@ -263,7 +263,7 @@ int32_t RdAdapter::TextInsert(uint32_t index, const std::string &content, const if (errCode != GRD_OK) { LOG_ERROR("RdTextInsert go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } int32_t RdAdapter::TextDelete(uint32_t index, uint32_t length) @@ -283,7 +283,7 @@ int32_t RdAdapter::TextDelete(uint32_t index, uint32_t length) if (errCode != GRD_OK) { LOG_ERROR("RdTextDelete go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } int32_t RdAdapter::TextFormat(uint32_t index, uint32_t length, const std::string &formatStr) @@ -304,7 +304,7 @@ int32_t RdAdapter::TextFormat(uint32_t index, uint32_t length, const std::string if (errCode != GRD_OK) { LOG_ERROR("RdTextFormat go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } std::pair RdAdapter::GetTextLength() @@ -325,7 +325,7 @@ std::pair RdAdapter::GetTextLength() if (errCode != GRD_OK) { LOG_ERROR("RdTextGetLength go wrong, errCode = %{public}d", errCode); } - return std::make_pair(TransferToNapiErrNo(errCode), textLength); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), textLength); } std::pair RdAdapter::ReadDeltaText(const std::string &snapshot, const std::string &snapshotPrev) @@ -349,7 +349,7 @@ std::pair RdAdapter::ReadDeltaText(const std::string &snap snapshotPrevPtr, &deltaText); if (errCode != GRD_OK) { LOG_ERROR("RdTextReadInDeltaMode go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), ""); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), ""); } std::string deltaString(deltaText == nullptr ? "" : deltaText); (void)RdUtils::RdFreeValue(deltaText); @@ -374,7 +374,7 @@ std::pair RdAdapter::ReadStringText() int32_t errCode = RdUtils::RdTextReadInStrMode(this->dbStore_->GetDB(), &position, &text); if (errCode != GRD_OK) { LOG_ERROR("RdTextReadInStrMode go wrong, errCode = %{public}d", errCode); - return std::make_pair(TransferToNapiErrNo(errCode), ""); + return std::make_pair(RdUtils::TransferToNapiErrNo(errCode), ""); } std::string str(text == nullptr ? "" : text); (void)RdUtils::RdFreeValue(text); @@ -396,7 +396,21 @@ int32_t RdAdapter::CreateUndoManager(uint64_t captureTimeout) if (errCode != GRD_OK) { LOG_ERROR("RdDocUndoManager go wrong, errCode = %{public}d", errCode); } - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); +} + +int32_t RdAdapter::CloseUndoManager() +{ + GRD_XmlOpPositionT position = { + .tableName = this->tableName_.c_str(), + .elementId = nullptr, + }; + + int32_t errCode = RdUtils::RdDocCloseUndoManager(this->dbStore_->GetDB(), &position); + if (errCode != GRD_OK) { + LOG_ERROR("RdDocCloseUndoManager go wrong, errCode = %{public}d", errCode); + } + return RdUtils::TransferToNapiErrNo(errCode); } int32_t RdAdapter::Undo() @@ -412,7 +426,7 @@ int32_t RdAdapter::Undo() LOG_ERROR("RdDocUndo go wrong, errCode = %{public}d", errCode); } (void)RdUtils::RdFreeValue(modify); - return TransferToNapiErrNo(errCode); + return RdUtils::TransferToNapiErrNo(errCode); } int32_t RdAdapter::Redo() @@ -428,16 +442,7 @@ int32_t RdAdapter::Redo() LOG_ERROR("RdDocRedo go wrong, errCode = %{public}d", errCode); } (void)RdUtils::RdFreeValue(modify); - return TransferToNapiErrNo(errCode); -} - -int32_t RdAdapter::TransferToNapiErrNo(int32_t originNo) -{ - auto it = g_errMap.find(originNo); - if (it == g_errMap.end()) { - return Status::DB_ERROR; - } - return it->second; + return RdUtils::TransferToNapiErrNo(errCode); } } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_utils.cpp b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_utils.cpp index 684a52f6..a5d85b36 100644 --- a/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_utils.cpp +++ b/data_object/frameworks/innerkitsimpl/collaboration_edit/src/rd_utils.cpp @@ -46,15 +46,73 @@ int RdUtils::RdDbClose(GRD_DB *db, uint32_t flags) return GRD_ApiInfo.DBCloseApi(db, flags); } -int RdUtils::RdRegisterEquipId(GRD_DB *db, GrdEquipIdGetFuncT func) +int RdUtils::RdSetLocalId(GRD_DB *db, const char *equipId) { - if (GRD_ApiInfo.RegisterEquipIdApi == nullptr) { + if (GRD_ApiInfo.SetLocalIdApi == nullptr) { GRD_ApiInfo = GetApiInfoInstance(); } - if (GRD_ApiInfo.RegisterEquipIdApi == nullptr) { + if (GRD_ApiInfo.SetLocalIdApi == nullptr) { return GRD_NOT_SUPPORT; } - return GRD_ApiInfo.RegisterEquipIdApi(db, func); + return GRD_ApiInfo.SetLocalIdApi(db, equipId); +} + +int RdUtils::RdGetLocalId(GRD_DB *db, char **localId) +{ + if (GRD_ApiInfo.GetLocalIdApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.GetLocalIdApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.GetLocalIdApi(db, localId); +} + +int RdUtils::RdApplyUpdate(GRD_DB *db, char **applyInfo) +{ + if (GRD_ApiInfo.ApplyUpdateApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.ApplyUpdateApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.ApplyUpdateApi(db, nullptr, applyInfo); +} + +int RdUtils::RdWriteUpdate( + GRD_DB *db, const char *equipId, const uint8_t *data, uint32_t size, const std::string &watermark) +{ + if (GRD_ApiInfo.WriteUpdateApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.WriteUpdateApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.WriteUpdateApi(db, equipId, data, size, watermark.c_str()); +} + +int RdUtils::RdGetRelativePos( + GRD_DB *db, const char *tableName, const char *nodeSize, uint32_t pos, char **relPos) +{ + if (GRD_ApiInfo.GetRelativePosApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.GetRelativePosApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.GetRelativePosApi(db, tableName, nodeSize, pos, relPos); +} + +int RdUtils::RdGetAbsolutePos( + GRD_DB *db, const char *tableName, const char *relPos, const char *nodeSize, uint32_t *pos) +{ + if (GRD_ApiInfo.GetAbsolutePosApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.GetAbsolutePosApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.GetAbsolutePosApi(db, tableName, relPos, nodeSize, pos); } // 2. Node operation interface encapsulation @@ -224,6 +282,18 @@ int32_t RdUtils::RdDocUndoManager(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, G return GRD_ApiInfo.DocUndoManagerApi(db, elementAddr, param); } +int32_t RdUtils::RdDocCloseUndoManager(GRD_DB *db, GRD_XmlOpPositionT *elementAddr) +{ + if (GRD_ApiInfo.DocCloseUndoManagerApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + if (GRD_ApiInfo.DocCloseUndoManagerApi == nullptr) { + return GRD_NOT_SUPPORT; + } + } + + return GRD_ApiInfo.DocCloseUndoManagerApi(db, elementAddr); +} + int32_t RdUtils::RdDocUndo(GRD_DB *db, GRD_XmlOpPositionT *elementAddr, char **modify) { if (GRD_ApiInfo.DocUndoApi == nullptr) { @@ -260,6 +330,29 @@ int32_t RdUtils::RdDocStopCapturing(GRD_DB *db, GRD_XmlOpPositionT *elementAddr) return GRD_ApiInfo.DocStopCapturingApi(db, elementAddr); } +// 6. Sync operation interface encapsulation +int32_t RdUtils::RdSync(GRD_DB *db, GRD_SyncConfig *config) +{ + if (GRD_ApiInfo.SyncApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.SyncApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.SyncApi(db, config); +} + +int32_t RdUtils::RdRegistryThreadPool(GRD_DB *db, GRD_ThreadPoolT *threadPool) +{ + if (GRD_ApiInfo.RegistryThreadPoolApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.RegistryThreadPoolApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.RegistryThreadPoolApi(db, threadPool); +} + // Last. Memory free and others void RdUtils::RdFreeElementId(GRD_ElementIdT *outElementId) { @@ -282,4 +375,24 @@ int32_t RdUtils::RdFreeValue(char *value) } return GRD_ApiInfo.FreeValueApi(value); } + +int32_t RdUtils::RdSetCloudDb(GRD_DB *db, GRD_ICloudDBT *iCloud) +{ + if (GRD_ApiInfo.SetCloudDbApi == nullptr) { + GRD_ApiInfo = GetApiInfoInstance(); + } + if (GRD_ApiInfo.SetCloudDbApi == nullptr) { + return GRD_NOT_SUPPORT; + } + return GRD_ApiInfo.SetCloudDbApi(db, iCloud); +} + +int32_t RdUtils::TransferToNapiErrNo(int32_t originNo) +{ + auto it = g_errMap.find(originNo); + if (it == g_errMap.end()) { + return Status::DB_ERROR; + } + return it->second; +} } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/asset_change_timer.h b/data_object/frameworks/innerkitsimpl/include/adaptor/asset_change_timer.h index 75772a36..fd45e258 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/asset_change_timer.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/asset_change_timer.h @@ -14,11 +14,9 @@ */ #ifndef ASSET_CHANGE_TIMER_H #define ASSET_CHANGE_TIMER_H -#include #include "executor_pool.h" #include "flat_object_store.h" -#include "object_types.h" namespace OHOS::ObjectStore { class AssetChangeTimer { @@ -47,8 +45,6 @@ private: static std::mutex instanceMutex; static AssetChangeTimer *instance; - static constexpr uint32_t WAIT_INTERVAL = 100; - static constexpr char ASSET_SEPARATOR = '#'; }; } // namespace OHOS::ObjectStore #endif // ASSET_CHANGE_TIMER_H \ No newline at end of file diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/client_adaptor.h b/data_object/frameworks/innerkitsimpl/include/adaptor/client_adaptor.h index 695cf3df..6fa8b1c6 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/client_adaptor.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/client_adaptor.h @@ -17,7 +17,6 @@ #define OBJECT_CLIENT_ADAPTOR_H #include "object_service_proxy.h" -#include "iobject_service.h" namespace OHOS::ObjectStore { class ObjectStoreDataServiceProxy; diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_object_impl.h b/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_object_impl.h index 16fc6bf0..279346b4 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_object_impl.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_object_impl.h @@ -15,9 +15,7 @@ #ifndef DISTRIBUTED_OBJECT_IMPL_H #define DISTRIBUTED_OBJECT_IMPL_H -#include -#include "distributed_object.h" #include "flat_object_store.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_objectstore_impl.h b/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_objectstore_impl.h index 7ce14d14..a191f6c3 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_objectstore_impl.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/distributed_objectstore_impl.h @@ -16,15 +16,12 @@ #ifndef DISTRIBUTED_OBJECTSTORE_IMPL_H #define DISTRIBUTED_OBJECTSTORE_IMPL_H -#include - #include #include "distributed_objectstore.h" -#include "flat_object_store.h" namespace OHOS::ObjectStore { class WatcherProxy; -enum SyncStatus { +enum SyncStatus : int8_t { SYNC_START, SYNCING, SYNC_SUCCESS, diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_storage_engine.h b/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_storage_engine.h index 3b7b6b32..6bec2b7d 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_storage_engine.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_storage_engine.h @@ -16,12 +16,6 @@ #ifndef FLAT_OBJECT_STORAGE_ENGINE_H #define FLAT_OBJECT_STORAGE_ENGINE_H -#include -#include -#include -#include - -#include "kv_store_delegate_manager.h" #include "object_storage_engine.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_store.h b/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_store.h index 3b37fc22..e06449fd 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_store.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/flat_object_store.h @@ -16,13 +16,9 @@ #ifndef FLAT_OBJECT_STORE_H #define FLAT_OBJECT_STORE_H -#include -#include - #include "bytes.h" #include "flat_object_storage_engine.h" #include "distributed_object.h" -#include "object_types.h" namespace OHOS::ObjectStore { class FlatObjectWatcher : public TableWatcher { diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/object_service.h b/data_object/frameworks/innerkitsimpl/include/adaptor/object_service.h index f56f5a29..3898d718 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/object_service.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/object_service.h @@ -16,11 +16,6 @@ #ifndef DISTRIBUTED_OBJECT_SERVICE_H #define DISTRIBUTED_OBJECT_SERVICE_H -#include -#include -#include - -#include "object_callback.h" namespace OHOS::ObjectStore { class ObjectService { public: diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/object_storage_engine.h b/data_object/frameworks/innerkitsimpl/include/adaptor/object_storage_engine.h index ef9a747e..7678e601 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/object_storage_engine.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/object_storage_engine.h @@ -16,11 +16,6 @@ #ifndef OBJECT_STORAGE_ENGINE_H #define OBJECT_STORAGE_ENGINE_H -#include -#include -#include - -#include "kv_store_observer.h" #include "watcher.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/adaptor/watcher.h b/data_object/frameworks/innerkitsimpl/include/adaptor/watcher.h index 61c02a23..b9855bf4 100644 --- a/data_object/frameworks/innerkitsimpl/include/adaptor/watcher.h +++ b/data_object/frameworks/innerkitsimpl/include/adaptor/watcher.h @@ -16,10 +16,7 @@ #ifndef WATCHER_H #define WATCHER_H -#include - #include "kv_store_delegate_manager.h" -#include "logger.h" namespace OHOS::ObjectStore { class Watcher : public DistributedDB::KvStoreObserver { diff --git a/data_object/frameworks/innerkitsimpl/include/common/bytes.h b/data_object/frameworks/innerkitsimpl/include/common/bytes.h index 5558bf14..ecfcbc25 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/bytes.h +++ b/data_object/frameworks/innerkitsimpl/include/common/bytes.h @@ -15,16 +15,15 @@ #ifndef BYTES_H #define BYTES_H - -#include +#include #include namespace OHOS::ObjectStore { using Bytes = std::vector; -static const char *FIELDS_PREFIX = "p_"; -static const int32_t FIELDS_PREFIX_LEN = 2; -static const std::string STRING_PREFIX = "[STRING]"; -static const int32_t STRING_PREFIX_LEN = STRING_PREFIX.length(); +static constexpr const char *FIELDS_PREFIX = "p_"; +static constexpr int32_t FIELDS_PREFIX_LEN = 2; +static constexpr const char STRING_PREFIX[] = "[STRING]"; +static constexpr int32_t STRING_PREFIX_LEN = sizeof(STRING_PREFIX) - 1; } // namespace OHOS::ObjectStore #endif // BYTES_H diff --git a/data_object/frameworks/innerkitsimpl/include/common/bytes_utils.h b/data_object/frameworks/innerkitsimpl/include/common/bytes_utils.h index 125e48e6..2a30cedf 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/bytes_utils.h +++ b/data_object/frameworks/innerkitsimpl/include/common/bytes_utils.h @@ -16,9 +16,6 @@ #ifndef BYTES_UTILS_H #define BYTES_UTILS_H -#include - -#include "bytes.h" #include "logger.h" #include "objectstore_errors.h" diff --git a/data_object/frameworks/innerkitsimpl/include/common/logger.h b/data_object/frameworks/innerkitsimpl/include/common/logger.h index 63f2a1e7..ef712797 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/logger.h +++ b/data_object/frameworks/innerkitsimpl/include/common/logger.h @@ -15,7 +15,6 @@ #ifndef OBJECT_STORE_LOGGER_H #define OBJECT_STORE_LOGGER_H -#include #ifdef HILOG_ENABLE #include "hilog/log.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/common/object_radar_reporter.h b/data_object/frameworks/innerkitsimpl/include/common/object_radar_reporter.h index 2b4ae3b4..ea4922fc 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/object_radar_reporter.h +++ b/data_object/frameworks/innerkitsimpl/include/common/object_radar_reporter.h @@ -21,39 +21,39 @@ #include "visibility.h" namespace OHOS::ObjectStore { -enum BizScene { +enum BizScene : int8_t { CREATE = 1, SAVE = 2, DATA_RESTORE = 3, }; -enum CreateStage { +enum CreateStage : int8_t { INIT_STORE = 1, CREATE_TABLE = 2, RESTORE = 3, TRANSFER = 4, }; -enum SaveStage { +enum SaveStage : int8_t { SAVE_TO_SERVICE = 1, SAVE_TO_STORE = 2, PUSH_ASSETS = 3, SYNC_DATA = 4, }; -enum DataRestoreStage { +enum DataRestoreStage : int8_t { DATA_RECV = 1, ASSETS_RECV = 2, NOTIFY = 3, }; -enum StageRes { +enum StageRes : int8_t { IDLE = 0, RADAR_SUCCESS = 1, RADAR_FAILED = 2, CANCELLED = 3, }; -enum BizState { +enum BizState : int8_t { START = 1, FINISHED = 2, }; -enum ErrorCode { +enum ErrorCode : int32_t { OFFSET = 27525120, DUPLICATE_CREATE = OFFSET, NO_MEMORY = OFFSET + 1, diff --git a/data_object/frameworks/innerkitsimpl/include/common/object_utils.h b/data_object/frameworks/innerkitsimpl/include/common/object_utils.h index f1137298..35e05dbd 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/object_utils.h +++ b/data_object/frameworks/innerkitsimpl/include/common/object_utils.h @@ -15,8 +15,6 @@ #ifndef OBJECT_UTILS_H #define OBJECT_UTILS_H -#include - namespace OHOS::ObjectStore { class ObjectUtils final { public: diff --git a/data_object/frameworks/innerkitsimpl/include/common/string_utils.h b/data_object/frameworks/innerkitsimpl/include/common/string_utils.h index de81a4fe..4cf30a10 100644 --- a/data_object/frameworks/innerkitsimpl/include/common/string_utils.h +++ b/data_object/frameworks/innerkitsimpl/include/common/string_utils.h @@ -16,11 +16,6 @@ #ifndef STRING_UTILS_H #define STRING_UTILS_H -#include -#include -#include -#include - #include "bytes.h" #include "distributed_object.h" #include "logger.h" diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/app_device_status_change_listener.h b/data_object/frameworks/innerkitsimpl/include/communicator/app_device_status_change_listener.h index 257a5e4f..a235a24b 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/app_device_status_change_listener.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/app_device_status_change_listener.h @@ -16,11 +16,9 @@ #ifndef APP_DEVICE_STATUS_CHANGE_LISTENER_H #define APP_DEVICE_STATUS_CHANGE_LISTENER_H -#include "app_types.h" - namespace OHOS { namespace ObjectStore { -enum class ChangeLevelType { +enum class ChangeLevelType : int8_t { HIGH, LOW, MIN, diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_handler.h b/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_handler.h index eb025774..33f31ae6 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_handler.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_handler.h @@ -16,13 +16,6 @@ #ifndef DISTRIBUTEDDATAFWK_SRC_PIPE_HANDLER_H #define DISTRIBUTEDDATAFWK_SRC_PIPE_HANDLER_H -#include -#include -#include -#include - -#include "app_data_change_listener.h" -#include "app_types.h" #include "logger.h" #include "softbus_adapter.h" diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_mgr.h b/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_mgr.h index 64d14ee8..5a681adf 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_mgr.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/app_pipe_mgr.h @@ -16,13 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_APP_PIPE_MGR_H #define DISTRIBUTEDDATAMGR_APP_PIPE_MGR_H -#include -#include - -#include "app_data_change_listener.h" #include "app_pipe_handler.h" -#include "app_types.h" -#include "logger.h" namespace OHOS { namespace ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/app_types.h b/data_object/frameworks/innerkitsimpl/include/communicator/app_types.h index ea02e192..dc8b8fb5 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/app_types.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/app_types.h @@ -18,11 +18,7 @@ #include -#include #include -#include - -#include "visibility.h" namespace OHOS { namespace ObjectStore { @@ -36,7 +32,7 @@ struct DeviceInfo { std::string deviceType; }; -enum class MessageType { +enum class MessageType : int8_t { DEFAULT = 0, }; @@ -44,7 +40,7 @@ struct MessageInfo { MessageType msgType; }; -enum class DeviceChangeType { +enum class DeviceChangeType : int8_t { DEVICE_OFFLINE = 0, DEVICE_ONLINE = 1, }; @@ -53,7 +49,7 @@ struct DeviceId { std::string deviceId; }; -enum RouteType : int32_t { +enum RouteType : int8_t { INVALID_ROUTE_TYPE = -1, ROUTE_TYPE_ALL = 0, WIFI_STA = 1, diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/ark_communication_provider.h b/data_object/frameworks/innerkitsimpl/include/communicator/ark_communication_provider.h index a6189dde..74939b53 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/ark_communication_provider.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/ark_communication_provider.h @@ -16,8 +16,6 @@ #ifndef DISTRIBUTEDDATAFWK_ARK_COMMUNICATION_PROVIDER_H #define DISTRIBUTEDDATAFWK_ARK_COMMUNICATION_PROVIDER_H -#include "app_device_handler.h" -#include "app_pipe_mgr.h" #include "communication_provider_impl.h" #include "nocopyable.h" diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider.h b/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider.h index 14acf451..e91f6b27 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider.h @@ -21,8 +21,6 @@ #include "app_data_change_listener.h" #include "app_device_status_change_listener.h" -#include "app_types.h" -#include "visibility.h" namespace OHOS { namespace ObjectStore { class CommunicationProvider { diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider_impl.h b/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider_impl.h index 6164e068..80401951 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider_impl.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/communication_provider_impl.h @@ -16,8 +16,6 @@ #ifndef DISTRIBUTEDDATA_SRC_COMMUNICATION_PROVIDER_IMPL_H #define DISTRIBUTEDDATA_SRC_COMMUNICATION_PROVIDER_IMPL_H -#include - #include "app_device_handler.h" #include "app_pipe_mgr.h" #include "communication_provider.h" diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/dev_manager.h b/data_object/frameworks/innerkitsimpl/include/communicator/dev_manager.h index 02c62ec7..32cf16e5 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/dev_manager.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/dev_manager.h @@ -14,8 +14,6 @@ */ #ifndef DATA_OBJECT_DEV_MANAGER_H #define DATA_OBJECT_DEV_MANAGER_H -#include -#include #include namespace OHOS { namespace ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/include/communicator/softbus_adapter.h b/data_object/frameworks/innerkitsimpl/include/communicator/softbus_adapter.h index 94b22916..a4be9d58 100644 --- a/data_object/frameworks/innerkitsimpl/include/communicator/softbus_adapter.h +++ b/data_object/frameworks/innerkitsimpl/include/communicator/softbus_adapter.h @@ -15,19 +15,11 @@ #ifndef DISTRIBUTEDDATAFWK_SRC_SOFTBUS_ADAPTER_H #define DISTRIBUTEDDATAFWK_SRC_SOFTBUS_ADAPTER_H -#include -#include -#include -#include -#include #include "app_data_change_listener.h" #include "app_device_status_change_listener.h" -#include "app_types.h" #include "concurrent_map.h" -#include "session.h" #include "socket.h" -#include "softbus_bus_center.h" #include "task_scheduler.h" namespace OHOS { diff --git a/data_object/frameworks/innerkitsimpl/include/distributeddata_object_store_ipc_interface_code.h b/data_object/frameworks/innerkitsimpl/include/distributeddata_object_store_ipc_interface_code.h index 1de2eb22..0523098b 100644 --- a/data_object/frameworks/innerkitsimpl/include/distributeddata_object_store_ipc_interface_code.h +++ b/data_object/frameworks/innerkitsimpl/include/distributeddata_object_store_ipc_interface_code.h @@ -20,7 +20,7 @@ namespace OHOS::DistributedObject { namespace ObjectStoreService { -enum class ObjectServiceInterfaceCode { +enum class ObjectServiceInterfaceCode : int8_t { OBJECTSTORE_SAVE, OBJECTSTORE_REVOKE_SAVE, OBJECTSTORE_RETRIEVE, @@ -33,7 +33,7 @@ enum class ObjectServiceInterfaceCode { OBJECTSTORE_SERVICE_CMD_MAX }; -enum class KvStoreServiceInterfaceCode { +enum class KvStoreServiceInterfaceCode : int8_t { GET_FEATURE_INTERFACE = 0, REGISTERCLIENTDEATHOBSERVER, }; diff --git a/data_object/frameworks/innerkitsimpl/include/iobject_service.h b/data_object/frameworks/innerkitsimpl/include/iobject_service.h index 1bc58610..cb911ff7 100644 --- a/data_object/frameworks/innerkitsimpl/include/iobject_service.h +++ b/data_object/frameworks/innerkitsimpl/include/iobject_service.h @@ -16,9 +16,6 @@ #ifndef DISTRIBUTEDDATAFWK_IOBJECT_SERVICE_H #define DISTRIBUTEDDATAFWK_IOBJECT_SERVICE_H -#include - -#include #include "object_service.h" #include "distributeddata_object_store_ipc_interface_code.h" diff --git a/data_object/frameworks/innerkitsimpl/include/object_callback.h b/data_object/frameworks/innerkitsimpl/include/object_callback.h index ecd209b1..9279a0ad 100644 --- a/data_object/frameworks/innerkitsimpl/include/object_callback.h +++ b/data_object/frameworks/innerkitsimpl/include/object_callback.h @@ -17,8 +17,6 @@ #define OBJECT_CALLBACK_H #include -#include -#include namespace OHOS { namespace DistributedObject { diff --git a/data_object/frameworks/innerkitsimpl/include/object_callback_stub.h b/data_object/frameworks/innerkitsimpl/include/object_callback_stub.h index f0603094..8a7c7109 100644 --- a/data_object/frameworks/innerkitsimpl/include/object_callback_stub.h +++ b/data_object/frameworks/innerkitsimpl/include/object_callback_stub.h @@ -15,7 +15,6 @@ #ifndef OBJECT_CALLBACK_STUB_H #define OBJECT_CALLBACK_STUB_H -#include #include #include "object_callback.h" diff --git a/data_object/frameworks/innerkitsimpl/include/object_service.h b/data_object/frameworks/innerkitsimpl/include/object_service.h index 0c5c8d57..d59cc95b 100644 --- a/data_object/frameworks/innerkitsimpl/include/object_service.h +++ b/data_object/frameworks/innerkitsimpl/include/object_service.h @@ -16,10 +16,6 @@ #ifndef DISTRIBUTED_OBJECT_SERVICE_H #define DISTRIBUTED_OBJECT_SERVICE_H -#include -#include -#include -#include "distributeddata_object_store_ipc_interface_code.h" #include "object_types.h" namespace OHOS::DistributedObject { diff --git a/data_object/frameworks/innerkitsimpl/include/object_service_proxy.h b/data_object/frameworks/innerkitsimpl/include/object_service_proxy.h index 366207fc..724aa688 100644 --- a/data_object/frameworks/innerkitsimpl/include/object_service_proxy.h +++ b/data_object/frameworks/innerkitsimpl/include/object_service_proxy.h @@ -16,8 +16,6 @@ #ifndef DISTRIBUTED_OBJECT_SERVICE_PROXY_H #define DISTRIBUTED_OBJECT_SERVICE_PROXY_H -#include -#include #include #include "iobject_service.h" diff --git a/data_object/frameworks/innerkitsimpl/src/adaptor/asset_change_timer.cpp b/data_object/frameworks/innerkitsimpl/src/adaptor/asset_change_timer.cpp index 0655c1f1..c65ad97c 100644 --- a/data_object/frameworks/innerkitsimpl/src/adaptor/asset_change_timer.cpp +++ b/data_object/frameworks/innerkitsimpl/src/adaptor/asset_change_timer.cpp @@ -24,6 +24,8 @@ std::mutex AssetChangeTimer::instanceMutex; AssetChangeTimer *AssetChangeTimer::instance = nullptr; static constexpr size_t MAX_THREADS = 3; static constexpr size_t MIN_THREADS = 0; +static constexpr uint32_t WAIT_INTERVAL = 100; +static constexpr char ASSET_SEPARATOR = '#'; AssetChangeTimer *AssetChangeTimer::GetInstance(FlatObjectStore *flatObjectStore) { diff --git a/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_impl.cpp b/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_impl.cpp index 9bc4246e..844cd2cc 100644 --- a/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_impl.cpp +++ b/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_impl.cpp @@ -17,10 +17,8 @@ #include "hitrace.h" #include "objectstore_errors.h" -#include "string_utils.h" #include "dev_manager.h" #include "bytes_utils.h" -#include "object_radar_reporter.h" namespace OHOS::ObjectStore { DistributedObjectImpl::~DistributedObjectImpl() diff --git a/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_store_impl.cpp b/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_store_impl.cpp index 5904e831..22cf1f78 100644 --- a/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_store_impl.cpp +++ b/data_object/frameworks/innerkitsimpl/src/adaptor/distributed_object_store_impl.cpp @@ -13,14 +13,13 @@ * limitations under the License. */ -#include #include +#include #include "hitrace.h" #include "distributed_object_impl.h" #include "distributed_objectstore_impl.h" #include "objectstore_errors.h" -#include "softbus_adapter.h" #include "string_utils.h" #include "asset_change_timer.h" #include "object_radar_reporter.h" @@ -141,11 +140,11 @@ uint32_t DistributedObjectStoreImpl::Get(const std::string &sessionId, Distribut uint32_t DistributedObjectStoreImpl::Watch(DistributedObject *object, std::shared_ptr watcher) { if (object == nullptr) { - LOG_ERROR("DistributedObjectStoreImpl::Sync object err "); + LOG_ERROR("Watch sync object err "); return ERR_NULL_OBJECT; } if (flatObjectStore_ == nullptr) { - LOG_ERROR("DistributedObjectStoreImpl::Sync object err "); + LOG_ERROR("Watch sync flatObjectStore err "); return ERR_NULL_OBJECTSTORE; } std::lock_guard lock(watchersLock_); @@ -172,11 +171,11 @@ uint32_t DistributedObjectStoreImpl::Watch(DistributedObject *object, std::share uint32_t DistributedObjectStoreImpl::UnWatch(DistributedObject *object) { if (object == nullptr) { - LOG_ERROR("DistributedObjectStoreImpl::Sync object err "); + LOG_ERROR("UnWatch sync object err "); return ERR_NULL_OBJECT; } if (flatObjectStore_ == nullptr) { - LOG_ERROR("DistributedObjectStoreImpl::Sync object err "); + LOG_ERROR("UnWatch sync flatObjectStore err "); return ERR_NULL_OBJECTSTORE; } uint32_t status = flatObjectStore_->UnWatch(object->GetSessionId()); @@ -202,6 +201,10 @@ uint32_t DistributedObjectStoreImpl::SetStatusNotifier(std::shared_ptrCheckRetrieveCache(sessionId); } @@ -242,8 +245,8 @@ void WatcherProxy::OnChanged( bool WatcherProxy::FindChangedAssetKey(const std::string &changedKey, std::string &assetKey) { std::size_t dotPos = changedKey.find(ASSET_DOT); - if ((changedKey.size() > MODIFY_TIME_SUFFIX.length() && changedKey.substr(dotPos) == MODIFY_TIME_SUFFIX) || - (changedKey.size() > SIZE_SUFFIX.length() && changedKey.substr(dotPos) == SIZE_SUFFIX)) { + if ((changedKey.size() > strlen(MODIFY_TIME_SUFFIX) && changedKey.substr(dotPos) == MODIFY_TIME_SUFFIX) || + (changedKey.size() > strlen(SIZE_SUFFIX) && changedKey.substr(dotPos) == SIZE_SUFFIX)) { assetKey = changedKey.substr(0, dotPos); return true; } diff --git a/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_storage_engine.cpp b/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_storage_engine.cpp index 9509b219..91123488 100644 --- a/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_storage_engine.cpp +++ b/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_storage_engine.cpp @@ -16,13 +16,10 @@ #include "accesstoken_kit.h" #include "ipc_skeleton.h" -#include "logger.h" #include "objectstore_errors.h" #include "process_communicator_impl.h" -#include "securec.h" #include "softbus_adapter.h" #include "string_utils.h" -#include "types_export.h" #include "object_radar_reporter.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_store.cpp b/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_store.cpp index aea237c5..4d257080 100644 --- a/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_store.cpp +++ b/data_object/frameworks/innerkitsimpl/src/adaptor/flat_object_store.cpp @@ -19,14 +19,9 @@ #include "block_data.h" #include "bytes_utils.h" #include "client_adaptor.h" -#include "distributed_objectstore_impl.h" #include "ipc_skeleton.h" -#include "logger.h" #include "object_callback_impl.h" #include "object_radar_reporter.h" -#include "object_service_proxy.h" -#include "objectstore_errors.h" -#include "softbus_adapter.h" #include "string_utils.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_handler.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_handler.cpp index 78da6b60..d0af0f1f 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_handler.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_handler.cpp @@ -15,10 +15,6 @@ #include "app_pipe_handler.h" -#include - -#include "logger.h" - namespace OHOS { namespace ObjectStore { using namespace std; diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_mgr.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_mgr.cpp index b3827ed0..97eb8a71 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_mgr.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/app_pipe_mgr.cpp @@ -17,7 +17,7 @@ namespace OHOS { namespace ObjectStore { -static const int MAX_TRANSFER_SIZE = 1024 * 1024 * 5; +static constexpr int MAX_TRANSFER_SIZE = 1024 * 1024 * 5; Status AppPipeMgr::StartWatchDataChange(const AppDataChangeListener *observer, const PipeInfo &pipeInfo) { LOG_INFO("begin"); diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/ark_communication_provider.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/ark_communication_provider.cpp index cdb8dba6..d0ed72f5 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/ark_communication_provider.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/ark_communication_provider.cpp @@ -15,8 +15,6 @@ #include "ark_communication_provider.h" -#include - namespace OHOS { namespace ObjectStore { CommunicationProvider &ArkCommunicationProvider::Init() diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider.cpp index ffa2a1c4..ec465cb1 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider.cpp @@ -13,8 +13,6 @@ * limitations under the License. */ -#include "communication_provider.h" - #include "ark_communication_provider.h" namespace OHOS { diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider_impl.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider_impl.cpp index 9a585cf1..90b81dfc 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider_impl.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/communication_provider_impl.cpp @@ -15,8 +15,6 @@ #include "communication_provider_impl.h" -#include - namespace OHOS { namespace ObjectStore { std::mutex CommunicationProviderImpl::mutex_; diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/dev_manager.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/dev_manager.cpp index 2e480ebf..94522125 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/dev_manager.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/dev_manager.cpp @@ -15,13 +15,9 @@ #include "dev_manager.h" #include -#include #include "device_manager.h" -#include "device_manager_callback.h" -#include "dm_device_info.h" #include "softbus_adapter.h" -#include "app_types.h" namespace OHOS { namespace ObjectStore { @@ -119,7 +115,7 @@ void DevManager::RegisterDevCallback() if (status == DM_OK) { return; } - LOG_INFO("register device callback failed, try again."); + LOG_INFO("register device callback failed, status: %{public}d, try again.", status); std::thread th = std::thread([this]() { pthread_setname_np(pthread_self(), "Data_Object_InitDevManager"); constexpr int RETRY_TIMES = 300; diff --git a/data_object/frameworks/innerkitsimpl/src/communicator/softbus_adapter_standard.cpp b/data_object/frameworks/innerkitsimpl/src/communicator/softbus_adapter_standard.cpp index 0e8c8711..04f9c381 100644 --- a/data_object/frameworks/innerkitsimpl/src/communicator/softbus_adapter_standard.cpp +++ b/data_object/frameworks/innerkitsimpl/src/communicator/softbus_adapter_standard.cpp @@ -19,12 +19,9 @@ #include "bundle_mgr_interface.h" #include "dev_manager.h" #include "dms_handler.h" -#include "iservice_registry.h" #include "logger.h" #include "securec.h" -#include "session.h" #include "softbus_bus_center.h" -#include "system_ability_definition.h" namespace OHOS { namespace ObjectStore { @@ -114,7 +111,7 @@ void SoftBusAdapter::NotifyAll(const DeviceInfo &deviceInfo, const DeviceChangeT } LOG_DEBUG("high"); std::string uuid = DevManager::GetInstance()->GetUuidByNodeId(deviceInfo.deviceId); - LOG_DEBUG("[Notify] to DB from: %{public}s, type:%{public}d", ToBeAnonymous(uuid).c_str(), type); + LOG_DEBUG("[Notify] to DB from: %{public}s, type:%{public}hhd", ToBeAnonymous(uuid).c_str(), type); UpdateRelationship(deviceInfo.deviceId, type); for (const auto &device : listeners) { if (device == nullptr) { diff --git a/data_object/frameworks/innerkitsimpl/src/object_callback_stub.cpp b/data_object/frameworks/innerkitsimpl/src/object_callback_stub.cpp index 0acee40b..754be02c 100644 --- a/data_object/frameworks/innerkitsimpl/src/object_callback_stub.cpp +++ b/data_object/frameworks/innerkitsimpl/src/object_callback_stub.cpp @@ -16,7 +16,6 @@ #define LOG_TAG "ObjectCallbackStub" #include "object_callback_stub.h" #include -#include #include "itypes_util.h" #include "log_print.h" diff --git a/data_object/frameworks/innerkitsimpl/src/object_service_proxy.cpp b/data_object/frameworks/innerkitsimpl/src/object_service_proxy.cpp index 715315cc..934b0333 100644 --- a/data_object/frameworks/innerkitsimpl/src/object_service_proxy.cpp +++ b/data_object/frameworks/innerkitsimpl/src/object_service_proxy.cpp @@ -159,12 +159,12 @@ int32_t ObjectServiceProxy::RegisterDataObserver(const std::string &bundleName, ZLOGE("write descriptor failed"); return ERR_IPC; } - + if (!ITypesUtil::Marshal(data, bundleName, sessionId, callback)) { ZLOGE("Marshalling failed, bundleName = %{public}s", bundleName.c_str()); return ERR_IPC; } - + MessageParcel reply; MessageOption mo { MessageOption::TF_SYNC }; sptr remoteObject = Remote(); @@ -188,12 +188,12 @@ int32_t ObjectServiceProxy::UnregisterDataChangeObserver(const std::string &bund ZLOGE("write descriptor failed"); return ERR_IPC; } - + if (!ITypesUtil::Marshal(data, bundleName, sessionId)) { ZLOGE("Marshalling failed, bundleName = %{public}s", bundleName.c_str()); return ERR_IPC; } - + MessageParcel reply; MessageOption mo { MessageOption::TF_SYNC }; sptr remoteObject = Remote(); @@ -218,7 +218,7 @@ int32_t ObjectServiceProxy::BindAssetStore(const std::string &bundleName, const ZLOGE("write descriptor failed"); return ERR_IPC; } - + if (!ITypesUtil::Marshal(data, bundleName, sessionId, asset, bindInfo)) { ZLOGE("Marshalling failed, bundleName = %{public}s", bundleName.c_str()); return ERR_IPC; diff --git a/data_object/frameworks/innerkitsimpl/test/unittest/BUILD.gn b/data_object/frameworks/innerkitsimpl/test/unittest/BUILD.gn index a79ea560..4f3f4b89 100644 --- a/data_object/frameworks/innerkitsimpl/test/unittest/BUILD.gn +++ b/data_object/frameworks/innerkitsimpl/test/unittest/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/data_object/data_object.gni") -module_output_path = "data_object/impl" +module_output_path = "data_object/data_object/impl" data_object_base_path = "//foundation/distributeddatamgr/data_object" diff --git a/data_object/frameworks/innerkitsimpl/test/unittest/src/asset_change_timer_test.cpp b/data_object/frameworks/innerkitsimpl/test/unittest/src/asset_change_timer_test.cpp index 7453ac4f..ed31f97c 100644 --- a/data_object/frameworks/innerkitsimpl/test/unittest/src/asset_change_timer_test.cpp +++ b/data_object/frameworks/innerkitsimpl/test/unittest/src/asset_change_timer_test.cpp @@ -33,6 +33,8 @@ using namespace OHOS; using namespace std; namespace { +static constexpr uint32_t WAIT_INTERVAL = 100; +static constexpr char ASSET_SEPARATOR = '#'; class AssetChangeTimerTest : public testing::Test { public: static void SetUpTestCase(void); @@ -77,9 +79,9 @@ HWTEST_F(AssetChangeTimerTest, StartTimer_001, TestSize.Level1) FlatObjectStore *flatObjectStore = nullptr; AssetChangeTimer assetChangeTimer(flatObjectStore); assetChangeTimer.StartTimer(sessionId, assetKey, watcherImpl); - EXPECT_EQ(assetChangeTimer.assetChangeTasks_[sessionId + assetChangeTimer.ASSET_SEPARATOR + assetKey], + EXPECT_EQ(assetChangeTimer.assetChangeTasks_[sessionId + ASSET_SEPARATOR + assetKey], assetChangeTimer.executor_->Reset(assetChangeTimer.assetChangeTasks_[sessionId + - assetChangeTimer.ASSET_SEPARATOR + assetKey], std::chrono::milliseconds(assetChangeTimer.WAIT_INTERVAL))); + ASSET_SEPARATOR + assetKey], std::chrono::milliseconds(WAIT_INTERVAL))); } /** @@ -129,9 +131,9 @@ HWTEST_F(AssetChangeTimerTest, OnAssetChanged_001, TestSize.Level1) FlatObjectStore *flatObjectStore = nullptr; AssetChangeTimer assetChangeTimer(flatObjectStore); assetChangeTimer.OnAssetChanged(sessionId, assetKey, watcherImpl); - EXPECT_EQ(assetChangeTimer.assetChangeTasks_[sessionId + assetChangeTimer.ASSET_SEPARATOR + assetKey], + EXPECT_EQ(assetChangeTimer.assetChangeTasks_[sessionId + ASSET_SEPARATOR + assetKey], assetChangeTimer.executor_->Reset(assetChangeTimer.assetChangeTasks_[sessionId + - assetChangeTimer.ASSET_SEPARATOR + assetKey], std::chrono::milliseconds(assetChangeTimer.WAIT_INTERVAL))); + ASSET_SEPARATOR + assetKey], std::chrono::milliseconds(WAIT_INTERVAL))); } /** diff --git a/data_object/frameworks/innerkitsimpl/test/unittest/src/object_store_test.cpp b/data_object/frameworks/innerkitsimpl/test/unittest/src/object_store_test.cpp index c95ddb0e..55c04bdc 100644 --- a/data_object/frameworks/innerkitsimpl/test/unittest/src/object_store_test.cpp +++ b/data_object/frameworks/innerkitsimpl/test/unittest/src/object_store_test.cpp @@ -71,12 +71,13 @@ void StatusNotifierImpl::OnChanged(const std::string &sessionId, void GrantPermissionNative() { - const char **perms = new const char *[2]; + const char **perms = new const char *[3]; perms[0] = "ohos.permission.DISTRIBUTED_DATASYNC"; perms[1] = "ohos.permission.DISTRIBUTED_SOFTBUS_CENTER"; + perms[2] = "ohos.permission.MONITOR_DEVICE_NETWORK_STATE"; // perms[2] is a permission parameter TokenInfoParams infoInstance = { .dcapsNum = 0, - .permsNum = 2, + .permsNum = 3, .aclsNum = 0, .dcaps = nullptr, .perms = perms, diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/BUILD.gn b/data_object/frameworks/jskitsimpl/collaboration_edit/BUILD.gn index 9e7291db..0a942f35 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/BUILD.gn +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/BUILD.gn @@ -16,11 +16,7 @@ import("//build/ohos/ace/ace.gni") config("collaboration_edit_config") { visibility = [ ":*" ] - cflags = [ - "-DHILOG_ENABLE", - "-fvisibility=hidden", - "-Oz", - ] + cflags = [ "-DHILOG_ENABLE" ] include_dirs = [ "include", @@ -39,20 +35,27 @@ ohos_shared_library("collaborationeditobject") { } sources = [ + "../../innerkitsimpl/collaboration_edit/src/cloud_db_proxy.cpp", "../../innerkitsimpl/collaboration_edit/src/db_common.cpp", "../../innerkitsimpl/collaboration_edit/src/db_store.cpp", "../../innerkitsimpl/collaboration_edit/src/db_store_config.cpp", "../../innerkitsimpl/collaboration_edit/src/db_store_manager.cpp", + "../../innerkitsimpl/collaboration_edit/src/db_thread_pool.cpp", "../../innerkitsimpl/collaboration_edit/src/grd_api_manager.cpp", "../../innerkitsimpl/collaboration_edit/src/rd_adapter.cpp", "../../innerkitsimpl/collaboration_edit/src/rd_utils.cpp", "src/entry_point.cpp", + "src/js_utils.cpp", "src/napi_abstract_type.cpp", + "src/napi_async_call.cpp", + "src/napi_cloud_db.cpp", "src/napi_collaboration_edit_object.cpp", + "src/napi_const_properties.cpp", "src/napi_edit_unit.cpp", "src/napi_error_utils.cpp", "src/napi_node.cpp", "src/napi_parser.cpp", + "src/napi_sync_service.cpp", "src/napi_text.cpp", "src/napi_undo_manager.cpp", "src/napi_utils.cpp", diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/js_utils.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/js_utils.h new file mode 100644 index 00000000..e5f8af38 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/js_utils.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_JS_UTILS_H +#define COLLABORATION_EDIT_JS_UTILS_H + +#include "napi/native_api.h" +#include "napi/native_common.h" +#include "napi/native_node_api.h" + +namespace OHOS::CollaborationEdit { +namespace JSUtils { +napi_value Convert2JSValue(napi_env env, int32_t value); +} // namespace JSUtils +} // namespace OHOS::CollaborationEdit +#endif // COLLABORATION_EDIT_JS_UTILS_H \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_async_call.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_async_call.h new file mode 100644 index 00000000..10dc29be --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_async_call.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATA_OBJ_NAPI_ASYNC_CALL_H +#define DATA_OBJ_NAPI_ASYNC_CALL_H + +#include + +#include "db_store.h" +#include "napi_errno.h" +#include "napi_utils.h" +#include "napi/native_api.h" +#include "napi/native_common.h" +#include "rd_type.h" + +namespace OHOS::CollaborationEdit { +struct ProgressDetailT { + ProgressCode code; +}; + +struct SyncCallbackParamT { + std::promise promise; + ProgressDetailT detail; +}; + +using InputAction = std::function; +using OutputAction = std::function; +using ExecuteAction = std::function; + +constexpr const char *SYNC_FUNCTION_NAME = "CollaborationAsyncCall"; +constexpr std::chrono::seconds TIME_THRESHOLD = std::chrono::seconds(30); + +class SyncContext { +public: + int SetAll(napi_env env, napi_callback_info info, InputAction input, ExecuteAction exec, OutputAction output); + void ReleaseInnerReference(); + virtual ~SyncContext(); + + napi_env env_ = nullptr; + napi_ref self_ = nullptr; + void *boundObj = nullptr; // unwrapped collaborationEdit object + const char *fun = nullptr; // function name in napi queue + + napi_async_work work_ = nullptr; + napi_value result_ = nullptr; + + int execCode_ = OK; + OutputAction output_ = nullptr; + ExecuteAction exec_ = nullptr; + std::shared_ptr keep_; // keep self alive + + uint64_t syncId = 0u; + std::mutex callbackMutex_; + napi_threadsafe_function callback_ = nullptr; + + int32_t syncMode_; + std::shared_ptr dbStore_ = nullptr; +}; + +class AsyncCall final { +public: + static int Call(napi_env env, std::shared_ptr context, const char *fun); + static void CloudSyncCallback(napi_env env, napi_value js_cb, void *context, void *data); + +private: + static void OnExecute(napi_env env, void *data); + static void OnComplete(napi_env env, void *data); + static void OnReturn(napi_env env, napi_status status, void *data); + static void OnComplete(napi_env env, napi_status status, void *data); +}; +} // namespace OHOS::CollaborationEdit +#endif \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_cloud_db.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_cloud_db.h new file mode 100644 index 00000000..ae41557e --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_cloud_db.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_CLOUD_DB_H +#define COLLABORATION_EDIT_CLOUD_DB_H + +#include +#include +#include "napi_error_utils.h" +#include "napi/native_node_api.h" +#include "rd_type.h" + +namespace OHOS::CollaborationEdit { + +struct CloudParamsAdapterT { + int64_t cursor = 0; + uint64_t timestamp = 0; + std::string id; + std::vector record; +}; + +struct QueryConditionT { + Predicate predicate; + std::string fieldName; + std::string fieldValue_str; + int64_t fieldValue_num; +}; + +struct AssetOpConfig { + std::string inputPath; +}; + +class NapiCloudDb { +public: + NapiCloudDb(); + ~NapiCloudDb(); + int32_t BatchInsert(const std::vector &cloudParamsAdapter); + int32_t Query(const std::vector &queryConditions, std::vector &extends); + int32_t DownloadAsset(const std::string equipId, const std::string path); + int32_t UploadAsset(const std::string path); + int32_t DeleteAsset(const std::string path); + int32_t DeleteLocalAsset(const std::string path); + + static void BatchInsertInner(napi_env env, napi_value js_cb, void *context, void *data); + static void QueryInner(napi_env env, napi_value js_cb, void *context, void *data); + static void DownloadAssetInner(napi_env env, napi_value js_cb, void *context, void *data); + static void UploadAssetInner(napi_env env, napi_value js_cb, void *context, void *data); + static void DeleteAssetInner(napi_env env, napi_value js_cb, void *context, void *data); + static void DeleteLocalAssetInner(napi_env env, napi_value js_cb, void *context, void *data); + + static napi_value BatchInsertResolvedCallback(napi_env env, napi_callback_info info); + static napi_value BatchInsertRejectedCallback(napi_env env, napi_callback_info info); + static napi_value HandleAssetResolvedCallback(napi_env env, napi_callback_info info); + static napi_value HandleAssetRejectedCallback(napi_env env, napi_callback_info info); + static napi_value HandleAssetAsyncResolvedCallback(napi_env env, napi_callback_info info); + static napi_value HandleAssetAsyncRejectedCallback(napi_env env, napi_callback_info info); + static napi_value QueryResolvedCallback(napi_env env, napi_callback_info info); + static napi_value QueryRejectedCallback(napi_env env, napi_callback_info info); + + napi_threadsafe_function batchInsertInnerFunc_ = nullptr; + napi_threadsafe_function queryInnerFunc_ = nullptr; + napi_threadsafe_function downloadAssetInnerFunc_ = nullptr; + napi_threadsafe_function uploadAssetInnerFunc_ = nullptr; + napi_threadsafe_function deleteAssetInnerFunc_ = nullptr; + napi_threadsafe_function deleteLocalAssetInnerFunc_ = nullptr; + +private: + static void GetCloudParamsVector(napi_env env, void *data, napi_value &arrayParams); + static void GetJsQueryParams(napi_env env, std::vector conditions, napi_value &arrayParams); + static void HandleAssetInner(napi_env env, napi_value jsCb, void *context, void *data); + static void HandleAssetAsyncInner(napi_env env, napi_value jsCb, void *context, void *data); + static void ClearLastException(const napi_env& env); +}; +} // namespace OHOS::CollaborationEdit +#endif // COLLABORATION_EDIT_CLOUD_DB_H diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_collaboration_edit_object.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_collaboration_edit_object.h index 6e16a2da..d7be9b21 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_collaboration_edit_object.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_collaboration_edit_object.h @@ -17,14 +17,17 @@ #define NAPI_COLLABORATION_EDIT_OBJECT_H #include "db_store.h" +#include "grd_type_export.h" #include "napi/native_api.h" #include "napi/native_common.h" #include "napi/native_node_api.h" +#include "napi_async_call.h" +#include "napi_cloud_db.h" #include "napi_undo_manager.h" #include "napi_utils.h" +#include "rd_type.h" namespace OHOS::CollaborationEdit { - class CollaborationEditObject { public: CollaborationEditObject(std::string docName, ContextParam param); @@ -33,15 +36,29 @@ public: static napi_value Delete(napi_env env, napi_callback_info info); void SetDBStore(std::shared_ptr dbStore); std::shared_ptr GetDBStore(); + static napi_value SetCloudDb(napi_env env, napi_callback_info info); private: static napi_value Constructor(napi_env env); static napi_value Initialize(napi_env env, napi_callback_info info); + static napi_status CreateHandlerFunc(napi_env env, std::vector &cloudDbFunc, NapiCloudDb *napiCloudDb); + static void ReleaseHandlerFunc(NapiCloudDb *napiCloudDb); static napi_value GetEditUnit(napi_env env, napi_callback_info info); static napi_value GetUndoRedoManager(napi_env env, napi_callback_info info); static napi_value DeleteUndoRedoManager(napi_env env, napi_callback_info info); static napi_value GetName(napi_env env, napi_callback_info info); + static napi_value CloudSync(napi_env env, napi_callback_info info); + static napi_value GetLocalId(napi_env env, napi_callback_info info); + static napi_value ApplyUpdate(napi_env env, napi_callback_info info); + static napi_value WriteUpdate(napi_env env, napi_callback_info info); + static void SyncCallbackFunc(GRD_SyncProcessT *syncProcess); + static GRD_SyncModeE GetGRDSyncMode(int32_t mode); + static ProgressCode GetProgressCode(int32_t errCode); + static int ParseThis(const napi_env &env, const napi_value &self, std::shared_ptr context); + static int ParseCloudSyncMode(const napi_env env, const napi_value arg, std::shared_ptr context); + static InputAction GetCloudSyncInput(std::shared_ptr context); + static ExecuteAction GetCloudSyncExec(std::shared_ptr context); std::string docName_; std::shared_ptr param_; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_const_properties.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_const_properties.h new file mode 100644 index 00000000..0c4b9d02 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_const_properties.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLLABORATION_EDIT_CONST_PROPERTIES_H +#define COLLABORATION_EDIT_CONST_PROPERTIES_H + +#include "napi/native_api.h" +#include "napi/native_common.h" +#include "napi/native_node_api.h" + +namespace OHOS::CollaborationEdit { +napi_status InitConstProperties(napi_env env, napi_value exports); +} +#endif // COLLABORATION_EDIT_CONST_PROPERTIES_H diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_edit_unit.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_edit_unit.h index 48ec6edb..068afa61 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_edit_unit.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_edit_unit.h @@ -39,6 +39,8 @@ private: static napi_value GetChildren(napi_env env, napi_callback_info info); static napi_value GetJsonResult(napi_env env, napi_callback_info info); static napi_value GetName(napi_env env, napi_callback_info info); + static napi_value GetRelativePos(napi_env env, napi_callback_info info); + static napi_value GetAbsolutePos(napi_env env, napi_callback_info info); std::string name_; }; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_errno.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_errno.h index be189483..8f70699f 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_errno.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_errno.h @@ -23,6 +23,7 @@ static constexpr const int EDIT_ERROR_OFFSET = 15410000; enum Status : int32_t { SUCCESS = 0, + NOT_SYSTEM_APP = 202, INVALID_ARGUMENT = 401, INTERNAL_ERROR = EDIT_ERROR_OFFSET, UNSUPPORTED_OPERATION = EDIT_ERROR_OFFSET + 1, // 15410001 diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_node.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_node.h index 946e8164..dafe3c49 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_node.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_node.h @@ -46,6 +46,7 @@ private: static napi_value SetAttributes(napi_env env, napi_callback_info info); static napi_value RemoveAttributes(napi_env env, napi_callback_info info); static napi_value GetAttributes(napi_env env, napi_callback_info info); + static napi_value SetAsset(napi_env env, napi_callback_info info); std::string name_; }; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_parser.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_parser.h index c2765ae7..4dc953fc 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_parser.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_parser.h @@ -23,11 +23,13 @@ #include #include "napi_abstract_type.h" +#include "napi_cloud_db.h" #include "napi/native_api.h" #include "napi/native_common.h" #include "napi/native_node_api.h" namespace OHOS::CollaborationEdit { +using json = nlohmann::json; class Parser { public: static void Stringsplit(std::string str, const char split, std::vector &res); @@ -38,6 +40,20 @@ public: static int ParseFromAttrsJsonStr(napi_env env, const std::string &jsonStr, napi_value &out); static int ParseJsFormatToStr(napi_env env, napi_value jsFormat, std::string &out); static int ParseVariantJsValueToStr(napi_env env, napi_value input, std::string &out); + static int CheckValueType(napi_env env, napi_value value); + static int ParseCloudDbFields(napi_env env, napi_value input, std::vector &cloudDbFuncVector); + static napi_value GetUniqueIdFromJsonStr(napi_env env, json &root); + static napi_value GetRelativePosFromJsonStr(napi_env env, std::string &relPos); + static void GetUniqueIdFromNapiValueToJsonStr(napi_env env, napi_value type, json &typeJson); + static int ParseJsonStrToJsUpdateNode( + napi_env env, std::string nodeJsonStr, std::shared_ptr dbStore, napi_value &out); + static napi_value ParseFromAssetOpConfig(napi_env env, const AssetOpConfig &config); + +private: + static int SetRelativePosType(napi_env env, json &root, napi_value &relativePos); + static int SetRelativePosItem(napi_env env, json &root, napi_value &relativePos); + static int SetRelativePosTname(napi_env env, json &root, napi_value &relativePos); + static int SetRelativePosAssoc(napi_env env, json &root, napi_value &relativePos); }; } // namespace OHOS::CollaborationEdit #endif // COLLABORATION_EDIT_PARSER_H diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_sync_service.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_sync_service.h new file mode 100644 index 00000000..5e6c3053 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_sync_service.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRD_SYNC_SERVICE_H +#define GRD_SYNC_SERVICE_H + +#include +#include +#include + +#include "db_store.h" +#include "napi_async_call.h" + +namespace OHOS::CollaborationEdit { +class SyncService { +public: + SyncService& operator=(SyncService const&) = delete; + + static std::shared_ptr GetInstance(); + uint64_t GetSyncId(); + std::shared_ptr GetSyncContext(uint64_t syncId); + void AddSyncContext(uint64_t syncId, std::shared_ptr syncContext); + void RemoveSyncContext(uint64_t syncId); +private: + std::atomic syncId_{}; + std::mutex contextMutex_; + std::map> syncContextMap_; +}; + +} // namespace OHOS::CollaborationEdit +#endif // GRD_SYNC_SERVICE_H \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_utils.h b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_utils.h index b7d7b1ab..9a7413e6 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_utils.h +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/include/napi_utils.h @@ -81,36 +81,6 @@ public: static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - /* napi_value <-> std::vector */ - static napi_status GetValue(napi_env env, napi_value input, std::vector &out); - - static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - - /* napi_value <-> std::vector */ - static napi_status GetValue(napi_env env, napi_value input, std::vector &out); - - static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - - /* napi_value <-> std::vector */ - static napi_status GetValue(napi_env env, napi_value input, std::vector &out); - - static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - - /* napi_value <-> std::vector */ - static napi_status GetValue(napi_env env, napi_value input, std::vector &out); - - static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - - /* napi_value <-> std::vector */ - static napi_status GetValue(napi_env env, napi_value input, std::vector &out); - - static napi_status SetValue(napi_env env, const std::vector &input, napi_value &out); - - /* napi_value <-> std::map */ - static napi_status GetValue(napi_env env, napi_value input, std::map &out); - - static napi_status SetValue(napi_env env, const std::map &input, napi_value &out); - static napi_status GetCurrentAbilityParam(napi_env env, ContextParam ¶m); static napi_status GetValue(napi_env env, napi_value input, ContextParam ¶m); @@ -133,6 +103,8 @@ public: return (jsValue == nullptr) ? status : GetValue(env, jsValue, value); }; + static std::string RemovePrefix(std::string str, std::string prefix); + private: enum { /* std::map to js::tuple */ diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/entry_point.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/entry_point.cpp index 7180c862..e4638db4 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/entry_point.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/entry_point.cpp @@ -16,6 +16,7 @@ #define LOG_TAG "EntryPoint" #include "log_print.h" +#include "napi_const_properties.h" #include "napi_collaboration_edit_object.h" #include "napi/native_api.h" #include "napi/native_common.h" @@ -34,6 +35,7 @@ static napi_value Init(napi_env env, napi_value exports) NAPI_CALL(env, napi_define_properties(env, exports, sizeof(desc) / sizeof(desc[0]), desc)); Node::Init(env, exports); Text::Init(env, exports); + InitConstProperties(env, exports); return exports; } diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/js_utils.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/js_utils.cpp new file mode 100644 index 00000000..236bcb2c --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/js_utils.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "js_utils.h" + +namespace OHOS::CollaborationEdit { +napi_value JSUtils::Convert2JSValue(napi_env env, int32_t value) +{ + napi_value jsValue = nullptr; + napi_status status = napi_create_int32(env, value, &jsValue); + if (status != napi_ok) { + return nullptr; + } + return jsValue; +} +} // namespace OHOS::CollaborationEdit \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_async_call.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_async_call.cpp new file mode 100644 index 00000000..0ed5c8de --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_async_call.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "AsyncCall" + +#include "napi_async_call.h" + +#include "log_print.h" +#include "napi_error_utils.h" + +namespace OHOS::CollaborationEdit { + +int SyncContext::SetAll( + napi_env env, napi_callback_info info, InputAction input, ExecuteAction exec, OutputAction output) +{ + env_ = env; + size_t argc = 2; + napi_value self = nullptr; + napi_value argv[2] = { nullptr }; + napi_status status = napi_get_cb_info(env, info, &argc, argv, &self, nullptr); + if (status != napi_ok) { + LOG_ERROR("get callback info go wrong, status: %{public}" PRIi32, static_cast(status)); + return ERR; + } + + napi_create_reference(env, self, 1, &self_); + if (status != napi_ok) { + LOG_ERROR("create self reference go wrong, status: %{public}" PRIi32, static_cast(status)); + return ERR; + } + + int ret = input(env, argc, argv, self); + ASSERT(ret == OK, "get input function go wrong", ret); + + output_ = std::move(output); + exec_ = std::move(exec); + return OK; +} + +void SyncContext::ReleaseInnerReference() +{ + if (keep_ != nullptr) { + keep_.reset(); + keep_ = nullptr; + } +} + +SyncContext::~SyncContext() +{ + LOG_DEBUG("SyncContext freed."); + if (env_ == nullptr) { + return; + } + if (work_ != nullptr) { + napi_delete_async_work(env_, work_); + work_ = nullptr; + } + if (self_ != nullptr) { + napi_delete_reference(env_, self_); + self_ = nullptr; + } + env_ = nullptr; +} + +int AsyncCall::Call(napi_env env, std::shared_ptr context, const char *fun) +{ + if (context->callback_ == nullptr) { + LOG_ERROR("create async task wrong, callback is null"); + return ERR; + } + context->fun = (fun == nullptr || fun[0] == '\0') ? SYNC_FUNCTION_NAME : fun; + context->keep_ = context; + napi_value resource = nullptr; + napi_status status = napi_create_string_utf8(env, context->fun, NAPI_AUTO_LENGTH, &resource); + if (status != napi_ok) { + LOG_ERROR("napi create string wrong, status: %{public}" PRIi32, static_cast(status)); + return ERR; + } + // create async work, execute function is OnExecute, complete function is OnComplete + status = napi_create_async_work(env, nullptr, resource, AsyncCall::OnExecute, AsyncCall::OnComplete, + reinterpret_cast(context.get()), &context->work_); + if (status != napi_ok) { + LOG_ERROR("create async work go wrong, status: %{public}" PRIi32, static_cast(status)); + return ERR; + } + // add async work to execute queue + status = napi_queue_async_work_with_qos(env, context->work_, napi_qos_user_initiated); + if (status != napi_ok) { + LOG_ERROR("add async work go wrong, status: %{public}" PRIi32, static_cast(status)); + return ERR; + } + return OK; +} + +void AsyncCall::OnExecute(napi_env env, void *data) +{ + SyncContext *context = reinterpret_cast(data); + if (context->exec_ != nullptr) { + context->execCode_ = context->exec_(); + } + context->exec_ = nullptr; +} + +void AsyncCall::OnComplete(napi_env env, void *data) +{ + SyncContext *context = reinterpret_cast(data); + if (context->output_) { + context->output_(env, context->result_); + } + context->output_ = nullptr; +} + +void AsyncCall::OnComplete(napi_env env, napi_status status, void *data) +{ + OnComplete(env, data); + if (status != napi_ok) { + LOG_ERROR("task execute go wrong, status: %{public}" PRIi32, static_cast(status)); + SyncContext *context = reinterpret_cast(data); + context->ReleaseInnerReference(); + } +} + +void AsyncCall::CloudSyncCallback(napi_env env, napi_value js_cb, void *context, void *data) +{ + SyncCallbackParamT *callbackParam = reinterpret_cast(data); + ASSERT_VOID(callbackParam != nullptr, "callbackParam is null"); + + napi_value param = nullptr; + napi_status status = napi_create_object(env, ¶m); + if (status != napi_ok) { + LOG_ERROR("create param object wrong, status: %{public}" PRIi32, static_cast(status)); + callbackParam->promise.set_value(0); + return; + } + + napi_value pCode = nullptr; + NapiUtils::SetValue(env, callbackParam->detail.code, pCode); + status = napi_set_named_property(env, param, "code", pCode); + if (status != napi_ok) { + LOG_ERROR("set named property wrong, status: %{public}" PRIi32, static_cast(status)); + callbackParam->promise.set_value(0); + return; + } + + auto start = std::chrono::steady_clock::now(); + status = napi_call_function(env, nullptr, js_cb, 1, ¶m, nullptr); + auto finish = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(finish - start); + if (duration > TIME_THRESHOLD) { + int64_t ms = duration.count(); + LOG_WARN("callback overtime, time: %{public}" PRIi64 " ms", ms); + } + if (status != napi_ok) { + LOG_ERROR("napi call js function wrong, status: %{public}" PRIi32, static_cast(status)); + } + callbackParam->promise.set_value(0); +} +} // namespace CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_cloud_db.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_cloud_db.cpp new file mode 100644 index 00000000..c6059300 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_cloud_db.cpp @@ -0,0 +1,651 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "NapiCloudDb" + +#include + +#include "napi_cloud_db.h" + +#include "napi_errno.h" +#include "napi_error_utils.h" +#include "napi_parser.h" +#include "napi_utils.h" +#include "napi/native_node_api.h" + +namespace OHOS::CollaborationEdit { +constexpr size_t HANDLER_PARAM_NUM = 1; +constexpr size_t PROMISE_PARAM_NUM = 2; +const std::string CURSOR = "cursor"; + +enum CallBackResult : int32_t { + CALL_BACK_RESULT_INVALID = -1, + CALL_BACK_RESULT_OK = 0, + CALL_BACK_RESULT_ERR, + CALL_BACK_RESULT_QUERY_END, +}; + +struct BatchInsertInfo { + std::vector inputParam; + std::promise promiseRes; + int32_t callBackErrCode; +}; + +struct AssetInfo { + AssetOpConfig config; + std::promise promiseRes; + int32_t callBackErrCode; +}; + +struct QueryInfo { + std::vector inputParam; + std::promise> promiseRes; + int32_t callBackErrCode; +}; + +NapiCloudDb::NapiCloudDb() {} + +NapiCloudDb::~NapiCloudDb() {} + +int32_t NapiCloudDb::BatchInsert(const std::vector &cloudParamsAdapter) +{ + napi_status ret = napi_acquire_threadsafe_function(batchInsertInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("[BatchInsert] acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + BatchInsertInfo *batchInsertInfo = new BatchInsertInfo(); + batchInsertInfo->inputParam = cloudParamsAdapter; + + auto future = batchInsertInfo->promiseRes.get_future(); + ret = napi_call_threadsafe_function(batchInsertInnerFunc_, reinterpret_cast(batchInsertInfo), + napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("[BatchInsert] call thread function go wrong, ret=%{public}d", ret); + delete batchInsertInfo; + return CALL_BACK_RESULT_ERR; + } + auto result = future.get(); + + int32_t errCode = batchInsertInfo->callBackErrCode; + LOG_INFO("[BatchInsert] batch insert complete, result = %{public}d, errCode = %{public}d", result, errCode); + delete batchInsertInfo; + return errCode; +} + +int32_t NapiCloudDb::Query(const std::vector &queryConditions, + std::vector &extends) +{ + napi_status ret = napi_acquire_threadsafe_function(queryInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + QueryInfo *queryInfo = new QueryInfo(); + queryInfo->inputParam = queryConditions; + + auto future = queryInfo->promiseRes.get_future(); + ret = napi_call_threadsafe_function(queryInnerFunc_, reinterpret_cast(queryInfo), napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("call thread function go wrong, ret=%{public}d", ret); + delete queryInfo; + return CALL_BACK_RESULT_ERR; + } + auto result = future.get(); + extends = std::move(result); + + int32_t errCode = queryInfo->callBackErrCode; + LOG_INFO("query complete, result size = %{public}zu, errCode = %{public}d", extends.size(), errCode); + delete queryInfo; + return errCode; +} + +int32_t NapiCloudDb::DownloadAsset(const std::string equipId, std::string path) +{ + napi_status ret = napi_acquire_threadsafe_function(downloadAssetInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("[DownloadAsset] acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + AssetInfo *downloadAssetInfo = new AssetInfo(); + downloadAssetInfo->config.inputPath = path; + ret = napi_call_threadsafe_function(downloadAssetInnerFunc_, (void*)downloadAssetInfo, napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("[DownloadAsset] call thread function go wrong, ret=%{public}d", ret); + delete downloadAssetInfo; + return CALL_BACK_RESULT_ERR; + } + + LOG_INFO("[DownloadAsset] download asset complete"); + return CALL_BACK_RESULT_OK; +} + +int32_t NapiCloudDb::UploadAsset(const std::string path) +{ + napi_status ret = napi_acquire_threadsafe_function(uploadAssetInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("[UploadAsset] acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + AssetInfo *assetInfo = new AssetInfo(); + assetInfo->config.inputPath = path; + auto future = assetInfo->promiseRes.get_future(); + ret = napi_call_threadsafe_function(uploadAssetInnerFunc_, (void*)assetInfo, napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("[UploadAsset] call thread function go wrong, ret=%{public}d", ret); + delete assetInfo; + return CALL_BACK_RESULT_ERR; + } + future.wait(); + + int32_t errCode = assetInfo->callBackErrCode; + LOG_INFO("[UploadAsset] upload asset complete, errCode=%{public}d", errCode); + delete assetInfo; + return errCode; +} + +int32_t NapiCloudDb::DeleteAsset(const std::string path) +{ + napi_status ret = napi_acquire_threadsafe_function(deleteAssetInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("[DeleteAsset] acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + AssetInfo *assetInfo = new AssetInfo(); + assetInfo->config.inputPath = path; + auto future = assetInfo->promiseRes.get_future(); + ret = napi_call_threadsafe_function(deleteAssetInnerFunc_, (void*)assetInfo, napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("[DeleteAsset] call thread function go wrong, ret=%{public}d", ret); + delete assetInfo; + return CALL_BACK_RESULT_ERR; + } + future.wait(); + + int32_t errCode = assetInfo->callBackErrCode; + LOG_INFO("[DeleteAsset] Delete asset complete, errCode=%{public}d", errCode); + delete assetInfo; + return errCode; +} + +int32_t NapiCloudDb::DeleteLocalAsset(const std::string path) +{ + napi_status ret = napi_acquire_threadsafe_function(deleteLocalAssetInnerFunc_); + if (ret != napi_ok) { + LOG_ERROR("[DeleteLocalAsset] acquire thread function go wrong, ret=%{public}d", ret); + return CALL_BACK_RESULT_ERR; + } + AssetInfo *assetInfo = new AssetInfo(); + assetInfo->config.inputPath = path; + ret = napi_call_threadsafe_function(deleteLocalAssetInnerFunc_, (void*)assetInfo, napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("[DeleteLocalAsset] call thread function go wrong, ret=%{public}d", ret); + delete assetInfo; + return CALL_BACK_RESULT_ERR; + } + + LOG_INFO("[DeleteLocalAsset] delete local asset complete"); + return CALL_BACK_RESULT_OK; +} + +void NapiCloudDb::GetCloudParamsVector(napi_env env, void *data, napi_value &arrayParams) +{ + int i = 0; + std::vector *cloudParamsAdapter = reinterpret_cast *>(data); + if (cloudParamsAdapter == nullptr) { + return; + } + for (const auto ¶msAdapter : *cloudParamsAdapter) { + napi_value cursor = nullptr; + NapiUtils::SetValue(env, paramsAdapter.cursor, cursor); + napi_value id = nullptr; + NapiUtils::SetValue(env, paramsAdapter.id, id); + napi_value record = nullptr; + NapiUtils::SetValue(env, paramsAdapter.record, record); + + napi_value editObjectRecord = nullptr; + napi_create_object(env, &editObjectRecord); + napi_set_named_property(env, editObjectRecord, "cursor", cursor); + napi_set_named_property(env, editObjectRecord, "id", id); + napi_set_named_property(env, editObjectRecord, "data", record); + + napi_set_element(env, arrayParams, i, editObjectRecord); + i++; + } +} + +void NapiCloudDb::GetJsQueryParams(napi_env env, std::vector conditions, napi_value &arrayParams) +{ + uint32_t i = 0; + for (QueryConditionT condition : conditions) { + napi_value predicate = nullptr; + napi_value fieldName = nullptr; + napi_value fieldValue = nullptr; + + NapiUtils::SetValue(env, static_cast(condition.predicate), predicate); + NapiUtils::SetValue(env, condition.fieldName, fieldName); + if (condition.fieldName == CURSOR) { + NapiUtils::SetValue(env, condition.fieldValue_num, fieldValue); + } else { + NapiUtils::SetValue(env, condition.fieldValue_str, fieldValue); + } + + napi_value queryCondition = nullptr; + napi_create_object(env, &queryCondition); + napi_set_named_property(env, queryCondition, "condition", predicate); + napi_set_named_property(env, queryCondition, "fieldName", fieldName); + napi_set_named_property(env, queryCondition, "fieldValue", fieldValue); + + napi_set_element(env, arrayParams, i, queryCondition); + i++; + } +} + +void NapiCloudDb::BatchInsertInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + BatchInsertInfo *batchInsertInfo = reinterpret_cast(data); + std::vector inputParam = batchInsertInfo->inputParam; + napi_value arrayParams = nullptr; + napi_status status = napi_create_array(env, &arrayParams); + if (status != napi_ok) { + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + return; + } + GetCloudParamsVector(env, &inputParam, arrayParams); + + napi_value promise = nullptr; + status = napi_call_function(env, nullptr, jsCb, HANDLER_PARAM_NUM, &arrayParams, &promise); + if (status != napi_ok) { + LOG_ERROR("[BatchInsertInner] napi call function go wrong, status = %{public}d", status); + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + return; + } + napi_value thenFunc = nullptr; + status = napi_get_named_property(env, promise, "then", &thenFunc); + if (status != napi_ok) { + LOG_ERROR("[BatchInsertInner] napi get named property fail, status = %{public}d", status); + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + return; + } + napi_value resolvedCallback; + napi_value rejectedCallback; + status = napi_create_function(env, "resolvedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::BatchInsertResolvedCallback, data, &resolvedCallback); + if (status != napi_ok) { + LOG_ERROR("[BatchInsertInner] resolvedCallback, status = %{public}d", status); + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + return; + } + status = napi_create_function(env, "rejectedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::BatchInsertRejectedCallback, data, &rejectedCallback); + if (status != napi_ok) { + LOG_ERROR("[BatchInsertInner] rejectedCallback, status = %{public}d", status); + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + return; + } + napi_value argv[2] = {resolvedCallback, rejectedCallback}; + batchInsertInfo->callBackErrCode = CALL_BACK_RESULT_INVALID; + status = napi_call_function(env, promise, thenFunc, PROMISE_PARAM_NUM, argv, nullptr); + if (batchInsertInfo->callBackErrCode == CALL_BACK_RESULT_INVALID) { + batchInsertInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + LOG_ERROR("[BatchInsertInner] napi call function go wrong"); + } + ClearLastException(env); +} + +void NapiCloudDb::QueryInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + QueryInfo *queryInfo = reinterpret_cast(data); + std::vector inputParam = queryInfo->inputParam; + napi_value arrayParams = nullptr; + napi_status status = napi_create_array(env, &arrayParams); + if (status != napi_ok) { + queryInfo->promiseRes.set_value(std::vector()); + return; + } + GetJsQueryParams(env, inputParam, arrayParams); + + napi_value promise = nullptr; + status = napi_call_function(env, nullptr, jsCb, HANDLER_PARAM_NUM, &arrayParams, &promise); + if (status != napi_ok) { + LOG_ERROR("napi call function go wrong, status = %{public}d", status); + queryInfo->promiseRes.set_value(std::vector()); + return; + } + + napi_value thenFunc = nullptr; + status = napi_get_named_property(env, promise, "then", &thenFunc); + if (status != napi_ok) { + LOG_ERROR("napi get named property fail, status = %{public}d", status); + queryInfo->promiseRes.set_value(std::vector()); + return; + } + napi_value resolvedCallback; + napi_value rejectedCallback; + status = napi_create_function(env, "resolvedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::QueryResolvedCallback, data, &resolvedCallback); + if (status != napi_ok) { + LOG_ERROR("resolvedCallback, status = %{public}d", status); + queryInfo->promiseRes.set_value(std::vector()); + return; + } + status = napi_create_function(env, "rejectedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::QueryRejectedCallback, data, &rejectedCallback); + if (status != napi_ok) { + LOG_ERROR("rejectedCallback, status = %{public}d", status); + queryInfo->promiseRes.set_value(std::vector()); + return; + } + napi_value argv[2] = {resolvedCallback, rejectedCallback}; + queryInfo->callBackErrCode = CALL_BACK_RESULT_INVALID; + status = napi_call_function(env, promise, thenFunc, PROMISE_PARAM_NUM, argv, nullptr); + if (queryInfo->callBackErrCode == CALL_BACK_RESULT_INVALID) { + queryInfo->promiseRes.set_value(std::vector()); + queryInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + LOG_ERROR("napi call function go wrong"); + } + ClearLastException(env); +} + +void NapiCloudDb::DownloadAssetInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + HandleAssetAsyncInner(env, jsCb, context, data); +} + +void NapiCloudDb::UploadAssetInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + HandleAssetInner(env, jsCb, context, data); +} + +void NapiCloudDb::DeleteAssetInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + HandleAssetInner(env, jsCb, context, data); +} + +void NapiCloudDb::DeleteLocalAssetInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + HandleAssetAsyncInner(env, jsCb, context, data); +} + +void NapiCloudDb::HandleAssetInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + AssetInfo *assetInfo = reinterpret_cast(data); + napi_value param = Parser::ParseFromAssetOpConfig(env, assetInfo->config); + napi_value promise = nullptr; + napi_status status = napi_call_function(env, nullptr, jsCb, HANDLER_PARAM_NUM, ¶m, &promise); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetInner] napi call function go wrong, status = %{public}d", status); + assetInfo->promiseRes.set_value(); + return; + } + + napi_value thenFunc = nullptr; + status = napi_get_named_property(env, promise, "then", &thenFunc); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetInner] napi get named property fail, status = %{public}d", status); + assetInfo->promiseRes.set_value(); + return; + } + napi_value resolvedCallback; + napi_value rejectedCallback; + status = napi_create_function(env, "resolvedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::HandleAssetResolvedCallback, data, &resolvedCallback); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetInner] napi create resolvedCallback go wrong, status = %{public}d", status); + assetInfo->promiseRes.set_value(); + return; + } + status = napi_create_function(env, "rejectedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::HandleAssetRejectedCallback, data, &rejectedCallback); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetInner] napi create rejectedCallback go wrong, status = %{public}d", status); + assetInfo->promiseRes.set_value(); + return; + } + napi_value argv[PROMISE_PARAM_NUM] = {resolvedCallback, rejectedCallback}; + assetInfo->callBackErrCode = CALL_BACK_RESULT_INVALID; + status = napi_call_function(env, promise, thenFunc, PROMISE_PARAM_NUM, argv, nullptr); + if (assetInfo->callBackErrCode == CALL_BACK_RESULT_INVALID) { + assetInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + LOG_ERROR("[HandleAssetInner] napi call function go wrong"); + } + assetInfo->promiseRes.set_value(); + ClearLastException(env); +} + +void NapiCloudDb::HandleAssetAsyncInner(napi_env env, napi_value jsCb, void *context, void *data) +{ + AssetInfo *assetInfo = reinterpret_cast(data); + napi_value param = Parser::ParseFromAssetOpConfig(env, assetInfo->config); + napi_value promise = nullptr; + napi_status status = napi_call_function(env, nullptr, jsCb, HANDLER_PARAM_NUM, ¶m, &promise); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetAsyncInner] napi call function go wrong, status = %{public}d", status); + return; + } + + napi_value thenFunc = nullptr; + status = napi_get_named_property(env, promise, "then", &thenFunc); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetAsyncInner] napi get named property fail, status = %{public}d", status); + return; + } + napi_value resolvedCallback; + napi_value rejectedCallback; + status = napi_create_function(env, "resolvedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::HandleAssetAsyncResolvedCallback, data, &resolvedCallback); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetAsyncInner] napi create resolvedCallback go wrong, status = %{public}d", status); + return; + } + status = napi_create_function(env, "rejectedCallback", NAPI_AUTO_LENGTH, + NapiCloudDb::HandleAssetAsyncRejectedCallback, data, &rejectedCallback); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetAsyncInner] napi create rejectedCallback go wrong, status = %{public}d", status); + return; + } + napi_value argv[PROMISE_PARAM_NUM] = {resolvedCallback, rejectedCallback}; + status = napi_call_function(env, promise, thenFunc, PROMISE_PARAM_NUM, argv, nullptr); + if (status != napi_ok) { + LOG_ERROR("[HandleAssetAsyncInner] napi call function go wrong, status: %{public}d", status); + } + ClearLastException(env); +} + +napi_value NapiCloudDb::BatchInsertResolvedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + size_t argc = 1; + napi_value argv[1]; + napi_status status = napi_get_cb_info(env, info, &argc, argv, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[ResolvedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + + int32_t num = 0; + NapiUtils::GetValue(env, argv[0], num); + BatchInsertInfo *batchInsertInfo = reinterpret_cast(data); + batchInsertInfo->promiseRes.set_value(num); + batchInsertInfo->callBackErrCode = CALL_BACK_RESULT_OK; + return nullptr; +} + +napi_value NapiCloudDb::BatchInsertRejectedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + napi_status status = napi_get_cb_info(env, info, nullptr, nullptr, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[RejectedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + BatchInsertInfo *batchInsertInfo = reinterpret_cast(data); + batchInsertInfo->promiseRes.set_value(CALL_BACK_RESULT_ERR); + batchInsertInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + return nullptr; +} + +napi_status GetQueryResult(napi_env env, napi_value input, std::vector &out) +{ + LOG_DEBUG("napi_value -> std::vector"); + out.clear(); + bool isArray = false; + napi_is_array(env, input, &isArray); + ASSERT(isArray, "not an array", napi_invalid_arg); + + uint32_t length = 0; + napi_status statusMsg = napi_get_array_length(env, input, &length); + ASSERT((statusMsg == napi_ok), "get_array go wrong!", napi_invalid_arg); + for (uint32_t i = 0; i < length; ++i) { + napi_value item = nullptr; + statusMsg = napi_get_element(env, input, i, &item); + ASSERT((item != nullptr) && (statusMsg == napi_ok), "no element", napi_invalid_arg); + CloudParamsAdapterT res; + napi_value cursor = nullptr; + napi_value id = nullptr; + napi_value data = nullptr; + statusMsg = napi_get_named_property(env, item, "cursor", &cursor); + ASSERT(statusMsg == napi_ok, "get cursor property go wrong", napi_invalid_arg); + statusMsg = napi_get_named_property(env, item, "id", &id); + ASSERT(statusMsg == napi_ok, "get id property go wrong", napi_invalid_arg); + statusMsg = napi_get_named_property(env, item, "data", &data); + ASSERT(statusMsg == napi_ok, "get data property go wrong", napi_invalid_arg); + + statusMsg = NapiUtils::GetValue(env, cursor, res.cursor); + ASSERT(statusMsg == napi_ok, "get cursor go wrong", napi_invalid_arg); + statusMsg = NapiUtils::GetValue(env, id, res.id); + ASSERT(statusMsg == napi_ok, "get id go wrong", napi_invalid_arg); + statusMsg = NapiUtils::GetValue(env, data, res.record); + ASSERT(statusMsg == napi_ok, "get record go wrong", napi_invalid_arg); + + out.push_back(res); + } + return statusMsg; +} + +napi_value NapiCloudDb::QueryResolvedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + size_t argc = 1; + napi_value argv[1]; + napi_status status = napi_get_cb_info(env, info, &argc, argv, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + + QueryInfo *queryInfo = reinterpret_cast(data); + std::vector queryResult; + napi_status ret = GetQueryResult(env, argv[0], queryResult); + if (ret != napi_ok) { + LOG_ERROR("get callback result go wrong, ret=%{public}d", ret); + queryResult.clear(); + queryInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + } else if (ret == napi_ok && queryResult.size() == 0) { + queryInfo->callBackErrCode = CALL_BACK_RESULT_QUERY_END; + } else { + queryInfo->callBackErrCode = CALL_BACK_RESULT_OK; + } + queryInfo->promiseRes.set_value(queryResult); + return nullptr; +} + +napi_value NapiCloudDb::QueryRejectedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + napi_status status = napi_get_cb_info(env, info, nullptr, nullptr, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + QueryInfo *queryInfo = reinterpret_cast(data); + queryInfo->promiseRes.set_value(std::vector()); + queryInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + return nullptr; +} + +napi_value NapiCloudDb::HandleAssetResolvedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + size_t argc = 1; + napi_value argv[1]; + napi_status status = napi_get_cb_info(env, info, &argc, argv, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[ResolvedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + + AssetInfo *assetInfo = reinterpret_cast(data); + assetInfo->callBackErrCode = CALL_BACK_RESULT_OK; + return nullptr; +} + +napi_value NapiCloudDb::HandleAssetRejectedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + napi_status status = napi_get_cb_info(env, info, nullptr, nullptr, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[RejectedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + AssetInfo *assetInfo = reinterpret_cast(data); + assetInfo->callBackErrCode = CALL_BACK_RESULT_ERR; + return nullptr; +} + +napi_value NapiCloudDb::HandleAssetAsyncResolvedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + size_t argc = 1; + napi_value argv[1]; + napi_status status = napi_get_cb_info(env, info, &argc, argv, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[ResolvedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + + AssetInfo *assetInfo = reinterpret_cast(data); + delete assetInfo; + return nullptr; +} + +napi_value NapiCloudDb::HandleAssetAsyncRejectedCallback(napi_env env, napi_callback_info info) +{ + void *data = nullptr; + napi_status status = napi_get_cb_info(env, info, nullptr, nullptr, nullptr, &data); + if (status != napi_ok) { + LOG_ERROR("[RejectedCallback] napi get callback info go wrong, status = %{public}d", status); + return nullptr; + } + AssetInfo *assetInfo = reinterpret_cast(data); + delete assetInfo; + return nullptr; +} + +void NapiCloudDb::ClearLastException(const napi_env& env) +{ + bool isExistException = false; + napi_is_exception_pending(env, &isExistException); + if (!isExistException) { + return; + } + napi_value exception = nullptr; + napi_status status = napi_get_and_clear_last_exception(env, &exception); + if (status != napi_ok) { + LOG_ERROR("[ClearLastException] clear last exception wrong, status: %{public}d", status); + } +} +} // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_collaboration_edit_object.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_collaboration_edit_object.cpp index e4f751ab..6370df8a 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_collaboration_edit_object.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_collaboration_edit_object.cpp @@ -14,14 +14,32 @@ */ #define LOG_TAG "CollaborationEditObject" +#include + #include "napi_collaboration_edit_object.h" #include "db_store_config.h" #include "db_store_manager.h" #include "napi_edit_unit.h" #include "napi_error_utils.h" +#include "napi_parser.h" +#include "napi_sync_service.h" +#include "napi/native_node_api.h" namespace OHOS::CollaborationEdit { +static constexpr const uint8_t BATCH_INSERT_FUNC_INDEX = 0; +static constexpr const uint8_t QUERY_FUNC_INDEX = 1; +static constexpr const uint8_t DOWNLOAD_ASSET_FUNC_INDEX = 2; +static constexpr const uint8_t UPLOAD_ASSET_FUNC_INDEX = 3; +static constexpr const uint8_t DELETE_ASSET_FUNC_INDEX = 4; +static constexpr const uint8_t DELETE_LOCAL_ASSET_FUNC_INDEX = 5; + +static constexpr const uint8_t CLOUD_SYNC_PARAM_NUMBER = 2; +static const std::map SYNC_MODE_MAP = { + {SyncMode::PUSH, GRD_SYNC_MODE_UPLOAD}, + {SyncMode::PULL, GRD_SYNC_MODE_DOWNLOAD_LOG}, + {SyncMode::PULL_PUSH, GRD_SYNC_MODE_UP_DOWN_LOG} +}; CollaborationEditObject::CollaborationEditObject(std::string docName, ContextParam param) : docName_(docName), param_(std::make_shared(std::move(param))) @@ -39,6 +57,7 @@ napi_value CollaborationEditObject::Initialize(napi_env env, napi_callback_info ContextParam context; napi_status status = NapiUtils::GetValue(env, argv[0], context); ASSERT_THROW_BASE(env, status == napi_ok, Status::INVALID_ARGUMENT, "read context param go wrong", self); + ASSERT_THROW_BASE(env, context.isSystemApp, Status::NOT_SYSTEM_APP, "Not a system app", self); std::string docName; status = NapiUtils::GetNamedProperty(env, argv[1], "name", docName); ASSERT_THROW_BASE(env, status == napi_ok, Status::INVALID_ARGUMENT, "read docName param go wrong", self); @@ -52,6 +71,7 @@ napi_value CollaborationEditObject::Initialize(napi_env env, napi_callback_info } CollaborationEditObject *editObject = new (std::nothrow) CollaborationEditObject(docName, context); + ASSERT_THROW_BASE(env, editObject != nullptr, Status::INTERNAL_ERROR, "Initialize: new editObject go wrong", self); editObject->SetDBStore(dbStore); auto finalize = [](napi_env env, void *data, void *hint) { CollaborationEditObject *editObject = reinterpret_cast(data); @@ -69,6 +89,11 @@ napi_value CollaborationEditObject::Constructor(napi_env env) DECLARE_NAPI_FUNCTION("getUndoRedoManager", GetUndoRedoManager), DECLARE_NAPI_FUNCTION("deleteUndoRedoManager", DeleteUndoRedoManager), DECLARE_NAPI_FUNCTION("getName", GetName), + DECLARE_NAPI_FUNCTION("cloudSync", CloudSync), + DECLARE_NAPI_FUNCTION("setCloudDB", SetCloudDb), + DECLARE_NAPI_FUNCTION("getLocalId", GetLocalId), + DECLARE_NAPI_FUNCTION("applyUpdate", ApplyUpdate), + DECLARE_NAPI_FUNCTION("writeUpdate", WriteUpdate), }; return properties; }; @@ -126,15 +151,28 @@ napi_value CollaborationEditObject::GetUndoRedoManager(napi_env env, napi_callba } undoManager->SetDBStore(editObject->GetDBStore()); int32_t retCode = undoManager->GetAdapter()->CreateUndoManager(undoManager->GetCaptureTimeout()); - if (retCode != SUCCESS) { - ThrowNapiError(env, retCode, "create undo manager go wrong."); - return nullptr; - } + ASSERT_THROW(env, retCode == SUCCESS, retCode, "create undo manager go wrong."); return jsUndoManager; } napi_value CollaborationEditObject::DeleteUndoRedoManager(napi_env env, napi_callback_info info) { + size_t argc = 1; + napi_value argv[1] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + CollaborationEditObject *editObject = nullptr; + NAPI_CALL(env, napi_unwrap(env, self, reinterpret_cast(&editObject))); + std::string editUnitName; + napi_status status = NapiUtils::GetValue(env, argv[0], editUnitName); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read editUnitName go wrong"); + std::shared_ptr adapter = std::make_shared(); + std::string tableName = std::to_string(LABEL_FRAGMENT) + "_" + editUnitName; + adapter->SetTableName(tableName); + adapter->SetDBStore(editObject->GetDBStore()); + int32_t retCode = adapter->CloseUndoManager(); + ASSERT_THROW(env, retCode == SUCCESS, retCode, "close undo manager go wrong."); + LOG_INFO("Close undo manager successfully."); return nullptr; } @@ -153,6 +191,114 @@ napi_value CollaborationEditObject::GetName(napi_env env, napi_callback_info inf return result; } +napi_value CollaborationEditObject::SetCloudDb(napi_env env, napi_callback_info info) +{ + size_t argc = 1; + napi_value argv[1] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + + std::vector cloudDbFunc; + int ret = Parser::ParseCloudDbFields(env, argv[0], cloudDbFunc); + if (ret != OK) { + ThrowNapiError(env, Status::INVALID_ARGUMENT, "parse cloudDb fields go wrong."); + return nullptr; + } + NapiCloudDb *napiCloudDb = new (std::nothrow) NapiCloudDb(); + ASSERT_THROW(env, napiCloudDb != nullptr, Status::INTERNAL_ERROR, "new cloud db instance go wrong."); + napi_status status = CollaborationEditObject::CreateHandlerFunc(env, cloudDbFunc, napiCloudDb); + if (status != napi_ok) { + ReleaseHandlerFunc(napiCloudDb); + delete napiCloudDb; + ThrowNapiError(env, Status::INTERNAL_ERROR, "create handler func go wrong."); + return nullptr; + } + + CollaborationEditObject *editObject = nullptr; + status = napi_unwrap(env, self, reinterpret_cast(&editObject)); + if (status != napi_ok) { + ReleaseHandlerFunc(napiCloudDb); + delete napiCloudDb; + ThrowNapiError(env, Status::INTERNAL_ERROR, "unwrap editObject go wrong."); + return nullptr; + } + ret = DBStoreManager::GetInstance().SetCloudDb(editObject->GetDBStore(), napiCloudDb); + if (ret != 0) { + ReleaseHandlerFunc(napiCloudDb); + delete napiCloudDb; + ThrowNapiError(env, ret, "set cloud db go wrong."); + } + return nullptr; +} + +napi_status CollaborationEditObject::CreateHandlerFunc(napi_env env, std::vector &cloudDbFunc, + NapiCloudDb *napiCloudDb) +{ + napi_value name = nullptr; + napi_status status = napi_create_string_utf8(env, "batchInsert", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create batchInsert string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[BATCH_INSERT_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::BatchInsertInner, &napiCloudDb->batchInsertInnerFunc_); + ASSERT(status == napi_ok, "create batchInsert func wrong", status); + + name = nullptr; + status = napi_create_string_utf8(env, "query", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create query string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[QUERY_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::QueryInner, &napiCloudDb->queryInnerFunc_); + ASSERT(status == napi_ok, "create query func wrong", status); + + name = nullptr; + status = napi_create_string_utf8(env, "downloadAsset", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create downloadAsset string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[DOWNLOAD_ASSET_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::DownloadAssetInner, &napiCloudDb->downloadAssetInnerFunc_); + ASSERT(status == napi_ok, "create downloadAsset func wrong", status); + + name = nullptr; + status = napi_create_string_utf8(env, "uploadAsset", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create uploadAsset string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[UPLOAD_ASSET_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::UploadAssetInner, &napiCloudDb->uploadAssetInnerFunc_); + ASSERT(status == napi_ok, "create uploadAsset func wrong", status); + + name = nullptr; + status = napi_create_string_utf8(env, "deleteAsset", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create deleteAsset string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[DELETE_ASSET_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::DeleteAssetInner, &napiCloudDb->deleteAssetInnerFunc_); + ASSERT(status == napi_ok, "create deleteAsset func wrong", status); + + name = nullptr; + status = napi_create_string_utf8(env, "deleteLocalAsset", NAPI_AUTO_LENGTH, &name); + ASSERT(status == napi_ok, "create deleteLocalAsset string wrong", status); + status = napi_create_threadsafe_function(env, cloudDbFunc[DELETE_LOCAL_ASSET_FUNC_INDEX], nullptr, name, 0, 1, + nullptr, nullptr, nullptr, NapiCloudDb::DeleteLocalAssetInner, &napiCloudDb->deleteLocalAssetInnerFunc_); + ASSERT(status == napi_ok, "create deleteLocalAsset func wrong", status); + return napi_ok; +} + +void CollaborationEditObject::ReleaseHandlerFunc(NapiCloudDb *napiCloudDb) +{ + if (napiCloudDb == nullptr) { + return; + } + napi_release_threadsafe_function(napiCloudDb->batchInsertInnerFunc_, napi_tsfn_release); + napi_release_threadsafe_function(napiCloudDb->queryInnerFunc_, napi_tsfn_release); + napi_release_threadsafe_function(napiCloudDb->downloadAssetInnerFunc_, napi_tsfn_release); + napi_release_threadsafe_function(napiCloudDb->uploadAssetInnerFunc_, napi_tsfn_release); + napi_release_threadsafe_function(napiCloudDb->deleteAssetInnerFunc_, napi_tsfn_release); + napi_release_threadsafe_function(napiCloudDb->deleteLocalAssetInnerFunc_, napi_tsfn_release); + + napiCloudDb->batchInsertInnerFunc_ = nullptr; + napiCloudDb->queryInnerFunc_ = nullptr; + napiCloudDb->downloadAssetInnerFunc_ = nullptr; + napiCloudDb->downloadAssetInnerFunc_ = nullptr; + napiCloudDb->uploadAssetInnerFunc_ = nullptr; + napiCloudDb->deleteAssetInnerFunc_ = nullptr; + napiCloudDb->deleteLocalAssetInnerFunc_ = nullptr; +} + napi_value CollaborationEditObject::Delete(napi_env env, napi_callback_info info) { size_t argc = 2; @@ -162,6 +308,7 @@ napi_value CollaborationEditObject::Delete(napi_env env, napi_callback_info info ContextParam context; napi_status status = NapiUtils::GetValue(env, argv[0], context); ASSERT_THROW_BASE(env, status == napi_ok, Status::INVALID_ARGUMENT, "read context param go wrong", self); + ASSERT_THROW_BASE(env, context.isSystemApp, Status::NOT_SYSTEM_APP, "Not a system app", self); std::string docName; status = NapiUtils::GetNamedProperty(env, argv[1], "name", docName); ASSERT_THROW_BASE(env, status == napi_ok, Status::INVALID_ARGUMENT, "read docName param go wrong", self); @@ -175,6 +322,191 @@ napi_value CollaborationEditObject::Delete(napi_env env, napi_callback_info info return nullptr; } +int CollaborationEditObject::ParseCloudSyncMode(const napi_env env, const napi_value arg, + std::shared_ptr context) +{ + int32_t mode = 0; + napi_status status = NapiUtils::GetValue(env, arg, mode); + if (status != napi_ok || mode < 0 || mode > SyncMode::PULL_PUSH) { + LOG_ERROR("CloudSync parse syncMode go wrong, mode: %d", mode); + return ERR; + } + context->syncMode_ = mode; + return OK; +} + +int CollaborationEditObject::ParseThis(const napi_env &env, const napi_value &self, + std::shared_ptr context) +{ + CollaborationEditObject *editObject = nullptr; + napi_status status = napi_unwrap(env, self, reinterpret_cast(&editObject)); + if (status != napi_ok || editObject == nullptr) { + LOG_ERROR("CloudSync unwrap object go wrong"); + return ERR; + } + context->dbStore_ = editObject->GetDBStore(); + context->boundObj = editObject; + return OK; +} + +InputAction CollaborationEditObject::GetCloudSyncInput(std::shared_ptr context) +{ + return [context](napi_env env, size_t argc, napi_value *argv, napi_value self) -> int { + LOG_DEBUG("get CloudSync input start."); + ASSERT(argc == CLOUD_SYNC_PARAM_NUMBER, "CloudSync read param go wrong", ERR); + // set params to sync context + ASSERT(OK == ParseThis(env, self, context), "CloudSync get editObject go wrong", ERR); + ASSERT(OK == ParseCloudSyncMode(env, argv[0], context), "parse syncMode go wrong", ERR); + + napi_value name = nullptr; + ASSERT(napi_ok == napi_create_string_utf8(env, "syncCallback", NAPI_AUTO_LENGTH, &name), + "create function name string wrong", ERR); + ASSERT(napi_ok == napi_create_threadsafe_function(env, argv[1], nullptr, name, 0, 1, nullptr, nullptr, nullptr, + AsyncCall::CloudSyncCallback, &context->callback_), "create syncCallback function wrong", ERR); + return OK; + }; +} + +ExecuteAction CollaborationEditObject::GetCloudSyncExec(std::shared_ptr context) +{ + return [context]() -> int { + LOG_INFO("CollaborationEditObject::CloudSync Async execute."); + auto *editObject = reinterpret_cast(context->boundObj); + ASSERT(editObject != nullptr && context->dbStore_ != nullptr, "editObject is null or dbStore is null", ERR); + + std::shared_ptr syncService = SyncService::GetInstance(); + uint64_t syncId = syncService->GetSyncId(); + context->syncId = syncId; + syncService->AddSyncContext(syncId, context); + int32_t ret = context->dbStore_->Sync(GetGRDSyncMode(context->syncMode_), syncId, + CollaborationEditObject::SyncCallbackFunc); + if (ret != GRD_OK) { + LOG_ERROR("dbStore sync go wrong, errCode: %{public}d", ret); + GRD_SyncProcessT process = {}; + process.status = GRD_SYNC_PROCESS_FINISHED; + process.errCode = ret; + process.mode = GRD_SYNC_MODE_INVALID; + process.cloudDB = nullptr; + process.syncId = context->syncId; + SyncCallbackFunc(&process); + return ERR; + } + return OK; + }; +} + +napi_value CollaborationEditObject::CloudSync(napi_env env, napi_callback_info info) +{ + LOG_DEBUG("CloudSync start."); + auto context = std::make_shared(); + auto input = CollaborationEditObject::GetCloudSyncInput(context); + // create cloudSync execute function + auto exec = GetCloudSyncExec(context); + // create cloudSync output function + auto output = [context](napi_env env, napi_value &result) { + LOG_DEBUG("CollaborationEditObject::CloudSync output."); + if (context->execCode_ != OK) { + SyncService::GetInstance()->RemoveSyncContext(context->syncId); + } + }; + // parse napi info and register exec/output functions to context + int ret = context->SetAll(env, info, input, exec, output); + if (ret != OK) { + ThrowNapiError(env, Status::INVALID_ARGUMENT, "parse params go wrong"); + return nullptr; + } + // register cloudSync task to napi queue + ret = AsyncCall::Call(env, context, SYNC_FUNCTION_NAME); + if (ret != OK) { + if (context->callback_ != nullptr) { + napi_release_threadsafe_function(context->callback_, napi_tsfn_release); + context->callback_ = nullptr; + } + context->ReleaseInnerReference(); + context.reset(); + ThrowNapiError(env, Status::INTERNAL_ERROR, "register sync task go wrong"); + } + return nullptr; +} + +GRD_SyncModeE CollaborationEditObject::GetGRDSyncMode(int32_t mode) +{ + SyncMode syncMode = static_cast(mode); + auto it = SYNC_MODE_MAP.find(syncMode); + if (it == SYNC_MODE_MAP.end()) { + return GRD_SYNC_MODE_INVALID; + } + return it->second; +} + +ProgressCode CollaborationEditObject::GetProgressCode(int32_t errCode) +{ + switch (errCode) { + case GRD_OK: + return ProgressCode::CLOUD_SYNC_SUCCESS; + case GRD_SYNC_PREREQUISITES_ABNORMAL: + return ProgressCode::CLOUD_NOT_SET; + case GRD_INNER_ERR: + return ProgressCode::SYNC_INTERNAL_ERROR; + default: + break; + } + return ProgressCode::SYNC_EXTERNAL_ERROR; +} + +void CollaborationEditObject::SyncCallbackFunc(GRD_SyncProcessT *syncProcess) +{ + LOG_INFO("syncId=%{public}" PRIu64 " status=%{public}d, errCode=%{public}d.", + syncProcess->syncId, syncProcess->status, syncProcess->errCode); + + // call arkTs callback + std::shared_ptr syncService = SyncService::GetInstance(); + std::shared_ptr context = syncService->GetSyncContext(syncProcess->syncId); + syncService->RemoveSyncContext(syncProcess->syncId); + if (context == nullptr) { + LOG_ERROR("context is null"); + return; + } + napi_threadsafe_function js_cb = context->callback_; + if (js_cb == nullptr) { + LOG_ERROR("callback is null"); + context->ReleaseInnerReference(); + return; + } + + napi_status ret = napi_acquire_threadsafe_function(js_cb); + if (ret != napi_ok) { + LOG_ERROR("acquire thread safe function failed, ret: %{public}d", static_cast(ret)); + napi_release_threadsafe_function(js_cb, napi_tsfn_release); + context->callback_ = nullptr; + context->ReleaseInnerReference(); + return; + } + SyncCallbackParamT param = {}; + param.detail.code = CollaborationEditObject::GetProgressCode(syncProcess->errCode); + + std::future future = param.promise.get_future(); + ret = napi_call_threadsafe_function(js_cb, ¶m, napi_tsfn_blocking); + if (ret != napi_ok) { + LOG_ERROR("call function go wrong, ret=%{public}d", static_cast(ret)); + napi_release_threadsafe_function(js_cb, napi_tsfn_release); + context->callback_ = nullptr; + context->ReleaseInnerReference(); + return; + } + std::future_status fstatus = future.wait_for(std::chrono::duration_cast(TIME_THRESHOLD)); + if (fstatus == std::future_status::ready) { + future.get(); + } else { + LOG_ERROR("wait for js callback timeout"); + } + + napi_release_threadsafe_function(js_cb, napi_tsfn_release); + context->callback_ = nullptr; + context->ReleaseInnerReference(); + LOG_INFO("syncCallback end"); +} + void CollaborationEditObject::SetDBStore(std::shared_ptr dbStore) { this->dbStore_ = dbStore; @@ -184,4 +516,62 @@ std::shared_ptr CollaborationEditObject::GetDBStore() { return this->dbStore_; } + +napi_value CollaborationEditObject::GetLocalId(napi_env env, napi_callback_info info) +{ + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, nullptr, nullptr, &self, nullptr)); + CollaborationEditObject *editObject = nullptr; + napi_status status = napi_unwrap(env, self, reinterpret_cast(&editObject)); + ASSERT_THROW(env, status == napi_ok, status, "unwrap object go wrong"); + std::string localId = (*editObject->dbStore_).GetLocalId(); + napi_value result; + NapiUtils::SetValue(env, localId, result); + return result; +} + +napi_value CollaborationEditObject::ApplyUpdate(napi_env env, napi_callback_info info) +{ + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, nullptr, nullptr, &self, nullptr)); + CollaborationEditObject *editObject = nullptr; + napi_status status = napi_unwrap(env, self, reinterpret_cast(&editObject)); + ASSERT_THROW(env, status == napi_ok, status, "unwrap object go wrong"); + + std::string applyInfo; + int32_t ret = (*editObject->dbStore_).ApplyUpdate(applyInfo); + ASSERT_THROW(env, ret == OK, ret, "ApplyUpdate go wrong"); + napi_value result; + ret = Parser::ParseJsonStrToJsUpdateNode(env, applyInfo, editObject->dbStore_, result); + ASSERT_THROW(env, ret == OK, ret, "ParseJsonStrToJsUpdateNode go wrong"); + return result; +} + +napi_value CollaborationEditObject::WriteUpdate(napi_env env, napi_callback_info info) +{ + size_t argc = 3; + napi_value argv[3] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + CollaborationEditObject *editObject = nullptr; + napi_status status = napi_unwrap(env, self, reinterpret_cast(&editObject)); + ASSERT_THROW(env, status == napi_ok, status, "unwrap object go wrong"); + + std::string equipId; + status = NapiUtils::GetNamedProperty(env, argv[0], "id", equipId); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read equipId param go wrong"); + + std::vector data; + status = NapiUtils::GetNamedProperty(env, argv[0], "data", data); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read data param go wrong"); + + int64_t watermark; + status = NapiUtils::GetNamedProperty(env, argv[0], "cursor", watermark); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read cursor param go wrong"); + + int32_t ret = + (*editObject->dbStore_).WriteUpdate(equipId.c_str(), data.data(), data.size(), std::to_string(watermark)); + ASSERT_THROW(env, ret == OK, ret, "WriteUpdate go wrong"); + return nullptr; +} } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_const_properties.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_const_properties.cpp new file mode 100644 index 00000000..1a224806 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_const_properties.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "NapiConstProperties" + +#include "napi_const_properties.h" + +#include "js_utils.h" +#include "napi_error_utils.h" +#include "napi_utils.h" +#include "rd_type.h" + +namespace OHOS::CollaborationEdit { +static napi_value ExportSyncMode(napi_env env) +{ + napi_value syncMode = nullptr; + napi_status status = napi_create_object(env, &syncMode); + ASSERT(status == napi_ok, "create syncMode object go wrong.", nullptr); + napi_set_named_property(env, syncMode, "SYNC_MODE_PUSH", + JSUtils::Convert2JSValue(env, static_cast(SyncMode::PUSH))); + napi_set_named_property(env, syncMode, "SYNC_MODE_PULL", + JSUtils::Convert2JSValue(env, static_cast(SyncMode::PULL))); + napi_set_named_property(env, syncMode, "SYNC_MODE_PULL_PUSH", + JSUtils::Convert2JSValue(env, static_cast(SyncMode::PULL_PUSH))); + napi_object_freeze(env, syncMode); + return syncMode; +} + +static napi_value ExportPredicate(napi_env env) +{ + napi_value predicate = nullptr; + napi_status status = napi_create_object(env, &predicate); + ASSERT(status == napi_ok, "create predicate object go wrong.", nullptr); + napi_set_named_property(env, predicate, "EQUAL_TO", + JSUtils::Convert2JSValue(env, static_cast(Predicate::EQUAL_TO))); + napi_set_named_property(env, predicate, "NOT_EQUAL_TO", + JSUtils::Convert2JSValue(env, static_cast(Predicate::NOT_EQUAL_TO))); + napi_set_named_property(env, predicate, "GREATER_THAN", + JSUtils::Convert2JSValue(env, static_cast(Predicate::GREATER_THAN))); + napi_set_named_property(env, predicate, "LESS_THAN", + JSUtils::Convert2JSValue(env, static_cast(Predicate::LESS_THAN))); + napi_set_named_property(env, predicate, "GREATER_THAN_OR_EQUAL_TO", + JSUtils::Convert2JSValue(env, static_cast(Predicate::GREATER_THAN_OR_EQUAL_TO))); + napi_set_named_property(env, predicate, "GREATER_THAN_OR_EQUAL_TO", + JSUtils::Convert2JSValue(env, static_cast(Predicate::LESS_THAN_OR_EQUAL_TO))); + napi_object_freeze(env, predicate); + return predicate; +} + +static napi_value ExportProgressCode(napi_env env) +{ + napi_value progressCode = nullptr; + napi_status status = napi_create_object(env, &progressCode); + ASSERT(status == napi_ok, "create progressCode object go wrong.", nullptr); + napi_set_named_property(env, progressCode, "CLOUD_SYNC_SUCCESS", + JSUtils::Convert2JSValue(env, static_cast(ProgressCode::CLOUD_SYNC_SUCCESS))); + napi_set_named_property(env, progressCode, "CLOUD_NOT_SET", + JSUtils::Convert2JSValue(env, static_cast(ProgressCode::CLOUD_NOT_SET))); + napi_set_named_property(env, progressCode, "INTERNAL_ERROR", + JSUtils::Convert2JSValue(env, static_cast(ProgressCode::SYNC_INTERNAL_ERROR))); + napi_set_named_property(env, progressCode, "EXTERNAL_ERROR", + JSUtils::Convert2JSValue(env, static_cast(ProgressCode::SYNC_EXTERNAL_ERROR))); + napi_object_freeze(env, progressCode); + return progressCode; +} + +napi_status InitConstProperties(napi_env env, napi_value exports) +{ + const napi_property_descriptor properties[] = { + DECLARE_NAPI_PROPERTY("SyncMode", ExportSyncMode(env)), + DECLARE_NAPI_PROPERTY("Predicate", ExportPredicate(env)), + DECLARE_NAPI_PROPERTY("ProgressCode", ExportProgressCode(env)), + }; + + size_t count = sizeof(properties) / sizeof(properties[0]); + return napi_define_properties(env, exports, count, properties); +} +} // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_edit_unit.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_edit_unit.cpp index 38317537..183d849a 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_edit_unit.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_edit_unit.cpp @@ -17,11 +17,14 @@ #include "napi_edit_unit.h" +#include + #include "napi_collaboration_edit_object.h" #include "napi_error_utils.h" #include "napi_parser.h" namespace OHOS::CollaborationEdit { +using json = nlohmann::json; EditUnit::EditUnit(std::string name) : AbstractType(), name_(name) {} @@ -40,6 +43,7 @@ napi_value EditUnit::Initialize(napi_env env, napi_callback_info info) ASSERT_THROW_BASE(env, !name.empty(), Status::INVALID_ARGUMENT, "Param Error: invalid name", self); name = std::to_string(LABEL_FRAGMENT) + "_" + name; EditUnit *editUnit = new (std::nothrow) EditUnit(name); + ASSERT_THROW_BASE(env, editUnit != nullptr, Status::INTERNAL_ERROR, "Initialize: new editunit go wrong", self); editUnit->SetTableName(name); auto finalize = [](napi_env env, void *data, void *hint) { EditUnit *editUnit = reinterpret_cast(data); @@ -58,6 +62,8 @@ napi_value EditUnit::Constructor(napi_env env) DECLARE_NAPI_FUNCTION("getChildren", GetChildren), DECLARE_NAPI_FUNCTION("getJsonResult", GetJsonResult), DECLARE_NAPI_FUNCTION("getName", GetName), + DECLARE_NAPI_FUNCTION("getRelativePos", GetRelativePos), + DECLARE_NAPI_FUNCTION("getAbsolutePos", GetAbsolutePos), }; return properties; }; @@ -89,7 +95,7 @@ napi_value EditUnit::GetName(napi_env env, napi_callback_info info) return nullptr; } std::string name = editUnit->name_; - if (name.compare(0, NUMBER_OF_CHARS_IN_LABEL_PREFIX, std::to_string(LABEL_FRAGMENT) + "_") == 0) { + if (name.compare(0, NUMBER_OF_CHARS_IN_LABEL_PREFIX, std::to_string(LABEL_FRAGMENT) + "_") == 0) { name = name.substr(NUMBER_OF_CHARS_IN_LABEL_PREFIX); } napi_value result; @@ -212,4 +218,82 @@ napi_value EditUnit::GetJsonResult(napi_env env, napi_callback_info info) } return output; } -} // namespace OHOS::CollaborationEdit + +napi_value EditUnit::GetRelativePos(napi_env env, napi_callback_info info) +{ + size_t argc = 1; + napi_value argv[1] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + + uint32_t pos; + napi_status status = NapiUtils::GetValue(env, argv[0], pos); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read pos param go wrong"); + + EditUnit *editUnit = nullptr; + NAPI_CALL(env, napi_unwrap(env, self, reinterpret_cast(&editUnit))); + ASSERT(editUnit != nullptr, "unwrap self go wrong.", nullptr); + + auto dbStore = editUnit->GetDBStore(); + std::string relPos; + int32_t ret = (*dbStore).GetRelativePos(editUnit->name_.c_str(), "{}", pos, relPos); + ASSERT_THROW(env, ret == OK, Status::INTERNAL_ERROR, "GetRelativePos go wrong"); + + return Parser::GetRelativePosFromJsonStr(env, relPos); +} + +napi_value EditUnit::GetAbsolutePos(napi_env env, napi_callback_info info) +{ + size_t argc = 1; + napi_value argv[1] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + EditUnit *editUnit = nullptr; + NAPI_CALL(env, napi_unwrap(env, self, reinterpret_cast(&editUnit))); + ASSERT(editUnit != nullptr, "unwrap self go wrong.", nullptr); + + json root; + napi_value type; + napi_status status = NapiUtils::GetNamedProperty(env, argv[0], "parentId", type); + json typeJson; + if (status == napi_ok && !NapiUtils::IsNull(env, type)) { + Parser::GetUniqueIdFromNapiValueToJsonStr(env, type, typeJson); + } else { + typeJson = nullptr; + } + root["type"] = typeJson; + + std::string typeName; + NapiUtils::GetNamedProperty(env, argv[0], "parentName", typeName); + if (!typeName.empty()) { + root["tname"] = typeName; + } else { + root["tname"] = nullptr; + } + + napi_value item; + status = NapiUtils::GetNamedProperty(env, argv[0], "id", item); + json itemJson; + if (status == napi_ok && !NapiUtils::IsNull(env, item)) { + Parser::GetUniqueIdFromNapiValueToJsonStr(env, item, itemJson); + } else { + itemJson = nullptr; + } + root["item"] = itemJson; + + int64_t assoc; + status = NapiUtils::GetNamedProperty(env, argv[0], "pos", assoc); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "read pos param go wrong"); + root["assoc"] = assoc; + + std::string relPos_str = root.dump(); + auto dbStore = editUnit->GetDBStore(); + uint32_t pos; + int32_t ret = (*dbStore).GetAbsolutePos(editUnit->name_.c_str(), relPos_str.c_str(), "{}", &pos); + ASSERT_THROW(env, ret == SUCCESS, ret, "GetAbsolutePos go wrong"); + + napi_value output = nullptr; + status = NapiUtils::SetValue(env, static_cast(pos), output); + return output; +} +} // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_node.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_node.cpp index f9f3d339..685339af 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_node.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_node.cpp @@ -50,6 +50,7 @@ napi_value Node::Constructor(napi_env env) DECLARE_NAPI_FUNCTION("setAttributes", SetAttributes), DECLARE_NAPI_FUNCTION("removeAttributes", RemoveAttributes), DECLARE_NAPI_FUNCTION("getAttributes", GetAttributes), + DECLARE_NAPI_FUNCTION("setAsset", SetAsset), }; napi_value cons = nullptr; NAPI_CALL(env, napi_define_class(env, "Node", NAPI_AUTO_LENGTH, New, nullptr, @@ -78,6 +79,7 @@ napi_value Node::New(napi_env env, napi_callback_info info) std::string name = ""; NapiUtils::GetValue(env, argv[0], name); Node *node = new (std::nothrow) Node(name); + ASSERT_THROW_BASE(env, node != nullptr, Status::INTERNAL_ERROR, "new: new node go wrong", self); auto finalize = [](napi_env env, void *data, void *hint) { Node *node = reinterpret_cast(data); delete node; @@ -376,4 +378,31 @@ napi_value Node::GetAttributes(napi_env env, napi_callback_info info) } return output; } + +napi_value Node::SetAsset(napi_env env, napi_callback_info info) +{ + size_t argc = 2; + napi_value argv[2] = {nullptr}; + napi_value self = nullptr; + NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); + Node *thisNode = nullptr; + NAPI_CALL(env, napi_unwrap(env, self, reinterpret_cast(&thisNode))); + ASSERT(thisNode != nullptr, "unwrap self go wrong.", nullptr); + ASSERT_THROW(env, thisNode->GetID().has_value(), Status::UNSUPPORTED_OPERATION, "empty id"); + + // convert input argument + std::string assetKey; + napi_status status = NapiUtils::GetValue(env, argv[0], assetKey); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "Param Error: get asset key go wrong."); + + std::string assetValue; + status = NapiUtils::GetValue(env, argv[1], assetValue); + ASSERT_THROW(env, status == napi_ok, Status::INVALID_ARGUMENT, "Param Error: get asset value go wrong."); + + int32_t retCode = thisNode->GetAdapter()->SetAttribute(assetKey, assetValue, 1); + if (retCode != SUCCESS) { + ThrowNapiError(env, retCode, "Set asset go wrong."); + } + return nullptr; +} } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_parser.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_parser.cpp index fc6b3e2c..7dd63b13 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_parser.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_parser.cpp @@ -28,10 +28,18 @@ #include "napi_utils.h" namespace OHOS::CollaborationEdit { -using json = nlohmann::json; static constexpr const uint8_t NUMBER_OF_FIELDS_IN_ID = 2; +static std::vector g_cloudDbFields = { + "batchInsert", + "query", + "downloadAsset", + "uploadAsset", + "deleteAsset", + "deleteLocalAsset" +}; + void Parser::Stringsplit(std::string str, const char split, std::vector &res) { std::istringstream iss(str); @@ -144,6 +152,53 @@ int Parser::ParseJsonStrToJsChildren( return OK; } +int Parser::ParseJsonStrToJsUpdateNode( + napi_env env, std::string nodeJsonStr, std::shared_ptr dbStore, napi_value &out) +{ + ASSERT(!nodeJsonStr.empty() && json::accept(nodeJsonStr), "invalid json str", ERR); + napi_status status = napi_create_array(env, &out); + ASSERT(status == napi_ok, "create array go wrong!", ERR); + json jsonArray = json::parse(nodeJsonStr); + ASSERT(jsonArray.is_array(), "result is not json array.", ERR); + int i = 0; + for (const auto &jsonObj : jsonArray) { + if (!jsonObj.contains("type") || !jsonObj.contains("name")) { + continue; + } + std::string type = jsonObj["type"]; + std::string tableName = NapiUtils::RemovePrefix(jsonObj["name"], std::to_string(LABEL_FRAGMENT) + "_"); + AbstractType parent; + parent.SetDBStore(dbStore); + parent.SetTableName(tableName); + + napi_value jsNode; + if (type.compare("XML_ELEMENT") == 0) { + int ret = ParseJsonToJsNode(env, jsonObj, &parent, jsNode); + ASSERT(ret == OK, "Parse json to node go wrong.", ERR); + } else if (type.compare("XML_TEXT") == 0) { + int ret = ParseJsonToJsText(env, jsonObj, &parent, jsNode); + ASSERT(ret == OK, "Parse json to text go wrong.", ERR); + } else { + LOG_ERROR("Unsupported type. type = %{public}s", type.c_str()); + continue; + } + + napi_value jsUpdateNode = nullptr; + status = napi_create_object(env, &jsUpdateNode); + ASSERT(status == napi_ok, "create object go wrong!", ERR); + napi_value jstableName; + NapiUtils::SetValue(env, tableName, jstableName); + status = napi_set_named_property(env, jsUpdateNode, "editUnitName", jstableName); + ASSERT(status == napi_ok, "set editUnitName go wrong.", ERR); + status = napi_set_named_property(env, jsUpdateNode, "node", jsNode); + ASSERT(status == napi_ok, "set node go wrong.", ERR); + status = napi_set_element(env, out, i, jsUpdateNode); + ASSERT(status == napi_ok, "set element go wrong.", ERR); + i++; + } + return OK; +} + int Parser::ParseFromAttrsJsonStr(napi_env env, const std::string &jsonStr, napi_value &out) { ASSERT(!jsonStr.empty() && json::accept(jsonStr), "invalid json str", ERR); @@ -237,4 +292,144 @@ int Parser::ParseVariantJsValueToStr(napi_env env, napi_value input, std::string return OK; } +int Parser::CheckValueType(napi_env env, napi_value value) +{ + napi_valuetype valueType; + napi_status status = napi_typeof(env, value, &valueType); + if (status != napi_ok) { + LOG_ERROR("type of args go wrong, status = %{public}d", status); + return ERR; + } + if (valueType != napi_function) { + LOG_ERROR("value type go wrong: %{public}d", valueType); + return ERR; + } + return OK; +} + +int Parser::ParseCloudDbFields(napi_env env, napi_value input, std::vector &cloudDbFuncVector) +{ + for (auto field : g_cloudDbFields) { + napi_value cloudDbFunc = nullptr; + napi_status status = NapiUtils::GetNamedProperty(env, input, field.c_str(), cloudDbFunc); + if (status != napi_ok) { + LOG_ERROR("get func go wrong, status = %{public}d", status); + return ERR; + } + int ret = CheckValueType(env, cloudDbFunc); + if (ret != OK) { + LOG_ERROR("check func type go wrong, status = %{public}d", ret); + return ret; + } + cloudDbFuncVector.push_back(cloudDbFunc); + } + return OK; +} + +napi_value Parser::GetUniqueIdFromJsonStr(napi_env env, json &root) +{ + ASSERT(root.contains("client"), "parse client from json str go wrong", nullptr); + ASSERT(root.contains("clock"), "parse clock from json str go wrong", nullptr); + + napi_value uniqueId = nullptr; + NAPI_CALL(env, napi_create_object(env, &uniqueId)); + napi_value jsDeviceId = nullptr; + std::string client = root["client"]; + NapiUtils::SetValue(env, client, jsDeviceId); + NAPI_CALL(env, napi_set_named_property(env, uniqueId, "id", jsDeviceId)); + napi_value jsClock = nullptr; + int64_t clock = root["clock"]; + NapiUtils::SetValue(env, clock, jsClock); + NAPI_CALL(env, napi_set_named_property(env, uniqueId, "clock", jsClock)); + + return uniqueId; +} + +int Parser::SetRelativePosType(napi_env env, json &root, napi_value &relativePos) +{ + if (!root.contains("type")) { + return OK; + } + napi_value jsType = Parser::GetUniqueIdFromJsonStr(env, root["type"]); + if (jsType == nullptr) { + return ERR; + } + napi_status status = napi_set_named_property(env, relativePos, "parentId", jsType); + return status == napi_ok ? OK : ERR; +} + +int Parser::SetRelativePosItem(napi_env env, json &root, napi_value &relativePos) +{ + if (!root.contains("item")) { + return OK; + } + napi_value jsItem = Parser::GetUniqueIdFromJsonStr(env, root["item"]); + if (jsItem == nullptr) { + return ERR; + } + napi_status status = napi_set_named_property(env, relativePos, "id", jsItem); + return status == napi_ok ? OK : ERR; +} + +int Parser::SetRelativePosTname(napi_env env, json &root, napi_value &relativePos) +{ + if (!root.contains("tname")) { + return OK; + } + std::string tnameStrTmp = root["tname"]; + auto tnameStr = NapiUtils::RemovePrefix(tnameStrTmp, std::to_string(LABEL_FRAGMENT) + "_"); + napi_value tname = nullptr; + NapiUtils::SetValue(env, tnameStr, tname); + napi_status status = napi_set_named_property(env, relativePos, "parentName", tname); + return status == napi_ok ? OK : ERR; +} + +int Parser::SetRelativePosAssoc(napi_env env, json &root, napi_value &relativePos) +{ + if (!root.contains("assoc")) { + return OK; + } + napi_value assoc; + int64_t assoc_num = root["assoc"]; + NapiUtils::SetValue(env, assoc_num, assoc); + napi_status status = napi_set_named_property(env, relativePos, "pos", assoc); + return status == napi_ok ? OK : ERR; +} + +napi_value Parser::GetRelativePosFromJsonStr(napi_env env, std::string &relPos) +{ + ASSERT_THROW(env, json::accept(relPos), Status::INTERNAL_ERROR, "parse relpos str go wrong"); + json root = json::parse(relPos); + + napi_value relativePos; + NAPI_CALL(env, napi_create_object(env, &relativePos)); + Parser::SetRelativePosType(env, root, relativePos); + Parser::SetRelativePosItem(env, root, relativePos); + Parser::SetRelativePosTname(env, root, relativePos); + Parser::SetRelativePosAssoc(env, root, relativePos); + return relativePos; +} + +void Parser::GetUniqueIdFromNapiValueToJsonStr(napi_env env, napi_value type, json &typeJson) +{ + std::string id; + int64_t clock; + napi_status status = NapiUtils::GetNamedProperty(env, type, "id", id); + ASSERT_THROW_VOID(env, status == napi_ok, Status::INVALID_ARGUMENT, "read id param from type go wrong"); + status = NapiUtils::GetNamedProperty(env, type, "clock", clock); + ASSERT_THROW_VOID(env, status == napi_ok, Status::INVALID_ARGUMENT, "read clock param from type go wrong"); + typeJson["client"] = id; + typeJson["clock"] = clock; +} + +napi_value Parser::ParseFromAssetOpConfig(napi_env env, const AssetOpConfig &config) +{ + napi_value jsConfig = nullptr; + NAPI_CALL(env, napi_create_object(env, &jsConfig)); + napi_value jsPath = nullptr; + NapiUtils::SetValue(env, config.inputPath, jsPath); + NAPI_CALL(env, napi_set_named_property(env, jsConfig, "path", jsPath)); + return jsConfig; +} + } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_sync_service.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_sync_service.cpp new file mode 100644 index 00000000..c00c2cd1 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_sync_service.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "napi_sync_service.h" + +namespace OHOS::CollaborationEdit { +std::shared_ptr SyncService::GetInstance() +{ + static std::shared_ptr serviceObj_ = std::make_shared(); + return serviceObj_; +} + +uint64_t SyncService::GetSyncId() +{ + uint64_t value = ++syncId_; + if (value == 0) { + value = ++syncId_; + } + return value; +} + +std::shared_ptr SyncService::GetSyncContext(uint64_t syncId) +{ + std::lock_guard lock(contextMutex_); + auto iter = syncContextMap_.find(syncId); + if (iter == syncContextMap_.end()) { + return nullptr; + } + return iter->second; +} + +void SyncService::AddSyncContext(uint64_t syncId, std::shared_ptr syncContext) +{ + std::lock_guard lock(contextMutex_); + syncContextMap_[syncId] = syncContext; +} + +void SyncService::RemoveSyncContext(uint64_t syncId) +{ + std::lock_guard lock(contextMutex_); + auto iter = syncContextMap_.find(syncId); + if (iter != syncContextMap_.end()) { + syncContextMap_.erase(iter); + } +} +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_text.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_text.cpp index b15640e5..69579ebd 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_text.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_text.cpp @@ -64,6 +64,7 @@ napi_value Text::New(napi_env env, napi_callback_info info) // create instance by 'new Node(name)' if (newTarget != nullptr) { Text *text = new (std::nothrow) Text(); + ASSERT_THROW_BASE(env, text != nullptr, Status::INTERNAL_ERROR, "text new: new text go wrong", self); auto finalize = [](napi_env env, void *data, void *hint) { Text *text = reinterpret_cast(data); delete text; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_undo_manager.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_undo_manager.cpp index 103878eb..f8b4a21d 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_undo_manager.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_undo_manager.cpp @@ -37,10 +37,7 @@ napi_value UndoManager::Initialize(napi_env env, napi_callback_info info) NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, &self, nullptr)); std::string tableName; napi_status status = NapiUtils::GetValue(env, argv[0], tableName); - if (status != napi_ok) { - ThrowNapiError(env, Status::INVALID_ARGUMENT, "read tableName go wrong"); - return self; - } + ASSERT_THROW_BASE(env, status == napi_ok, Status::INVALID_ARGUMENT, "read tableName go wrong", self); int64_t captureTimeout = 0; status = NapiUtils::GetNamedProperty(env, argv[1], "captureTimeout", captureTimeout); if (status != napi_ok) { diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_utils.cpp b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_utils.cpp index c3e7d3f7..1e011468 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_utils.cpp +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/src/napi_utils.cpp @@ -20,7 +20,6 @@ #include "napi_error_utils.h" namespace OHOS::CollaborationEdit { -constexpr int32_t STR_MAX_LENGTH = 4096; constexpr size_t STR_TAIL_LENGTH = 1; // second param is the base64 code of data.collaborationEditObject @@ -118,24 +117,21 @@ napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::string &out napi_status ret = napi_typeof(env, input, &type); ASSERT((ret == napi_ok) && (type == napi_string), "invalid type", napi_invalid_arg); - size_t maxLen = STR_MAX_LENGTH; - ret = napi_get_value_string_utf8(env, input, NULL, 0, &maxLen); - if (maxLen <= 0) { + size_t maxLen = 0; + ret = napi_get_value_string_utf8(env, input, nullptr, 0, &maxLen); + if (ret != napi_ok) { return ret; } LOG_DEBUG("napi_value -> std::string get length %{public}d", (int)maxLen); char *buf = new (std::nothrow) char[maxLen + STR_TAIL_LENGTH]; - if (buf != nullptr) { - size_t len = 0; - ret = napi_get_value_string_utf8(env, input, buf, maxLen + STR_TAIL_LENGTH, &len); - if (ret == napi_ok) { - buf[len] = 0; - out = std::string(buf); - } - delete[] buf; - } else { - ret = napi_generic_failure; - } + ASSERT(buf != nullptr, "buffer is nullptr", napi_generic_failure); + size_t len = 0; + ret = napi_get_value_string_utf8(env, input, buf, maxLen + STR_TAIL_LENGTH, &len); + if (ret == napi_ok) { + buf[len] = 0; + out = std::string(buf); + } + delete[] buf; return ret; } @@ -159,7 +155,7 @@ napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - out.assign((uint8_t *)data, ((uint8_t *)data) + length); + out.assign(static_cast(data), static_cast(data) + length); return statusMsg; } @@ -181,293 +177,6 @@ napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, return statusMsg; } -template -void TypedArray2Vector(uint8_t *dataPtr, size_t length, napi_typedarray_type type, std::vector &out) -{ - auto convert = [&out](auto *dataPtr, size_t elements) { - for (size_t index = 0; index < elements; index++) { - out.push_back(static_cast(dataPtr[index])); - } - }; - - switch (type) { - case napi_int8_array: - convert(reinterpret_cast(dataPtr), length); - break; - case napi_uint8_array: - convert(dataPtr, length); - break; - case napi_uint8_clamped_array: - convert(dataPtr, length); - break; - case napi_int16_array: - convert(reinterpret_cast(dataPtr), length / sizeof(int16_t)); - break; - case napi_uint16_array: - convert(reinterpret_cast(dataPtr), length / sizeof(uint16_t)); - break; - case napi_int32_array: - convert(reinterpret_cast(dataPtr), length / sizeof(int32_t)); - break; - case napi_uint32_array: - convert(reinterpret_cast(dataPtr), length / sizeof(uint32_t)); - break; - case napi_float32_array: - convert(reinterpret_cast(dataPtr), length / sizeof(float)); - break; - case napi_float64_array: - convert(reinterpret_cast(dataPtr), length / sizeof(double)); - break; - case napi_bigint64_array: - convert(reinterpret_cast(dataPtr), length / sizeof(int64_t)); - break; - case napi_biguint64_array: - convert(reinterpret_cast(dataPtr), length / sizeof(uint64_t)); - break; - default: - ASSERT_VOID(false, "[FATAL] invalid napi_typedarray_type!"); - } -} - -/* napi_value <-> std::vector */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector &out) -{ - out.clear(); - LOG_DEBUG("napi_value -> std::vector "); - napi_typedarray_type type = napi_biguint64_array; - napi_value buffer = nullptr; - uint8_t *data = nullptr; - size_t length = 0; - size_t offset = 0; - napi_status status = - napi_get_typedarray_info(env, input, &type, &length, reinterpret_cast(&data), &buffer, &offset); - LOG_DEBUG("array type=%{public}d length=%{public}d offset=%{public}d", (int)type, (int)length, (int)offset); - ASSERT(status == napi_ok, "napi_get_typedarray_info go wrong!", napi_invalid_arg); - ASSERT(type <= napi_int32_array, "is not int32 supported typed array!", napi_invalid_arg); - ASSERT((length > 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - TypedArray2Vector(data, length, type, out); - return status; -} - -napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::vector "); - size_t bytes = input.size() * sizeof(int32_t); - ASSERT(bytes > 0, "invalid std::vector", napi_invalid_arg); - void *data = nullptr; - napi_value buffer = nullptr; - napi_status status = napi_create_arraybuffer(env, bytes, &data, &buffer); - ASSERT((status == napi_ok), "invalid buffer", status); - - if (memcpy_s(data, bytes, input.data(), bytes) != EOK) { - LOG_ERROR("napi_value <- std::vector: memcpy_s go wrong, vector size:%{public}zd", input.size()); - return napi_invalid_arg; - } - status = napi_create_typedarray(env, napi_int32_array, input.size(), buffer, 0, &out); - ASSERT((status == napi_ok), "invalid buffer", status); - return status; -} - -/* napi_value <-> std::vector */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector &out) -{ - out.clear(); - LOG_DEBUG("napi_value -> std::vector "); - napi_typedarray_type type = napi_biguint64_array; - napi_value buffer = nullptr; - uint8_t *data = nullptr; - size_t length = 0; - size_t offset = 0; - napi_status status = - napi_get_typedarray_info(env, input, &type, &length, reinterpret_cast(&data), &buffer, &offset); - LOG_DEBUG("napi_get_typedarray_info type=%{public}d", (int)type); - ASSERT(status == napi_ok, "napi_get_typedarray_info go wrong!", napi_invalid_arg); - ASSERT((type <= napi_uint16_array) || (type == napi_uint32_array), "invalid type!", napi_invalid_arg); - ASSERT((length > 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - TypedArray2Vector(data, length, type, out); - return status; -} - -napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::vector "); - size_t bytes = input.size() * sizeof(uint32_t); - ASSERT(bytes > 0, "invalid std::vector", napi_invalid_arg); - void *data = nullptr; - napi_value buffer = nullptr; - napi_status status = napi_create_arraybuffer(env, bytes, &data, &buffer); - ASSERT((status == napi_ok), "invalid buffer", status); - - if (memcpy_s(data, bytes, input.data(), bytes) != EOK) { - LOG_ERROR("napi_value <- std::vector: memcpy_s go wrong, vector size:%{public}zd", input.size()); - return napi_invalid_arg; - } - status = napi_create_typedarray(env, napi_uint32_array, input.size(), buffer, 0, &out); - ASSERT((status == napi_ok), "invalid buffer", status); - return status; -} - -/* napi_value <-> std::vector */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector &out) -{ - out.clear(); - LOG_DEBUG("napi_value -> std::vector "); - napi_typedarray_type type = napi_biguint64_array; - napi_value buffer = nullptr; - uint8_t *data = nullptr; - size_t length = 0; - size_t offset = 0; - napi_status status = - napi_get_typedarray_info(env, input, &type, &length, reinterpret_cast(&data), &buffer, &offset); - LOG_DEBUG("array type=%{public}d length=%{public}d offset=%{public}d", (int)type, (int)length, (int)offset); - ASSERT(status == napi_ok, "napi_get_typedarray_info go wrong!", napi_invalid_arg); - ASSERT((type <= napi_uint32_array) || (type == napi_bigint64_array), "invalid type!", napi_invalid_arg); - ASSERT((length > 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - TypedArray2Vector(data, length, type, out); - return status; -} - -napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::vector "); - size_t bytes = input.size() * sizeof(int64_t); - ASSERT(bytes > 0, "invalid std::vector", napi_invalid_arg); - void *data = nullptr; - napi_value buffer = nullptr; - napi_status status = napi_create_arraybuffer(env, bytes, &data, &buffer); - ASSERT((status == napi_ok), "invalid buffer", status); - - if (memcpy_s(data, bytes, input.data(), bytes) != EOK) { - LOG_ERROR("napi_value <- std::vector: memcpy_s go wrong, vector size:%{public}zd", input.size()); - return napi_invalid_arg; - } - status = napi_create_typedarray(env, napi_bigint64_array, input.size(), buffer, 0, &out); - ASSERT((status == napi_ok), "invalid buffer", status); - return status; -} - -/* napi_value <-> std::vector */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector &out) -{ - out.clear(); - bool isTypedArray = false; - napi_status status = napi_is_typedarray(env, input, &isTypedArray); - LOG_DEBUG("napi_value -> std::vector input %{public}s a TypedArray", isTypedArray ? "is" : "is not"); - ASSERT((status == napi_ok), "napi_is_typedarray go wrong!", status); - if (isTypedArray) { - LOG_DEBUG("napi_value -> std::vector "); - napi_typedarray_type type = napi_biguint64_array; - napi_value buffer = nullptr; - uint8_t *data = nullptr; - size_t length = 0; - size_t offset = 0; - status = - napi_get_typedarray_info(env, input, &type, &length, reinterpret_cast(&data), &buffer, &offset); - LOG_DEBUG("napi_get_typedarray_info status=%{public}d type=%{public}d", status, (int)type); - ASSERT(status == napi_ok, "napi_get_typedarray_info go wrong!", napi_invalid_arg); - ASSERT((length > 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - TypedArray2Vector(data, length, type, out); - } else { - bool isArray = false; - status = napi_is_array(env, input, &isArray); - LOG_DEBUG("napi_value -> std::vector input %{public}s an Array", isArray ? "is" : "is not"); - ASSERT((status == napi_ok) && isArray, "invalid data!", napi_invalid_arg); - uint32_t length = 0; - status = napi_get_array_length(env, input, &length); - ASSERT((status == napi_ok) && (length > 0), "invalid data!", napi_invalid_arg); - for (uint32_t i = 0; i < length; ++i) { - napi_value item = nullptr; - status = napi_get_element(env, input, i, &item); - ASSERT((item != nullptr) && (status == napi_ok), "no element", napi_invalid_arg); - double vi = 0.0; - status = napi_get_value_double(env, item, &vi); - ASSERT(status == napi_ok, "element not a double", napi_invalid_arg); - out.push_back(vi); - } - } - return status; -} - -napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::vector "); - (void)(env); - (void)(input); - (void)(out); - ASSERT(false, "std::vector to napi_value, unsupported!", napi_invalid_arg); - return napi_invalid_arg; -} - -/* napi_value <-> std::vector */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::vector &out) -{ - LOG_DEBUG("napi_value -> std::vector"); - out.clear(); - bool isArray = false; - napi_is_array(env, input, &isArray); - ASSERT(isArray, "not an array", napi_invalid_arg); - - uint32_t length = 0; - napi_status statusMsg = napi_get_array_length(env, input, &length); - ASSERT((statusMsg == napi_ok) && (length > 0), "get_array go wrong!", napi_invalid_arg); - for (uint32_t i = 0; i < length; ++i) { - napi_value item = nullptr; - statusMsg = napi_get_element(env, input, i, &item); - ASSERT((item != nullptr) && (statusMsg == napi_ok), "no element", napi_invalid_arg); - std::string value; - statusMsg = GetValue(env, item, value); - ASSERT(statusMsg == napi_ok, "not a string", napi_invalid_arg); - out.push_back(value); - } - return statusMsg; -} - -napi_status NapiUtils::SetValue(napi_env env, const std::vector &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::vector"); - napi_status status = napi_create_array_with_length(env, input.size(), &out); - ASSERT(status == napi_ok, "create array go wrong!", status); - int index = 0; - for (auto &item : input) { - napi_value element = nullptr; - SetValue(env, item, element); - status = napi_set_element(env, out, index++, element); - ASSERT((status == napi_ok), "napi_set_element go wrong!", status); - } - return status; -} - -/* napi_value <-> std::map */ -napi_status NapiUtils::GetValue(napi_env env, napi_value input, std::map &out) -{ - LOG_DEBUG("napi_value -> std::map "); - (void)(env); - (void)(input); - (void)(out); - ASSERT(false, "std::map from napi_value, unsupported!", napi_invalid_arg); - return napi_invalid_arg; -} - -napi_status NapiUtils::SetValue(napi_env environment, const std::map &input, napi_value &out) -{ - LOG_DEBUG("napi_value <- std::map "); - napi_status status = napi_create_array_with_length(environment, input.size(), &out); - ASSERT((status == napi_ok), "invalid object", status); - int index = 0; - for (const auto &[key, value] : input) { - napi_value element = nullptr; - napi_create_array_with_length(environment, TUPLE_SIZE, &element); - napi_value jsKey = nullptr; - napi_create_string_utf8(environment, key.c_str(), key.size(), &jsKey); - napi_set_element(environment, element, TUPLE_KEY, jsKey); - napi_value jsValue = nullptr; - napi_create_int32(environment, static_cast(value), &jsValue); - napi_set_element(environment, element, TUPLE_VALUE, jsValue); - napi_set_element(environment, out, index++, element); - } - return status; -} - napi_status NapiUtils::GetCurrentAbilityParam(napi_env env, ContextParam ¶m) { auto ability = AbilityRuntime::GetCurrentAbility(env); @@ -518,7 +227,7 @@ napi_status NapiUtils::GetValue(napi_env env, napi_value input, ContextParam &pa } napi_value appInfo = nullptr; - GetNamedProperty(env, input, "applicationInfo", hapInfo); + GetNamedProperty(env, input, "applicationInfo", appInfo); if (appInfo != nullptr) { status = GetNamedProperty(env, appInfo, "name", param.bundleName); ASSERT(status == napi_ok, "get applicationInfo.name go wrong", napi_invalid_arg); @@ -652,4 +361,14 @@ std::pair NapiUtils::GetInnerValue( return std::make_pair(napi_ok, inner); } +std::string NapiUtils::RemovePrefix(std::string str, std::string prefix) +{ + size_t prefixLength = prefix.length(); + size_t foundPos = str.find_first_of(prefix); + if (foundPos == std::string::npos) { + return str; + } + return str.erase(foundPos, prefixLength); +} + } // namespace OHOS::CollaborationEdit diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/BUILD.gn b/data_object/frameworks/jskitsimpl/collaboration_edit/test/BUILD.gn index 8c6395d9..c6bf0ace 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/BUILD.gn +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/BUILD.gn @@ -17,6 +17,6 @@ import("//build/test.gni") group("unittest") { testonly = true deps = [] - deps += [ "unittest/src:unittest" ] + deps += [ "unittest/collaboration_edit_js_test:CollaborationEditJsTest" ] } ############################################################################### diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/app.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/app.json new file mode 100644 index 00000000..327a4541 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/app.json @@ -0,0 +1,21 @@ +{ + "app": { + "bundleName": "com.collaborationedit.test", + "vendor": "huawei", + "versionCode": 1000000, + "versionName": "1.0.0", + "debug": false, + "icon": "$media:icon", + "label": "$string:app_name", + "description": "$string:description_application", + "distributedNotificationEnabled": true, + "keepAlive": true, + "singleUser": true, + "minAPIVersion": 10, + "targetAPIVersion": 10, + "car": { + "apiCompatibleVersion": 10, + "singleUser": false + } + } +} \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/resources/base/element/string.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/resources/base/element/string.json new file mode 100644 index 00000000..27aa6beb --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/AppScope/resources/base/element/string.json @@ -0,0 +1,8 @@ +{ + "string":[ + { + "name":"app_name", + "value":"ohosProject" + } + ] +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/permission/BUILD.gn b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/BUILD.gn similarity index 35% rename from datamgr_service/services/distributeddataservice/adapter/permission/BUILD.gn rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/BUILD.gn index f4f3a0e1..c7b4be3c 100644 --- a/datamgr_service/services/distributeddataservice/adapter/permission/BUILD.gn +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Huawei Device Co., Ltd. +# Copyright (c) 2025 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,42 +10,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import("//build/ohos.gni") -import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -ohos_static_library("distributeddata_permission_static") { - branch_protector_ret = "pac_ret" - sanitize = { - cfi = true - cfi_cross_dso = true - debug = false - boundary_sanitize = true - ubsan = true - } - sources = [ "src/permission_validator.cpp" ] +import("//build/test.gni") - cflags_cc = [ "-fvisibility=hidden" ] +module_out_path = "data_object/data_object/collaboration_edit" - include_dirs = [ - "../include/permission", - "../include/utils", - "${kv_store_common_path}", - "${kv_store_path}/interfaces/innerkits/distributeddata/include", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/framework/include", +ohos_js_stage_unittest("CollaborationEditJsTest") { + hap_profile = "entry/src/main/module.json" + deps = [ + ":collaborationeditjstest_js_assets", + ":collaborationeditjstest_resources", ] + ets2abc = true + certificate_profile = "signature/openharmony_sx.p7b" + hap_name = "CollaborationEditJsTest" + subsystem_name = "distributeddatamgr" + part_name = "data_object" + module_out_path = module_out_path +} - if (build_public_version) { - cflags_cc += [ "-DCONFIG_PUBLIC_VERSION" ] - } - ldflags = [ "-Wl,--exclude-libs,ALL" ] - deps = [ "../utils:distributeddata_utils_static" ] +ohos_app_scope("collaborationeditjstest_app_profile") { + app_profile = "AppScope/app.json" + sources = [ "AppScope/resources" ] +} - external_deps = [ - "access_token:libaccesstoken_sdk", - "c_utils:utils", - "hilog:libhilog", - ] - subsystem_name = "distributeddatamgr" - part_name = "datamgr_service" - defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +ohos_js_assets("collaborationeditjstest_js_assets") { + source_dir = "entry/src/main/ets" +} + +ohos_resources("collaborationeditjstest_resources") { + sources = [ "entry/src/main/resources" ] + deps = [ ":collaborationeditjstest_app_profile" ] + hap_profile = "entry/src/main/module.json" } diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/Test.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/Test.json new file mode 100644 index 00000000..c467f4f8 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/Test.json @@ -0,0 +1,26 @@ +{ + "description": "Configuration for collaboration edit js tests", + "driver": { + "type": "OHJSUnitTest", + "test-timeout": "180000", + "bundle-name": "com.collaborationedit.test", + "module-name": "testModule", + "shell-timeout": "600000", + "testcase-timeout": 70000 + }, + "kits": [ + { + "test-file-name": [ + "CollaborationEditJsTest.hap" + ], + "type": "AppInstallKit", + "cleanup-apps": true + }, + { + "type": "ShellKit", + "teardown-command": [ + "bm uninstall -n com.collaborationedit.test" + ] + } + ] +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/Application/AbilityStage.ts b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/Application/AbilityStage.ts new file mode 100644 index 00000000..13d01e24 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/Application/AbilityStage.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import AbilityStage from "@ohos.app.ability.AbilityStage" + +export default class MyAbilityStage extends AbilityStage { + onCreate() { + console.log('[Demo] MyAbilityStage onCreate'); + globalThis.stageOnCreateRun = 1; + globalThis.stageContext = this.context; + } +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/MainAbility.ts b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/MainAbility.ts new file mode 100644 index 00000000..b5dc855a --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/MainAbility.ts @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import UIAbilit from '@ohos.app.ability.UIAbility' + +export default class MainAbility extends UIAbilit { + onCreate(want, launchParam){ + // Ability is creating, initialize resources for this ability + console.log('[Demo] MainAbility onCreate'); + globalThis.abilityWant = want; + } + + onDestroy() { + // Ability is destroying, release resources for this ability + console.log('[Demo] MainAbility onDestroy'); + } + + onWindowStageCreate(windowStage) { + // Main window is created, set main page for this ability + console.log('[Demo] MainAbility onWindowStageCreate'); + globalThis.abilityContext = this.context; + windowStage.setUIContent(this.context, "MainAbility/pages/index/index", null); + } + + onWindowStageDestroy() { + //Main window is destroyed, release UI related resources + console.log('[Demo] MainAbility onWindowStageDestroy'); + } + + onForeground() { + // Ability has brought to foreground + console.log('[Demo] MainAbility onForeground'); + } + + onBackground() { + // Ability has back to background + console.log('[Demo] MainAbility onBackground'); + } +}; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/pages/index/index.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/pages/index/index.ets new file mode 100644 index 00000000..b4da38eb --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/MainAbility/pages/index/index.ets @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import router from '@ohos.router'; + +import AbilityDelegatorRegistry from '@ohos.application.abilityDelegatorRegistry' +import { Hypium } from '@ohos/hypium' +import testsuite from '../../../test/List.test' + + +@Entry +@Component +struct Index { + + aboutToAppear(){ + console.info("start run testcase!!!!") + var abilityDelegator: any + abilityDelegator = AbilityDelegatorRegistry.getAbilityDelegator() + var abilityDelegatorArguments: any + abilityDelegatorArguments = AbilityDelegatorRegistry.getArguments() + console.info('start run testcase!!!') + Hypium.hypiumTest(abilityDelegator, abilityDelegatorArguments, testsuite) + } + + build() { + Flex({ direction:FlexDirection.Column, alignItems:ItemAlign.Center, justifyContent: FlexAlign.Center }) { + Text('Hello World') + .fontSize(50) + .fontWeight(FontWeight.Bold) + Button() { + Text('next page') + .fontSize(25) + .fontWeight(FontWeight.Bold) + }.type(ButtonType.Capsule) + .margin({ + top: 20 + }) + .backgroundColor('#0D9FFB') + .onClick(() => {}) + } + .width('100%') + .height('100%') + } +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/TestAbility.ts b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/TestAbility.ts new file mode 100644 index 00000000..97971945 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/TestAbility.ts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//OpenHarmary\vendor\unionman\unionpi_tiger\sample\app\distra-game\entry\src\ohosTest\ets\testability\TestAbility.ets +import UIAbility from '@ohos.app.ability.UIAbility' + +export default class TestAbility extends UIAbility { + onCreate(want, launchParam) { + console.log('TestAbility context' + this.context); + } + + onDestroy() { + console.log('TestAbility onDestroy'); + } + + onWindowStageCreate(windowStage) { + console.log('TestAbility onWindowStageCreate'); + windowStage.setUIContent(this.context, 'TestAbility/pages/index', null); + } + + onWindowStageDestroy() { + console.log('TestAbility onWindowStageDestroy'); + } + + onForeground() { + console.log('TestAbility onForeground'); + } + + onBackground() { + console.log('TestAbility onBackground'); + } +}; \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/pages/index.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/pages/index.ets new file mode 100644 index 00000000..84a0667e --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestAbility/pages/index.ets @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import router from '@ohos.router'; + +@Entry +@Component +struct Index { + aboutToAppear() { + console.info('TestAbility index aboutToAppear') + } + @State message: string = 'Hello World' + build() { + Row() { + Column() { + Text(this.message) + .fontSize(50) + .fontWeight(FontWeight.Bold) + Button() { + Text('next page') + .fontSize(20) + .fontWeight(FontWeight.Bold) + } + .type(ButtonType.Capsule) + .margin({ + top: 20 + }) + .backgroundColor('#0D9FFB') + .width('35%') + .height('5%') + .onClick(()=>{ }) + } + .width('100%') + } + .height('100%') + } +} \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestRunner/OpenHarmonyTestRunner.ts b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestRunner/OpenHarmonyTestRunner.ts new file mode 100644 index 00000000..4c838962 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/TestRunner/OpenHarmonyTestRunner.ts @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import TestRunner from '@ohos.application.testRunner' +import AbilityDelegatorRegistry from '@ohos.application.abilityDelegatorRegistry' + +var abilityDelegator = undefined +var abilityDelegatorArguments = undefined + +function translateParamsToString(parameters) { + const keySet = new Set([ + '-s class', '-s notClass', '-s suite', '-s it', + '-s level', '-s testType', '-s size', '-s timeout' + ]) + let targetParams = ''; + for (const key in parameters) { + if (keySet.has(key)) { + targetParams = `${targetParams} ${key} ${parameters[key]}` + } + } + return targetParams.trim() +} + +async function onAbilityCreateCallback() { + console.log('onAbilityCreateCallback'); +} + +async function addAbilityMonitorCallback(err: any) { + console.info('addAbilityMonitorCallback : ' + JSON.stringify(err)); +} + +export default class OpenHarmonyTestRunner implements TestRunner { + constructor() { + } + + onPrepare() { + console.info("OpenHarmonyTestRunner OnPrepare "); + } + + async onRun() { + console.log('OpenHarmonyTestRunner onRun run') + abilityDelegatorArguments = AbilityDelegatorRegistry.getArguments() + abilityDelegator = AbilityDelegatorRegistry.getAbilityDelegator() + var testAbilityName = abilityDelegatorArguments.bundleName + '.MainAbility' + let lMonitor = { + abilityName: testAbilityName, + onAbilityCreate: onAbilityCreateCallback, + }; + abilityDelegator.addAbilityMonitor(lMonitor, addAbilityMonitorCallback) + var cmd = 'aa start -d 0 -a com.collaborationedit.test.MainAbility' + ' -b ' + abilityDelegatorArguments.bundleName + cmd += ' '+translateParamsToString(abilityDelegatorArguments.parameters) + console.info('cmd : '+ cmd) + abilityDelegator.executeShellCommand(cmd, + (err: any, d: any) => { + console.info('executeShellCommand : err : ' + JSON.stringify(err)); + console.info('executeShellCommand : data : ' + d.stdResult); + console.info('executeShellCommand : data : ' + d.exitCode); + }) + console.info('OpenHarmonyTestRunner onRun end'); + } +}; diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/CloudDbMock.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/CloudDbMock.ets new file mode 100644 index 00000000..07dc09a2 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/CloudDbMock.ets @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file expect in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import collaboration_edit from "@ohos.data.collaborationEditObject" + +const QRY_CURSOR_IDX = 0; +const QRY_EQUIP_ID_IDX = 1; + +export default class CloudDbMock { + static g_batchUploadIndex = 0; + static g_cursor = 0; + static g_records: Array = new Array(); + static g_assets: Array = new Array(); + + static batchInsertTestWrap(records: Array) : number { + for (let record of records) { + record.cursor = ++CloudDbMock.g_cursor; + } + CloudDbMock.g_records = CloudDbMock.g_records.concat(records); + return records.length; + } + + static queryTestWrap(conditions: Array) : Array { + let cursor: number = conditions[QRY_CURSOR_IDX].fieldValue as number; + let equipId = conditions[QRY_EQUIP_ID_IDX].fieldValue; + + const result = CloudDbMock.g_records.filter((item) => item.cursor > cursor).filter((item) => item.id != equipId); + return result; + } + + static uploadAssetTestWrap(path: string) : boolean { + CloudDbMock.g_assets.push(path); + return true; + } + + static deleteAssetTestWrap(path: string) : boolean { + let index = CloudDbMock.g_assets.indexOf(path); + if (index < 0) { + console.log("no matching asset"); + return false; + } + CloudDbMock.g_assets.splice(index, 1); + return true; + } + + static deleteLocalAssetTestWrap() : boolean { + return true; + } + + static getCloudRecords() { + return CloudDbMock.g_records; + } + + static getCloudAssets() { + return CloudDbMock.g_assets; + } + + static resetEnv() : void { + CloudDbMock.g_records = new Array(); + CloudDbMock.g_assets = new Array(); + CloudDbMock.g_cursor = 0; + } +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationEditUnit.test.js b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/EditUnit.ets similarity index 48% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationEditUnit.test.js rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/EditUnit.ets index 3341c988..a3df06da 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationEditUnit.test.js +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/EditUnit.ets @@ -12,60 +12,61 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { describe, beforeEach, afterEach, beforeAll, afterAll, it, expect } from 'deccjsunit/index' + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' import collaboration_edit from "@ohos.data.collaborationEditObject" -import ability_featureAbility from '@ohos.ability.featureAbility' - -const TAG = "[CollaborationEdit_JsTest]" -const DOC_CONFIG = {name: "doc_test"} -const EDIT_UNIT_NAME = "top" - -var context = ability_featureAbility.getContext() -var editObject = undefined; -var editUnit = undefined; - -describe('collaborationEditUnitTest', () => { - beforeAll(async () => { +import common from '@ohos.app.ability.common'; + +const TAG = "[CollaborationEdit_JsTest_EditUnit]" +const DOC_CONFIG: collaboration_edit.CollaborationConfig = {name: "doc_test"} +const EDIT_UNIT_NAME: string = "top" +let context: common.UIAbilityContext = globalThis.abilityContext; +let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; +let editUnit: collaboration_edit.EditUnit | undefined = undefined; + +export default function collaborationEditUnitTest() { + describe('collaborationEditUnitTest', () => { + beforeAll(() => { console.log(TAG + "beforeAll"); }) - - beforeEach(async () => { + + beforeEach(() => { console.log(TAG + "beforeEach"); try { editObject = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG); editUnit = editObject.getEditUnit(EDIT_UNIT_NAME); } catch (err) { - console.log(TAG + "get edit object failed. err: %s", err.message); + console.error(TAG + "get edit object failed."); } }) - - afterEach(async () => { + + afterEach(() => { console.log(TAG + "afterEach"); try { collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG); console.log(TAG + "delete edit object successfully"); } catch (err) { - console.log(TAG + "delete edit object failed. err: %s", err.message); + console.error(TAG + `delete edit object failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } }) - - afterAll(async () => { + + afterAll(() => { console.log(TAG + "afterAll"); }) /** * @tc.number CollaborationEdit_EditUnit_0001 * @tc.name getEditUnit by invalid empty name - * @tc.desc + * @tc.desc * 1. getEditUnit by empty input string * 2. check 401 error code */ - it("CollaborationEdit_EditUnit_0001", 0, async () => { + it("CollaborationEdit_EditUnit_0001", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_EditUnit_0001 Start*****************"); - let editUnit = undefined; - let errCode = ""; + expect(editObject !== undefined).assertTrue(); + let editUnit: collaboration_edit.EditUnit | undefined; + let errCode: string = ""; try { editUnit = editObject?.getEditUnit(""); } catch (err) { @@ -73,42 +74,40 @@ describe('collaborationEditUnitTest', () => { } expect(editUnit).assertUndefined(); expect("401").assertEqual(errCode); - console.log(TAG + "*****************CollaborationEdit_EditUnit_0001 End*****************"); }) - + /** * @tc.number CollaborationEdit_EditUnit_0002 * @tc.name EditUnit.insertNodes by null array - * @tc.desc + * @tc.desc * 1. check EditUnit.getName * 2. insert null node array and check 401 error code */ - it("CollaborationEdit_EditUnit_0002", 0, async () => { + it("CollaborationEdit_EditUnit_0002", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_EditUnit_0002 Start*****************"); expect(editUnit !== undefined).assertTrue(); expect(EDIT_UNIT_NAME).assertEqual(editUnit?.getName()); - let errCode = ""; + let errCode: string = ""; try { editUnit?.insertNodes(0, null); } catch (err) { - console.log(TAG + "insert node failed. err: %s", err); + console.error(TAG + `insert node failed. code:${err.code}, message:${err.message}`); errCode = err.code; } expect("401").assertEqual(errCode); - console.log(TAG + "*****************CollaborationEdit_EditUnit_0002 End*****************"); }) - + /** * @tc.number CollaborationEdit_EditUnit_0003 * @tc.name Normal test case of methods in EditUnit - * @tc.desc + * @tc.desc * 1. construct node list * 2. EditUnit.insertNodes * 3. check the id of inserted nodes * 4. EditUnit.getChildren and check result * 5. EditUnit.getJsonResult and check result */ - it("CollaborationEdit_EditUnit_0003", 0, async () => { + it("CollaborationEdit_EditUnit_0003", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_EditUnit_0003 Start*****************"); expect(editUnit !== undefined).assertTrue(); try { @@ -130,98 +129,9 @@ describe('collaborationEditUnitTest', () => { expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[]}},{\"ele\":{\"name\":\"p2\",\"attr\":{},\"children\":[]}}]}"); } catch (err) { - console.log(TAG + "CollaborationEdit_EditUnit_0003 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_EditUnit_0003 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } - console.log(TAG + "*****************CollaborationEdit_EditUnit_0003 End*****************"); - }) - - /** - * @tc.number CollaborationEdit_EditUnit_0004 - * @tc.name EditUnit.getChildren by invalid input - * @tc.desc - * 1. start is negative - * 2. end is negative - * 3. start is greater than end - */ - it("CollaborationEdit_EditUnit_0004", 0, async () => { - console.log(TAG + "*****************CollaborationEdit_EditUnit_0004 Start*****************"); - expect(editUnit !== undefined).assertTrue(); - - let errCode = ""; - try { - editUnit.getChildren(-1, 2); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - - errCode = ""; - try { - editUnit.getChildren(0, -1); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - - errCode = ""; - try { - editUnit.getChildren(1, 0); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - - console.log(TAG + "*****************CollaborationEdit_EditUnit_0004 End*****************"); - }) - - /** - * @tc.number CollaborationEdit_EditUnit_0005 - * @tc.name EditUnit.delete by invalid input - * @tc.desc - * 1. index is negative - * 2. length is negative or zero - */ - it("CollaborationEdit_EditUnit_0005", 0, async () => { - console.log(TAG + "*****************CollaborationEdit_EditUnit_0005 Start*****************"); - expect(editUnit !== undefined).assertTrue(); - - let errCode = ""; - try { - editUnit.delete(-1, 2); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - - errCode = ""; - try { - editUnit.delete(0, 0); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - - console.log(TAG + "*****************CollaborationEdit_EditUnit_0005 End*****************"); - }) - - /** - * @tc.number CollaborationEdit_EditUnit_0006 - * @tc.name EditUnit.insertNodes by invalid index - * @tc.desc - * 1. index is invalid and check 401 error code - */ - it("CollaborationEdit_EditUnit_0006", 0, async () => { - console.log(TAG + "*****************CollaborationEdit_EditUnit_0006 Start*****************"); - expect(editUnit !== undefined).assertTrue(); - - let errCode = ""; - try { - editUnit.insertNodes(-1, []); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("401"); - console.log(TAG + "*****************CollaborationEdit_EditUnit_0006 End*****************"); }) -}) + }) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationGetEditObject.test.js b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/GetEditObject.ets similarity index 66% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationGetEditObject.test.js rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/GetEditObject.ets index f28418e1..b9c6d6f7 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationGetEditObject.test.js +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/GetEditObject.ets @@ -12,40 +12,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { describe, beforeEach, afterEach, beforeAll, afterAll, it, expect } from 'deccjsunit/index' + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' import collaboration_edit from "@ohos.data.collaborationEditObject" -import ability_featureAbility from '@ohos.ability.featureAbility' - -const TAG = "[CollaborationEdit_JsTest]" -const DOC_CONFIG = {name: "doc_test"} - -var context = ability_featureAbility.getContext() - -describe('collaborationGetEditObjectTest', () => { - beforeAll(async () => { - console.log(TAG + "beforeAll"); +import common from '@ohos.app.ability.common'; + +const TAG = "[CollaborationEdit_JsTest_GetEditObject]" +const DOC_CONFIG: collaboration_edit.CollaborationConfig = {name: "doc_test"} +let context: common.UIAbilityContext = globalThis.abilityContext; + +export default function collaborationGetEditObjectTest() { + describe('collaborationGetEditObjectTest', () => { + beforeAll(() => { + console.log(TAG + "GetEditObject beforeAll"); }) - - beforeEach(async () => { + + beforeEach(() => { console.log(TAG + "beforeEach"); }) - - afterEach(async () => { + + afterEach(() => { console.log(TAG + "afterEach"); try { collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG); console.log(TAG + "delete edit object successfully"); - } catch (err) { - console.log(TAG + "delete edit object failed. err: %s", err.message); + } catch (error) { + console.error(TAG + "delete edit object failed"); expect().assertFail(); } }) - - afterAll(async () => { - console.log(TAG + "afterAll"); + + afterAll(() => { + console.log(TAG + "GetEditObject afterAll"); }) - + /** * @tc.number CollaborationEdit_GetEditObject_0001 * @tc.name Normal case of getCollaborationEditObject @@ -53,18 +53,18 @@ describe('collaborationGetEditObjectTest', () => { * 1. get collaboration edit object correctly * 2. delete edit object by empty name, then check 401 code */ - it("CollaborationEdit_GetEditObject_0001", 0, async () => { + it("CollaborationEdit_GetEditObject_0001", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_GetEditObject_0001 Start*****************"); - let editObject = undefined; + let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; try { editObject = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG); expect(editObject !== undefined).assertTrue(); } catch (err) { - console.log(TAG + "CollaborationEdit_GetEditObject_0001 failed."); + console.log(TAG + `CollaborationEdit_GetEditObject_0001 failed. code:${err.code}`); expect().assertFail(); } - let errCode = ""; + let errCode: string = ""; try { collaboration_edit.deleteCollaborationEditObject(context, {name: ""}); } catch (err) { @@ -73,7 +73,7 @@ describe('collaborationGetEditObjectTest', () => { expect(errCode).assertEqual("401"); console.log(TAG + "*****************CollaborationEdit_GetEditObject_0001 End*****************"); }) - + /** * @tc.number CollaborationEdit_GetEditObject_0002 * @tc.name getCollaborationEditObject by null context @@ -81,20 +81,21 @@ describe('collaborationGetEditObjectTest', () => { * 1. get collaboration edit object by null context * 2. check 401 error code */ - it("CollaborationEdit_GetEditObject_0002", 0, async () => { + it("CollaborationEdit_GetEditObject_0002", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_GetEditObject_0002 Start*****************"); - let editObject = undefined; - let errCode = ""; + let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; + let errCode: string = ""; try { editObject = collaboration_edit.getCollaborationEditObject(null, DOC_CONFIG); } catch (err) { + console.error(TAG + `get edit object failed. code:${err.code}, message:${err.message}`); errCode = err.code; } expect(editObject).assertUndefined(); expect("401").assertEqual(errCode); console.log(TAG + "*****************CollaborationEdit_GetEditObject_0002 End*****************"); }) - + /** * @tc.number CollaborationEdit_GetEditObject_0003 * @tc.name getCollaborationEditObject by empty name @@ -102,17 +103,19 @@ describe('collaborationGetEditObjectTest', () => { * 1. get collaboration edit object by empty name * 2. check 401 error code */ - it("CollaborationEdit_GetEditObject_0003", 0, async () => { + it("CollaborationEdit_GetEditObject_0003", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_GetEditObject_0003 Start*****************"); - let editObject = undefined; - let errCode = ""; + let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; + let errCode: string = ""; try { editObject = collaboration_edit.getCollaborationEditObject(context, {name: ""}); } catch (err) { + console.error(TAG + `get edit object failed. code:${err.code}, message:${err.message}`); errCode = err.code; } expect(editObject).assertUndefined(); expect("401").assertEqual(errCode); console.log(TAG + "*****************CollaborationEdit_GetEditObject_0003 End*****************"); }) -}) + }) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/List.test.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/List.test.ets new file mode 100644 index 00000000..1da6e6c6 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/List.test.ets @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import collaborationGetEditObjectTest from './GetEditObject.ets'; +import collaborationEditUnitTest from './EditUnit.ets'; +import collaborationNodeTest from './Node.ets'; +import collaborationTextTest from './Text.ets'; +import collaborationUndoRedoTest from './UndoRedo.ets'; +import collaborationSetCloudDbTest from './SetCloudDb.ets'; + +export default function testsuite() { + collaborationGetEditObjectTest(); + collaborationEditUnitTest(); + collaborationNodeTest(); + collaborationTextTest(); + collaborationUndoRedoTest(); + collaborationSetCloudDbTest(); +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationNode.test.js b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Node.ets similarity index 52% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationNode.test.js rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Node.ets index d517b88e..6bb407a8 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationNode.test.js +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Node.ets @@ -12,72 +12,72 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { describe, beforeEach, afterEach, beforeAll, afterAll, it, expect } from 'deccjsunit/index' + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' import collaboration_edit from "@ohos.data.collaborationEditObject" -import ability_featureAbility from '@ohos.ability.featureAbility' - -const TAG = "[CollaborationEdit_JsTest]" -const DOC_CONFIG = {name: "doc_test"} -const EDIT_UNIT_NAME = "top" - -var context = ability_featureAbility.getContext() -var editObject = undefined; -var editUnit = undefined; - -describe('collaborationNodeTest', () => { - beforeAll(async () => { - console.log(TAG + "beforeAll"); +import common from '@ohos.app.ability.common'; + +const TAG = "[CollaborationEdit_JsTest_Node]" +const DOC_CONFIG: collaboration_edit.CollaborationConfig = {name: "doc_test"} +const EDIT_UNIT_NAME: string = "top" +let context: common.UIAbilityContext = globalThis.abilityContext; +let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; +let editUnit: collaboration_edit.EditUnit | undefined = undefined; + +export default function collaborationNodeTest() { + describe('collaborationNodeTest', () => { + beforeAll(() => { + console.log(TAG + "beforeAll"); }) - - beforeEach(async () => { + + beforeEach(() => { console.log(TAG + "beforeEach"); try { editObject = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG); editUnit = editObject.getEditUnit(EDIT_UNIT_NAME); } catch (err) { - console.log(TAG + "get edit object failed. err: %s", err.message); + console.error(TAG + "get edit object failed."); } }) - - afterEach(async () => { + + afterEach(() => { console.log(TAG + "afterEach"); try { collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG); - console.log(TAG + "delete edit object failed. err: %s", err.message); + console.log(TAG + "delete edit object successfully"); } catch (err) { - console.log(TAG + "delete edit object failed"); + console.error(TAG + `delete edit object failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } }) - - afterAll(async () => { - console.log(TAG + "afterAll"); + + afterAll(() => { + console.log(TAG + "afterAll"); }) /** * @tc.number CollaborationEdit_Node_0001 * @tc.name Normal case of attribute-related methods - * @tc.desc + * @tc.desc * 1. construct a node list and insert them into edit unit - * 2. set diffrent attributes + * 2. set different attributes * 3. get attributes and check result * 4. get json result and check result * 5. remove some attributes * 6. again check result by getAttributes and getJsonResult */ - it("CollaborationEdit_Node_0001", 0, async () => { + it("CollaborationEdit_Node_0001", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_Node_0001 Start*****************"); expect(editUnit !== undefined).assertTrue(); try { let node1 = new collaboration_edit.Node("p1"); editUnit?.insertNodes(0, [node1]); node1.setAttributes({"align": "left", "width": 36, "italics": true}); - let attrs = node1.getAttributes(); + let attrs: collaboration_edit.AttributesRecord = node1.getAttributes(); expect(attrs["align"]).assertEqual("left"); expect(attrs["width"]).assertEqual("36"); expect(attrs["italics"]).assertEqual("true"); - let jsonStr = node1.getJsonResult(); + let jsonStr: string = node1.getJsonResult(); expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{\"align\":\"left\",\"italics\":\"true\",\"width\":\"36\"},\"children\":[]}}]}"); node1.removeAttributes(["align", "italics"]); @@ -89,7 +89,7 @@ describe('collaborationNodeTest', () => { expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{\"width\":\"36\"},\"children\":[]}}]}"); } catch (err) { - console.log(TAG + "CollaborationEdit_Node_0001 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_Node_0001 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } console.log(TAG + "*****************CollaborationEdit_Node_0001 End*****************"); @@ -98,22 +98,22 @@ describe('collaborationNodeTest', () => { /** * @tc.number CollaborationEdit_Node_0002 * @tc.name Normal case of insertTexts and delete - * @tc.desc + * @tc.desc * 1. construct a node and insert it into edit unit * 2. construct a text list and insert them into the node * 3. check the unique id of the texts - * 4. getChildren and check unique id of the result list + * 4. getChildren and check unique id of the result list * 5. getJsonResult and check the result * 6. delete the first child * 7. again check result by getChildren and getJsonResult */ - it("CollaborationEdit_Node_0002", 0, async () => { + it("CollaborationEdit_Node_0002", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_Node_0002 Start*****************"); expect(editUnit !== undefined).assertTrue(); try { let node1 = new collaboration_edit.Node("p1"); editUnit?.insertNodes(0, [node1]); - + // insert Texts let text1 = new collaboration_edit.Text(); let text2 = new collaboration_edit.Text(); @@ -122,17 +122,17 @@ describe('collaborationNodeTest', () => { expect(text2.getId() !== undefined).assertTrue(); expect(text1.getId().clock).assertEqual(1); expect(text2.getId().clock).assertEqual(2); - let nodeList = node1.getChildren(0, 2); + let nodeList: Array | undefined = node1.getChildren(0, 2); expect(nodeList !== undefined).assertTrue(); expect(2).assertEqual(nodeList.length); if (nodeList !== undefined) { expect(nodeList[0].getId().clock).assertEqual(1); expect(nodeList[1].getId().clock).assertEqual(2); } - let jsonStr = node1.getJsonResult(); + let jsonStr: string | undefined = node1.getJsonResult(); expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"text\":\"[]\"},{\"text\":\"[]\"}]}}]}"); - + // delete first child node1.delete(0, 1); nodeList = node1.getChildren(0, 2); // if end out of length, return all @@ -141,7 +141,7 @@ describe('collaborationNodeTest', () => { jsonStr = node1.getJsonResult(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"text\":\"[]\"}]}}]}"); } catch (err) { - console.log(TAG + "CollaborationEdit_Node_0002 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_Node_0002 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } console.log(TAG + "*****************CollaborationEdit_Node_0002 End*****************"); @@ -150,22 +150,22 @@ describe('collaborationNodeTest', () => { /** * @tc.number CollaborationEdit_Node_0003 * @tc.name Normal case of insertNode and delete - * @tc.desc + * @tc.desc * 1. construct a node p1 and insert it into edit unit * 2. construct a node list and insert them into the node p1 * 3. check the unique id of the node list - * 4. getChildren and check unique id of the result list + * 4. getChildren and check unique id of the result list * 5. getJsonResult and check the result * 6. delete the second child * 7. again check result by getChildren and getJsonResult */ - it("CollaborationEdit_Node_0003", 0, async () => { + it("CollaborationEdit_Node_0003", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_Node_0003 Start*****************"); expect(editUnit !== undefined).assertTrue(); try { let node1 = new collaboration_edit.Node("p1"); editUnit?.insertNodes(0, [node1]); - + // insert Nodes let node2 = new collaboration_edit.Node("p2"); let node3 = new collaboration_edit.Node("p3"); @@ -174,17 +174,17 @@ describe('collaborationNodeTest', () => { expect(node3.getId() != undefined).assertTrue(); expect(node2.getId().clock).assertEqual(1); expect(node3.getId().clock).assertEqual(2); - let nodeList = node1?.getChildren(0, 2); + let nodeList: Array | undefined = node1?.getChildren(0, 2); expect(nodeList !== undefined).assertTrue(); expect(2).assertEqual(nodeList?.length); if (nodeList !== undefined) { expect(nodeList[0].getId().clock).assertEqual(1); expect(nodeList[1].getId().clock).assertEqual(2); } - let jsonStr = node1?.getJsonResult(); + let jsonStr: string | undefined = node1?.getJsonResult(); expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"ele\":{\"name\":\"p2\",\"attr\":{},\"children\":[]}},{\"ele\":{\"name\":\"p3\",\"attr\":{},\"children\":[]}}]}}]}"); - + // delete second child node1.delete(1, 1); nodeList = node1.getChildren(0, 2); // if end out of length, return all @@ -193,85 +193,86 @@ describe('collaborationNodeTest', () => { jsonStr = node1.getJsonResult(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"ele\":{\"name\":\"p2\",\"attr\":{},\"children\":[]}}]}}]}"); } catch (err) { - console.log(TAG + "CollaborationEdit_Node_0003 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_Node_0003 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } + console.log(TAG + "*****************CollaborationEdit_Node_0003 End*****************"); }) /** * @tc.number CollaborationEdit_Node_0004 * @tc.name test getChildren by index out of range - * @tc.desc + * @tc.desc * 1. construct a node list and insert them into edit unit * 2. construct a text list and insert them into the node * 3. check the unique id of the texts * 4. getChildren if start and end are out of range * 5. check error code */ - it("CollaborationEdit_Node_0004", 0, async () => { - console.log(TAG + "*****************CollaborationEdit_Node_0004 Start*****************"); - expect(editUnit !== undefined).assertTrue(); - let errCode = ""; - let nodeList = undefined; - try { - let node1 = new collaboration_edit.Node("p1"); - editUnit?.insertNodes(0, [node1]); - - // insert Texts - let text1 = new collaboration_edit.Text(); - let text2 = new collaboration_edit.Text(); - node1.insertTexts(0, [text1, text2]); - expect(text1.getId() !== undefined).assertTrue(); - expect(text2.getId() !== undefined).assertTrue(); - expect(text1.getId().clock).assertEqual(1); - expect(text2.getId().clock).assertEqual(2); - nodeList = node1.getChildren(2, 4); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("15410002"); - expect(nodeList).assertUndefined(); - console.log(TAG + "*****************CollaborationEdit_Node_0004 End*****************"); + it("CollaborationEdit_Node_0004", Level.LEVEL0, () => { + console.log(TAG + "*****************CollaborationEdit_Node_0004 Start*****************"); + expect(editUnit !== undefined).assertTrue(); + let errCode: string = ""; + let nodeList: Array | undefined; + try { + let node1 = new collaboration_edit.Node("p1"); + editUnit?.insertNodes(0, [node1]); + + // insert Texts + let text1 = new collaboration_edit.Text(); + let text2 = new collaboration_edit.Text(); + node1.insertTexts(0, [text1, text2]); + expect(text1.getId() !== undefined).assertTrue(); + expect(text2.getId() !== undefined).assertTrue(); + expect(text1.getId().clock).assertEqual(1); + expect(text2.getId().clock).assertEqual(2); + nodeList = node1.getChildren(2, 4); + } catch (err) { + errCode = err.code; + } + expect(errCode).assertEqual("15410002"); + expect(nodeList).assertUndefined(); + console.log(TAG + "*****************CollaborationEdit_Node_0004 End*****************"); }) /** * @tc.number CollaborationEdit_Node_0005 * @tc.name Invalid operation if node is not inserted - * @tc.desc + * @tc.desc * 1. construct a node * 2. call getId/setAttributes of the node * 3. check the invalid operation error code */ - it("CollaborationEdit_Node_0005", 0, async () => { - console.log(TAG + "*****************CollaborationEdit_Node_0005 Start*****************"); - let node = new collaboration_edit.Node("p1"); - let errCode = ""; - let id = undefined; - try { - id = node.getId(); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("15410001"); - expect(id).assertUndefined(); - - errCode = ""; - try { - node.setAttributes({ - "align": "left" - }); - } catch (err) { - errCode = err.code; - } - expect(errCode).assertEqual("15410001"); - console.log(TAG + "*****************CollaborationEdit_Node_0005 End*****************"); + it("CollaborationEdit_Node_0005", Level.LEVEL0, () => { + console.log(TAG + "*****************CollaborationEdit_Node_0005 Start*****************"); + let node = new collaboration_edit.Node("p1"); + let errCode: string = ""; + let id: collaboration_edit.UniqueId | undefined; + try { + id = node.getId(); + } catch (err) { + errCode = err.code; + } + expect(errCode).assertEqual("15410001"); + expect(id).assertUndefined(); + + errCode = ""; + try { + node.setAttributes({ + "align": "left" + }); + } catch (err) { + errCode = err.code; + } + expect(errCode).assertEqual("15410001"); + console.log(TAG + "*****************CollaborationEdit_Node_0005 End*****************"); }) /** * @tc.number CollaborationEdit_Node_0006 * @tc.name Invalid index input when call insertXXXs/delete/getChildren - * @tc.desc + * @tc.desc * 1. insertNodes - index is negative * 2. insertTexts - index is negative * 3. delete - index is negative @@ -281,26 +282,26 @@ describe('collaborationNodeTest', () => { */ it("CollaborationEdit_Node_0006", 0, async () => { console.log(TAG + "*****************CollaborationEdit_Node_0006 Start*****************"); - let node = undefined; + let node: collaboration_edit.Node | undefined = undefined; try { node = new collaboration_edit.Node("p1"); - editUnit.insertNodes(0, [node]); + editUnit?.insertNodes(0, [node]); } catch (err) { - console.log(TAG + "CollaborationEdit_Node_0006 insertNodes failed, err: %s", err); + console.error(TAG + `CollaborationEdit_Node_0006 insertNodes failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } expect(node !== undefined).assertTrue(); let errCode = ""; try { - node.insertNodes(-1, []); + node?.insertNodes(-1, []); } catch (err) { errCode = err.code; } expect(errCode).assertEqual("401"); - + errCode = ""; try { - node.insertTexts(-1, []); + node?.insertTexts(-1, []); } catch (err) { errCode = err.code; } @@ -308,7 +309,7 @@ describe('collaborationNodeTest', () => { errCode = ""; try { - node.delete(-1, 1); + node?.delete(-1, 1); } catch (err) { errCode = err.code; } @@ -316,7 +317,7 @@ describe('collaborationNodeTest', () => { errCode = ""; try { - node.delete(0, 0); + node?.delete(0, 0); } catch (err) { errCode = err.code; } @@ -324,7 +325,7 @@ describe('collaborationNodeTest', () => { errCode = ""; try { - node.getChildren(-1, 1); + node?.getChildren(-1, 1); } catch (err) { errCode = err.code; } @@ -332,11 +333,105 @@ describe('collaborationNodeTest', () => { errCode = ""; try { - node.getChildren(1, 0); + node?.getChildren(1, 0); } catch (err) { errCode = err.code; } expect(errCode).assertEqual("401"); console.log(TAG + "*****************CollaborationEdit_Node_0006 End*****************"); + }) + + it("CollaborationEdit_Node_0007", Level.LEVEL0, () => { + console.log(TAG + "*****************CollaborationEdit_Node_0007 Start*****************"); + expect(editUnit !== undefined).assertTrue(); + try { + let node1 = new collaboration_edit.Node("p1"); + editUnit?.insertNodes(0, [node1]); + + // insert Texts + let text1 = new collaboration_edit.Text(); + node1.insertTexts(0, [text1]); + text1.insert(0,'你好'); + expect(text1.getId() !== undefined).assertTrue(); + expect(text1.getId().clock).assertEqual(1); + let relPos = editUnit?.getRelativePos(2); + expect(relPos !== undefined).assertTrue(); + if (relPos !== undefined) { + if (relPos.parentId != null) { + expect(relPos.parentId.clock === 1).assertTrue(); + } + if (relPos.id != null) { + expect(relPos.id.clock === 3).assertTrue(); + } + expect(relPos.pos === 0).assertTrue(); + } + } catch (err) { + console.error(TAG + `CollaborationEdit_Node_0007 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_Node_0007 End*****************"); + }) + + it("CollaborationEdit_Node_0008", Level.LEVEL0, () => { + console.log(TAG + "*****************CollaborationEdit_Node_0008 Start*****************"); + expect(editUnit !== undefined).assertTrue(); + try { + let node1 = new collaboration_edit.Node("p1"); + editUnit?.insertNodes(0, [node1]); + + let relPos = editUnit?.getRelativePos(2); + expect(relPos !== undefined).assertTrue(); + if (relPos !== undefined) { + expect(relPos.parentName != null).assertTrue(); + if (relPos.parentName != null) { + expect(relPos.parentName === "top").assertTrue(); + } + expect(relPos.pos === 0).assertTrue(); + } + } catch (err) { + console.error(TAG + `CollaborationEdit_Node_0008 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_Node_0008 End*****************"); + }) + + it("CollaborationEdit_Node_0009", Level.LEVEL0, () => { + console.log(TAG + "*****************CollaborationEdit_Node_0009 Start*****************"); + expect(editUnit !== undefined).assertTrue(); + try { + let node1 = new collaboration_edit.Node("p1"); + editUnit?.insertNodes(0, [node1]); + + // insert Texts + let text1 = new collaboration_edit.Text(); + node1.insertTexts(0, [text1]); + text1.insert(0,'followRedone'); + expect(text1.getId() !== undefined).assertTrue(); + expect(text1.getId().clock).assertEqual(1); + + let node2 = new collaboration_edit.Node("p2"); + editUnit?.insertNodes(1, [node2]); + + let pos11 = editUnit?.getRelativePos(2); + console.log("pos11 :" + JSON.stringify(pos11)); + let pos = editUnit?.getAbsolutePos({parentId: null, parentName:"top", id:null, pos:0}); + expect(pos === 16).assertTrue(); + + let localId = editObject?.getLocalId(); + expect(localId !== undefined).assertTrue(); + console.log("localId :" + localId); + if (localId !== undefined) { + pos = editUnit?.getAbsolutePos({parentId: {id:localId, clock:1}, parentName:null, id:{id:localId, clock:4}, pos:0}); + expect(pos === 3).assertTrue(); + + pos = editUnit?.getAbsolutePos({parentId: {id:localId, clock:14}, parentName:null, id:null, pos:0}); + expect(pos === 15).assertTrue(); + } + } catch (err) { + console.error(TAG + `CollaborationEdit_Node_0009 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_Node_0009 End*****************"); + }) }) -}) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/SetCloudDb.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/SetCloudDb.ets new file mode 100644 index 00000000..3fa52a73 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/SetCloudDb.ets @@ -0,0 +1,441 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file expect in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' +import collaboration_edit from "@ohos.data.collaborationEditObject" +import common from '@ohos.app.ability.common'; + +import CloudDbMock from "./CloudDbMock"; +import TestUtils from './TestUtils'; + +const TAG = "[CollaborationEdit_JsTest_UndoRedo]" +const DOC_CONFIG_A: collaboration_edit.CollaborationConfig = {name: "doc_test_A"} +const DOC_CONFIG_B: collaboration_edit.CollaborationConfig = {name: "doc_test_B"} +let context: common.UIAbilityContext = globalThis.abilityContext; + +const EDIT_UNIT_NAME = "top"; +let editObject_A: collaboration_edit.CollaborationEditObject | undefined = undefined; +let editObject_B: collaboration_edit.CollaborationEditObject | undefined = undefined; + +function uploadAssetHandler(config : collaboration_edit.AssetOperationConfig) : Promise { + return new Promise((resolve, reject) => { + console.log(TAG + "upload asset handler start, path=" + config.path); + let ret = CloudDbMock.uploadAssetTestWrap(config.path); + console.log("uploadAsset result: " + ret); + ret ? resolve() : reject(); + console.log(TAG + "upload asset handler end"); + }); +} + +function downloadAssetHandler(config : collaboration_edit.AssetOperationConfig) : Promise { + return new Promise((resolve, reject) => { + console.log(TAG + "downloadAssetHandler start, path: " + config.path); + resolve(); + }); +} + +function deleteAssetHandler(config : collaboration_edit.AssetOperationConfig) : Promise { + return new Promise((resolve, reject) => { + console.log(TAG + "deleteAssetHandler start, path: " + config.path); + let ret = CloudDbMock.deleteAssetTestWrap(config.path); + ret ? resolve() : reject(); + console.log(TAG + "deleteAssetHandler end"); + }); +} + +function deleteLocalAssetHandler(config : collaboration_edit.AssetOperationConfig) : Promise { + return new Promise((resolve, reject) => { + console.log(TAG + "deleteLocalAssetHandler start, path: " + config.path); + resolve(); + }); +} + +function batchInsertHandler(updates: Array) : Promise { + return new Promise((resolve, reject) => { + console.log(TAG + "batchInsertHandler start"); + if (updates.length == 0) { + resolve(0); + } + let success_num = CloudDbMock.batchInsertTestWrap(updates); + console.log("success num: " + success_num); + resolve(success_num); + console.log(TAG + "batchInsertHandler end"); + }); +} + +function queryHandler(queryCondition: Array) : Promise> { + return new Promise((resolve, reject) => { + console.log(TAG + "queryHandler promise start"); + try { + let res = CloudDbMock.queryTestWrap(queryCondition); + resolve(res); + } catch (err) { + console.error("query err: " + err); + reject(); + } + + console.log(TAG + "queryHandler promise end"); + }); +} + +function batchInsertSyncCallback(progress: collaboration_edit.ProgressDetail) : void { + console.log("batchInsert sync callback, progress code: " + progress.code.toString()); +} + +function querySyncCallback_B(progress: collaboration_edit.ProgressDetail) : void { + console.log("called query sync callback, progress code: " + progress.code.toString()); + editObject_B?.applyUpdate(); +} + +function querySyncCallback_A(progress: collaboration_edit.ProgressDetail) : void { + console.log("called query sync callback, progress code: " + progress.code.toString()); + editObject_A?.applyUpdate(); +} + +function sync(editObject: collaboration_edit.CollaborationEditObject, syncMode: collaboration_edit.SyncMode, pCode: collaboration_edit.ProgressCode): Promise { + return new Promise((resolve, reject) => { + const callback = (progress: collaboration_edit.ProgressDetail) => { + console.log(TAG + "batchInsert sync callback, progress code: " + progress.code.toString()); + expect(pCode).assertEqual(progress.code); + resolve(); + } + editObject?.cloudSync(syncMode, callback); + }); +} + +const CLOUD_DB_FUNC: collaboration_edit.CloudDB = { + batchInsert: batchInsertHandler, + query: queryHandler, + downloadAsset: downloadAssetHandler, + uploadAsset: uploadAssetHandler, + deleteAsset: deleteAssetHandler, + deleteLocalAsset: deleteLocalAssetHandler, +}; + +export default function collaborationSetCloudDbTest() { + describe('collaborationSetCloudDbTest', () => { + beforeAll(() => { + console.log(TAG + "beforeAll"); + }) + + beforeEach(() => { + console.log(TAG + "beforeEach"); + try { + editObject_A = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG_A); + editObject_B = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG_B); + } catch (err) { + console.error(TAG + `get edit object failed. code:${err.code}, message:${err.message}`); + } + }) + + afterEach(() => { + console.log(TAG + "afterEach"); + try { + CloudDbMock.resetEnv(); + collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG_A); + collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG_B); + console.log(TAG + "delete edit object successfully"); + } catch (err) { + expect().assertFail(); + console.error(TAG + `delete edit object failed. code:${err.code}, message:${err.message}`); + } + }) + + afterAll(() => { + console.log(TAG + "afterAll"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0001 + * @tc.name test cloud sync in pull mode + * @tc.desc + * 1. equip A insert 1 node + * 2. equip A push to cloud + * 3. equip B pull from cloud + * 4. equip B reads one record + */ + it("CollaborationEdit_SetCloudDb_001", Level.LEVEL0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_001 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + expect(editObject_B != undefined).assertTrue(); + editObject_A?.setCloudDB(CLOUD_DB_FUNC); + editObject_B?.setCloudDB(CLOUD_DB_FUNC); + + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + let editUnit_B = editObject_B?.getEditUnit(EDIT_UNIT_NAME); + let node1 = new collaboration_edit.Node("p1"); // insert xmlText + editUnit_A?.insertNodes(0, [node1]); + let nodeList = editUnit_A?.getChildren(0, 1); + expect(nodeList !== undefined).assertTrue(); + expect(1).assertEqual(nodeList?.length); + + // equip A and equip B sync node + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + editObject_B?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PULL, querySyncCallback_B); + await TestUtils.sleep(500); + + let nodes = editUnit_B?.getChildren(0, 1); + expect(nodes !== undefined).assertTrue(); + expect(1).assertEqual(nodes?.length); + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_001 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_001 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0002 + * @tc.name test cloud sync in push mode + * @tc.desc + * 1. equip A insert 1 node + * 2. equip A push to cloud + * 3. verify cloud record + */ + it("CollaborationEdit_SetCloudDb_0002", 0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_002 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + expect(editObject_B != undefined).assertTrue(); + editObject_A?.setCloudDB(CLOUD_DB_FUNC); + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + + // equip A edit + let node1 = new collaboration_edit.Node("p1"); + editUnit_A?.insertNodes(0, [node1]); + let nodeList = editUnit_A?.getChildren(0, 1); + expect(nodeList !== undefined).assertTrue(); + expect(1).assertEqual(nodeList?.length); + + // equip A push to cloud + console.log("equip A start cloudSync"); + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + + let records = CloudDbMock.getCloudRecords(); + expect(1).assertEqual(records.length); + expect(1).assertEqual(records[0].cursor); + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_0002 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_002 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0003 + * @tc.name test cloud sync with asset + * @tc.desc + * 1. equip A insert 1 node then set as asset + * 2. equip A push to cloud + * 3. equip B pull from cloud + * 4. equip B read asset node normally + * 5. equip B delete asset node, then sync + * 6. equip A reads updated data + */ + it("CollaborationEdit_SetCloudDb_0003", 0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_003 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + expect(editObject_B != undefined).assertTrue(); + editObject_A?.setCloudDB(CLOUD_DB_FUNC); + editObject_B?.setCloudDB(CLOUD_DB_FUNC); + + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + let editUnit_B = editObject_B?.getEditUnit(EDIT_UNIT_NAME); + + let node1 = new collaboration_edit.Node("p1"); + let node2 = new collaboration_edit.Node("p2"); + editUnit_A?.insertNodes(0, [node1, node2]); + let nodeList = editUnit_A?.getChildren(0, 2); + expect(nodeList !== undefined).assertTrue(); + expect(2).assertEqual(nodeList?.length); + + node1.setAsset("src", "app/equip_A/1.jpeg"); + let node_A = node1.getJsonResult(); + + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + editObject_B?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PULL, querySyncCallback_B); + await TestUtils.sleep(500); + + let nodes = editUnit_B?.getChildren(0, 2); + expect(nodes !== undefined).assertTrue(); + expect(2).assertEqual(nodes?.length); + + let node_B: string = ""; + if (nodes !== undefined) { + node_B = nodes[0].getJsonResult() + } + expect(node_B).assertEqual(node_A); + + editUnit_B?.delete(0, 1); + editObject_B?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PULL, querySyncCallback_A); + await TestUtils.sleep(500); + + let nodes_A = editUnit_A?.getChildren(0, 1); + expect(1).assertEqual(nodes_A?.length); + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_0003 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_003 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0004 + * @tc.name test cloud sync with SYNC_MODE_PULL_PUSH mode + * @tc.desc + * 1. equip A insert 1 node locally + * 2. equip A push to cloud + * 3. equip B insert 1 node locally + * 4. equip B sync with SYNC_MODE_PULL_PUSH mode + * 5. cloud contains record from both devices, and equip B has equip A's data + */ + it("CollaborationEdit_SetCloudDb_0004", Level.LEVEL0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_004 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + expect(editObject_B != undefined).assertTrue(); + editObject_A?.setCloudDB(CLOUD_DB_FUNC); + editObject_B?.setCloudDB(CLOUD_DB_FUNC); + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + let editUnit_B = editObject_B?.getEditUnit(EDIT_UNIT_NAME); + + // equip A edit + let node_A = new collaboration_edit.Node("p1"); + editUnit_A?.insertNodes(0, [node_A]); + let nodeList_A = editUnit_A?.getChildren(0, 1); + expect(nodeList_A !== undefined).assertTrue(); + expect(1).assertEqual(nodeList_A?.length); + + // equip A push to cloud + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + + // equip B edit + let node_B = new collaboration_edit.Node("p2"); + editUnit_B?.insertNodes(0, [node_B]); + let nodeList_B = editUnit_B?.getChildren(0, 1); + expect(nodeList_B !== undefined).assertTrue(); + expect(1).assertEqual(nodeList_B?.length); + + // equip B sync with SYNC_MODE_PULL_PUSH mode + editObject_B?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PULL_PUSH, querySyncCallback_B); + await TestUtils.sleep(500); + + // cloud has 2 records + let records = CloudDbMock.getCloudRecords(); + expect(2).assertEqual(records.length); + expect(2).assertEqual(records[1].cursor); + + // equip B has 2 nodes + nodeList_B = editUnit_B?.getChildren(0, 2); + expect(nodeList_B !== undefined).assertTrue(); + expect(2).assertEqual(nodeList_B?.length); + + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_0004 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_004 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0005 + * @tc.name test write update from cloud to local device + * @tc.desc + * 1. equip A insert 1 node locally + * 2. equip A push to cloud + * 3. write cloud records to equip B + * 4. equip B contains equip A's data + */ + it("CollaborationEdit_SetCloudDb_0005", 0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_005 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + expect(editObject_B != undefined).assertTrue(); + editObject_A?.setCloudDB(CLOUD_DB_FUNC); + editObject_B?.setCloudDB(CLOUD_DB_FUNC); + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + let editUnit_B = editObject_B?.getEditUnit(EDIT_UNIT_NAME); + + // equip A insert 1 node + let node_A = new collaboration_edit.Node("p1"); + editUnit_A?.insertNodes(0, [node_A]); + let nodeList_A = editUnit_A?.getChildren(0, 1); + expect(nodeList_A !== undefined).assertTrue(); + expect(1).assertEqual(nodeList_A?.length); + + // equip A push to cloud + editObject_A?.cloudSync(collaboration_edit.SyncMode.SYNC_MODE_PUSH, batchInsertSyncCallback); + await TestUtils.sleep(500); + + // write update to equip B + let records = CloudDbMock.getCloudRecords(); + for (let record of records) { + editObject_B?.writeUpdate(record); + } + editObject_B?.applyUpdate(); + + // equip B has data from device A + let nodeList_B = editUnit_B?.getChildren(0, 1); + expect(nodeList_B !== undefined).assertTrue(); + expect(1).assertEqual(nodeList_B?.length); + + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_0005 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_005 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_SetCloudDb_0006 + * @tc.name test cloud sync when cloud db not set + * @tc.desc + * 1. equip A insert 1 node and push to cloud + * 2. get CLOUD_NOT_SET error code in callback + */ + it("CollaborationEdit_SetCloudDb_0006", 0, async () => { + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_006 Start*****************"); + try { + expect(editObject_A !== undefined).assertTrue(); + let editUnit_A = editObject_A?.getEditUnit(EDIT_UNIT_NAME); + + // equip A insert 1 node + let node_A = new collaboration_edit.Node("p1"); + editUnit_A?.insertNodes(0, [node_A]); + let nodeList_A = editUnit_A?.getChildren(0, 1); + expect(nodeList_A !== undefined).assertTrue(); + expect(1).assertEqual(nodeList_A?.length); + + // equip A push to cloud + if (editObject_A !== undefined) { + await sync(editObject_A, collaboration_edit.SyncMode.SYNC_MODE_PUSH, + collaboration_edit.ProgressCode.CLOUD_NOT_SET); + } + } catch (err) { + console.error(TAG + `CollaborationEdit_SetCloudDb_0006 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_SetCloudDb_006 End*****************"); + }) + }) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/TestUtils.ets b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/TestUtils.ets new file mode 100644 index 00000000..1a40e56a --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/TestUtils.ets @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file expect in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default class TestUtils { + static sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationText.test.js b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Text.ets similarity index 75% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationText.test.js rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Text.ets index 69ee3724..75f1d31b 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationText.test.js +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/Text.ets @@ -12,53 +12,53 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { describe, beforeEach, afterEach, beforeAll, afterAll, it, expect } from 'deccjsunit/index' + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' import collaboration_edit from "@ohos.data.collaborationEditObject" -import ability_featureAbility from '@ohos.ability.featureAbility' - -const TAG = "[CollaborationEdit_JsTest]" -const DOC_CONFIG = {name: "doc_test"} -const EDIT_UNIT_NAME = "top" - -var context = ability_featureAbility.getContext() -var editObject = undefined; -var editUnit = undefined; - -describe('collaborationTextTest', () => { - beforeAll(async () => { +import common from '@ohos.app.ability.common'; + +const TAG = "[CollaborationEdit_JsTest_Text]" +const DOC_CONFIG: collaboration_edit.CollaborationConfig = {name: "doc_test"} +const EDIT_UNIT_NAME: string = "top" +let context: common.UIAbilityContext = globalThis.abilityContext; +let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; +let editUnit: collaboration_edit.EditUnit | undefined = undefined; + +export default function collaborationTextTest() { + describe('collaborationTextTest', () => { + beforeAll(() => { console.log(TAG + "beforeAll"); }) - - beforeEach(async () => { + + beforeEach(() => { console.log(TAG + "beforeEach"); try { editObject = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG); editUnit = editObject.getEditUnit(EDIT_UNIT_NAME); } catch (err) { - console.log(TAG + "get edit object failed. err: %s", err.message); + console.error(TAG + "get edit object failed."); } }) - - afterEach(async () => { + + afterEach(() => { console.log(TAG + "afterEach"); try { collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG); console.log(TAG + "delete edit object successfully"); } catch (err) { - console.log(TAG + "delete edit object failed. err: %s", err.message); + console.error(TAG + `delete edit object failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } }) - - afterAll(async () => { + + afterAll(() => { console.log(TAG + "afterAll"); }) /** * @tc.number CollaborationEdit_Text_0001 * @tc.name Normal test case of methods in Text - * @tc.desc + * @tc.desc * 1. construct a node and insert it into edit unit * 2. construct a text and insert it into the node * 3. insert strings into text @@ -66,33 +66,33 @@ describe('collaborationTextTest', () => { * 5. format some characters and check result by getJsonResult * 6. delete some characters and check result by getPlainText and getJsonResult */ - it("CollaborationEdit_Text_0001", 0, async () => { + it("CollaborationEdit_Text_0001", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_Text_0001 Start*****************"); expect(editUnit !== undefined).assertTrue(); try { - let node = new collaboration_edit.Node("p1"); + let node: collaboration_edit.Node = new collaboration_edit.Node("p1"); editUnit?.insertNodes(0, [node]); - + // insert Text - let text = new collaboration_edit.Text(); + let text: collaboration_edit.Text = new collaboration_edit.Text(); node.insertTexts(0, [text]); expect(text.getId() !== undefined).assertTrue(); expect(text.getId().clock).assertEqual(1); - + // insert string into text text.insert(0, "abc"); text.insert(3, "def", {"color":"red", "isBold":true}); - let plainText = text.getPlainText(); + let plainText: string = text.getPlainText(); expect(plainText).assertEqual("abcdef"); - let jsonStr = text.getJsonResult(); + let jsonStr: string = text.getJsonResult(); expect(jsonStr).assertEqual("[{\"insert\":\"abc\"},{\"insert\":\"def\",\"attributes\":{\"color\":\"red\",\"isBold\":\"true\"}}]"); - + // format text text.format(1, 2, {"font-size": 12}); jsonStr = text.getJsonResult(); console.log(TAG + "json str = %s", jsonStr); expect(jsonStr).assertEqual("[{\"insert\":\"a\"},{\"insert\":\"bc\",\"attributes\":{\"font-size\":\"12\"}},{\"insert\":\"def\",\"attributes\":{\"color\":\"red\",\"isBold\":\"true\"}}]"); - + // delete text.delete(2, 3); plainText = text.getPlainText(); @@ -100,7 +100,7 @@ describe('collaborationTextTest', () => { jsonStr = text.getJsonResult(); expect(jsonStr).assertEqual("[{\"insert\":\"a\"},{\"insert\":\"b\",\"attributes\":{\"font-size\":\"12\"}},{\"insert\":\"f\",\"attributes\":{\"color\":\"red\",\"isBold\":\"true\"}}]"); } catch (err) { - console.log(TAG + "CollaborationEdit_Text_0001 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_Text_0001 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } console.log(TAG + "*****************CollaborationEdit_Text_0001 End*****************"); @@ -114,11 +114,11 @@ describe('collaborationTextTest', () => { * 2. call getId/insert of the text * 3. check the invalid operation error code */ - it("CollaborationEdit_Text_0002", 0, async () => { + it("CollaborationEdit_Text_0002", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_Text_0002 Start*****************"); let text = new collaboration_edit.Text(); let errCode = ""; - let id = undefined; + let id: collaboration_edit.UniqueId | undefined; try { id = text.getId(); } catch (err) { @@ -126,7 +126,7 @@ describe('collaborationTextTest', () => { } expect(errCode).assertEqual("15410001"); expect(id).assertUndefined(); - + errCode = ""; try { text.insert(0, "abc"); @@ -140,7 +140,7 @@ describe('collaborationTextTest', () => { /** * @tc.number CollaborationEdit_Text_0003 * @tc.name Invalid index input when call insert/delete/format - * @tc.desc + * @tc.desc * 1. construct a node and insert it into edit unit * 2. construct a text and insert it into the node * 3. insert string if the index is negative @@ -149,29 +149,29 @@ describe('collaborationTextTest', () => { * 6. format if the index is negative * 7. format if the length is negative or zero */ - it("CollaborationEdit_Text_0003", 0, async () => { + it("CollaborationEdit_Text_0003", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_Text_0003 Start*****************"); - let text = undefined; + let text: collaboration_edit.Text | undefined; try { let node = new collaboration_edit.Node("p1"); editUnit?.insertNodes(0, [node]); text = new collaboration_edit.Text(); node.insertTexts(0, [text]); } catch (err) { - console.log(TAG + "CollaborationEdit_Text_0003 failed, err: %s", err); + console.error(TAG + `CollaborationEdit_Text_0003 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } let errCode = ""; try { - text.insert(-1, "abc"); + text?.insert(-1, "abc"); } catch (err) { errCode = err.code; } expect(errCode).assertEqual("401"); - + errCode = ""; try { - text.delete(-1, 1); + text?.delete(-1, 1); } catch (err) { errCode = err.code; } @@ -179,7 +179,7 @@ describe('collaborationTextTest', () => { errCode = ""; try { - text.delete(0, 0); + text?.delete(0, 0); } catch (err) { errCode = err.code; } @@ -187,7 +187,7 @@ describe('collaborationTextTest', () => { errCode = ""; try { - text.format(-1, 1, {"color":"red"}); + text?.format(-1, 1, {"color":"red"}); } catch (err) { errCode = err.code; } @@ -195,11 +195,12 @@ describe('collaborationTextTest', () => { errCode = ""; try { - text.format(0, 0, {"color":"red"}); + text?.format(0, 0, {"color":"red"}); } catch (err) { errCode = err.code; } expect(errCode).assertEqual("401"); console.log(TAG + "*****************CollaborationEdit_Text_0003 End*****************"); }) -}) + }) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationUndoRedo.test.js b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/UndoRedo.ets similarity index 64% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationUndoRedo.test.js rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/UndoRedo.ets index 4748c516..aae7d63e 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/CollaborationUndoRedo.test.js +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/ets/test/UndoRedo.ets @@ -12,98 +12,97 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { describe, beforeEach, afterEach, beforeAll, afterAll, it, expect } from 'deccjsunit/index' + +import { describe, beforeEach, afterEach, beforeAll, it, afterAll, expect, Level } from '@ohos/hypium' import collaboration_edit from "@ohos.data.collaborationEditObject" -import ability_featureAbility from '@ohos.ability.featureAbility' - -const TAG = "[CollaborationEdit_JsTest]" -const DOC_CONFIG = {name: "doc_test"} -const EDIT_UNIT_NAME = "top" - -var context = ability_featureAbility.getContext() -var editObject = undefined; -var editUnit = undefined; - -function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -describe('collaborationUndoRedoTest', () => { - beforeAll(async () => { +import common from '@ohos.app.ability.common'; + +import TestUtils from './TestUtils' + +const TAG = "[CollaborationEdit_JsTest_UndoRedo]" +const DOC_CONFIG: collaboration_edit.CollaborationConfig = {name: "doc_test"} +const EDIT_UNIT_NAME: string = "top" +let context: common.UIAbilityContext = globalThis.abilityContext; +let editObject: collaboration_edit.CollaborationEditObject | undefined = undefined; +let editUnit: collaboration_edit.EditUnit | undefined = undefined; + +export default function collaborationUndoRedoTest() { + describe('collaborationUndoRedoTest', () => { + beforeAll(() => { console.log(TAG + "beforeAll"); }) - - beforeEach(async () => { + + beforeEach(() => { console.log(TAG + "beforeEach"); try { editObject = collaboration_edit.getCollaborationEditObject(context, DOC_CONFIG); editUnit = editObject.getEditUnit(EDIT_UNIT_NAME); } catch (err) { - console.log(TAG + "get edit object failed. err: %s", err.message); + console.error(TAG + "get edit object failed."); } }) - - afterEach(async () => { + + afterEach(() => { console.log(TAG + "afterEach"); try { collaboration_edit.deleteCollaborationEditObject(context, DOC_CONFIG); console.log(TAG + "delete edit object successfully"); } catch (err) { - console.log(TAG + "delete edit object failed. err: %s", err.message); + console.error(TAG + `delete edit object failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } }) - - afterAll(async () => { + + afterAll(() => { console.log(TAG + "afterAll"); }) /** * @tc.number CollaborationEdit_UndoRedo_0001 * @tc.name getUndoRedoManager when editUnitName is non-empty, captureTimeout is float - * @tc.desc + * @tc.desc * 1. editUnitName is any non-empty string * 2. captureTimeout is float */ - it("CollaborationEdit_UndoRedo_0001", 0, async () => { + it("CollaborationEdit_UndoRedo_0001", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_UndoRedo_0001 Start*****************"); expect(editUnit !== undefined).assertTrue(); - let undoManager = undefined; + let undoManager: collaboration_edit.UndoRedoManager | undefined; try { - // captureTimeout非整数，向下取整 undoManager = editObject?.getUndoRedoManager("notFound", {captureTimeout: 500.95}); } catch (err) { - console.log(TAG + "getUndoRedoManager failed. err: %s", err); + console.error(TAG + `getUndoRedoManager failed. code:${err.code}, message:${err.message}`); } expect(undoManager !== undefined).assertTrue(); + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0001 End*****************"); }) /** * @tc.number CollaborationEdit_UndoRedo_0002 * @tc.name getUndoRedoManager when captureTimeout is negative - * @tc.desc + * @tc.desc * 1. captureTimeout is negative, then check 401 error code */ - it("CollaborationEdit_UndoRedo_0002", 0, async () => { + it("CollaborationEdit_UndoRedo_0002", Level.LEVEL0, () => { console.log(TAG + "*****************CollaborationEdit_UndoRedo_0002 Start*****************"); expect(editUnit !== undefined).assertTrue(); - let undoManager = undefined; - let errCode = ""; + let undoManager: collaboration_edit.UndoRedoManager | undefined; + let errCode: string = ""; try { undoManager = editObject?.getUndoRedoManager("top", {captureTimeout: -1}); } catch (err) { - console.log(TAG + "getUndoRedoManager failed. err: %s", err); + console.error(TAG + `getUndoRedoManager failed. code:${err.code}, message:${err.message}`); errCode = err.code; } expect(undoManager).assertUndefined(); expect(errCode).assertEqual("401"); + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0002 End*****************"); }) /** * @tc.number CollaborationEdit_UndoRedo_0003 * @tc.name Noraml case of undo/redo after setAttributes - * @tc.desc + * @tc.desc * 1. get undoRedoManager * 2. construct a node and insert it into edit unit * 3. wait for 500ms @@ -111,10 +110,10 @@ describe('collaborationUndoRedoTest', () => { * 5. call undo, then check attributes undefined * 6. call redo, then check attributes */ - it("CollaborationEdit_UndoRedo_0003", 0, async () => { + it("CollaborationEdit_UndoRedo_0003", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_UndoRedo_0003 Start*****************"); expect(editUnit !== undefined).assertTrue(); - let undoManager = undefined; + let undoManager: collaboration_edit.UndoRedoManager | undefined; try { undoManager = editObject?.getUndoRedoManager("top", {captureTimeout: 500}); expect(undoManager !== undefined).assertTrue(); @@ -123,22 +122,22 @@ describe('collaborationUndoRedoTest', () => { let nodeList = editUnit?.getChildren(0, 1); expect(nodeList !== undefined).assertTrue(); expect(1).assertEqual(nodeList?.length); - + // sleep for 500ms then set attributes - await sleep(500); + await TestUtils.sleep(500); node1.setAttributes({"align": "left", "width": 36, "italics": true}); let attrs = node1.getAttributes(); expect(attrs["align"]).assertEqual("left"); expect(attrs["width"]).assertEqual("36"); expect(attrs["italics"]).assertEqual("true"); - + // undo undoManager?.undo(); attrs = node1.getAttributes(); expect(attrs["align"]).assertUndefined(); expect(attrs["width"]).assertUndefined(); expect(attrs["italics"]).assertUndefined(); - + // redo undoManager?.redo(); attrs = node1.getAttributes(); @@ -146,15 +145,16 @@ describe('collaborationUndoRedoTest', () => { expect(attrs["width"]).assertEqual("36"); expect(attrs["italics"]).assertEqual("true"); } catch (err) { - console.log(TAG + "CollaborationEdit_UndoRedo_0003 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_UndoRedo_0003 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0003 End*****************"); }) /** * @tc.number CollaborationEdit_UndoRedo_0004 * @tc.name Noraml case of undo/redo after insertTexts - * @tc.desc + * @tc.desc * 1. get undoRedoManager * 2. construct a node and insert it into edit unit * 3. wait for 500ms @@ -162,10 +162,10 @@ describe('collaborationUndoRedoTest', () => { * 5. call undo, then check result * 6. call redo, then check clock of the texts children */ - it("CollaborationEdit_UndoRedo_0004", 0, async () => { + it("CollaborationEdit_UndoRedo_0004", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_UndoRedo_0004 Start*****************"); expect(editUnit !== undefined).assertTrue(); - let undoManager = undefined; + let undoManager: collaboration_edit.UndoRedoManager | undefined; try { undoManager = editObject?.getUndoRedoManager("top", {captureTimeout: 500}); expect(undoManager !== undefined).assertTrue(); @@ -174,9 +174,9 @@ describe('collaborationUndoRedoTest', () => { let nodeList = editUnit?.getChildren(0, 1); expect(nodeList !== undefined).assertTrue(); expect(1).assertEqual(nodeList?.length); - + // sleep for 500ms then insert Texts - await sleep(500); + await TestUtils.sleep(500); let text1 = new collaboration_edit.Text(); let text2 = new collaboration_edit.Text(); node1.insertTexts(0, [text1, text2]); @@ -190,13 +190,13 @@ describe('collaborationUndoRedoTest', () => { let jsonStr = node1.getJsonResult(); expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"text\":\"[]\"},{\"text\":\"[]\"}]}}]}"); - + // undo undoManager?.undo(); jsonStr = node1.getJsonResult(); expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[]}}]}"); - + // redo undoManager?.redo(); nodeList = node1.getChildren(0, 2); @@ -210,15 +210,16 @@ describe('collaborationUndoRedoTest', () => { expect(jsonStr !== undefined).assertTrue(); expect(jsonStr).assertEqual("{\"array\":[{\"ele\":{\"name\":\"p1\",\"attr\":{},\"children\":[{\"text\":\"[]\"},{\"text\":\"[]\"}]}}]}"); } catch (err) { - console.log(TAG + "CollaborationEdit_UndoRedo_0004 failed. err: %s", err); + console.error(TAG + `CollaborationEdit_UndoRedo_0004 failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0004 End*****************"); }) /** * @tc.number CollaborationEdit_UndoRedo_0005 * @tc.name Noraml case of undo/redo after inserting strings into text - * @tc.desc + * @tc.desc * 1. get undoRedoManager * 2. construct a node and insert it into edit unit * 3. construct a text and insert it into the node @@ -227,10 +228,10 @@ describe('collaborationUndoRedoTest', () => { * 5. call undo, then check the content of the text is empty * 6. call redo, then check the content of the text */ - it("CollaborationEdit_UndoRedo_0005", 0, async () => { + it("CollaborationEdit_UndoRedo_0005", Level.LEVEL0, async () => { console.log(TAG + "*****************CollaborationEdit_UndoRedo_0005 Start*****************"); expect(editUnit !== undefined).assertTrue(); - let undoManager = undefined; + let undoManager: collaboration_edit.UndoRedoManager | undefined; try { undoManager = editObject?.getUndoRedoManager("top", {captureTimeout: 500}); expect(undoManager !== undefined).assertTrue(); @@ -239,37 +240,93 @@ describe('collaborationUndoRedoTest', () => { let nodeList = editUnit?.getChildren(0, 1); expect(nodeList !== undefined).assertTrue(); expect(1).assertEqual(nodeList?.length); - + // insert Texts - let text = new collaboration_edit.Text(); + let text: collaboration_edit.Text = new collaboration_edit.Text(); node1.insertTexts(0, [text]); - + // sleep for 500ms then insert strings into text - await sleep(500); + await TestUtils.sleep(500); text.insert(0, "abc"); text.insert(3, "def", {"color": "red", "font-size":12}); - let plainText = text.getPlainText(); + let plainText: string = text.getPlainText(); expect(plainText).assertEqual("abcdef"); - let jsonStr = text.getJsonResult(); + let jsonStr: string = text.getJsonResult(); expect(jsonStr).assertEqual("[{\"insert\":\"abc\"},{\"insert\":\"def\",\"attributes\":{\"color\":\"red\",\"font-size\":\"12\"}}]"); - + // undo undoManager?.undo(); plainText = text.getPlainText(); expect(plainText).assertEqual(""); jsonStr = text.getJsonResult(); expect(jsonStr).assertEqual("[]"); - + // redo undoManager?.redo(); plainText = text.getPlainText(); expect(plainText).assertEqual("abcdef"); jsonStr = text.getJsonResult(); expect(jsonStr).assertEqual("[{\"insert\":\"abc\"},{\"insert\":\"def\",\"attributes\":{\"color\":\"red\",\"font-size\":\"12\"}}]"); - + + } catch (err) { + console.error(TAG + `CollaborationEdit_UndoRedo_0005 failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0005 End*****************"); + }) + + /** + * @tc.number CollaborationEdit_UndoRedo_0006 + * @tc.name test DeleteUndoRedoManager + * @tc.desc + * 1. get undo redo manager + * 2. delete undo redo manager + */ + it("CollaborationEdit_UndoRedo_0006", 0, async () => { + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0006 Start*****************"); + expect(editUnit !== undefined).assertTrue(); + let undoManager: collaboration_edit.UndoRedoManager | undefined; + try { + undoManager = editObject?.getUndoRedoManager("top", {captureTimeout: 500}); + expect(undoManager !== undefined).assertTrue(); + let node1 = new collaboration_edit.Node("p1"); + editUnit?.insertNodes(0, [node1]); + + // sleep for 500ms then set attributes + await TestUtils.sleep(500); + node1.setAttributes({"align": "left"}); + let attrs = node1.getAttributes(); + expect(attrs["align"]).assertEqual("left"); + + // undo + undoManager?.undo(); + attrs = node1.getAttributes(); + expect(attrs["align"]).assertUndefined(); + + // redo + undoManager?.redo(); + attrs = node1.getAttributes(); + expect(attrs["align"]).assertEqual("left"); + } catch (err) { + console.error(TAG + `undo redo failed. code:${err.code}, message:${err.message}`); + expect().assertFail(); + } + + try { + editObject?.deleteUndoRedoManager("top"); } catch (err) { - console.log(TAG + "CollaborationEdit_UndoRedo_0005 failed. err: %s", err); + console.error(TAG + `deleteUndoRedoManager failed. code:${err.code}, message:${err.message}`); expect().assertFail(); } + + let errCode: string = ""; + try { + undoManager?.undo(); + } catch (err) { + errCode = err.code; + } + expect("15410003").assertEqual(errCode); + console.log(TAG + "*****************CollaborationEdit_UndoRedo_0006 End*****************"); }) -}) + }) +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/module.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/module.json new file mode 100644 index 00000000..510a5f06 --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/module.json @@ -0,0 +1,52 @@ +{ + "module": { + "name": "testModule", + "type": "entry", + "srcEntrance": "./ets/Application/AbilityStage.ts", + "description": "$string:testModule_entry_dsc", + "mainElement": "com.collaborationedit.test.MainAbility", + "deviceTypes": [ + "default", + "tablet", + "2in1", + "phone" + ], + "deliveryWithInstall": true, + "installationFree": false, + "uiSyntax": "ets", + "pages": "$profile:main_pages", + "metadata": [ + { + "name": "ArkTSPartialUpdate", + "value": "false" + }, + { + "name": "MetaData1", + "value": "MetaDataValue", + "resource": "$profile:shortcuts_config" + } + ], + "abilities": [ + { + "name": "com.collaborationedit.test.MainAbility", + "srcEntrance": "./ets/MainAbility/MainAbility.ts", + "description": "$string:testModule_entry_main", + "icon": "$media:icon", + "label": "$string:entry_label", + "visible": true, + "launchType": "singleton", + "orientation": "portrait", + "skills": [ + { + "actions": [ + "action.system.home" + ], + "entities": [ + "entity.system.home" + ] + } + ] + } + ] + } +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/element/string.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/element/string.json new file mode 100644 index 00000000..b8cfd8dd --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/element/string.json @@ -0,0 +1,52 @@ +{ + "string": [ + { + "name": "testModule_entry_dsc", + "value": "i am an entry for testModule" + }, + { + "name": "testModule_entry_main", + "value": "the testModule entry ability" + }, + { + "name": "entry_label", + "value": "CollaborationEditJsTest" + }, + { + "name": "form_description", + "value": "my form" + }, + { + "name": "serviceability_description", + "value": "my whether" + }, + { + "name": "description_application", + "value": "demo for test" + }, + { + "name": "app_name", + "value": "Demo" + }, + { + "name": "form_FormAbility_desc", + "value": "form_description" + }, + { + "name": "form_FormAbility_label", + "value": "form_label" + }, + { + "name": "permission_reason", + "value": "Need permission" + }, + { + "name": "reason_hid_ddk", + "value": "hid_ddk ndk test" + }, + { + "name": "reason_usb_ddk", + "value": "usb_ddk ndk test" + } + ] +} diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/media/icon.png b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/media/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..205ad8b5a8a42e8762fbe4899b8e5e31ce822b8b GIT binary patch literal 20093 zcmV)JK)b(*P)AsI{U0tD9;7S&f z3`9H(<`G*WCN>bN493AFOi{!!!L|afI7%o`6&6lXK&2`L1YumJiZTQ+5doQ^Fu|gz zI6Nvw1cME>!8`;4iI*N+z3;u_gZtzG5&vyF~^*1 z?S1yyXYbweAFzGO*PdLxe&gE9j&{c{J=rY}9i1#6cCzdq+ASx~UzXhiC(H6orN{Ar zj;qq$yDTU7NWP@ws1J2_*G}Ykx7%{iE$G@-7-eF^Y3#}`(v#ySiIZdTj}`y+a>=Im9Vq=f1W5yxR*!@kj+Rxz&v=+4_?qb>2v z^P8^zTt$BB=j8B|JpIS7`QY>Jz4z#w<>ZT>lB09T6nS2-t-LNa`Yg!ixr}^gvZsB` z{B;rQ@uVEqwOt7oA8%Sn=e2VBs;^`dNc~|xx$^LKH+*6BuO8<1`K9&UDuw8t_%!FY zoV0NZ!^eH~qhBH?uakr4K4~ZC5VHnAA|L9#J5r^|-)7;Y zUl$mM>pDMqeipwr+7#N+YO&F-3t!twD#tH9_S*S{wQ+C`@f*(uNuw}s=xXMh&DI;Q z;_u$0c(3`5*FEq(O?pz@6#ee_pZMDAFS)(D{hdnlGw+UhHaZ&vMC3y~_HorR=oT!) zD&Jv0*w5!@vBS?MX~$>r(d*!xjZ=9%U3__Gl0?W|%cDAF&TIVSk@)+3cqc!3boGhhYzil=`)k_5%wL2pqQz`Ju@50G)sNfVj zoXGZ|Q(f3+@xx0`O2~K<`L6lJ-SXStp$#*Nk@$Du%RKJ9@n>4_fX zCq4RXG{SB86?4nquk-Hy-E#B;AN86?zpBs|J16`d(I5ZXNB^!~KL7eV0uKN-_1L$Q zfhXMkzP+y=*8|%=cJL*vJ8JS$i*h!V@e z?gp)OZL3q^qPRQ$mTS*l z!1Lo9sgwA)pzOQd7ry0nSAP)8dF^z>J#;@|{wb*sK5UU+HV4!!`0VEJLKou6^E1;q z{-F(t{g8gMTs+F%4CL8B(dE++Be1u} zQa1d_@^?2B{4?(K#G2gBZ2YKxYj^wS1vv8wb2h-K`rtLS+C4j5oS5zZQT6pjk(( zJ4B5)x)C<~DS-Jn#3lX27u>p0yp_M+jn)mGYaUy>+T%Nnb1#0!>tbyAQ%)nklRSgJ z&7=Ic?ks-hoA@5fJ^x~JiY`PYkDmW0C(plGd!Q$Ex;t|N@d~qieC9rdJUa(Jbmg%% zxJoLcUW^RY7oUugb$iXkOVyLI8AJG+ zNchYly!4G7Y^6~5nrXo&e$8p}lUVB0m<1UOEOBY-ht5+)-??6hPx|GZjRV(b``>-$ zM|{PjUt-09)0*964ZWy4qG3A!iZuCL5J4vSq$?ol?wO2=1e&!;9t z{HK#&d2T{`aKZSSV$8nw`5IF+b?d?_&_RB2Nn@S=KEJHRZ&{wfFD-HANt+d!8=g@V${FeVy<@Q=p|RCl}k1iW;RIY+rXYw+ro1J ztScYrS3bq4R+FlcH(!!*-yB2t`NcV#59x0CP?FiqC-VdG1vMIuAg3o=Td=#P|3Z0B%|-@17rLGk-6p<6~!$6~POh1kU3(XXZO`=|>$d z!lw$=5_RyEi#Jr~RP#^%iC^4A^2m;K+VClBHe2;z6Z14*Mk&|$%X0f<_lmdugY8>E zPThfcKaZ0b)2b2Pn1`Dkmvb_pUZ*zC08jjo)ep|hccB`;;R{6kL;Ts-DL%Zk@M}Ec zYe??S-~5VIlRb~$9A!25WQb$>P5#6re$4=RZ7!m^$ICJHQwLq8^3qO zSIW*0ziJfhY2#Np#+5qaD29V6USiSHHu0r%dVQte1>d!Te30L9h<8T(gM1~;2HMmK zAIaG=K2h~u$+A`Ao#yL~^C@rnmi3*Dn>*0%_Q|VFij#Is9D-CUfq|-t52LPSO>Mf;|h8QzG9r>i*kxj)D&%wf12-@hxpQE(boL;`OLW% z&4ra*97R9KXL{m{MVR>LH~jeO-Z?hkb&`yq#K-O6lT$@0DD?-g)^Uzc7T&5n8gw__ z0DpXP`45D@vQE5>CYLA9MXJba02$ioVhjTWVS5bZ6(4zN`ENe`p5>!H^k})NKh(Lb zKhik@lUA-Xx~smjY)TJqEB4J>%kshNC(AGX&hhfC|NQ3id+))>f~iYr%eBS5L6diS z0c(T7VNUk2yzB*+mM{H`dzO#=6GzJf`m=$1G@nblG}%hD(09V$W~@UCQLSS;5BqEV zWae*vfSYo>EH@?Gc;aOFp#GTWmw)f}@_j#ZYkBJ*Le`;RxE%9>G%3oHFxKHSfF_;E zFF&fw_1jO}dg1SWTfI@g(_fZ9_1ee&mj2x4J1a|pX>wLqgaW;Whu>GnNZR9Y^4s;%W zx4i1NzvUU8TZ6Uq$a?oX>%J5^9jAU9em|0;-_C;e(1}uEYG}e zr$t+qTP`-spu!U-M~AgevS79|o^g>`wAc>y@e7Vk`?z91a^qxq>GOBXzxbc8ET8gX z-7Xxv6CigTGJZUUv*`9=vmA1gzg4h49N+Y^ODZ8#@KI9`q-_X zaPu5;fuSS!*@le$mhP;#HK&jK(B1NbUvXvmPhY0_kiYDk{5AHRoIkT@vw@Z8z;F1q z7l7fCCi(MA@@nf@5q}|i{jv8-IsM&M6%o3LI{BfEQREKp4HG$@wUJ1eYx}Q!%BAIh z`K$LWk8838tEq&7|H$p$UeKq__MwZg*U!9Rnw3=(J#1>imzU))z3%$*uKvrZuZ{Wd>ES!5dgNmrfBPTZ zSl;rks&UNFhD?$g9J)KT33%MPXFTyAfBeSP=e+&fch`Iedi2_(FPHhgB&G`tFhZFY^iGZTPO8%A6S;JedWE&6Z7VgKJMLTtbV@Au;oe}a$|fo@8QFpeTE;~ z=(!{4cwATZ_x+vv)3p?oK6COMai}`b-FNw9`G;R}pRW2^Ajgt*_)SjojgA<};ZV-D zH)q&q4iEL*eWU|BFmM=S?>NY;&)5I;`<6?(5sl{jyXGx}^8>dxQX%Vtv5PEo8w6JK zToHH6efQkYp6Q3Mqvhz+s$i(tXF7XpLn?CV%Z6Oqu_p_+nw!5{zT;K*3%heMNzF;f zzun5oTzGVll(CU?9of+U+nP1y(OpU zvv~w9Sr;nLG5?3p<|70ueyyDbUY}Yd!E0=`V+1F2S@%7DUU z!+3G5v_Yp@FhhD(9o{OXys6YM@?dLP0LotS!( zZ~o{ThY!62s*m!Sg&e-XdU0#<$S=0*Pb|w{eYqaXoLkS+K6Rp~Y^EN+{G*Qi6P;tq z8XuKI#YV0>%Nz^2?6yhv9fh2b=evx?JV#`6&=bQOMZM+dz(~P{OOO4g=JV%2_LA3t zIWdLGe~6_L*6U?ZoidN$t=;E~mp$XEY0L*5)a)#9%C_**_ejXj1}SaGL~lF&7ro-L z5_Il{V)fCw*fu?YZqYMj%cgB7z3S~eAahn{_@cQMlFic3)%3UY#Noj!JH4cEvRr#S z^9EDCiHH1&FTSjo9Q4r{^K&2ha-QnFK^=vKuFYqvdxW=7K2uz)M)&XO4}*2S)oU;32*?s`tzhPoNdy zMK~{~T*=4;PVlC()T`0MfB8pTs;kbv+GgKHr(Rq!;3+S|5(B&y+n5*@z^5dLrcGjDVs3` zF=w9B8T=Q$;LA>~9`X4+qVFJ-liI=f8qb5;adlP9$i*t%;M>z~dBL;M7jh(|v1O@a za}jzx7Y{1+b#a=fVe#WfJ$C)~F&^GD!hg8&3xD97hwY{wLOxnA2;wJqo|?br07>n| zdc9}P-SQkmio~mhtX%z&MJycY7!O^|^}~~L*w+vLY!DscBm0>6jPaAr#6u#lPtl}a zn^g8A4RF_SY<9BpclX?P?PZtsH(oFGD^X@u>A2cxb^Xba#{f#>E7Bp? ztFxkR`P@dmpq)Vyx9`@uFnA8e#&tpr-DGb_G^IYIlqLQGW*i-bW1&6e29O6Y4AR#5 zvw3QcRQo|aIrZklmvExE$M4X$oUyA07_9mhM=sXuWE_~5;nT=?xmN7c}VZTZ(}?rL~jVuDCHDd zW0I>4RkJL)P{rpZ{mdS{51lA{3Pf+T`jPlbs|k>vbZN6ZbRkPI+fmPp0DeI6t7Nc~ z$NhZ%nT)>k;6(Zz50&~yf1iG^fs4sKviK#}-Dl{r>Bu~hY2DR;F}T*pmL9|4wUTbw z@xnlPQdFhr&E%R&<~6QfTI+#VgCJrYF+`(acGqTfD_@rASLH)IiT<#`a<+xCqjpL` z>#D>_%Q%UnL=``~nBcrnhfBLfp$0UGM~}`pY-%%xL2Su?1!0>O+=jhV^Q|SHHsi~S zD~0ov1zlYjfNIlt^GFNNb-;qpg1EPAM(ME^ps)?4i@M~QXic5q&!wGA8~zyJ#}kr& z^`4JJ%2R4dCKVL9!V%6$c5)Gv^*q_xt7|K06))bGDUPP7^FtSfX;?h<0|XKb062A zIY|b0!pj0C)Y$7;i^P=d-~9Mh&zQKh^`h&1%>hsw!5hUsnpx4t z<}nU3;cAnu{B7X&Vn5^sgN95?k&<*Nw-dMSz$p_Pc^$xvIFk*X^*T}DEO_*uml7(B z&nEcAJ#m?Xu}#P#5u(vuOElFSM`G;J(?_?d0s0skGYz4+p=0BMwY@=f?C04B`6n16 z7Y+?9wH$J zAxS-==YiY@80*`{n1+s)KEk056AV77g?$%2H0xq(Q))9XS&VWbRL_G=l_J9>UJl0D zL}N3`NDj2QCw^L+J)AKpGPZ04N*&EdoH2o<_uVvg5ExqK?h8cD!pAn(v{$fP*#~QU zh>wrmGmlPAjvv4qPUcCCWLhX|Ka2&~1>W*WY1;yK(tBoXnGCEf#s(&kaR8=O7&`Rb z4)NokexjR!kF~8MOFmU5aQ$lW3aOlWOo#8pn)8ot^lQLVQZO5XoZ}x``u%x;$Cmjs zwt{}jE1RV@QuzczTVvNF(%{QMY#aX3$pievr_W(l1ZA{3C6z9Llh!WOKW`#3*AYhq z-tucRhL5MYjUq^yq;P4yz(j=;Uhu<*6tg}0;12PFp$~4~hxPm_+Zg8Ct>f7*BneZNsSb8?%&Jh@KlZTTrOg zc*d4a&)A=--&QSt^&=aCKtMfi2RM(tjY0_3lN)$zC%(pMOo(G{xaW#VQD)ml*8}*( zn%f398D{+~2NGYgRbLr0gOY-ta%{uQ8}bVGoMs=E!xb*`2zR1d+}H1qgGY~B`-@YJ z>*a;j$od&444i_t&M>U#WibY2>CmtI+6%Qc>JFq&fKMxFac!J|LFhSyp@oAfvh|$Q!ky#K zhS(4BtuuI=bE{5uez>A2b4!3M+hm`g$1$&w|CB6iS~rUj(~}eO8bJK3dJ?_67ebx{ zSHS|R%y8%`=YQMnAR>?_}JgGOix59Mum~lwBBOj7l{Dr%(^B9~CeuB#Ukb0`^qvuU*Y(62BICR)&Tg!A&&-M+!2eTcS zQp|kcb?_I5@TRuW`$zm0SeN?*o>tHfJx!tLIT3p}glz!EcCx$YvH;wLhF24aiOPLh zoyM4vMhXD7pn%KA%I|SJ3pjFVbc&HshPKa%R-zM#w$p3fhA+q*C$x=DN^`o8SMD%{ zlYy6XyKVf(AvWYbX0=U|B7A&%L$qy^lSpgCbq?mNVK#inCYah3&VIO?=1DXw=#`qC zbt3TAho;;JwjNhLV1kW_T;f+5&f5zw$zb{>8{!V`+%h~%KVy-DqlO+=H=VZ=FkY%TPJGOKbO-eUMZb@k`Qw5*kXQI4 zNn-VY-V}k{dvi=NgDj)aFv2b;9&Lhj62jH0Xgt5%4NV`a$nS9VFeZ8jwL3ZT-35mn zvUwAUQ9a=cgBJ%U^%9B`*>UXEt~NPJ9a#K=jILPgIq5_LF4);`bivL2J}%hVmz_pI z&(zfWn4ASNsVrtA?CTky6@SLgnCP>dnQ&s$k2bCduV@v=0M<$2v&?X_w&f?0 zdVL4q!ob4O|06wo;ixOrj>l#y;~Gg=-=WAx*pV-hTSqte=+)3!U&FCJJ(R7IGj_tH zSk_m_@)csRD}7KQl3@|As*N?`C_c!U@vo=O(oUUM9HYTXr$fev>%5uanu%NzjR zCb4pse%58Ff_FbT99ZTs=22SCWBp8Il>D>{j4u>gKeWxhWg0&$HJ{gkdPXCf61P@& ztiI#OvjYd~D)hvhL4pdPanYqKH?T(AS0xsJjcpoa4(T1TJw`VIoTCqRpI?P*;>dsN z5f0BOf=znyxkaZ2tJWn8N$N>lK}c;lWS?W5vOBR=JKko}KC|$3Z%PH$J5|jKJ-NqE z_ZknrZ7W~D$^f(y8P~onU3Oty2J4NY*@llDx%i|JpU9&wHDK(xtG@VU#^kYat*h>i zdSLC^jL7(-#cz$a=M=p%&kPDtW4)wR`B-^()-G4{E(m^LY+5LRq%6%7l<6vOPNhVCyvY=4yUI zIx&MxLE28(nmXlm7viLOLSs$b4|GCD7I{^>sJ)bo<7qB^r=YAS^^JFY6;xwEh zZpDM~;ZEeb0~BvkTQTEG0U3VZL5j9H_mXvxdHwoPMGk8H%GZ$DSUoG};o!Bp*+kXX z`qy7&0LlzDGC5UnIv&!hC5g%LKEG*AaEI$`J|`zF9*~_UC6v2ef%Yt=w?iGS=`x{m`*tc1v}Pz zf~slY{K=p-7He#u7L@_cNMwKhd*f^(-Vaneam*r{gTf>LelwEqaEL>^IXTI3UTi}^ zZkltHCYX)!fRgkGlZFWF0F?CZ*bebcbNh5(fov2_4=P{4lkUMPb=`l~2uhFxu>7&DseW}mFpI(L7m<98w3m<&s^gYwzKLS`@ ziH2UU5yjHI=Sa0E5;z6n)mm>R$Iaaa0HpF2H=cyKrST)6aY5j>Y2EFa4KyaOJpi`Y z0cR0NFVNX;eH&s&2RLs_Wk`!X1Ktl5EXMuVY^M5^Na4ay{PgzMr(hU*GqwVm<`|tx zHqpMHc}$IYj}CnPhO8RSa9ryZ-xY7p0CWe2u`wOua|f#J0CPySsjO015zUoj^|=$R z&P!8a>m2?Q`plg2TfXWox!mch;lqB)b!%4}(i&%-8hjt^C)?8v8krgXwGp&JSbXUmUuKNKj;seLQ@+i{*gD4%I@RALNg?5Nv zHQN3d?-dcg{ZuEQo!};N-E}JHlr|#Z=D+=Y^?ah~?(8cL)5{VsbD?G)a@Zyct*NHxP>~FNNVt39Nz-u{udkt;$vC~g<^Q~(o z@!$ErW946qkAsrqYR=YH5b{$F!kam>41*1>C($G?Qu;QuA8=!KcHIVdWNDr-8-7uK zNuNiULdrZEx{d!~v71dXW?a|C=vhDe#uyuYWb4hW)6k0ypF8ER{BAwTAx;YE-wb!) zU;16Was^(;$OUp5dXvkJY0hDAS|8fn=gyP6&xSuan8cZ0vW)z(=x@DiJPDG%HphC= z- zpYdSh-(EFF=R=BYI@>x#_%jYWdLEjhM|USaBzVpNLG3+y_(R$BD_RmMas$MWs~oG^0ClV~+&9ED$w?cD|Yz+=nu2k$xd2U}uu6PP0V zCo+iBf#`{lqWxs#{-;()(J&9)cV& z*MIxg+j{>(@hd`~jcXbH;1z zth?n%0u(-3tD58KJI#tQPuPp_{T#@NnLsv#(utmIWON>=r)G}FN{F5lNBD@6U;Bn9 z>MqnKn+0+&Jbe!0Sg#XY1|IL>WT_VXUT;oA+Kv6ir{@DlMjpC8`1rDX*N^ifn3Oa- zP>v=r{|3wSjsMrp<+?rvZ1#&IQ%o*?Q%fUy9{OfIvd7w82leqs-`IVe19y5!^8?p+ z%lE(O);9mymq@O`lr{MH-Gap%a!lvK(+9_5!wv_d}s`<0wzR2F;-6sG^f)1 zfAhBE<$Hhn)^a}|--)B-fGBwkg|A}DfUPxB;ADB-k7x(+!4Wu(Z^V|l+qB6&n>1q*9dcD_jHBlT z*vR|+hTp{?KmT(AyX9Nn__#hpI{B~9Yw%ik6(uW2wP}cuI}>`1H0k-6=fBTqX`C$v zyXpzH+GeRX%|8xjW>_S<&=S+Pnr``~H$Jia)W5&2PruNUE@20Cie;tIvIjt59r&b0 zjV=c|+__#ALk??qI+k=+1B_gv^QeSsUl&j? z;p|tZ|KgJ`FMscq_bfcG=0&dhz{tYj7c4!e`8Av9+C(?nNM0J_+A`~hL2+5Y%lGV- zcj`{^cVGXwo}+cX;<;dQvT7u2?0R+qYFq{XM198e*L=}E%d_>lL3~zo=0om&Voy%^ z%h9>f^lD0ytPpr zg~{1jZAiO~^T97J@yeh09w`1xwSh24F`NSEhCjRLSXJn`%mH@4#+$x@;up2ebwIl&_3snm%EJ(YEoj{-clclgY{Q#$UL- z{G^^VuQM1Gu)n(U2vif97a;}2J2D&cm4Ei0<mZtf?9#n|`tkjxXn6KX&EI1=R@*$+Kyw>;|^ zN6TfsKa#H^pu#R*_}$O*#n-X_6q!ggu8IzGT!q@a0d4&GoYsxW{s08 zxcb6`!zl91*VjDiv#}r4pKJ1goci!UFDRc`2%OJ$tT_0@2dCnL<$j-qr9L&M`lL5D z(Jg%h*(2AFmk(S^Onhux>cB?H;>YJE=cKZwR~3}pmJcYob}zo~KupBx=(Nh~M4*nz zFreXsw&7fy?>G)Rb7uLh_>fd0az4fHf;q3Jlg~yVw=Ucr;=5V{Uqw2b-#L3OowL9U z9j+Ix`1q<;8v}WtQ-xXig+I)9(3;nXc|pGNB1^pvR0~0A$kl-?YrweTR}h1GVi

c)ijgxDm}8EsRXFt3h@+Ufr7@DN z^55r2UpdZvo*$)c`MJ_3zXBARbH%T}ifygzYy6g*WBtspGU<*Ccb`wpyW!Ui$gZ}y zo>MwK`K>f-62KfvO2{S zXF|ni6T=gB=C>=mF~5ojWS?I%DBt!ouB^&}v*S8G>5&(6>bM<0W9)PIeSXbv;v2lq zgZx&0)nJZqzUPEz=3RZouldy~VSciFe9|fxrs_KoD#u$hYz3BTu8Twxs@yt>*lp{< zm_XbpVEfL5#v}%x;+@AY<0*cV$ZF-248A&7CXCUG-9e@z7Va=V8J*&{q4I$n{~M-~K{qUmg-Y{N~tC__Y!6wZ`uS zAN=8SKnb`wARia}P{>}4q*mFJ2rt$xz9z}40>2@prKgMpJ4y?1MK zsu;8LLY(s8tNKp-L`??i35r}^567PuI=u8S&*EdFoy9Nf;48%{S#m8d=h|q*N!*Hw zE&QzCc2jn4u4(uar*pTPKCQ7DC)&Cs49?>3$7+X~)XJA`!=HT>p7`~r%@S~FvIWT% zL)t28t$h|BY!xpHnSQNXihG*>p${(0U;hi2mrwZcOUrZh0ee^UiT1oYO{3$5Hop*u zLXEN0l1qM=vD`rN)XOLJdon_5oHz3`AzpsrE1f=|*Mk1={U^)6{EcJ3kodUYZmX=p z&l4~2a)h&L*mG4|<3d+3_?Prr)`vgu$Y1U7EWIl2?@iUEd5K>;n9zxxlFNU^0vTLl zH@o9AcfQkuuVr{d?>6N1tv`70$?|*eKGqA1!uC8^rS(s+P1LOQ9lYFac+7nk_^^=}_9|LQHrRm;gm z#jgtmwd-2xd;fSm;rGSZd-@wbDeXS|)%sP&lv@b1qs`Sf43!0V?3qvsHeeF4^Q(*h z^}o7zxuRcU@`@_U0N4FIMxo}rPTLvJc{K#}XhYWmowJJ2$Yjbl`u)zkPnNIv?#GvR zeQ>x@oZ)FOm|m&l>_ivC(ek;URCk@4f5BINBIPcJedSknv#$7sL09O4r%@qb_M zz2et2d?)PSD|vhJv?jf^coe^7;*5D_(i{GoNjc@GFgNZjMJ5=HK91L-#6s_k5ZsDS zGS%RQ&sF+5eNE*3{W~3);ByDsjH9O)4$S@$?yR>?gy?){V`EPI$n>{$7kZJt&E|jq z@9tl&>KhB0wjiX?fvux_ph<@^P`xU#l~@YcVmvoP|52 zFCDST=db-|m-UT`(xE24+%n&4gZ%FnLi&Yo)!)!<`8*?XqEn@~PlG4oI{hPQc|SBA-3UqQo@Ok7n} zIAZ21l@78Rn`X^sw|ukiJP&AnypS?sjm)BYgRrvd_2vm*-zj>cKd@`Ab&91Yp=>6{)F%4)7auKu@lUJhnvWozKNZb^uG+`E@Y3=U zeK~|@uUf1nf;jWRpXQgYuqA_|MTZQJmcB;TNR^GlS{T8}iC6rO{IH|tWqO{uY5h}C zK^05FmfvX7IMk$1hE*ehH{+tKyHIa1DdB;;rJvHi z@XysN8q8vy7k-&z&tLr~zqICPT-#vO+|kk)bI{UP%}!$rHS^6TDD1uXt~a|@W*~+c z8vo^wJW;Rw34f4ZJkG`2_D~Yj%WRNd2O^Mwn=s<$0*s{9@EYCPT5v)bA~e(n|~6M0EUxGtnrcN&$s(s zzN8S(XWAcol9+ za@NCPqQw`HsBTqo#8>DWj&U^~+CTP~&69^IHqX$ty#E|%_>m7|XO7~asM|V+|Xy_l(fh&fm#RNST>VcoN?=6S_DPi%0~BG=sQt4-78)-@|b)lahBHa~PL<9jHj zNE~dl9PG02qUPM@QPu+cEDu-Af8%z}zB%Ihfge*{9Wd$&G+)E(=&9+o!^CjO`cwNdjVRH+WU`h_MXAOitJp5x3ifW{$igPf9iBj$(b=HI#x==`-hy-E&gI#->XR(BW&pMdcoR19-nNcPkY4s2bR7uK27u z;T-wi{Jv$d3tg^Khr|3zu!D-f$3GV1rd-BjB{h8+psmB&uHFO}3e<>-KnIym}P_oSC zslstp61Dm&1NiV|^pEbaNt}ZX!rh1GA<@OoA~K`yhAgd{@foOROsg!`F}gM(u1!jB zP-&PeM7Vk8W1#d^)-p1e`o(13g|c~w?dj`;4_bZu^_E|g3d=E{cLES;rdxmDH283uG=7WUKG<2~ea{IxU4q0( zBCeM((XD0e;O571>R|^u&Ev*jpsQGwzvm-2(K$^ICifY)?_e`E(umG-isbY(H;sFS z_TV{-u;uIR9OWMt?$V=eCxZbQ9k$3lC>2^A@xz~@XvD&(_uWN31AO=Zpf(=jB!lHh zOT3|j8)NsuFr00(J`~5*Aa@-yCcZDeY#2MK^7+byjE?yuYo4B|14zoWZPTeh8BIOF zi#LZ9-0pPpQq1&2arSg`YF@vQoGhb26RLwnlb*1L_^M-Vlx>giHItHpV-y+pt6ZEK z556G7lZ4?GS?qbNp_S;OAM&IlDs9+mIL@;^vinA)D6z3H9OHAVWxzHP_n^luSJ#<< zbsIty2lS^g(Tp%sL>_Jx%DMrbLPR&IRuN*2au@Mv3b3wQaDyVnmOp4Ma3Q*l1@}l- z7!@6xqcC>X;&3#^WC@2>d~Pt-WCFI;DSS*he8-yHfN>hl!&k7gZRoJWX*}IU_<3Dv zFh%O=_d;$wPTu#$88_QzeaYlJH`gOD^~u}%0AtVi0{v!P<5awgzdH2uJ`V|wUL*2lawezA2~fq&{P;mfB?8T6HUC*4h6A&Uoa8O-j$RT~z$aZBVg6 zzF?cyl6N zdHw?sJ7Tp$XXHMr#>SS7hWS(q4Vv|F6FxR`qoAKa__u1W&%AQI4T^VKan^IyU>zfs zE|$R$NQPNwnbWKcmi{dLjG5%b9r@2i8f!K??SvY4H+*lPY@EblJRiC1P#E;CqroIW z@amJ2xy(A56v{9|GuaTpMMj+DK>H#%Xah4-!k=}#^ zneQH-ALI49-brtya+(0Rs?MoH;W4xa=7q~HKFb7Z1nBuy5&@vrkTKXDY=saRII;oP z3R%&P2^nF-NYearIVR*J3O2Ys934KH3%!qF8Ezacu`vg0S*Oab^yt!p+xLq-xy5gM z#Kw5jI=`XA!CkZ&zAqE&VEj1=NFmPhl*4MSO=PEas`~e2-T71-1sApc|fu*Q}= zsYFnC_DZcy+zSDb@&j)&>t^-n;oK7;%>Y=GI zf;q6^#lf=W>#ky4S#ll)lVVQT_DO*_|C(c%5cIB9nT$1w zdZdwu#x~{=-+@S!Al?*`YqRX_$W)w|mL<42l`iKk-%cwYqIN?eH8`i)kL=}d1?JZx ztLCs2KGwvGug#(X==ud4yo;s5T!B+uNNV9YMyc!;d~C+efEeaJa{IVw7aDzJFOkR6 zSlJt<<>?A3vyx@)YW!;#RD~3cJ<+yt$FWi*K*_8K6|i@y5t3Ja zJ+H|ads>I+vjj95MRGK=^x>=qv2joEMXBp_IFN4`AdHaye#ZCSN+T3ki zEEWhGJ-%>&Q^eAnKgqhuJba{|Jl+AxddOr{Cxi+(@50!IbHi4?hjyY5LQ=XVPTEpb zyqVjwx1@vOf~d3GC@cCi=V6PSGqd|Ua>`SZ|JP5mkUUL?=|EPi{@-nlH?JLkAw z*sMbLgtgvL+o_1?*wJfZjcXpC5>GR~M4yu?y`l7N54Pg1hB01ME2+8Z!14qfU-Yz@ zpP&@C_lf&Q^@(4j;1EbkPV$`KhCay2t@XoalE&DO(HG;)bGsV$(1$|8a365@r{WKw zNW$FkEp^Sm<|7b9uV3Ad{N#D~L@0goVuYqx6L^T_<{Zg#=0otZT7J0Sg93< zJ_mX2IquB#Bm6s#^rsweb>du#$y5q2icb}=oNpi;{UA7T{^iK)*yGw5d6=pq_?*D>mRC&iQRDaItw;A9 zUwyN}YMcO55)^&3H9%p>YklyFuHBgRqrZ5o{^}Fg-RyE2Q&BkPr4P7!;2dsBBY5kZ z6MOo=-HSke#!JD&S`O^!e_!8v^T8YV)+p1?{L!gB{K1puy1vT%sWe=-JBLXqC(&~o zh8QdS8g_rYT88wPo<6+$(H>5CKO8#&q^#c>*j4hprAvR9e{%Kyt8YGf`?u>?8Tz14 zS1k!Et{sV(!ehcu#U^0M9yMmukRS`=W<1D5*Xuj%0?f#3B#i1AuV%Dk0a#p(np`Z z@Ny<>{{ZDV5+@v)mOs>&&;9Vv>-)pHaOkS3YygE%;ePHnZ!h`bKx(H9HZuLnZ`piM z2ii=ClLN3rsu>=c{+jNjKd(=0rLpid^!u4*y(mWJPG6kjm0Yv8i=0jt@0q$c?3SO6 zo`T_+i0(Myt98b;JQvD(PJ8@c_^spR4R6xbATVp;gA^fWJoolt6Viy=aHkR(bL6>a z0*u#QIOR-CHs#1eI_@gp{LgMJH~1i?ZcMM{ufkCb2He+@V%l*Br$@ccN`(OGk)9u)8Cl^IS$70>cnNtJOD;^adIv1mfzOH@{j*A zpUGT+)Iu&-&YD8$81J|E-`Afpo?Sod(=~-f1KG?W4N<>A4H|trX(W)6k{Oa&+m(#9NV~FpO<-jgq5FpLo=R80h%`t-tc094&kfl2?<-(g>J|r?=r^r}OA> zmp&f(`pX~wSI3@L@|*kMoPV!t)up3lQ3afNHGkNJ?ukAA%&S+P!*d|=aQo0Nz5YfK zKR4s_UId|>uzYyqbjJt5=GTt(Ez-yS$U9G{Cqm(9+ajN> zgT~ide(a0*RMefm>R_qQXttNTKUJiWa#G(o>gibbxL(-&eO>l^>-4Yw{;}#f=Ndog zTpjgwLr5GKkp=Bm^VjU9%39U~*@|iCk3RCfSN<|`f4G7d?}tSDTy`AIwQL?;#$97+ ztSvnwvYK=4p}Io0?fv>@g@5oyeJpBc$rtZF^xS26hCWZ4#Yok->p2VeHu^YSPUGG2k^A|XtmgmW>+a9E=9)4OCk5TSW^(Rd;pI_JfySLre zQLOv*sbCN46V?6wuS}=FN|eBT_p(bFq*`MXpIA`Vg(EMp(umI{;a4t?=!xmyYV?&H2P7PMKv=d+vjRBWh(As6Lj0Qcn$#3?!%y6`&&<3aj!!;n$@xk0 z*`QFf2~yb7*ZgYBR84)J;s=KZ&x_vE!tWtII60`G5(@|IFyHPr=5zVG<@(X_<1hTc z_kGCwAo)o&!Uw+XL*A!{f;S*LxN;y5=0e-ZrK)pdNED2liw(!iVbw-%n7!XMpG8kA zGUJMmr0RBj5-MyJddQOpL{O*s7%s{`6u+WXrgQwlI?smCIg$&Q{AYgqCt0wKb7$_% zm%{TugWsEv_{Fa|uJO;}cZ_9uLpG0)>jq*Vhu`WPlbLjiH(IU~Fm-o{X+n|rIebs+ zBK*FBMohVN%r4@=_@qH>4)KXqe5CL#cK)Tu;+Dei@z-rsKEYOe;uO{W-~*^lGv{e} zg4af91r84J?WZul<4pXy&Q9bMAD7uEiayKu@j6WtFdw~+#;%<5b$dDfR;X#?4us;} z-~EhV6zs>~=Rof`?o~=VM~9%M_?8J+n!&AcCV)?AP=;fE71{~UeEA>#S{QucDki=r zzHybu$j{hvT>Nr&n2+r=zY;+&dlw*cHh$KbFJ$UN=-6jIG7AR2vDH_c$iN1FmhpRt z?{%2s!?BZglURd~-k|DP8~&9Flv)o?mLI$Jz3h>-Z8i{UeJRS<(K9vL#!-~$F*1Sp z9>4-|wb7EC2gB>kF9$2`EI#_O(HBeOdGZy+=Ze2BPH_+Mi?qgP47=j(>kB=mJ%oMS z9r<0iE@an9F`Z)KGra&4x%#2EIrCiSSMf=2pI?~4w>$UPbpC{gT;8zlrl=Bb2 zc!MuoiVfHWSDf^|NDlF(^ZW;&*`LSHX6X1EeyW$cIeN{P*pA<}=H;OUB#~>P2l%!Y z!u69#KlsSz*U2UJ{M*;+{q-Mwz4pdlJGFtZ-+TGiS1Ql<#B&y|xO2F8BP#-G95X!= zS3AtF&0v5*jT?Lk8~!j1%0_T}otooBko6is#Sgz&6@Aj7$ONp`$^7Ks*zOGN$=Vl+ z!3WfQyRB%BY(65Ff(S*v1=yWtyJ{I0gB$4W-~OP!g>&~BlI$ss{JeWJ0Y~lvE4La}LgwmJ{B^=-^LrxrR*K+!NY34Y z%M z<9FfUS32e(gAJbEtbl5ub8iasSIo+HYW6cI2(;PPCVrX9hj6>)HIID%gYPzH@6^%v zv^{*@-@5)2n!;y#NN$bBu|)+fn^0}89(_q=8AGE|lG!A3qm}-*G$sPd@g2 zSN`*ry_F8$fdaX8yu3>5_^=Mm3a>SxDq|(W496V3gthog+!l-+gI^0x3>K~U0B9_I z@g1v9#%%cbQY(J<)|7{e%NhR$c6@0R)3;{wt|Y5hT-qAn?23((Ie*Is_;P_4Gx3j1 z3^!RMCcZ=O#~*wM_}}BBm6H6+W|(D1K9`SA_)O&v{7zZehxLm7tBQH}eC`H%|3AL+ zwv$WC=ZSiwBbOHn*aasRMW->jDp-wcQfvqt$sDPv&GGOq`KuGkd^o;c>O`@?JJE_` zdU788%6;TNa;;()znFK!uf=i(n|UXb!}$}T5F5S&N6!Fu`(`Au^2Zij=Z|V?HNBZ# z{Jg_J&>P3Qlh3>HhAVHIXs5)?*?J{TB9TPPY-Gp32p`^F3!lv=`TY2MT!#Dn_EX5YDwXjm4@%zo zyA%j0dpPZ8aUi>rp!dHqyG~d+l6Q>+x9T-*oC&4dQmFv;TYcH~Spj>DJ0esIt zzWNO+#A`{>E5i(Xk;Z0`sjgNLsQM^ePYfMu`tZTDpWqGSgiZetwnduxeT7P8ynTsi zel~9SC}kpn5&t6m<~Z?*-@e9Xw_7%@1cxGiwOUv!*ZAgV{^YpI;WyoHSsAi`#H6j9 zt$aSe;%xY&tQ7Q@%CCLw|GfH*c7B0V=63;TLHuy07aBFXpK@e@kz6>#YSGcv3{ghz zzVXF3=^Q@()T&z5KP7&Q>i!XZTNu&$kfkNQnO!8-_aDL+?R~C8sjF4t! z6x@c9tB)3F@nK85F<=By?G&Gi4}X@LiXJ2XmM&tvDMDVeZJcH{s6W+y1bgFn`9~ZXTFjEjziZ(}(o3vn z`%X>ZGshK%2W48h%Jnqix>9=bSGbGC-{Va~Hp{r_k-l2)R5e=9GXJFTue#GuTPtHLO_kpoE;{;<|N8ou=yCIP zN<{A~WY5T@7mLhsKlK)EER*b9LF?v{dT-&+=Hpvd_~PVB{13->Hs|DD_AU++MKR^? zVbs#s_)ceV^X6!`7vaB08NBAP@4xarcZzYI{jMLv_MN@||G4r!x9+?3(b^}k&qm0m zIJo%3!Mf<)XVROminu6NX7e>E)#+h2O$}L)eu$)~=3}XaGUgyZ_V8KMnK#)7zjPHp z_Ts=j%wK(OAJ%4maf|Pa51wLAKZDR6(r+-k<@J}An;-pDHxE9y+0Rj)g#6$aUwirP zX!kYxQ0mVy-QN2yL-92;)+QS*i|kvrv|fAPK+-?Jmin%y1ZS6N0LGw(w2!|y(vgZ*y#F}>^b>-1db)Nj=f;xC|Ft8@YI zMIq1nn~#0+?)d1{!hey9e+8a5izk@{Oplez2GHqrSUlSN&@^wrvVyP!giSlmuO%9r zW`jOGD83?gYTjdlCEZT%G_f_YKb`yp!)N?Qcc8y6-5c~LFW-9YpKRX@b^v?Vs?#fW z*DlT`JnOH$|Jl3C_q|fP=kqnu&(d`7^YSrkS5(VraZMu&zIv_2t3qXyto_-1d=_pk z^vbJk!~$p|XLVszAW2V_Pv+Y=r{jaEb~--#@C&o@YkYyT{(x!uak=@SdyXFer}KN5 zFTlMk$hvZOMZ0@2f4q3@#*LTjFKs?eK|fUioJEMtmjUO-<02&yOE|p|V-%X=6Xv@X(oCxjr1jf2;npdQ$tQM<2QW z=azp~pZ|S`@O0`r&8O4l#eLPLy7n@?{`u15<>(>(HP?sj)ax^gp0C0^Q@=iWK*f2c zD)fL#sXs~F-K&MVM;neWi6M8@tERwteOT%%cv{JMqtu2a&-F?ld~arKwAH@y=LKKw z#h-2EA?L&VSjQ(K-_mq$Dl8u&b4}hKRXUGo8jtD{dqj15STlZy(C<7sI)2CQ_~fnE k9@EG3{4s5ok?kb>|H;3ubeVRY^#A|>07*qoM6N<$f~C=$asU7T literal 0 HcmV?d00001 diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/main_pages.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/main_pages.json new file mode 100644 index 00000000..ceb075cd --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/main_pages.json @@ -0,0 +1,5 @@ +{ + "src": [ + "MainAbility/pages/index/index" + ] +} \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/shortcuts_config.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/shortcuts_config.json new file mode 100644 index 00000000..ceb075cd --- /dev/null +++ b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/entry/src/main/resources/base/profile/shortcuts_config.json @@ -0,0 +1,5 @@ +{ + "src": [ + "MainAbility/pages/index/index" + ] +} \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/openharmony_sx.p7b b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/signature/openharmony_sx.p7b similarity index 50% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/openharmony_sx.p7b rename to data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/collaboration_edit_js_test/signature/openharmony_sx.p7b index 9be1e98fa4c0c28ca997ed660112fa16b194f0f5..085a8c2abaa7f8bd232ff29a9999e205eef13cf6 100644 GIT binary patch delta 967 zcmZ9L&2QUe0LIBjJKD7;(3r|3iV+fnHhJyDYf?1O>yOwDbsV?OheHOl9mh`Mymn%z zc2kK;0aX)1Wg|k9w#(3@o#<)1@bw=|+kpe(!Wm7VaoYh+B2uM-I6gl;dQQ*p&y_D; zwZUZh+Wf-Xhy3BQ<=C0{oyqdv42|Nk7`_b7&M#a&gZ z+&;Ez4*iac){)jXu}zFfffT^jPO;&(W|+9q9ll29v$;zoLFU#l_Wn6>Ow&*|Ef@+Ev;vK_3d`ohMvWBIdNI%mk`xNtj#>lJ2Mj#upOi$UiFok88e0erF7i~O z0JK_uBovB{&Q)!LsDw2IDwJgm>|R$Y%K}AN)}V<_l1NcuiH1*`%jabXXYT7#4b-$OM|8V3NSrhN*$0OgXchw zBr+QfkUsTcA&Fzj)#)eVh3kKCU*G-cz39=WzrOX^&Nt8Re)sfZ;tHRBSiAZTSe$=x zX4<36$wWNK0XF{n3%`7PYkTcjf1Fni6YARPvnQ)xJ-i*9UphKI+Q0Yw$@lTq#rB1t j2Gg6rTe!RN%1hwKAI^I}fB7x`m)H8O@cDyV<(vNi``|!n delta 974 zcmY+DONiri9LKkv%&a3*1sC_AhgrJ^!FeT3AL+un{_~h5ok`lXNgtuYBx%~Td9`WN zJX|UCFbFQf(2MLr*Tae@UC@gsS#QFkpo@AFLD7pY=wb07*d5sskH63N`-R`(`@Qk} zZ_mx(d8)s*{?ajX{4kX`pS(3s$xBpECKAZgaB6M+=$J|9D^N1AdJdkiuKj0C+?qoV zp*i&DYw4-(jr*>%<0!VCzLG|E;axbreTFr+v&6`DS{V*^I{op)>#P3cQD73Cy#DHD zK7$rA*#d%XryX~)qjV>_xAf<8xlA4|ASZvJDc5kv9et`>?ojs@g~YXsWs)F(n(2Xn z#X*my#A-#B;>C@``!jw(A#H~v*_an4v|(sG7gNEK#%FAR`DrSMdGtVkTro<=~(9jYj9skUKgHZoNmGiZoF#CzJvFbCzfQxRAk@AXcv ziooG`z!Lye!9lG1B0^CDHO&XKk?Hzsz{F9p>C2PA6#XVUV4ARz3)_y)>pfKy2)rYC zofe6NEutwCcuaX_>%hTWfn^=O<7W<|c;7|XKq5gG@9}k}#!)m08f~YCi>l#i{l$C* zo3)*NA})qqCT_x3)HE5sis&BBNI3w34A?+Kwhm1k zW802xgx$c2^4YMCPOQk6q_J_JR6R3P04D9xmhX7e@vzv4^h&M5MpIkiLbp@{1Dt>- z9#6L&zsvwW>so!dQBztYyuvWOJ;~^5B(`N%H8SEQ>tIr@J08lIhj9}t6+5}HrxoMz z)x~uv^Q;~!_R!LI?P%FctFI}KD!KRuy8Q8l)x!Bz0-h(XEy^3Cr~bc!#l4N)83N~# z9GpQh6qDh-vr()=NE*5n+`p)OfZT08NPcpA>*&g6^X7Hw#}DtTf1G^&(RbJzZ^BQm zJ+U0zmsg>cBt*kh@{(63FK=xA{>22fr?0&0@x=5i`R~#@>ZtU?gSR$bd^!2TxlglN l@a+os{mpdhnavA#+@F7W@96GP<@Vcm?u|eD_3P$O{{UIsMt}eS diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/config.json b/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/config.json deleted file mode 100644 index 63170377..00000000 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/config.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "app": { - "bundleName": "com.example.myapplication", - "vendor": "example", - "version": { - "code": 1, - "name": "1.0" - }, - "apiVersion": { - "compatible": 9, - "target": 14 - } - }, - "deviceConfig": {}, - "module": { - "package": "com.example.myapplication", - "name": ".MyApplication", - "deviceType": [ - "tablet", - "2in1", - "default", - "phone" - ], - "distro": { - "deliveryWithInstall": true, - "moduleName": "entry", - "moduleType": "entry" - }, - "abilities": [ - { - "visible": true, - "skills": [ - { - "entities": [ - "entity.system.home" - ], - "actions": [ - "action.system.home" - ] - } - ], - "name": "com.example.myapplication.MainAbility", - "icon": "$media:icon", - "description": "$string:mainability_description", - "label": "MyApplication", - "type": "page", - "launchType": "standard" - } - ], - "js": [ - { - "pages": [ - "pages/index/index" - ], - "name": "default", - "window": { - "designWidth": 720, - "autoDesignWidth": false - } - } - ] - } - } \ No newline at end of file diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/js_common.h b/data_object/frameworks/jskitsimpl/include/adaptor/js_common.h index 7eeb7dc9..b4d3d348 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/js_common.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/js_common.h @@ -15,7 +15,6 @@ #ifndef JS_COMMON_H #define JS_COMMON_H -#include "hilog/log.h" namespace OHOS::ObjectStore { class Constants { public: diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobject.h b/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobject.h index 91fd9f9e..c0b6f0b5 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobject.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobject.h @@ -16,9 +16,6 @@ #ifndef JS_DISTRIBUTEDOBJECT_H #define JS_DISTRIBUTEDOBJECT_H -#include - -#include "distributed_objectstore.h" #include "js_object_wrapper.h" namespace OHOS::ObjectStore { struct ConstructContext { diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobjectstore.h b/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobjectstore.h index 98afe9e0..4e7a1cbe 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobjectstore.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/js_distributedobjectstore.h @@ -16,13 +16,8 @@ #ifndef JS_DISTRIBUTEDDATAOBJECTSTORE_H #define JS_DISTRIBUTEDDATAOBJECTSTORE_H -#include - #include "concurrent_map.h" -#include "distributed_objectstore.h" -#include "js_native_api.h" #include "js_object_wrapper.h" -#include "node_api.h" namespace OHOS::ObjectStore { class JSDistributedObjectStore { public: diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/js_object_wrapper.h b/data_object/frameworks/jskitsimpl/include/adaptor/js_object_wrapper.h index c6acf0b9..3730ec8b 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/js_object_wrapper.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/js_object_wrapper.h @@ -16,10 +16,6 @@ #ifndef JS_OBJECT_WRAPPER_H #define JS_OBJECT_WRAPPER_H -#include - -#include "distributed_object.h" -#include "distributed_objectstore.h" #include "js_watcher.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/js_watcher.h b/data_object/frameworks/jskitsimpl/include/adaptor/js_watcher.h index 147e69fd..f3782796 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/js_watcher.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/js_watcher.h @@ -17,9 +17,6 @@ #define JSWATCHER_H #include "distributed_objectstore.h" -#include "flat_object_store.h" -#include "napi/native_api.h" -#include "napi/native_node_api.h" #include "uv_queue.h" namespace OHOS::ObjectStore { @@ -38,7 +35,7 @@ public: virtual ~EventListener() { } - + bool IsEmpty() { return handlers_ == nullptr; @@ -97,7 +94,7 @@ public: void Emit(const char *type, const std::string &sessionId, const std::vector &changeData); void Emit(const char *type, const std::string &sessionId, const std::string &networkId, const std::string &status); - + bool IsEmpty(); void SetListener(ChangeEventListener *changeEventListener, StatusEventListener *statusEventListener); diff --git a/data_object/frameworks/jskitsimpl/include/adaptor/notifier_impl.h b/data_object/frameworks/jskitsimpl/include/adaptor/notifier_impl.h index c4f6219a..4b1d2c6f 100644 --- a/data_object/frameworks/jskitsimpl/include/adaptor/notifier_impl.h +++ b/data_object/frameworks/jskitsimpl/include/adaptor/notifier_impl.h @@ -16,9 +16,6 @@ #ifndef JS_NOTIFIER_IMPL_H #define JS_NOTIFIER_IMPL_H -#include - -#include "distributed_objectstore.h" #include "js_watcher.h" namespace OHOS::ObjectStore { class NotifierImpl : public StatusNotifier { diff --git a/data_object/frameworks/jskitsimpl/include/common/js_ability.h b/data_object/frameworks/jskitsimpl/include/common/js_ability.h index b7f318a2..3ab824e0 100644 --- a/data_object/frameworks/jskitsimpl/include/common/js_ability.h +++ b/data_object/frameworks/jskitsimpl/include/common/js_ability.h @@ -15,13 +15,8 @@ #ifndef DISTRIBUTEDDATAMGR_OBJECT_JSABILITY_H #define DISTRIBUTEDDATAMGR_OBJECT_JSABILITY_H -#include -#include #include "ability.h" -#include "napi/native_api.h" -#include "napi/native_common.h" -#include "napi/native_node_api.h" #include "napi_base_context.h" namespace OHOS { diff --git a/data_object/frameworks/jskitsimpl/include/common/js_util.h b/data_object/frameworks/jskitsimpl/include/common/js_util.h index a2bff54b..0b96f402 100644 --- a/data_object/frameworks/jskitsimpl/include/common/js_util.h +++ b/data_object/frameworks/jskitsimpl/include/common/js_util.h @@ -14,13 +14,7 @@ */ #ifndef OHOS_JS_UTIL_H #define OHOS_JS_UTIL_H -#include -#include -#include -#include -#include -#include "napi/native_api.h" #include "napi/native_node_api.h" #include "object_types.h" @@ -38,7 +32,7 @@ public: /* napi_value <-> std::string */ static napi_status GetValue(napi_env env, napi_value in, std::string &out); static napi_status SetValue(napi_env env, const std::string &in, napi_value &out); - + /* napi_value <-> int32_t */ static napi_status GetValue(napi_env env, napi_value in, int32_t& out); static napi_status SetValue(napi_env env, const int32_t& in, napi_value& out); @@ -59,11 +53,11 @@ public: static napi_status SetValue(napi_env env, const std::vector &in, napi_value &out); static napi_status GetValue(napi_env env, napi_value in, Assets &assets); - + static napi_status GetValue(napi_env env, napi_value in, Asset &asset); - + static napi_status GetValue(napi_env env, napi_value in, AssetBindInfo &out); - + static napi_status GetValue(napi_env env, napi_value in, ValuesBucket &out); static napi_status GetValue(napi_env env, napi_value jsValue, std::monostate &out); diff --git a/data_object/frameworks/jskitsimpl/include/common/napi_queue.h b/data_object/frameworks/jskitsimpl/include/common/napi_queue.h index 02decd5f..2e5b0d9b 100644 --- a/data_object/frameworks/jskitsimpl/include/common/napi_queue.h +++ b/data_object/frameworks/jskitsimpl/include/common/napi_queue.h @@ -18,8 +18,6 @@ #include #include -#include "napi/native_api.h" -#include "napi/native_common.h" #include "napi/native_node_api.h" #include "object_error.h" diff --git a/data_object/frameworks/jskitsimpl/include/common/object_error.h b/data_object/frameworks/jskitsimpl/include/common/object_error.h index 0a640e97..ddffef30 100644 --- a/data_object/frameworks/jskitsimpl/include/common/object_error.h +++ b/data_object/frameworks/jskitsimpl/include/common/object_error.h @@ -20,7 +20,7 @@ namespace OHOS { namespace ObjectStore { -static const int EXCEPTION_INNER = 0; +static constexpr int EXCEPTION_INNER = 0; class Error { public: diff --git a/data_object/frameworks/jskitsimpl/include/common/uv_queue.h b/data_object/frameworks/jskitsimpl/include/common/uv_queue.h index 8042e955..7cc84ff5 100644 --- a/data_object/frameworks/jskitsimpl/include/common/uv_queue.h +++ b/data_object/frameworks/jskitsimpl/include/common/uv_queue.h @@ -14,15 +14,12 @@ */ #ifndef UV_QUEUE_H #define UV_QUEUE_H -#include #include #include #include #include -#include "napi/native_api.h" #include "napi/native_node_api.h" -#include "uv.h" namespace OHOS::ObjectStore { typedef void (*Process)(napi_env env, std::list &); diff --git a/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobject.cpp b/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobject.cpp index c932c2fc..96aedc2d 100644 --- a/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobject.cpp +++ b/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobject.cpp @@ -18,11 +18,9 @@ #include #include "js_common.h" -#include "js_object_wrapper.h" #include "js_util.h" #include "logger.h" #include "napi_queue.h" -#include "object_error.h" #include "objectstore_errors.h" namespace OHOS::ObjectStore { diff --git a/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobjectstore.cpp b/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobjectstore.cpp index 5b52811f..2ea6174d 100644 --- a/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobjectstore.cpp +++ b/data_object/frameworks/jskitsimpl/src/adaptor/js_distributedobjectstore.cpp @@ -15,17 +15,12 @@ #include "js_distributedobjectstore.h" -#include #include -#include "ability_context.h" #include "accesstoken_kit.h" -#include "application_context.h" -#include "distributed_objectstore.h" #include "js_ability.h" #include "js_common.h" #include "js_distributedobject.h" -#include "js_object_wrapper.h" #include "js_util.h" #include "logger.h" #include "object_error.h" @@ -33,7 +28,6 @@ namespace OHOS::ObjectStore { constexpr size_t TYPE_SIZE = 10; -const std::string DISTRIBUTED_DATASYNC = "ohos.permission.DISTRIBUTED_DATASYNC"; static ConcurrentMap> g_statusCallBacks; static ConcurrentMap> g_changeCallBacks; bool JSDistributedObjectStore::AddCallback(napi_env env, ConcurrentMap> &callbacks, @@ -121,10 +115,10 @@ napi_value JSDistributedObjectStore::NewDistributedObject( return; } auto objectWrapper = static_cast(data); - + JSDistributedObjectStore::DelCallback(env, g_changeCallBacks, objectWrapper->GetObjectId()); JSDistributedObjectStore::DelCallback(env, g_statusCallBacks, objectWrapper->GetObjectId()); - + if (objectWrapper->GetObject() == nullptr) { delete objectWrapper; return; @@ -140,7 +134,7 @@ napi_value JSDistributedObjectStore::NewDistributedObject( delete objectWrapper; return nullptr; } - + RestoreWatchers(env, objectWrapper, objectId); objectStore->NotifyCachedStatus(object->GetSessionId()); NOT_MATCH_RETURN_NULL(status == napi_ok); @@ -336,9 +330,9 @@ bool JSDistributedObjectStore::GetBundleNameWithContext(napi_env env, napi_value std::string JSDistributedObjectStore::GetBundleName(napi_env env) { static std::string bundleName; - if (bundleName.empty()) { - bundleName = AbilityRuntime::Context::GetApplicationContext()->GetBundleName(); - } +// if (bundleName.empty()) { +// bundleName = AbilityRuntime::Context::GetApplicationContext()->GetBundleName(); +// } return bundleName; } @@ -494,11 +488,11 @@ napi_value JSDistributedObjectStore::JSEquenceNum(napi_env env, napi_callback_in // don't create distributed data object while this application is sandbox bool JSDistributedObjectStore::IsSandBox() { - int32_t dlpFlag = Security::AccessToken::AccessTokenKit::GetHapDlpFlag( - AbilityRuntime::Context::GetApplicationContext()->GetApplicationInfo()->accessTokenId); - if (dlpFlag != 0) { - return true; - } +// int32_t dlpFlag = Security::AccessToken::AccessTokenKit::GetHapDlpFlag( +// AbilityRuntime::Context::GetApplicationContext()->GetApplicationInfo()->accessTokenId); +// if (dlpFlag != 0) { +// return true; +// } return false; } } // namespace OHOS::ObjectStore diff --git a/data_object/frameworks/jskitsimpl/src/adaptor/js_module_init.cpp b/data_object/frameworks/jskitsimpl/src/adaptor/js_module_init.cpp index 88c757cd..f212cdda 100644 --- a/data_object/frameworks/jskitsimpl/src/adaptor/js_module_init.cpp +++ b/data_object/frameworks/jskitsimpl/src/adaptor/js_module_init.cpp @@ -16,7 +16,6 @@ #include "js_common.h" #include "js_distributedobjectstore.h" #include "logger.h" -#include "notifier_impl.h" using namespace OHOS::ObjectStore; diff --git a/data_object/frameworks/jskitsimpl/src/adaptor/js_object_wrapper.cpp b/data_object/frameworks/jskitsimpl/src/adaptor/js_object_wrapper.cpp index c024035a..cfa505f8 100644 --- a/data_object/frameworks/jskitsimpl/src/adaptor/js_object_wrapper.cpp +++ b/data_object/frameworks/jskitsimpl/src/adaptor/js_object_wrapper.cpp @@ -15,6 +15,7 @@ #include "js_object_wrapper.h" +#include #include "logger.h" namespace OHOS::ObjectStore { JSObjectWrapper::JSObjectWrapper(DistributedObjectStore *objectStore, DistributedObject *object) diff --git a/data_object/frameworks/jskitsimpl/src/adaptor/notifier_impl.cpp b/data_object/frameworks/jskitsimpl/src/adaptor/notifier_impl.cpp index 9b7f4f4a..8e8fe84c 100644 --- a/data_object/frameworks/jskitsimpl/src/adaptor/notifier_impl.cpp +++ b/data_object/frameworks/jskitsimpl/src/adaptor/notifier_impl.cpp @@ -28,7 +28,12 @@ std::shared_ptr NotifierImpl::GetInstance() std::lock_guard lockGuard(instanceLock); if (instance == nullptr) { instance = std::make_shared(); - uint32_t ret = DistributedObjectStore::GetInstance()->SetStatusNotifier(instance); + DistributedObjectStore *storeInstance = DistributedObjectStore::GetInstance(); + if (storeInstance == nullptr) { + LOG_ERROR("Get store instance nullptr"); + return instance; + } + auto ret = storeInstance->SetStatusNotifier(instance); if (ret != SUCCESS) { LOG_ERROR("SetStatusNotifier %{public}d error", ret); } else { diff --git a/data_object/frameworks/jskitsimpl/src/common/js_ability.cpp b/data_object/frameworks/jskitsimpl/src/common/js_ability.cpp index 2ecaeba8..611ef901 100644 --- a/data_object/frameworks/jskitsimpl/src/common/js_ability.cpp +++ b/data_object/frameworks/jskitsimpl/src/common/js_ability.cpp @@ -15,7 +15,6 @@ #include "js_ability.h" -#include "extension_context.h" #include "logger.h" namespace OHOS { diff --git a/data_object/frameworks/jskitsimpl/src/common/js_util.cpp b/data_object/frameworks/jskitsimpl/src/common/js_util.cpp index 3e29ffaa..f3360b46 100644 --- a/data_object/frameworks/jskitsimpl/src/common/js_util.cpp +++ b/data_object/frameworks/jskitsimpl/src/common/js_util.cpp @@ -14,11 +14,9 @@ */ #include "js_util.h" -#include #include #include "logger.h" -#include "common_types.h" namespace OHOS::ObjectStore { constexpr int32_t STR_MAX_LENGTH = 4096; diff --git a/data_object/frameworks/jskitsimpl/src/common/object_error.cpp b/data_object/frameworks/jskitsimpl/src/common/object_error.cpp index 425763b9..3440ca83 100644 --- a/data_object/frameworks/jskitsimpl/src/common/object_error.cpp +++ b/data_object/frameworks/jskitsimpl/src/common/object_error.cpp @@ -17,10 +17,10 @@ namespace OHOS { namespace ObjectStore { -static const int EXCEPTION_DEVICE_NOT_SUPPORT = 801; -static const int EXCEPTION_PARAMETER_CHECK = 401; -static const int EXCEPTION_NO_PERMISSION = 201; -static const int EXCEPTION_DB_EXIST = 15400001; +static constexpr int EXCEPTION_DEVICE_NOT_SUPPORT = 801; +static constexpr int EXCEPTION_PARAMETER_CHECK = 401; +static constexpr int EXCEPTION_NO_PERMISSION = 201; +static constexpr int EXCEPTION_DB_EXIST = 15400001; std::string ParametersType::GetMessage() { diff --git a/data_object/frameworks/jskitsimpl/src/common/uv_queue.cpp b/data_object/frameworks/jskitsimpl/src/common/uv_queue.cpp index ca2b1880..bad83162 100644 --- a/data_object/frameworks/jskitsimpl/src/common/uv_queue.cpp +++ b/data_object/frameworks/jskitsimpl/src/common/uv_queue.cpp @@ -13,6 +13,7 @@ * limitations under the License. */ #include "uv_queue.h" + #include #include "logger.h" diff --git a/data_object/frameworks/jskitsimpl/test/unittest/src/BUILD.gn b/data_object/frameworks/jskitsimpl/test/unittest/src/BUILD.gn index aca50301..c71fb5fc 100644 --- a/data_object/frameworks/jskitsimpl/test/unittest/src/BUILD.gn +++ b/data_object/frameworks/jskitsimpl/test/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "data_object/napi" +module_output_path = "data_object/data_object/napi" ohos_js_unittest("ObjectStoreJsTest") { module_out_path = module_output_path diff --git a/data_object/frameworks/jskitsimpl/test/unittest/src/ObjectStoreJsunit.test.js b/data_object/frameworks/jskitsimpl/test/unittest/src/ObjectStoreJsunit.test.js index aed6d72f..d6fc7c0c 100644 --- a/data_object/frameworks/jskitsimpl/test/unittest/src/ObjectStoreJsunit.test.js +++ b/data_object/frameworks/jskitsimpl/test/unittest/src/ObjectStoreJsunit.test.js @@ -100,7 +100,7 @@ describe('objectStoreTest',function () { * @tc.name: testOn001 * @tc.desc: object join session and on,object can receive callback when data has been changed * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOn001', 0, function () { console.log(TAG + "************* testOn001 start *************"); @@ -138,7 +138,7 @@ describe('objectStoreTest',function () { * @tc.name: testOn002 * @tc.desc object join session and no on,obejct can not receive callback when data has been changed * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOn002', 0, function () { console.log(TAG + "************* testOn002 start *************"); @@ -165,7 +165,7 @@ describe('objectStoreTest',function () { * @tc.name: testOn003 * @tc.desc: object join session and on,then object change data twice,object can receive two callbacks when data has been changed * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOn003', 0, function () { console.log(TAG + "************* testOn003 start *************"); @@ -199,7 +199,7 @@ describe('objectStoreTest',function () { * @tc.name: testOn004 * @tc.desc object join session and on,then object do not change data,object can not receive callbacks * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOn004', 0, function () { console.log(TAG + "************* testOn004 start *************"); @@ -218,7 +218,7 @@ describe('objectStoreTest',function () { * @tc.name testOff001 * @tc.desc object join session and on&off,object can not receive callback after off * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOff001', 0, function () { console.log(TAG + "************* testOff001 start *************"); @@ -260,7 +260,7 @@ describe('objectStoreTest',function () { * @tc.name:testOff002 * @tc.desc object join session and off,object can not receive callback * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testOff002', 0, function () { console.log(TAG + "************* testOff002 start *************"); @@ -289,7 +289,7 @@ describe('objectStoreTest',function () { * @tc.name: testMultiObjectOn001 * @tc.desc: two objects join session and on,then object change data,user can receive two callbacks from two objects * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testMultiObjectOn001', 0, function () { console.log(TAG + "************* testMultiObjectOn001 start *************"); @@ -331,7 +331,7 @@ describe('objectStoreTest',function () { * @tc.name: testMultiObjectOff001 * @tc.desc: two objects join session and on&off,then two objects can not receive callbacks * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testMultiObjectOff001', 0, function () { console.log(TAG + "************* testMultiObjectOff001 start *************"); @@ -402,7 +402,7 @@ describe('objectStoreTest',function () { * @tc.name: testChangeSession001 * @tc.desc: objects join session and on,then change sessionId * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testChangeSession001', 0, function () { console.log(TAG + "************* testChangeSession001 start *************"); @@ -446,7 +446,7 @@ describe('objectStoreTest',function () { * @tc.name: testUndefinedType001 * @tc.desc: object use undefined type,can not join session * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testUndefinedType001', 0, function () { console.log(TAG + "************* testUndefinedType001 start *************"); @@ -468,7 +468,7 @@ describe('objectStoreTest',function () { * @tc.name: testGenSessionId001 * @tc.desc: object generate random sessionId * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testGenSessionId001', 0, function () { console.log(TAG + "************* testGenSessionId001 start *************"); @@ -482,7 +482,7 @@ describe('objectStoreTest',function () { * @tc.name: testGenSessionId002 * @tc.desc: object generate 2 random sessionId and not equal * @tc.type: FUNC - * @tc.require: I4H3LS + * @tc.require: */ it('testGenSessionId002', 0, function () { console.log(TAG + "************* testGenSessionId002 start *************"); @@ -497,7 +497,7 @@ describe('objectStoreTest',function () { * @tc.name: testOnStatus001 * @tc.desc: object set a listener to watch another object online/offline * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testOnStatus001', 0, function () { console.log(TAG + "************* testOnStatus001 start *************"); @@ -514,7 +514,7 @@ describe('objectStoreTest',function () { * @tc.name: testOnStatus002 * @tc.desc: object set several listener and can unset specified listener * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testOnStatus002', 0, function () { console.log(TAG + "************* testOnStatus002 start *************"); @@ -539,7 +539,7 @@ describe('objectStoreTest',function () { * @tc.name: testOnStatus003 * @tc.desc: object set several listener and can unWatch all watcher * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testOnStatus003', 0, function () { console.log(TAG + "************* testOnStatus003 start *************"); @@ -565,7 +565,7 @@ describe('objectStoreTest',function () { * @tc.name: testComplex001 * @tc.desc: object can get/set complex data * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testComplex001', 0, function () { console.log(TAG + "************* testComplex001 start *************"); @@ -600,7 +600,7 @@ describe('objectStoreTest',function () { * @tc.name: testMaxSize001 * @tc.desc: object can get/set data under 4MB size * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testMaxSize001', 0, function () { console.log(TAG + "************* testMaxSize001 start *************"); @@ -630,7 +630,7 @@ describe('objectStoreTest',function () { * @tc.name: testPerformance001 * @tc.desc: performanceTest for set/get data * @tc.type: FUNC - * @tc.require: I4H3M8 + * @tc.require: */ it('testPerformance001', 0, function () { console.log(TAG + "************* testPerformance001 start *************"); @@ -752,7 +752,7 @@ describe('objectStoreTest',function () { * @tc.name: testRevokeSave001 * @tc.desc: test save local * @tc.type: FUNC - * @tc.require: I4WDAK + * @tc.require: */ it('testRevokeSave001', 0, async function (done) { console.log(TAG + "************* testRevokeSave001 start *************"); @@ -846,7 +846,7 @@ describe('objectStoreTest',function () { * @tc.name: OnstatusRestored * @tc.desc: test local device data restored * @tc.type: FUNC - * @tc.require: I5OXHH + * @tc.require: */ it('OnstatusRestored001', 0, async function () { console.log(TAG + "************* OnstatusRestored001 start *************"); diff --git a/data_object/frameworks/jskitsimpl/test/unittest/src/config.json b/data_object/frameworks/jskitsimpl/test/unittest/src/config.json index 68d3d755..917dfaf8 100644 --- a/data_object/frameworks/jskitsimpl/test/unittest/src/config.json +++ b/data_object/frameworks/jskitsimpl/test/unittest/src/config.json @@ -78,4 +78,4 @@ } ] } -} +} \ No newline at end of file diff --git a/data_object/interfaces/innerkits/distributed_object.h b/data_object/interfaces/innerkits/distributed_object.h index 71a931f0..a17b0597 100644 --- a/data_object/interfaces/innerkits/distributed_object.h +++ b/data_object/interfaces/innerkits/distributed_object.h @@ -31,7 +31,7 @@ enum Type : uint8_t { class DistributedObject { public: virtual ~DistributedObject(){}; - + /** * @brief Put or update the data whose value type is double into the database, which means that the data of * objects in the same sessionId is put or updated. @@ -42,7 +42,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t PutDouble(const std::string &key, double value) = 0; - + /** * @brief Put or update the data whose value type is bool into the database, which means that the data of * objects in the same sessionId is put or updated. @@ -53,7 +53,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t PutBoolean(const std::string &key, bool value) = 0; - + /** * @brief Put or update the data whose value type is string into the database, which means that the data of * objects in the same sessionId is put or updated. @@ -64,7 +64,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t PutString(const std::string &key, const std::string &value) = 0; - + /** * @brief Put or update the data whose value type is bytes stream into the database, which means that the data of * objects in the same sessionId is put or updated. @@ -75,7 +75,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t PutComplex(const std::string &key, const std::vector &value) = 0; - + /** * @brief Get the data whose value type is double from the database according to the key, * which means that the data of objects in the same sessionId is get. @@ -86,7 +86,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t GetDouble(const std::string &key, double &value) = 0; - + /** * @brief Get the data whose value type is bool from the database according to the key, * which means that the data of objects in the same sessionId is get. @@ -97,7 +97,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t GetBoolean(const std::string &key, bool &value) = 0; - + /** * @brief Get the data whose value type is string from the database according to the key, * which means that the data of objects in the same sessionId is get. @@ -108,7 +108,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t GetString(const std::string &key, std::string &value) = 0; - + /** * @brief Get the data whose value type is complex from the database according to the key, * which means that the data of objects in the same sessionId is get. @@ -119,7 +119,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t GetComplex(const std::string &key, std::vector &value) = 0; - + /** * @brief Get the value type of key-value data by the key * @@ -129,7 +129,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t GetType(const std::string &key, Type &type) = 0; - + /** * @brief Save the data to local device. * @@ -138,14 +138,14 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t Save(const std::string &deviceId) = 0; - + /** * @brief Revoke save data. * * @return Returns 0 for success, others for failure. */ virtual uint32_t RevokeSave() = 0; - + /** * @brief Get the sessionId of the object. * diff --git a/data_object/interfaces/innerkits/distributed_objectstore.h b/data_object/interfaces/innerkits/distributed_objectstore.h index dab7f78c..305a4772 100644 --- a/data_object/interfaces/innerkits/distributed_objectstore.h +++ b/data_object/interfaces/innerkits/distributed_objectstore.h @@ -15,10 +15,7 @@ #ifndef DISTRIBUTED_OBJECTSTORE_H #define DISTRIBUTED_OBJECTSTORE_H -#include #include -#include -#include #include "distributed_object.h" @@ -31,7 +28,7 @@ public: class DistributedObjectStore { public: virtual ~DistributedObjectStore(){}; - + /** * @brief Get the instance to handle the object, such as create the object. * @@ -40,7 +37,7 @@ public: * @return Returns the pointer to the DistributedObjectStore class. */ static DistributedObjectStore *GetInstance(const std::string &bundleName = ""); - + /** * @brief Create a object according to the sessionId. * @@ -49,7 +46,7 @@ public: * @return Returns the pointer to the DistributedObject class. */ virtual DistributedObject *CreateObject(const std::string &sessionId) = 0; - + /** * @brief Create a object according to the sessionId. * @@ -60,7 +57,7 @@ public: * @return Returns the pointer to the DistributedObject class. */ virtual DistributedObject *CreateObject(const std::string &sessionId, uint32_t &status) = 0; - + /** * @brief Get the double pointer to the object. * @@ -70,7 +67,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t Get(const std::string &sessionId, DistributedObject **object) = 0; - + /** * @brief Delete the object according to the sessionId. * @@ -79,7 +76,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t DeleteObject(const std::string &sessionId) = 0; - + /** * @brief Set listening for data changes. * @@ -89,7 +86,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t Watch(DistributedObject *object, std::shared_ptr objectWatcher) = 0; - + /** * @brief Undo listening for data changes. * @@ -98,7 +95,7 @@ public: * @return Returns the pointer to the DistributedObject class. */ virtual uint32_t UnWatch(DistributedObject *object) = 0; - + /** * @brief Set listening for device online and offline . * @@ -107,7 +104,7 @@ public: * @return Returns 0 for success, others for failure. */ virtual uint32_t SetStatusNotifier(std::shared_ptr notifier) = 0; - + /** * @brief Notify the status of the local device from the cached callback function according to the sessionId. * diff --git a/data_object/interfaces/innerkits/object_types.h b/data_object/interfaces/innerkits/object_types.h index 2bccc8a4..93286743 100644 --- a/data_object/interfaces/innerkits/object_types.h +++ b/data_object/interfaces/innerkits/object_types.h @@ -34,14 +34,14 @@ using ValuesBucket = CommonType::ValuesBucket; using ValueObject = CommonType::Value; using Assets = std::vector; -static const std::string STATUS_SUFFIX = ".status"; -static const std::string NAME_SUFFIX = ".name"; -static const std::string URI_SUFFIX = ".uri"; -static const std::string PATH_SUFFIX = ".path"; -static const std::string CREATE_TIME_SUFFIX = ".createTime"; -static const std::string MODIFY_TIME_SUFFIX = ".modifyTime"; -static const std::string SIZE_SUFFIX = ".size"; -static const std::string ASSET_DOT = "."; +static constexpr const char* STATUS_SUFFIX = ".status"; +static constexpr const char* NAME_SUFFIX = ".name"; +static constexpr const char* URI_SUFFIX = ".uri"; +static constexpr const char* PATH_SUFFIX = ".path"; +static constexpr const char* CREATE_TIME_SUFFIX = ".createTime"; +static constexpr const char* MODIFY_TIME_SUFFIX = ".modifyTime"; +static constexpr const char* SIZE_SUFFIX = ".size"; +static constexpr const char* ASSET_DOT = "."; static const std::string DEVICEID_KEY = "__deviceId"; } // namespace ObjectStore } // namespace OHOS diff --git a/data_object/interfaces/innerkits/objectstore_errors.h b/data_object/interfaces/innerkits/objectstore_errors.h index 8794d5eb..883393be 100644 --- a/data_object/interfaces/innerkits/objectstore_errors.h +++ b/data_object/interfaces/innerkits/objectstore_errors.h @@ -15,8 +15,6 @@ #ifndef OBJECTSTORE_ERRORS_H #define OBJECTSTORE_ERRORS_H -#include - namespace OHOS::ObjectStore { constexpr uint32_t BASE_ERR_OFFSET = 1650; diff --git a/data_share/CMakeLists.txt b/data_share/CMakeLists.txt index 682611a5..7f729357 100644 --- a/data_share/CMakeLists.txt +++ b/data_share/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.2) project(data_share) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "-std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") diff --git a/data_share/frameworks/js/napi/common/src/datashare_js_utils.cpp b/data_share/frameworks/js/napi/common/src/datashare_js_utils.cpp index ea3d66c4..3b0471bf 100644 --- a/data_share/frameworks/js/napi/common/src/datashare_js_utils.cpp +++ b/data_share/frameworks/js/napi/common/src/datashare_js_utils.cpp @@ -539,11 +539,8 @@ Template DataShareJSUtils::Convert2Template(napi_env env, napi_value value) LOG_ERROR("Convert2Template error, value is not object"); return {}; } - std::string update; - if (!UnwrapStringByPropertyName(env, value, "update", update)) { - LOG_INFO("Parameter update undefined"); - update = ""; - } + std::string update = ""; + UnwrapStringByPropertyName(env, value, "update", update); napi_value jsPredicates; auto status = napi_get_named_property(env, value, "predicates", &jsPredicates); diff --git a/data_share/frameworks/js/napi/dataShare/BUILD.gn b/data_share/frameworks/js/napi/dataShare/BUILD.gn index 70a6fb3f..6a2e9ef4 100644 --- a/data_share/frameworks/js/napi/dataShare/BUILD.gn +++ b/data_share/frameworks/js/napi/dataShare/BUILD.gn @@ -99,6 +99,7 @@ ohos_shared_library("datashare") { "ability_runtime:extensionkit_native", "ability_runtime:napi_base_context", "ability_runtime:napi_common", + "access_token:libtokenid_sdk", "c_utils:utils", "common_event_service:cesfwk_innerkits", "hilog:libhilog", diff --git a/data_share/frameworks/js/napi/dataShare/src/napi_datashare_helper.cpp b/data_share/frameworks/js/napi/dataShare/src/napi_datashare_helper.cpp index 884da5ff..531c5a8d 100644 --- a/data_share/frameworks/js/napi/dataShare/src/napi_datashare_helper.cpp +++ b/data_share/frameworks/js/napi/dataShare/src/napi_datashare_helper.cpp @@ -24,14 +24,24 @@ #include "napi_base_context.h" #include "napi_common_util.h" #include "napi_datashare_values_bucket.h" +#include "tokenid_kit.h" using namespace OHOS::AAFwk; using namespace OHOS::AppExecFwk; +using namespace OHOS::Security::AccessToken; namespace OHOS { namespace DataShare { static constexpr int MAX_ARGC = 6; +static constexpr int EXCEPTION_SYSTEMAPP_CHECK = 202; static thread_local napi_ref constructor_ = nullptr; + +static bool IsSystemApp() +{ + uint64_t tokenId = IPCSkeleton::GetSelfTokenID(); + return TokenIdKit::IsSystemAppByFullTokenID(tokenId); +} + static bool GetSilentUri(napi_env env, napi_value jsValue, std::string &uri) { napi_valuetype valuetype = napi_undefined; @@ -137,10 +147,10 @@ void NapiDataShareHelper::ExecuteCreator(std::shared_ptr ctxI if (ctxInfo->options.enabled_) { ctxInfo->options.token_ = ctxInfo->contextS->GetToken(); ctxInfo->dataShareHelper = DataShareHelper::Creator(ctxInfo->strUri, ctxInfo->options, "", - ctxInfo->options.waitTime_); + ctxInfo->options.waitTime_, true); } else { ctxInfo->dataShareHelper = DataShareHelper::Creator(ctxInfo->contextS->GetToken(), ctxInfo->strUri, "", - ctxInfo->options.waitTime_); + ctxInfo->options.waitTime_, true); } } @@ -148,6 +158,9 @@ napi_value NapiDataShareHelper::Napi_CreateDataShareHelper(napi_env env, napi_ca { auto ctxInfo = std::make_shared(); auto input = [ctxInfo](napi_env env, size_t argc, napi_value *argv, napi_value self) -> napi_status { + NAPI_ASSERT_CALL_ERRCODE(env, IsSystemApp(), + ctxInfo->error = std::make_shared(EXCEPTION_SYSTEMAPP_CHECK, "not system app"), + napi_generic_failure); NAPI_ASSERT_CALL_ERRCODE(env, argc == 2 || argc == 3 || argc == 4, ctxInfo->error = std::make_shared("2 or 3 or 4"), napi_invalid_arg); ctxInfo->contextS = OHOS::AbilityRuntime::GetStageModeContext(env, argv[0]); @@ -367,14 +380,8 @@ napi_value NapiDataShareHelper::Napi_Query(napi_env env, napi_callback_info info return napi_ok; }; auto output = [context](napi_env env, napi_value *result) -> napi_status { - if (context->businessError.GetCode() != 0) { + if (context->resultObject == nullptr || context->businessError.GetCode() != 0) { LOG_DEBUG("query failed, errorCode : %{public}d", context->businessError.GetCode()); - context->error = std::make_shared(context->businessError.GetCode(), - context->businessError.GetMessage()); - return napi_generic_failure; - } - - if (context->resultObject == nullptr) { context->error = std::make_shared(); return napi_generic_failure; } @@ -1302,11 +1309,20 @@ napi_value NapiDataShareHelper::SetSilentSwitch(napi_env env, napi_callback_info return napi_ok; }; auto output = [context](napi_env env, napi_value *result) -> napi_status { + if (context->error == nullptr) { + return napi_ok; + } + NAPI_ASSERT_CALL_ERRCODE(env, context->error->GetCode() != EXCEPTION_SYSTEMAPP_CHECK, + context->error = std::make_shared(EXCEPTION_SYSTEMAPP_CHECK, "not system app"), + napi_generic_failure); return napi_ok; }; auto exec = [context](AsyncCall::Context *ctx) { OHOS::Uri uri(context->strUri); - DataShareHelper::SetSilentSwitch(uri, context->silentSwitch); + int res = DataShareHelper::SetSilentSwitch(uri, context->silentSwitch, true); + if (res == E_NOT_SYSTEM_APP) { + context->error = std::make_shared(EXCEPTION_SYSTEMAPP_CHECK, "not system app"); + } }; context->SetAction(std::move(input), std::move(output)); AsyncCall asyncCall(env, info, context); diff --git a/data_share/frameworks/js/napi/observer/include/napi_observer.h b/data_share/frameworks/js/napi/observer/include/napi_observer.h index 18bede72..d5d2cca2 100644 --- a/data_share/frameworks/js/napi/observer/include/napi_observer.h +++ b/data_share/frameworks/js/napi/observer/include/napi_observer.h @@ -28,7 +28,7 @@ namespace DataShare { class NapiObserver { public: NapiObserver(napi_env env, napi_value callback); - ~NapiObserver(); + virtual ~NapiObserver(); virtual bool operator==(const NapiObserver &rhs) const; virtual bool operator!=(const NapiObserver &rhs) const; NapiObserver& operator=(NapiObserver &&rhs) = default; diff --git a/data_share/frameworks/js/napi/observer/include/napi_subscriber_manager.h b/data_share/frameworks/js/napi/observer/include/napi_subscriber_manager.h index 16068300..3c06ef06 100644 --- a/data_share/frameworks/js/napi/observer/include/napi_subscriber_manager.h +++ b/data_share/frameworks/js/napi/observer/include/napi_subscriber_manager.h @@ -19,6 +19,7 @@ #include #include +#include "concurrent_map.h" #include "napi_callbacks_manager.h" #include "datashare_helper.h" #include "napi/native_api.h" @@ -69,7 +70,7 @@ public: private: void Emit(const std::vector &keys, const std::shared_ptr &observer); std::weak_ptr dataShareHelper_; - std::map lastChangeNodeMap_; + ConcurrentMap lastChangeNodeMap_; }; struct NapiPublishedObserverMapKey { @@ -114,7 +115,7 @@ public: private: void Emit(const std::vector &keys, const std::shared_ptr &observer); std::weak_ptr dataShareHelper_; - std::map lastChangeNodeMap_; + ConcurrentMap lastChangeNodeMap_; }; } // namespace DataShare } // namespace OHOS diff --git a/data_share/frameworks/js/napi/observer/src/napi_observer.cpp b/data_share/frameworks/js/napi/observer/src/napi_observer.cpp index 484583c3..3b5fbfcf 100644 --- a/data_share/frameworks/js/napi/observer/src/napi_observer.cpp +++ b/data_share/frameworks/js/napi/observer/src/napi_observer.cpp @@ -62,7 +62,13 @@ void NapiObserver::CallbackFunc(ObserverWorker *observerWorker) NapiObserver::~NapiObserver() { if (ref_ != nullptr) { - napi_delete_reference(env_, ref_); + auto task = [env = env_, ref = ref_]() { + napi_delete_reference(env, ref); + }; + int ret = napi_send_event(env_, task, napi_eprio_immediate); + if (ret != 0) { + LOG_ERROR("napi_send_event failed: %{public}d", ret); + } ref_ = nullptr; } } diff --git a/data_share/frameworks/js/napi/observer/src/napi_subscriber_manager.cpp b/data_share/frameworks/js/napi/observer/src/napi_subscriber_manager.cpp index 2745d003..493d4544 100644 --- a/data_share/frameworks/js/napi/observer/src/napi_subscriber_manager.cpp +++ b/data_share/frameworks/js/napi/observer/src/napi_subscriber_manager.cpp @@ -81,7 +81,7 @@ std::vector NapiRdbSubscriberManager::DelObservers(napi_env env const std::shared_ptr &observer, std::vector &opResult) { std::vector lastDelUris; std::for_each(lastDelKeys.begin(), lastDelKeys.end(), [&lastDelUris, this](auto &result) { - lastChangeNodeMap_.erase(result); + lastChangeNodeMap_.Erase(result); lastDelUris.emplace_back(result); }); if (lastDelUris.empty()) { @@ -95,7 +95,7 @@ std::vector NapiRdbSubscriberManager::DelObservers(napi_env env void NapiRdbSubscriberManager::Emit(const RdbChangeNode &changeNode) { Key key(changeNode.uri_, changeNode.templateId_); - lastChangeNodeMap_[key] = changeNode; + lastChangeNodeMap_.InsertOrAssign(key, changeNode); auto callbacks = BaseCallbacks::GetEnabledObservers(key); for (auto &obs : callbacks) { if (obs != nullptr) { @@ -107,9 +107,15 @@ void NapiRdbSubscriberManager::Emit(const RdbChangeNode &changeNode) void NapiRdbSubscriberManager::Emit(const std::vector &keys, const std::shared_ptr &observer) { for (auto const &key : keys) { - auto it = lastChangeNodeMap_.find(key); - if (it != lastChangeNodeMap_.end()) { - observer->OnChange(it->second); + bool isExist = false; + RdbChangeNode node; + lastChangeNodeMap_.ComputeIfPresent(key, [&node, &isExist](const Key &, const RdbChangeNode &value) { + node = value; + isExist = true; + return true; + }); + if (isExist) { + observer->OnChange(node); } } } @@ -176,7 +182,7 @@ std::vector NapiPublishedSubscriberManager::DelObservers(napi_e const std::shared_ptr &observer, std::vector &opResult) { std::vector lastDelUris; std::for_each(lastDelKeys.begin(), lastDelKeys.end(), [&lastDelUris, this](auto &result) { - lastChangeNodeMap_.erase(result); + lastChangeNodeMap_.Erase(result); lastDelUris.emplace_back(result); }); if (lastDelUris.empty()) { @@ -191,7 +197,10 @@ void NapiPublishedSubscriberManager::Emit(const PublishedDataChangeNode &changeN { for (auto &data : changeNode.datas_) { Key key(data.key_, data.subscriberId_); - lastChangeNodeMap_[key].datas_.clear(); + lastChangeNodeMap_.Compute(key, [](const Key &, PublishedDataChangeNode &value) { + value.datas_.clear(); + return true; + }); } std::map, PublishedDataChangeNode> results; for (auto &data : changeNode.datas_) { @@ -201,8 +210,11 @@ void NapiPublishedSubscriberManager::Emit(const PublishedDataChangeNode &changeN LOG_WARN("%{private}s nobody subscribe, but still notify", data.key_.c_str()); continue; } - lastChangeNodeMap_[key].datas_.emplace_back(data.key_, data.subscriberId_, data.GetData()); - lastChangeNodeMap_[key].ownerBundleName_ = changeNode.ownerBundleName_; + lastChangeNodeMap_.Compute(key, [&data, &changeNode](const Key &, PublishedDataChangeNode &value) { + value.datas_.emplace_back(data.key_, data.subscriberId_, data.GetData()); + value.ownerBundleName_ = changeNode.ownerBundleName_; + return true; + }); for (auto const &obs : callbacks) { results[obs].datas_.emplace_back(data.key_, data.subscriberId_, data.GetData()); } @@ -217,14 +229,13 @@ void NapiPublishedSubscriberManager::Emit(const std::vector &keys, const st { PublishedDataChangeNode node; for (auto &key : keys) { - auto it = lastChangeNodeMap_.find(key); - if (it == lastChangeNodeMap_.end()) { - continue; - } - for (auto &data : it->second.datas_) { - node.datas_.emplace_back(data.key_, data.subscriberId_, data.GetData()); - } - node.ownerBundleName_ = it->second.ownerBundleName_; + lastChangeNodeMap_.ComputeIfPresent(key, [&node](const Key &, PublishedDataChangeNode &value) { + for (auto &data : value.datas_) { + node.datas_.emplace_back(data.key_, data.subscriberId_, data.GetData()); + } + node.ownerBundleName_ = value.ownerBundleName_; + return true; + }); } observer->OnChange(node); } diff --git a/data_share/frameworks/native/common/include/callbacks_manager.h b/data_share/frameworks/native/common/include/callbacks_manager.h index 052b1969..3c7fcd3f 100644 --- a/data_share/frameworks/native/common/include/callbacks_manager.h +++ b/data_share/frameworks/native/common/include/callbacks_manager.h @@ -294,7 +294,6 @@ std::vector CallbacksManager::DisableObservers(c for (auto &item : callbacks_[key]) { if (item.subscriber_ == subscriber) { item.enabled_ = false; - item.isNotifyOnEnabled_ = false; hasDisabled = true; } } diff --git a/data_share/frameworks/native/common/include/datashare_log.h b/data_share/frameworks/native/common/include/datashare_log.h index 24eb67c3..28c18b27 100644 --- a/data_share/frameworks/native/common/include/datashare_log.h +++ b/data_share/frameworks/native/common/include/datashare_log.h @@ -16,7 +16,6 @@ #ifndef DATASHARE_LOG_PRINT_H #define DATASHARE_LOG_PRINT_H -#include #include "hilog/log.h" namespace OHOS::DataShare { diff --git a/data_share/frameworks/native/common/include/datashare_radar_reporter.h b/data_share/frameworks/native/common/include/datashare_radar_reporter.h index 6860f54c..868e971a 100644 --- a/data_share/frameworks/native/common/include/datashare_radar_reporter.h +++ b/data_share/frameworks/native/common/include/datashare_radar_reporter.h @@ -138,8 +138,6 @@ class RadarReport final { public: RadarReport(int32_t bizScene, int32_t bizStage, const std::string funcName) { - RADAR_REPORT(funcName, bizScene, bizStage, RadarReporter::SUCCESS, - RadarReporter::BIZ_STATE, RadarReporter::START); bizScene_ = bizScene; bizStage_ = bizStage; funcName_ = funcName; @@ -150,9 +148,6 @@ public: if (errorCode_ != 0) { RADAR_REPORT(funcName_, bizScene_, bizStage_, RadarReporter::FAILED, RadarReporter::BIZ_STATE, RadarReporter::FINISHED, RadarReporter::ERROR_CODE, errorCode_); - } else { - RADAR_REPORT(funcName_, bizScene_, bizStage_, RadarReporter::SUCCESS, - RadarReporter::BIZ_STATE, RadarReporter::FINISHED); } } diff --git a/data_share/frameworks/native/common/include/distributeddata_data_share_ipc_interface_code.h b/data_share/frameworks/native/common/include/distributeddata_data_share_ipc_interface_code.h index 83f03e38..2b35d14c 100644 --- a/data_share/frameworks/native/common/include/distributeddata_data_share_ipc_interface_code.h +++ b/data_share/frameworks/native/common/include/distributeddata_data_share_ipc_interface_code.h @@ -74,6 +74,8 @@ enum class ISharedResultInterfaceCode { FUNC_BUTT, }; +static constexpr int DATA_SHARE_CMD_SYSTEM_CODE = 100; + enum class DataShareServiceInterfaceCode { DATA_SHARE_SERVICE_CMD_QUERY, DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE, @@ -97,7 +99,30 @@ enum class DataShareServiceInterfaceCode { DATA_SHARE_SERVICE_CMD_INSERTEX, DATA_SHARE_SERVICE_CMD_DELETEEX, DATA_SHARE_SERVICE_CMD_UPDATEEX, - DATA_SHARE_SERVICE_CMD_MAX + DATA_SHARE_SERVICE_CMD_MAX, + DATA_SHARE_SERVICE_CMD_QUERY_SYSTEM = DATA_SHARE_CMD_SYSTEM_CODE, + DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE_SYSTEM, + DATA_SHARE_SERVICE_CMD_DEL_TEMPLATE_SYSTEM, + DATA_SHARE_SERVICE_CMD_PUBLISH_SYSTEM, + DATA_SHARE_SERVICE_CMD_GET_DATA_SYSTEM, + DATA_SHARE_SERVICE_CMD_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_NOTIFY_SYSTEM, + DATA_SHARE_SERVICE_CMD_NOTIFY_OBSERVERS_SYSTEM, + DATA_SHARE_SERVICE_CMD_SET_SILENT_SWITCH_SYSTEM, + DATA_SHARE_SERVICE_CMD_GET_SILENT_PROXY_STATUS_SYSTEM, + DATA_SHARE_SERVICE_CMD_REGISTER_OBSERVER_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNREGISTER_OBSERVER_SYSTEM, + DATA_SHARE_SERVICE_CMD_INSERTEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_DELETEEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_UPDATEEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_MAX_SYSTEM, }; enum class IKvStoreDataInterfaceCode { diff --git a/data_share/frameworks/native/common/src/datashare_itypes_utils.cpp b/data_share/frameworks/native/common/src/datashare_itypes_utils.cpp index 8c76985b..a59643b7 100644 --- a/data_share/frameworks/native/common/src/datashare_itypes_utils.cpp +++ b/data_share/frameworks/native/common/src/datashare_itypes_utils.cpp @@ -925,12 +925,12 @@ bool MarshalPredicates(const Predicates &predicates, MessageParcel &parcel) bool UnmarshalPredicates(Predicates &predicates, MessageParcel &parcel) { - size_t length = parcel.ReadInt32(); + int32_t length = parcel.ReadInt32(); if (length < 1) { LOG_ERROR("Length of predicates is invalid."); return false; } - const char *buffer = reinterpret_cast(parcel.ReadRawData((size_t)length)); + const char *buffer = reinterpret_cast(parcel.ReadRawData(static_cast(length))); if (buffer == nullptr) { LOG_ERROR("ReadRawData failed."); return false; @@ -961,12 +961,12 @@ bool MarshalValuesBucketVec(const std::vector &values, Me bool UnmarshalValuesBucketVec(std::vector &values, MessageParcel &parcel) { - size_t length = parcel.ReadInt32(); + int32_t length = parcel.ReadInt32(); if (length < 1) { LOG_ERROR("Length of ValuesBucketVec is invalid."); return false; } - const char *buffer = reinterpret_cast(parcel.ReadRawData((size_t)length)); + const char *buffer = reinterpret_cast(parcel.ReadRawData(static_cast(length))); if (buffer == nullptr) { LOG_ERROR("ReadRawData failed."); return false; diff --git a/data_share/frameworks/native/common/src/ishared_result_set_stub.cpp b/data_share/frameworks/native/common/src/ishared_result_set_stub.cpp index a837ade9..08e102be 100644 --- a/data_share/frameworks/native/common/src/ishared_result_set_stub.cpp +++ b/data_share/frameworks/native/common/src/ishared_result_set_stub.cpp @@ -13,15 +13,18 @@ * limitations under the License. */ +#include #include "ishared_result_set_stub.h" #include "datashare_log.h" #include "datashare_errno.h" +#include "ipc_skeleton.h" #include "string_ex.h" namespace OHOS::DataShare { std::function(std::shared_ptr, MessageParcel &)> ISharedResultSet::providerCreator_ = ISharedResultSetStub::CreateStub; constexpr ISharedResultSetStub::Handler ISharedResultSetStub::handlers[static_cast(ResultCode::FUNC_BUTT)]; +const std::chrono::milliseconds TIME_THRESHOLD = std::chrono::milliseconds(500); sptr ISharedResultSetStub::CreateStub(std::shared_ptr result, OHOS::MessageParcel &parcel) @@ -60,16 +63,26 @@ int ISharedResultSetStub::OnRemoteRequest(uint32_t code, OHOS::MessageParcel &da return INVALID_FD; } + auto callingPid = IPCSkeleton::GetCallingPid(); if (code >= static_cast(ResultCode::FUNC_BUTT)) { - LOG_ERROR("method code(%{public}d) out of range", code); + LOG_ERROR("method code(%{public}d) out of range, callingPid:%{public}d", code, callingPid); return IPCObjectStub::OnRemoteRequest(code, data, reply, option); } Handler handler = handlers[code]; if (handler == nullptr) { - LOG_ERROR("method code(%{public}d) is not support", code); + LOG_ERROR("method code(%{public}d) is not support, callingPid:%{public}d", code, callingPid); return IPCObjectStub::OnRemoteRequest(code, data, reply, option); } - return (this->*handler)(data, reply); + auto start = std::chrono::steady_clock::now(); + int ret = (this->*handler)(data, reply); + auto finish = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(finish - start); + if (duration >= TIME_THRESHOLD) { + int64_t milliseconds = duration.count(); + LOG_WARN("over time, code:%{public}u callingPid:%{public}d, cost:%{public}" PRIi64 "ms", + code, callingPid, milliseconds); + } + return ret; } int ISharedResultSetStub::HandleGetRowCountRequest(MessageParcel &data, MessageParcel &reply) diff --git a/data_share/frameworks/native/consumer/include/datashare_helper_impl.h b/data_share/frameworks/native/consumer/include/datashare_helper_impl.h index 6ab631ba..d26fc000 100644 --- a/data_share/frameworks/native/consumer/include/datashare_helper_impl.h +++ b/data_share/frameworks/native/consumer/include/datashare_helper_impl.h @@ -26,8 +26,8 @@ namespace OHOS::DataShare { class DataShareHelperImpl : public DataShareHelper { public: DataShareHelperImpl(const Uri &uri, const sptr &token, - std::shared_ptr connection); - DataShareHelperImpl(std::string extUri = ""); + std::shared_ptr connection, bool isSystem = false); + DataShareHelperImpl(std::string extUri = "", bool isSystem = false); ~DataShareHelperImpl() override; @@ -113,6 +113,7 @@ private: std::shared_ptr generalCtl_ = nullptr; std::shared_ptr persistentDataCtl_ = nullptr; std::shared_ptr publishedDataCtl_ = nullptr; + bool isSystem_; }; } // namespace OHOS::DataShare #endif // DATA_SHARE_HELPER_IMPL_H diff --git a/data_share/frameworks/native/consumer/src/datashare_helper.cpp b/data_share/frameworks/native/consumer/src/datashare_helper.cpp index c2a71b2e..f0243cb7 100644 --- a/data_share/frameworks/native/consumer/src/datashare_helper.cpp +++ b/data_share/frameworks/native/consumer/src/datashare_helper.cpp @@ -74,8 +74,8 @@ std::string DataShareHelper::TransferUriPrefix(const std::string &originPrefix, * * @return Returns the created DataShareHelper instance. */ -std::shared_ptr DataShareHelper::Creator( - const sptr &token, const std::string &strUri, const std::string &extUri, const int waitTime) +std::shared_ptr DataShareHelper::Creator(const sptr &token, const std::string &strUri, + const std::string &extUri, const int waitTime, bool isSystem) { DISTRIBUTED_DATA_HITRACE(std::string(LOG_TAG) + "::" + std::string(__FUNCTION__)); if (token == nullptr) { @@ -87,23 +87,23 @@ std::shared_ptr DataShareHelper::Creator( Uri uri(replacedUriStr); std::shared_ptr helper = nullptr; if (uri.GetQuery().find("Proxy=true") != std::string::npos) { - auto result = CreateServiceHelper(extUri, ""); - if (result != nullptr && GetSilentProxyStatus(strUri) == E_OK) { + auto result = CreateServiceHelper(extUri, "", isSystem); + if (result != nullptr && GetSilentProxyStatus(strUri, isSystem) == E_OK) { return result; } if (extUri.empty()) { return nullptr; } Uri ext(extUri); - helper = CreateExtHelper(ext, token, waitTime); + helper = CreateExtHelper(ext, token, waitTime, isSystem); } else { - helper = CreateExtHelper(uri, token, waitTime); + helper = CreateExtHelper(uri, token, waitTime, isSystem); } return helper; } std::shared_ptr DataShareHelper::Creator(const string &strUri, const CreateOptions &options, - const std::string &bundleName, const int waitTime) + const std::string &bundleName, const int waitTime, bool isSystem) { DISTRIBUTED_DATA_HITRACE(std::string(LOG_TAG) + "::" + std::string(__FUNCTION__)); Uri uri(strUri); @@ -111,7 +111,8 @@ std::shared_ptr DataShareHelper::Creator(const string &strUri, LOG_ERROR("token is nullptr"); return nullptr; } - return options.isProxy_ ? CreateServiceHelper("", bundleName) : CreateExtHelper(uri, options.token_, waitTime); + return options.isProxy_ ? CreateServiceHelper("", bundleName, isSystem) + : CreateExtHelper(uri, options.token_, waitTime, isSystem); } std::pair> DataShareHelper::Create(const sptr &token, @@ -142,7 +143,8 @@ std::pair> DataShareHelper::Create(const s } uri = Uri(extUri); } - auto helper = CreateExtHelper(uri, token, waitTime); + // this create func is inner api, do not check system permission + auto helper = CreateExtHelper(uri, token, waitTime, false); if (helper != nullptr) { return std::make_pair(E_OK, helper); } @@ -150,7 +152,7 @@ std::pair> DataShareHelper::Create(const s } std::shared_ptr DataShareHelper::CreateServiceHelper(const std::string &extUri, - const std::string &bundleName) + const std::string &bundleName, bool isSystem) { auto manager = DataShareManagerImpl::GetInstance(); if (manager == nullptr) { @@ -162,21 +164,24 @@ std::shared_ptr DataShareHelper::CreateServiceHelper(const std: LOG_ERROR("Service proxy is nullptr."); return nullptr; } - return std::make_shared(extUri); + return std::make_shared(extUri, isSystem); } -int DataShareHelper::GetSilentProxyStatus(const std::string &uri) +int DataShareHelper::GetSilentProxyStatus(const std::string &uri, bool isSystem) { auto proxy = DataShareManagerImpl::GetServiceProxy(); if (proxy == nullptr) { LOG_ERROR("Service proxy is nullptr."); return E_ERROR; } - return proxy->GetSilentProxyStatus(uri); + DataShareServiceProxy::SetSystem(isSystem); + auto res = proxy->GetSilentProxyStatus(uri); + DataShareServiceProxy::CleanSystem(); + return res; } std::shared_ptr DataShareHelper::CreateExtHelper(Uri &uri, const sptr &token, - const int waitTime) + const int waitTime, bool isSystem) { if (uri.GetQuery().find("appIndex=") != std::string::npos) { LOG_ERROR("ExtHelper do not support appIndex. Uri:%{public}s", @@ -203,7 +208,7 @@ std::shared_ptr DataShareHelper::CreateExtHelper(Uri &uri, cons LOG_ERROR("connect failed"); return nullptr; } - return std::make_shared(uri, token, dataShareConnection); + return std::make_shared(uri, token, dataShareConnection, isSystem); } /** @@ -364,14 +369,17 @@ bool ObserverImpl::DeleteObserver(const Uri& uri, const std::shared_ptrSetSilentSwitch(uri, enable); + DataShareServiceProxy::SetSystem(isSystem); + auto res = proxy->SetSilentSwitch(uri, enable); + DataShareServiceProxy::CleanSystem(); + return res; } bool DataShareHelper::IsProxy(Uri &uri) @@ -382,7 +390,7 @@ bool DataShareHelper::IsProxy(Uri &uri) std::pair> DataShareHelper::CreateProxyHelper(const std::string &strUri, const std::string &extUri) { - int ret = GetSilentProxyStatus(strUri); + int ret = GetSilentProxyStatus(strUri, false); auto helper = ret == E_OK ? CreateServiceHelper(extUri) : nullptr; return std::make_pair(ret, helper); } diff --git a/data_share/frameworks/native/consumer/src/datashare_helper_impl.cpp b/data_share/frameworks/native/consumer/src/datashare_helper_impl.cpp index 167cf08c..a205f452 100644 --- a/data_share/frameworks/native/consumer/src/datashare_helper_impl.cpp +++ b/data_share/frameworks/native/consumer/src/datashare_helper_impl.cpp @@ -30,17 +30,21 @@ namespace OHOS { namespace DataShare { using namespace AppExecFwk; +// non-silent access DataShareHelperImpl::DataShareHelperImpl(const Uri &uri, const sptr &token, - std::shared_ptr connection) + std::shared_ptr connection, bool isSystem) { LOG_DEBUG("starts"); + isSystem_ = isSystem; generalCtl_ = std::make_shared(connection, uri, token); extSpCtl_ = std::make_shared(connection, uri, token); } -DataShareHelperImpl::DataShareHelperImpl(std::string extUri) +// silent access +DataShareHelperImpl::DataShareHelperImpl(std::string extUri, bool isSystem) { LOG_DEBUG("starts"); + isSystem_ = isSystem; generalCtl_ = std::make_shared(extUri); persistentDataCtl_ = std::make_shared(); publishedDataCtl_ = std::make_shared(); @@ -99,7 +103,10 @@ int DataShareHelperImpl::Insert(Uri &uri, const DataShareValuesBucket &value) LOG_ERROR("generalCtl_ is nullptr"); return DATA_SHARE_ERROR; } - return generalCtl->Insert(uri, value); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = generalCtl->Insert(uri, value); + DataShareServiceProxy::CleanSystem(); + return res; } int DataShareHelperImpl::InsertExt(Uri &uri, const DataShareValuesBucket &value, std::string &result) @@ -121,7 +128,10 @@ int DataShareHelperImpl::Update(Uri &uri, const DataSharePredicates &predicates, LOG_ERROR("generalCtl is nullptr"); return DATA_SHARE_ERROR; } - return generalCtl->Update(uri, predicates, value); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = generalCtl->Update(uri, predicates, value); + DataShareServiceProxy::CleanSystem(); + return res; } int DataShareHelperImpl::BatchUpdate(const UpdateOperations &operations, std::vector &results) @@ -142,7 +152,10 @@ int DataShareHelperImpl::Delete(Uri &uri, const DataSharePredicates &predicates) LOG_ERROR("generalCtl is nullptr"); return DATA_SHARE_ERROR; } - return generalCtl->Delete(uri, predicates); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = generalCtl->Delete(uri, predicates); + DataShareServiceProxy::CleanSystem(); + return res; } std::pair DataShareHelperImpl::InsertEx(Uri &uri, const DataShareValuesBucket &value) @@ -153,7 +166,9 @@ std::pair DataShareHelperImpl::InsertEx(Uri &uri, const DataSh LOG_ERROR("generalCtl_ is nullptr"); return std::make_pair(DATA_SHARE_ERROR, 0); } + DataShareServiceProxy::SetSystem(isSystem_); auto [errCode, status] = generalCtl->InsertEx(uri, value); + DataShareServiceProxy::CleanSystem(); if (errCode != E_OK) { LOG_ERROR("generalCtl insert failed, errCode = %{public}d", errCode); } @@ -169,7 +184,9 @@ std::pair DataShareHelperImpl::UpdateEx( LOG_ERROR("generalCtl is nullptr"); return std::make_pair(DATA_SHARE_ERROR, 0); } + DataShareServiceProxy::SetSystem(isSystem_); auto [errCode, status] = generalCtl->UpdateEx(uri, predicates, value); + DataShareServiceProxy::CleanSystem(); if (errCode != E_OK) { LOG_ERROR("generalCtl update failed, errCode = %{public}d", errCode); } @@ -184,7 +201,9 @@ std::pair DataShareHelperImpl::DeleteEx(Uri &uri, const DataSh LOG_ERROR("generalCtl is nullptr"); return std::make_pair(DATA_SHARE_ERROR, 0); } + DataShareServiceProxy::SetSystem(isSystem_); auto [errCode, status] = generalCtl->DeleteEx(uri, predicates); + DataShareServiceProxy::CleanSystem(); if (errCode != E_OK) { LOG_ERROR("generalCtl delete failed, errCode = %{public}d", errCode); } @@ -201,7 +220,9 @@ std::shared_ptr DataShareHelperImpl::Query(Uri &uri, const D return nullptr; } DatashareBusinessError error; + DataShareServiceProxy::SetSystem(isSystem_); auto resultSet = generalCtl->Query(uri, predicates, columns, error); + DataShareServiceProxy::CleanSystem(); if (businessError != nullptr) { *businessError = error; } @@ -283,7 +304,10 @@ void DataShareHelperImpl::NotifyChange(const Uri &uri) LOG_ERROR("extSpCtl is nullptr"); return; } - return generalCtl->NotifyChange(uri); + DataShareServiceProxy::SetSystem(isSystem_); + generalCtl->NotifyChange(uri); + DataShareServiceProxy::CleanSystem(); + return; } Uri DataShareHelperImpl::NormalizeUri(Uri &uri) @@ -318,7 +342,10 @@ int DataShareHelperImpl::AddQueryTemplate(const std::string &uri, int64_t subscr report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return DATA_SHARE_ERROR; } - return persistentDataCtl->AddQueryTemplate(uri, subscriberId, tpl); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->AddQueryTemplate(uri, subscriberId, tpl); + DataShareServiceProxy::CleanSystem(); + return res; } int DataShareHelperImpl::DelQueryTemplate(const std::string &uri, int64_t subscriberId) @@ -331,7 +358,10 @@ int DataShareHelperImpl::DelQueryTemplate(const std::string &uri, int64_t subscr report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return DATA_SHARE_ERROR; } - return persistentDataCtl->DelQueryTemplate(uri, subscriberId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->DelQueryTemplate(uri, subscriberId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::Publish(const Data &data, const std::string &bundleName) @@ -342,7 +372,10 @@ std::vector DataShareHelperImpl::Publish(const Data &data, cons LOG_ERROR("publishedDataCtl is nullptr"); return std::vector(); } - return publishedDataCtl->Publish(data, bundleName); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->Publish(data, bundleName); + DataShareServiceProxy::CleanSystem(); + return res; } Data DataShareHelperImpl::GetPublishedData(const std::string &bundleName, int &resultCode) @@ -353,7 +386,10 @@ Data DataShareHelperImpl::GetPublishedData(const std::string &bundleName, int &r LOG_ERROR("publishedDataCtl is nullptr"); return Data(); } - return publishedDataCtl->GetPublishedData(bundleName, resultCode); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->GetPublishedData(bundleName, resultCode); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::SubscribeRdbData(const std::vector &uris, @@ -368,7 +404,10 @@ std::vector DataShareHelperImpl::SubscribeRdbData(const std::ve report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return std::vector(); } - return persistentDataCtl->SubscribeRdbData(this, uris, templateId, callback); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->SubscribeRdbData(this, uris, templateId, callback); + DataShareServiceProxy::CleanSystem(); + return res; } __attribute__((no_sanitize("cfi"))) std::vector DataShareHelperImpl::UnsubscribeRdbData( @@ -383,7 +422,10 @@ __attribute__((no_sanitize("cfi"))) std::vector DataShareHelper report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return std::vector(); } - return persistentDataCtl->UnSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->UnSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::EnableRdbSubs(const std::vector &uris, @@ -395,7 +437,10 @@ std::vector DataShareHelperImpl::EnableRdbSubs(const std::vecto LOG_ERROR("persistentDataCtl is nullptr"); return std::vector(); } - return persistentDataCtl->EnableSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->EnableSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::DisableRdbSubs(const std::vector &uris, @@ -407,7 +452,10 @@ std::vector DataShareHelperImpl::DisableRdbSubs(const std::vect LOG_ERROR("persistentDataCtl is nullptr"); return std::vector(); } - return persistentDataCtl->DisableSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = persistentDataCtl->DisableSubscribeRdbData(this, uris, templateId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::SubscribePublishedData(const std::vector &uris, @@ -422,7 +470,10 @@ std::vector DataShareHelperImpl::SubscribePublishedData(const s report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return std::vector(); } - return publishedDataCtl->SubscribePublishedData(this, uris, subscriberId, callback); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->SubscribePublishedData(this, uris, subscriberId, callback); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::UnsubscribePublishedData(const std::vector &uris, @@ -437,7 +488,10 @@ std::vector DataShareHelperImpl::UnsubscribePublishedData(const report.SetError(RadarReporter::DATA_SHARE_DIED_ERROR); return std::vector(); } - return publishedDataCtl->UnSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->UnSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::EnablePubSubs(const std::vector &uris, @@ -449,7 +503,10 @@ std::vector DataShareHelperImpl::EnablePubSubs(const std::vecto LOG_ERROR("publishedDataCtl is nullptr"); return std::vector(); } - return publishedDataCtl->EnableSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->EnableSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::CleanSystem(); + return res; } std::vector DataShareHelperImpl::DisablePubSubs(const std::vector &uris, @@ -461,7 +518,10 @@ std::vector DataShareHelperImpl::DisablePubSubs(const std::vect LOG_ERROR("publishedDataCtl is nullptr"); return std::vector(); } - return publishedDataCtl->DisableSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::SetSystem(isSystem_); + auto res = publishedDataCtl->DisableSubscribePublishedData(this, uris, subscriberId); + DataShareServiceProxy::CleanSystem(); + return res; } int32_t DataShareHelperImpl::UserDefineFunc( diff --git a/data_share/frameworks/native/provider/include/datashare_uv_queue.h b/data_share/frameworks/native/provider/include/datashare_uv_queue.h index dff6a09a..8b8d35a6 100644 --- a/data_share/frameworks/native/provider/include/datashare_uv_queue.h +++ b/data_share/frameworks/native/provider/include/datashare_uv_queue.h @@ -16,7 +16,6 @@ #ifndef DATASHARE_UV_QUEUE_H #define DATASHARE_UV_QUEUE_H -#include #include #include #include diff --git a/data_share/frameworks/native/provider/include/js_datashare_ext_ability.h b/data_share/frameworks/native/provider/include/js_datashare_ext_ability.h index 1951ed8d..06582fe5 100644 --- a/data_share/frameworks/native/provider/include/js_datashare_ext_ability.h +++ b/data_share/frameworks/native/provider/include/js_datashare_ext_ability.h @@ -29,7 +29,62 @@ namespace OHOS { namespace DataShare { using namespace AbilityRuntime; +class JsResult { +public: + JsResult() = default; + bool GetRecvReply() const + { + return isRecvReply_; + } + void GetResult(int &value) + { + value = callbackResultNumber_; + } + + void GetResult(std::string &value) + { + std::lock_guard lock(asyncLock_); + value = callbackResultString_; + } + + void GetResult(std::vector &value) + { + std::lock_guard lock(asyncLock_); + value = callbackResultStringArr_; + } + + void GetResult(std::vector &results) + { + std::lock_guard lock(asyncLock_); + results = updateResults_; + } + + void GetResultSet(std::shared_ptr &value) + { + std::lock_guard lock(asyncLock_); + value = callbackResultObject_; + } + + void GetBusinessError(DatashareBusinessError &businessError) + { + std::lock_guard lock(asyncLock_); + businessError = businessError_; + } + + void SetAsyncResult(napi_env env, DatashareBusinessError &businessError, napi_value result); + void CheckAndSetAsyncResult(napi_env env); +private: + bool UnwrapBatchUpdateResult(napi_env env, napi_value &info, std::vector &results); + bool isRecvReply_ = false; + int callbackResultNumber_ = -1; + std::string callbackResultString_ = ""; + std::vector callbackResultStringArr_ = {}; + std::mutex asyncLock_; + std::shared_ptr callbackResultObject_ = nullptr; + DatashareBusinessError businessError_; + std::vector updateResults_ = {}; +}; /** * @brief Basic datashare extension ability components. */ @@ -242,94 +297,7 @@ public: */ Uri DenormalizeUri(const Uri &uri) override; - bool GetRecvReply() const - { - return isRecvReply_; - } - void SetRecvReply(bool recvReply) - { - isRecvReply_ = recvReply; - } - - napi_value GetAsyncResult() const - { - return callbackData_; - } - - void SetAsyncResult(napi_value asyncResult) - { - callbackData_ = asyncResult; - } - - void GetResult(int &value) - { - value = callbackResultNumber_; - } - - void SetResult(const int value) - { - callbackResultNumber_ = value; - } - - void GetResult(std::string &value) - { - std::lock_guard lock(asyncLock_); - value = callbackResultString_; - } - - void SetResult(const std::string value) - { - std::lock_guard lock(asyncLock_); - callbackResultString_ = value; - } - - void GetResult(std::vector &value) - { - std::lock_guard lock(asyncLock_); - value = callbackResultStringArr_; - } - - void SetResult(const std::vector &results) - { - std::lock_guard lock(asyncLock_); - updateResults_ = results; - } - - void GetResult(std::vector &results) - { - std::lock_guard lock(asyncLock_); - results = updateResults_; - } - - void SetResult(const std::vector value) - { - std::lock_guard lock(asyncLock_); - callbackResultStringArr_ = value; - } - - void GetResultSet(std::shared_ptr &value) - { - std::lock_guard lock(asyncLock_); - value = callbackResultObject_; - } - - void SetResultSet(const std::shared_ptr value) - { - std::lock_guard lock(asyncLock_); - callbackResultObject_ = value; - } - - void GetBusinessError(DatashareBusinessError &businessError) - { - std::lock_guard lock(asyncLock_); - businessError = businessError_; - } - - void SetBusinessError(DatashareBusinessError &businessError) - { - std::lock_guard lock(asyncLock_); - businessError_ = businessError; - } + void InitResult(std::shared_ptr result); struct AsyncContext { bool isNeedNotify_ = false; }; @@ -338,7 +306,7 @@ private: std::shared_ptr context; }; struct AsyncCallBackPoint { - std::weak_ptr extAbility; + std::shared_ptr result; }; napi_value CallObjectMethod(const char *name, napi_value const *argv = nullptr, size_t argc = 0, bool isAsync = true); @@ -355,22 +323,14 @@ private: static void UnWrapBusinessError(napi_env env, napi_value info, DatashareBusinessError &businessError); static napi_valuetype UnWrapPropertyType(napi_env env, napi_value info, const std::string &key); - static bool UnwrapBatchUpdateResult(napi_env env, napi_value &info, std::vector &results); + static std::string UnWrapProperty(napi_env env, napi_value info, const std::string &key); int32_t InitAsyncCallParams(size_t argc, napi_env &env, napi_value *args); static constexpr int ACTIVE_INVOKER = 1; JsRuntime& jsRuntime_; std::unique_ptr jsObj_; - bool isRecvReply_ = false; - napi_value callbackData_ = nullptr; - int callbackResultNumber_ = -1; - std::string callbackResultString_ = ""; - std::vector callbackResultStringArr_ = {}; - std::mutex asyncLock_; - std::shared_ptr callbackResultObject_ = nullptr; - DatashareBusinessError businessError_; - std::vector updateResults_ = {}; + std::shared_ptr result_; }; } // namespace DataShare } // namespace OHOS diff --git a/data_share/frameworks/native/provider/src/datashare_stub_impl.cpp b/data_share/frameworks/native/provider/src/datashare_stub_impl.cpp index 28073839..49dafcef 100644 --- a/data_share/frameworks/native/provider/src/datashare_stub_impl.cpp +++ b/data_share/frameworks/native/provider/src/datashare_stub_impl.cpp @@ -14,6 +14,7 @@ */ #include "datashare_stub_impl.h" +#include #include "accesstoken_kit.h" #include "datashare_log.h" @@ -54,16 +55,18 @@ std::vector DataShareStubImpl::GetFileTypes(const Uri &uri, const s if (extension == nullptr) { return ret; } - std::function syncTaskFunc = [extension, info, uri, mimeTypeFilter]() { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, mimeTypeFilter, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->GetFileTypes(uri, mimeTypeFilter); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -80,17 +83,19 @@ int DataShareStubImpl::OpenFile(const Uri &uri, const std::string &mode) if (extension == nullptr) { return DEFAULT_NUMBER; } + auto result = std::make_shared(); int ret = -1; - std::function syncTaskFunc = [extension, info, uri, mode]() { + std::function syncTaskFunc = [extension, info, uri, mode, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->OpenFile(uri, mode); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -134,17 +139,19 @@ int DataShareStubImpl::Insert(const Uri &uri, const DataShareValuesBucket &value return PERMISSION_ERROR_NUMBER; } + auto result = std::make_shared(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, value]() { + std::function syncTaskFunc = [extension, info, uri, value, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Insert(uri, value); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -169,17 +176,19 @@ int DataShareStubImpl::Update(const Uri &uri, const DataSharePredicates &predica return PERMISSION_ERROR_NUMBER; } + auto result = std::make_shared(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, predicates, value]() { + std::function syncTaskFunc = [extension, info, uri, predicates, value, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Update(uri, predicates, value); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -200,18 +209,20 @@ int DataShareStubImpl::BatchUpdate(const UpdateOperations &operations, std::vect LOG_ERROR("Check calling permission failed."); return PERMISSION_ERROR_NUMBER; } + auto result = std::make_shared(); std::shared_ptr ret = std::make_shared(0); - std::function syncTaskFunc = [extension, ret, operations, info]() { + std::function syncTaskFunc = [extension, ret, operations, info, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); std::vector tmp; *ret = extension->BatchUpdate(operations, tmp); }; - std::function getRetFunc = [&results, extension]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [&results, result]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(results); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(results); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -235,17 +246,19 @@ int DataShareStubImpl::Delete(const Uri &uri, const DataSharePredicates &predica return PERMISSION_ERROR_NUMBER; } + auto result = std::make_shared(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, predicates]() { + std::function syncTaskFunc = [extension, info, uri, predicates, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Delete(uri, predicates); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -269,17 +282,19 @@ std::pair DataShareStubImpl::InsertEx(const Uri &uri, const Da return std::make_pair(PERMISSION_ERROR_NUMBER, 0); } + auto result = std::make_shared(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, value]() { + std::function syncTaskFunc = [extension, info, uri, value, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Insert(uri, value); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -304,17 +319,19 @@ std::pair DataShareStubImpl::UpdateEx(const Uri &uri, const Da return std::make_pair(PERMISSION_ERROR_NUMBER, 0); } + auto result = std::make_shared(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, predicates, value]() { + std::function syncTaskFunc = [extension, info, uri, predicates, value, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Update(uri, predicates, value); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -339,16 +356,18 @@ std::pair DataShareStubImpl::DeleteEx(const Uri &uri, const Da } int ret = 0; - std::function syncTaskFunc = [extension, info, uri, predicates]() { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, predicates, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->Delete(uri, predicates); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -370,21 +389,23 @@ std::shared_ptr DataShareStubImpl::Query(const Uri &uri, if (!CheckCallingPermission(extension->abilityInfo_->readPermission)) { LOG_ERROR("Check calling permission failed."); + businessError.SetCode(PERMISSION_ERROR_NUMBER); return resultSet; } - - std::function syncTaskFunc = [extension, info, uri, predicates, columns]() mutable { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, predicates, columns, result]() mutable { extension->SetCallingInfo(info); + extension->InitResult(result); DatashareBusinessError businessErr; extension->Query(uri, predicates, columns, businessErr); }; - std::function getRetFunc = [extension, &resultSet, &businessError]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &resultSet, &businessError]() -> bool { + if (result == nullptr) { return false; } - auto isRecvReply = extension->GetRecvReply(); - extension->GetResultSet(resultSet); - extension->GetBusinessError(businessError); + auto isRecvReply = result->GetRecvReply(); + result->GetResultSet(resultSet); + result->GetBusinessError(businessError); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -402,19 +423,21 @@ std::string DataShareStubImpl::GetType(const Uri &uri) if (extension == nullptr) { return ret; } - std::function syncTaskFunc = [extension, info, uri]() { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, result]() { if (extension == nullptr) { return; } extension->SetCallingInfo(info); + extension->InitResult(result); extension->GetType(uri); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -437,17 +460,19 @@ int DataShareStubImpl::BatchInsert(const Uri &uri, const std::vector(); int ret = 0; - std::function syncTaskFunc = [extension, info, uri, values]() { + std::function syncTaskFunc = [extension, info, uri, values, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->BatchInsert(uri, values); }; - std::function getRetFunc = [extension, &ret]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &ret]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); - extension->GetResult(ret); + bool isRecvReply = result->GetRecvReply(); + result->GetResult(ret); return isRecvReply; }; std::lock_guard lock(mutex_); @@ -464,7 +489,7 @@ bool DataShareStubImpl::RegisterObserver(const Uri &uri, const sptrabilityInfo_->readPermission)) { LOG_ERROR("Register observer check permission failed. uri: %{public}s", DataShareStringUtils::Anonymous(uri.ToString()).c_str()); - return PERMISSION_ERROR_NUMBER; + return false; } return extension->RegisterObserver(uri, dataObserver); } @@ -478,7 +503,7 @@ bool DataShareStubImpl::UnregisterObserver(const Uri &uri, const sptrabilityInfo_->readPermission)) { LOG_ERROR("UnRegister observer check permission failed. uri: %{public}s", DataShareStringUtils::Anonymous(uri.ToString()).c_str()); - return PERMISSION_ERROR_NUMBER; + return false; } return extension->UnregisterObserver(uri, dataObserver); } @@ -511,17 +536,19 @@ Uri DataShareStubImpl::NormalizeUri(const Uri &uri) return normalizeUri; } - std::function syncTaskFunc = [extension, info, uri]() { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->NormalizeUri(uri); }; - std::function getRetFunc = [extension, &normalizeUri]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &normalizeUri]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); + bool isRecvReply = result->GetRecvReply(); std::string ret; - extension->GetResult(ret); + result->GetResult(ret); Uri tmp(ret); normalizeUri = tmp; return isRecvReply; @@ -541,17 +568,19 @@ Uri DataShareStubImpl::DenormalizeUri(const Uri &uri) if (extension == nullptr) { return denormalizedUri; } - std::function syncTaskFunc = [extension, info, uri]() { + auto result = std::make_shared(); + std::function syncTaskFunc = [extension, info, uri, result]() { extension->SetCallingInfo(info); + extension->InitResult(result); extension->DenormalizeUri(uri); }; - std::function getRetFunc = [extension, &denormalizedUri]() -> bool { - if (extension == nullptr) { + std::function getRetFunc = [result, &denormalizedUri]() -> bool { + if (result == nullptr) { return false; } - bool isRecvReply = extension->GetRecvReply(); + bool isRecvReply = result->GetRecvReply(); std::string ret; - extension->GetResult(ret); + result->GetResult(ret); Uri tmp(ret); denormalizedUri = tmp; return isRecvReply; diff --git a/data_share/frameworks/native/provider/src/js_datashare_ext_ability.cpp b/data_share/frameworks/native/provider/src/js_datashare_ext_ability.cpp index 036c37b5..9260f69e 100644 --- a/data_share/frameworks/native/provider/src/js_datashare_ext_ability.cpp +++ b/data_share/frameworks/native/provider/src/js_datashare_ext_ability.cpp @@ -14,6 +14,7 @@ */ #include "js_datashare_ext_ability.h" +#include #include "ability_info.h" #include "dataobs_mgr_client.h" @@ -46,6 +47,72 @@ constexpr const char ASYNC_CALLBACK_NAME[] = "AsyncCallback"; constexpr int CALLBACK_LENGTH = sizeof(ASYNC_CALLBACK_NAME) - 1; } + +void JsResult::SetAsyncResult(napi_env env, DatashareBusinessError &businessError, napi_value result) +{ + std::lock_guard lock(asyncLock_); + napi_valuetype type = napi_undefined; + napi_typeof(env, result, &type); + if (type == napi_valuetype::napi_number) { + int32_t value = OHOS::AppExecFwk::UnwrapInt32FromJS(env, result); + callbackResultNumber_ = value; + } else if (type == napi_valuetype::napi_string) { + std::string value = OHOS::AppExecFwk::UnwrapStringFromJS(env, result); + callbackResultString_ = std::move(value); + } else if (type == napi_valuetype::napi_object) { + JSProxy::JSCreator *proxy = nullptr; + napi_unwrap(env, result, reinterpret_cast(&proxy)); + if (proxy == nullptr) { + if (UnwrapBatchUpdateResult(env, result, updateResults_)) { + isRecvReply_ = true; + return; + } + OHOS::AppExecFwk::UnwrapArrayStringFromJS(env, result, callbackResultStringArr_); + } else { + std::shared_ptr value = proxy->Create(); + callbackResultObject_ = std::make_shared(value); + } + } + businessError_= businessError; + isRecvReply_ = true; +} + +bool JsResult::UnwrapBatchUpdateResult(napi_env env, napi_value &info, + std::vector &results) +{ + napi_value keys = 0; + if (napi_get_property_names(env, info, &keys) != napi_ok) { + LOG_ERROR("napi_get_property_names failed"); + return false; + } + + uint32_t arrLen = 0; + if (napi_get_array_length(env, keys, &arrLen) != napi_ok) { + LOG_ERROR("napi_get_array_length failed"); + return false; + } + for (size_t i = 0; i < arrLen; i++) { + napi_value key = 0; + if (napi_get_element(env, keys, i, &key) != napi_ok) { + LOG_ERROR("napi_get_element failed"); + return false; + } + BatchUpdateResult batchUpdateResult; + batchUpdateResult.uri = DataShareJSUtils::UnwrapStringFromJS(env, key); + napi_value value = 0; + if (napi_get_property(env, info, key, &value) != napi_ok) { + LOG_ERROR("napi_get_property failed"); + return false; + } + if (!UnwrapArrayInt32FromJS(env, value, batchUpdateResult.codes)) { + LOG_ERROR("UnwrapArrayInt32FromJS failed"); + return false; + } + results.push_back(std::move(batchUpdateResult)); + } + return true; +} + bool MakeNapiColumn(napi_env env, napi_value &napiColumns, const std::vector &columns); using namespace OHOS::AppExecFwk; @@ -146,78 +213,6 @@ sptr JsDataShareExtAbility::OnConnect(const AAFwk::Want &want) return remoteObject->AsObject(); } -bool JsDataShareExtAbility::UnwrapBatchUpdateResult(napi_env env, napi_value &info, - std::vector &results) -{ - napi_value keys = 0; - if (napi_get_property_names(env, info, &keys) != napi_ok) { - LOG_ERROR("napi_get_property_names failed"); - return false; - } - - uint32_t arrLen = 0; - if (napi_get_array_length(env, keys, &arrLen) != napi_ok) { - LOG_ERROR("napi_get_array_length failed"); - return false; - } - for (size_t i = 0; i < arrLen; i++) { - napi_value key = 0; - if (napi_get_element(env, keys, i, &key) != napi_ok) { - LOG_ERROR("napi_get_element failed"); - return false; - } - BatchUpdateResult batchUpdateResult; - batchUpdateResult.uri = DataShareJSUtils::UnwrapStringFromJS(env, key); - napi_value value = 0; - if (napi_get_property(env, info, key, &value) != napi_ok) { - LOG_ERROR("napi_get_property failed"); - return false; - } - if (!UnwrapArrayInt32FromJS(env, value, batchUpdateResult.codes)) { - LOG_ERROR("UnwrapArrayInt32FromJS failed"); - return false; - } - results.push_back(std::move(batchUpdateResult)); - } - return true; -} - -void JsDataShareExtAbility::CheckAndSetAsyncResult(napi_env env) -{ - napi_valuetype type = napi_undefined; - auto result = GetAsyncResult(); - napi_typeof(env, result, &type); - if (type == napi_valuetype::napi_number) { - int32_t value = OHOS::AppExecFwk::UnwrapInt32FromJS(env, result); - SetResult(value); - } else if (type == napi_valuetype::napi_string) { - std::string value = OHOS::AppExecFwk::UnwrapStringFromJS(env, result); - SetResult(value); - } else if (type == napi_valuetype::napi_object) { - JSProxy::JSCreator *proxy = nullptr; - napi_unwrap(env, result, reinterpret_cast(&proxy)); - if (proxy == nullptr) { - std::vector results; - if (UnwrapBatchUpdateResult(env, result, results)) { - SetResult(results); - return; - } - std::vector value; - OHOS::AppExecFwk::UnwrapArrayStringFromJS(env, result, value); - SetResult(value); - } else { - std::shared_ptr value = proxy->Create(); - std::shared_ptr resultSet = std::make_shared(value); - SetResultSet(resultSet); - } - } else { - callbackResultNumber_ = -1; - callbackResultString_ = ""; - callbackResultStringArr_ = {}; - SetResultSet(nullptr); - } -} - napi_value JsDataShareExtAbility::AsyncCallback(napi_env env, napi_callback_info info) { if (env == nullptr || info == nullptr) { @@ -239,8 +234,8 @@ napi_value JsDataShareExtAbility::AsyncCallback(napi_env env, napi_callback_info } AsyncCallBackPoint* point = static_cast(data); - auto instance = point->extAbility.lock(); - if (!instance) { + auto result = point->result; + if (!result) { LOG_ERROR("extension ability has been destroyed."); return CreateJsUndefined(env); } @@ -251,11 +246,8 @@ napi_value JsDataShareExtAbility::AsyncCallback(napi_env env, napi_callback_info LOG_INFO("Error in callback"); UnWrapBusinessError(env, argv[0], businessError); } - if (instance != nullptr) { - instance->SetBusinessError(businessError); - instance->SetAsyncResult(argv[1]); - instance->CheckAndSetAsyncResult(env); - instance->SetRecvReply(true); + if (result != nullptr) { + result->SetAsyncResult(env, businessError, argv[1]); } return CreateJsUndefined(env); } @@ -393,6 +385,11 @@ napi_value JsDataShareExtAbility::CallObjectMethod(const char* name, napi_value return handleEscape.Escape(remoteNapi); } +void JsDataShareExtAbility::InitResult(std::shared_ptr result) +{ + result_ = result; +} + void JsDataShareExtAbility::SaveNewCallingInfo(napi_env &env) { auto newCallingInfo = GetCallingInfo(); @@ -415,12 +412,8 @@ int32_t JsDataShareExtAbility::InitAsyncCallParams(size_t argc, napi_env &env, n if (point == nullptr) { return E_ERROR; } - callbackResultNumber_ = -1; - callbackResultString_ = ""; - callbackResultStringArr_ = {}; - SetResultSet(nullptr); - SetRecvReply(false); - point->extAbility = std::static_pointer_cast(shared_from_this()); + + point->result = std::move(result_); napi_create_function(env, ASYNC_CALLBACK_NAME, CALLBACK_LENGTH, JsDataShareExtAbility::AsyncCallback, point, &args[argc]); napi_add_finalizer(env, args[argc], point, diff --git a/data_share/frameworks/native/proxy/include/data_share_service_proxy.h b/data_share/frameworks/native/proxy/include/data_share_service_proxy.h index ab1028ee..75f1d3d1 100644 --- a/data_share/frameworks/native/proxy/include/data_share_service_proxy.h +++ b/data_share/frameworks/native/proxy/include/data_share_service_proxy.h @@ -88,8 +88,18 @@ public: std::pair DeleteEx(const Uri &uri, const Uri &extUri, const DataSharePredicates &predicate) override; + static void SetSystem(bool isSystem); + + static bool IsSystem(); + + static bool& GetSystem(); + + static void CleanSystem(); + private: static inline BrokerDelegator delegator_; + + uint32_t CastIPCCode(DistributedShare::DataShare::DataShareServiceInterfaceCode code); }; } // namespace OHOS::DataShare #endif diff --git a/data_share/frameworks/native/proxy/include/rdb_subscriber_manager.h b/data_share/frameworks/native/proxy/include/rdb_subscriber_manager.h index 814e1248..7e9ba769 100644 --- a/data_share/frameworks/native/proxy/include/rdb_subscriber_manager.h +++ b/data_share/frameworks/native/proxy/include/rdb_subscriber_manager.h @@ -19,6 +19,7 @@ #include #include "callbacks_manager.h" +#include "concurrent_map.h" #include "data_proxy_observer.h" #include "data_proxy_observer_stub.h" #include "datashare_template.h" @@ -97,7 +98,7 @@ private: void EmitOnEnable(std::map> &obsMap); RdbSubscriberManager(); sptr serviceCallback_; - std::map lastChangeNodeMap_; + ConcurrentMap lastChangeNodeMap_; }; } // namespace DataShare } // namespace OHOS diff --git a/data_share/frameworks/native/proxy/src/data_share_service_proxy.cpp b/data_share/frameworks/native/proxy/src/data_share_service_proxy.cpp index 15f5950c..ff0a2859 100644 --- a/data_share/frameworks/native/proxy/src/data_share_service_proxy.cpp +++ b/data_share/frameworks/native/proxy/src/data_share_service_proxy.cpp @@ -17,9 +17,11 @@ #include #include "data_ability_observer_interface.h" +#include "datashare_errno.h" #include "datashare_itypes_utils.h" #include "datashare_log.h" #include "datashare_string_utils.h" +#include "distributeddata_data_share_ipc_interface_code.h" #include "ishared_result_set.h" namespace OHOS { @@ -30,6 +32,35 @@ DataShareServiceProxy::DataShareServiceProxy(const sptr &object) { } +void DataShareServiceProxy::CleanSystem() +{ + GetSystem() = false; +} + +bool DataShareServiceProxy::IsSystem() +{ + return GetSystem(); +} + +void DataShareServiceProxy::SetSystem(bool isSystem) +{ + GetSystem() = isSystem; +} + +bool& DataShareServiceProxy::GetSystem() +{ + static thread_local bool isSystem = false; + return isSystem; +} + +uint32_t DataShareServiceProxy::CastIPCCode(DistributedShare::DataShare::DataShareServiceInterfaceCode code) +{ + if (IsSystem()) { + return static_cast(code) + DistributedShare::DataShare::DATA_SHARE_CMD_SYSTEM_CODE; + } + return static_cast(code); +} + int32_t DataShareServiceProxy::Insert(const Uri &uri, const Uri &extUri, const DataShareValuesBucket &value) { auto [errCode, status] = InsertEx(uri, extUri, value); @@ -86,7 +117,7 @@ std::pair DataShareServiceProxy::InsertEx(const Uri &uri, cons MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_INSERTEX), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_INSERTEX), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("InsertEx fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uriStr).c_str(), err); @@ -118,7 +149,7 @@ std::pair DataShareServiceProxy::UpdateEx(const Uri &uri, cons MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_UPDATEEX), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_UPDATEEX), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("UpdateEx fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uriStr).c_str(), err); @@ -150,7 +181,7 @@ std::pair DataShareServiceProxy::DeleteEx(const Uri &uri, cons MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_DELETEEX), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_DELETEEX), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("DeleteEx fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uriStr).c_str(), err); @@ -181,7 +212,7 @@ std::shared_ptr DataShareServiceProxy::Query(const Uri &uri, MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_QUERY), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_QUERY), data, reply, option); auto result = ISharedResultSet::ReadFromParcel(reply); businessError.SetCode(reply.ReadInt32()); @@ -219,7 +250,7 @@ int DataShareServiceProxy::AddQueryTemplate(const std::string &uri, int64_t subs MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("AddTemplate fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uri).c_str(), err); @@ -243,7 +274,7 @@ int DataShareServiceProxy::DelQueryTemplate(const std::string &uri, int64_t subs MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_DEL_TEMPLATE), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_DEL_TEMPLATE), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Delete template fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uri).c_str(), err); @@ -268,7 +299,7 @@ std::vector DataShareServiceProxy::Publish(const Data &data, co MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_PUBLISH), parcel, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_PUBLISH), parcel, reply, option); if (err != NO_ERROR) { LOG_ERROR("Publish fail to sendRequest. err: %{public}d", err); return results; @@ -294,7 +325,7 @@ Data DataShareServiceProxy::GetPublishedData(const std::string &bundleName, int MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_GET_DATA), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_GET_DATA), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Get published data fail to sendRequest, err: %{public}d", err); return results; @@ -329,7 +360,7 @@ std::vector DataShareServiceProxy::SubscribeRdbData(const std:: MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_SUBSCRIBE_RDB), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_SUBSCRIBE_RDB), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("SubscribeRdbData fail to sendRequest. err: %{public}d", err); return results; @@ -356,7 +387,7 @@ std::vector DataShareServiceProxy::UnSubscribeRdbData( MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_RDB), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_RDB), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Fail to sendRequest. err: %{public}d", err); return results; @@ -383,7 +414,7 @@ std::vector DataShareServiceProxy::EnableSubscribeRdbData( MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_RDB), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_RDB), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Fail to sendRequest. err: %{public}d", err); return results; @@ -410,7 +441,7 @@ std::vector DataShareServiceProxy::DisableSubscribeRdbData( MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_RDB), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_RDB), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Disable subscribe RdbData fail to sendRequest. err: %{public}d", err); return results; @@ -444,7 +475,7 @@ std::vector DataShareServiceProxy::SubscribePublishedData( MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_SUBSCRIBE_PUBLISHED), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_SUBSCRIBE_PUBLISHED), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Subscribe published data fail to sendRequest. err: %{public}d", err); return results; @@ -470,7 +501,7 @@ std::vector DataShareServiceProxy::UnSubscribePublishedData( MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_PUBLISHED), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_PUBLISHED), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("UnSubscribe published data fail to sendRequest. err: %{public}d", err); return results; @@ -496,7 +527,7 @@ std::vector DataShareServiceProxy::EnableSubscribePublishedData MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_PUBLISHED), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_PUBLISHED), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Enable subscribe published data fail to sendRequest. err: %{public}d", err); return results; @@ -522,7 +553,7 @@ std::vector DataShareServiceProxy::DisableSubscribePublishedDat MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_PUBLISHED), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_PUBLISHED), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Disable subscribe published data fail to sendRequest. err: %{public}d", err); return results; @@ -546,7 +577,7 @@ void DataShareServiceProxy::Notify(const std::string &uri) MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_NOTIFY_OBSERVERS), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_NOTIFY_OBSERVERS), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Notify fail to sendRequest. err: %{public}d", err); return; @@ -569,10 +600,13 @@ int DataShareServiceProxy::SetSilentSwitch(const Uri &uri, bool enable) MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_SET_SILENT_SWITCH), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_SET_SILENT_SWITCH), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("SetSilentSwitch fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uriStr).c_str(), err); + if (err == E_NOT_SYSTEM_APP) { + return E_NOT_SYSTEM_APP; + } return DATA_SHARE_ERROR; } return reply.ReadInt32(); @@ -593,7 +627,7 @@ int DataShareServiceProxy::GetSilentProxyStatus(const std::string &uri) MessageParcel reply; MessageOption option; int32_t err = Remote()->SendRequest( - static_cast(InterfaceCode::DATA_SHARE_SERVICE_CMD_GET_SILENT_PROXY_STATUS), data, reply, option); + CastIPCCode(InterfaceCode::DATA_SHARE_SERVICE_CMD_GET_SILENT_PROXY_STATUS), data, reply, option); if (err != NO_ERROR) { LOG_ERROR("Is silent proxy enable fail to sendRequest. uri: %{public}s, err: %{public}d", DataShareStringUtils::Anonymous(uri).c_str(), err); diff --git a/data_share/frameworks/native/proxy/src/published_data_subscriber_manager.cpp b/data_share/frameworks/native/proxy/src/published_data_subscriber_manager.cpp index 80ee18c4..9ecc1866 100644 --- a/data_share/frameworks/native/proxy/src/published_data_subscriber_manager.cpp +++ b/data_share/frameworks/native/proxy/src/published_data_subscriber_manager.cpp @@ -259,7 +259,7 @@ void PublishedDataSubscriberManager::EmitOnEnable(std::map, PublishedDataChangeNode> results; for (auto &[key, obsVector] : obsMap) { uint32_t num = 0; - lastChangeNodeMap_.ComputeIfPresent(key, [obsVector = obsVector, &results, &num](const Key &, + lastChangeNodeMap_.ComputeIfPresent(key, [&obsVector = obsVector, &results, &num](const Key &, PublishedDataChangeNode &value) { for (auto &data : value.datas_) { PublishedObserverMapKey mapKey(data.key_, data.subscriberId_); @@ -268,6 +268,7 @@ void PublishedDataSubscriberManager::EmitOnEnable(std::map RdbSubscriberManager::DelObservers(void *subscriber std::vector lastDelUris; std::for_each(lastDelKeys.begin(), lastDelKeys.end(), [&lastDelUris, this](auto &result) { lastDelUris.emplace_back(result); - lastChangeNodeMap_.erase(result); + lastChangeNodeMap_.Erase(result); }); if (lastDelUris.empty()) { return; @@ -119,7 +119,7 @@ std::vector RdbSubscriberManager::DelObservers(void *subscriber [&proxy, this](const std::vector &lastDelKeys, std::vector &opResult) { // delete all obs by subscriber for (const auto &key : lastDelKeys) { - lastChangeNodeMap_.erase(key); + lastChangeNodeMap_.Erase(key); auto unsubResult = proxy->UnSubscribeRdbData(std::vector(1, key.uri_), key.templateId_); opResult.insert(opResult.end(), unsubResult.begin(), unsubResult.end()); } @@ -216,7 +216,7 @@ void RdbSubscriberManager::RecoverObservers(std::shared_ptr &keys, const std::shared_ptr &observer) { for (auto const &key : keys) { - auto it = lastChangeNodeMap_.find(key); - if (it != lastChangeNodeMap_.end()) { - observer->OnChange(it->second); + bool isExist = false; + RdbChangeNode node; + lastChangeNodeMap_.ComputeIfPresent(key, [&node, &isExist](const Key &, const RdbChangeNode &value) { + node = value; + isExist = true; + return true; + }); + if (isExist) { + observer->OnChange(node); } } } @@ -241,13 +247,20 @@ void RdbSubscriberManager::Emit(const std::vector &keys, const std::shared_ void RdbSubscriberManager::EmitOnEnable(std::map> &obsMap) { for (auto &[key, obsVector] : obsMap) { - auto it = lastChangeNodeMap_.find(key); - if (it == lastChangeNodeMap_.end()) { + bool isExist = false; + RdbChangeNode node; + lastChangeNodeMap_.ComputeIfPresent(key, [&node, &isExist](const Key &, const RdbChangeNode &value) { + node = value; + isExist = true; + return true; + }); + if (!isExist) { continue; } for (auto &obs : obsVector) { if (obs.isNotifyOnEnabled_) { - obs.observer_->OnChange(it->second); + obs.isNotifyOnEnabled_ = false; + obs.observer_->OnChange(node); } } } diff --git a/data_share/interfaces/inner_api/common/include/datashare_errno.h b/data_share/interfaces/inner_api/common/include/datashare_errno.h index 9703c73d..d0e4bb04 100644 --- a/data_share/interfaces/inner_api/common/include/datashare_errno.h +++ b/data_share/interfaces/inner_api/common/include/datashare_errno.h @@ -188,6 +188,11 @@ constexpr int E_PROVIDER_CONN_NULL = (E_BASE + 67); * @brief The error code for passing invalid form of user id. */ constexpr int E_INVALID_USER_ID = (E_BASE + 68); + +/** +* @brief The error code for not system app. +*/ +constexpr int E_NOT_SYSTEM_APP = (E_BASE + 69); } // namespace DataShare } // namespace OHOS diff --git a/data_share/interfaces/inner_api/consumer/include/datashare_helper.h b/data_share/interfaces/inner_api/consumer/include/datashare_helper.h index 4a26bbe7..9d1a7c59 100644 --- a/data_share/interfaces/inner_api/consumer/include/datashare_helper.h +++ b/data_share/interfaces/inner_api/consumer/include/datashare_helper.h @@ -64,7 +64,7 @@ public: [[deprecated( "Use Create(const sptr &, const std::string &, const std::string &, const int &) instead.")]] static std::shared_ptr Creator(const sptr &token, - const std::string &strUri, const std::string &extUri = "", const int waitTime = 2); + const std::string &strUri, const std::string &extUri = "", const int waitTime = 2, bool isSystem = false); /** * @brief Creates a DataShareHelper instance with the Uri and {@link #CreateOptions} . @@ -78,7 +78,7 @@ public: [[deprecated( "Use Create(const sptr &, const std::string &,const std::string &, const int &) instead.")]] static std::shared_ptr Creator(const std::string &strUri, const CreateOptions &options, - const std::string &bundleName = "", const int waitTime = 2); + const std::string &bundleName = "", const int waitTime = 2, bool isSystem = false); /** * @brief Creates a DataShareHelper instance, priority silent access, use non-silent access when silent is not @@ -426,9 +426,10 @@ public: * @brief Set default switch for silent access. * @param uri, the uri to disable/enable. * @param enable, the enable of silent switch. + * @param isSystem, is system app or not. * @return Returns the error code. */ - static int SetSilentSwitch(Uri &uri, bool enable); + static int SetSilentSwitch(Uri &uri, bool enable, bool isSystem = false); /** * @brief Inserts a single data record into the database. @@ -477,12 +478,12 @@ public: private: static std::shared_ptr CreateServiceHelper(const std::string &extUri = "", - const std::string &bundleName = ""); + const std::string &bundleName = "", bool isSystem = false); - static int GetSilentProxyStatus(const std::string &uri); + static int GetSilentProxyStatus(const std::string &uri, bool isSystem); static std::shared_ptr CreateExtHelper(Uri &uri, const sptr &token, - const int waitTime = 2); + const int waitTime = 2, bool isSystem = false); static std::string TransferUriPrefix(const std::string &originPrefix, const std::string &replacedPrefix, const std::string &originUriStr); diff --git a/data_share/test/js/data_share/unittest/config.json b/data_share/test/js/data_share/unittest/config.json index 6e089ad3..8179c0c3 100644 --- a/data_share/test/js/data_share/unittest/config.json +++ b/data_share/test/js/data_share/unittest/config.json @@ -61,4 +61,4 @@ } ] } -} +} \ No newline at end of file diff --git a/data_share/test/js/data_share/unittest/src/BUILD.gn b/data_share/test/js/data_share/unittest/src/BUILD.gn index 2da1e9e6..157a6d38 100644 --- a/data_share/test/js/data_share/unittest/src/BUILD.gn +++ b/data_share/test/js/data_share/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "data_share/data_share" +module_output_path = "data_share/data_share/js_tests" ohos_js_unittest("DataShareSilentAccessJsTest") { module_out_path = module_output_path diff --git a/datamgr_service/OAT.xml b/datamgr_service/OAT.xml index f50b452b..6bed6e36 100644 --- a/datamgr_service/OAT.xml +++ b/datamgr_service/OAT.xml @@ -64,6 +64,13 @@ Note:If the text contains special characters, please escape them according to th + + + + + diff --git a/datamgr_service/bundle.json b/datamgr_service/bundle.json index 3283ee04..b1a8472d 100644 --- a/datamgr_service/bundle.json +++ b/datamgr_service/bundle.json @@ -94,7 +94,7 @@ "json", "dmsfwk", "data_object", - "window_manager" + "init" ], "third_party": [ "libuv", @@ -129,6 +129,8 @@ "cloud/schema_meta.h", "cloud/subscription.h", "cloud/sync_event.h", + "dfx/dfx_types.h", + "dfx/reporter.h", "directory/directory_manager.h", "error/general_error.h", "eventcenter/event.h", @@ -166,16 +168,6 @@ ], "header_base": "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/framework/include" } - }, - { - "name": "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/dfx:distributeddata_dfx_fault_static", - "header": { - "header_files": [ - "dfx_types.h", - "reporter.h" - ], - "header_base": "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/dfx" - } } ], "test": [ diff --git a/datamgr_service/conf/config.json b/datamgr_service/conf/config.json index 7790030d..b1d053eb 100644 --- a/datamgr_service/conf/config.json +++ b/datamgr_service/conf/config.json @@ -72,7 +72,8 @@ { "version": 50331653, "pattern": "/data/{type}/{area}/{userId}/database/{bundleName}/{hapName}/{store}/{customDir}", - "metaPath": "/data/service/el1/public/database/distributeddata/meta" + "metaPath": "/data/service/el1/public/database/distributeddata/meta", + "clonePath": "/data/service/el2/{userId}/database/distributeddata/secret_key_backup.conf" } ] }, diff --git a/datamgr_service/services/distributeddataservice/adapter/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/BUILD.gn deleted file mode 100644 index 8797b303..00000000 --- a/datamgr_service/services/distributeddataservice/adapter/BUILD.gn +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2021 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import("//build/ohos.gni") -import("//build/ohos_var.gni") -import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") - -config("distributeddata_adapter_private_config") { - visibility = [ ":*" ] - include_dirs = [] - - cflags = [ - "-Werror", - "-Wno-multichar", - "-D_LIBCPP_HAS_COND_CLOCKWAIT", - ] - cflags_cc = [ "-fvisibility=hidden" ] - defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] -} - -config("distributeddata_adapter_public_config") { - visibility = [ ":*" ] - - include_dirs = [ - "include/log", - "include/dfx", - "include/communicator", - "include/autils", - "include/utils", - "include", - "${kv_store_path}/interfaces/innerkits/distributeddata/include/", - "${kv_store_common_path}", - "${data_service_path}/framework/include", - ] -} - -ohos_shared_library("distributeddata_adapter") { - branch_protector_ret = "pac_ret" - sanitize = { - cfi = true - cfi_cross_dso = true - debug = false - boundary_sanitize = true - integer_overflow = true - ubsan = true - } - sources = [] - configs = [ ":distributeddata_adapter_private_config" ] - deps = [ - "${data_service_path}/adapter/account:distributeddata_account_static", - "${data_service_path}/adapter/communicator:distributeddata_communicator_static", - "${data_service_path}/adapter/dfx:distributeddata_dfx_static", - "${data_service_path}/adapter/permission:distributeddata_permission_static", - "${data_service_path}/framework:distributeddatasvcfwk", - ] - - if (defined(global_parts_info) && - defined(global_parts_info.theme_screenlock_mgr)) { - deps += [ "${data_service_path}/adapter/screenlock:distributeddata_screenlock_static" ] - } - - external_deps = [ - "c_utils:utils", - "hilog:libhilog", - "hisysevent:libhisysevent", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", - ] - public_external_deps = [ "device_manager:devicemanagersdk" ] - public_configs = [ ":distributeddata_adapter_public_config" ] - - subsystem_name = "distributeddatamgr" - part_name = "datamgr_service" -} diff --git a/datamgr_service/services/distributeddataservice/adapter/CMakeLists.txt b/datamgr_service/services/distributeddataservice/adapter/CMakeLists.txt index 3b7c83fd..83d33118 100644 --- a/datamgr_service/services/distributeddataservice/adapter/CMakeLists.txt +++ b/datamgr_service/services/distributeddataservice/adapter/CMakeLists.txt @@ -18,7 +18,9 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/behaviour adapterSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/fault adapterSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/statistic adapterSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src adapterSrc) -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/permission/src adapterSrc) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/network/src adapterSrc) +list(REMOVE_ITEM adapterSrc "${CMAKE_CURRENT_SOURCE_DIR}/network/src/network_delegate_default_impl.cpp") +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/schema_helper/src adapterSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/screenlock/src adapterSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/utils/src adapterSrc) list(REMOVE_ITEM adapterSrc "${CMAKE_CURRENT_SOURCE_DIR}/account/src/account_delegate_default_impl.cpp") @@ -31,6 +33,10 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/behaviour) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/fault) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src/statistic) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/dfx/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/network/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/schema_helper/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/screenlock/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/utils/src) #include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) #include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/account) #include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/broadcaster) @@ -51,5 +57,4 @@ target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/br target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/communicator) target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/dfx) target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/permission) -target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/screenlock) target_include_directories(adapter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/utils) diff --git a/datamgr_service/services/distributeddataservice/adapter/account/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/account/BUILD.gn index 85697b11..fc643f99 100644 --- a/datamgr_service/services/distributeddataservice/adapter/account/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/account/BUILD.gn @@ -35,7 +35,10 @@ ohos_source_set("distributeddata_account") { "${data_service_path}/framework/include/account", ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] remove_configs = [ "//build/config/compiler:no_exceptions" ] diff --git a/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.cpp b/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.cpp index 7f6d37e3..f57bfb74 100644 --- a/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.cpp @@ -154,7 +154,7 @@ bool AccountDelegateNormalImpl::IsVerified(int userId) void AccountDelegateNormalImpl::SubscribeAccountEvent() { ZLOGI("Subscribe account event listener start."); - if (AccountSubscriber_ == nullptr) { + if (accountSubscriber_ == nullptr) { std::set states; states.insert(AccountSA::REMOVED); states.insert(AccountSA::SWITCHED); @@ -162,8 +162,8 @@ void AccountDelegateNormalImpl::SubscribeAccountEvent() states.insert(AccountSA::STOPPING); states.insert(AccountSA::STOPPED); OsAccountSubscribeInfo info(states, true); - AccountSubscriber_ = std::make_shared(info, executors_); - AccountSubscriber_->SetEventCallback([this](AccountEventInfo &account, int32_t timeout) { + accountSubscriber_ = std::make_shared(info, executors_); + accountSubscriber_->SetEventCallback([this](AccountEventInfo &account, int32_t timeout) { UpdateUserStatus(account); account.harmonyAccountId = GetCurrentAccountId(); NotifyAccountChanged(account, timeout); @@ -195,8 +195,8 @@ void AccountDelegateNormalImpl::UpdateUserStatus(const AccountEventInfo &account ExecutorPool::Task AccountDelegateNormalImpl::GetTask(uint32_t retry) { return [this, retry] { - auto result = OsAccountManager::SubscribeOsAccount(AccountSubscriber_); - if (result) { + auto result = OsAccountManager::SubscribeOsAccount(accountSubscriber_); + if (result == ERR_OK) { ZLOGI("success to register subscriber."); return; } @@ -213,16 +213,16 @@ ExecutorPool::Task AccountDelegateNormalImpl::GetTask(uint32_t retry) AccountDelegateNormalImpl::~AccountDelegateNormalImpl() { ZLOGD("destruct"); - auto res = OsAccountManager::UnsubscribeOsAccount(AccountSubscriber_); - if (!res) { + auto res = OsAccountManager::UnsubscribeOsAccount(accountSubscriber_); + if (res != ERR_OK) { ZLOGW("unregister account event fail res:%d", res); } } void AccountDelegateNormalImpl::UnsubscribeAccountEvent() { - auto res = OsAccountManager::UnsubscribeOsAccount(AccountSubscriber_); - if (!res) { + auto res = OsAccountManager::UnsubscribeOsAccount(accountSubscriber_); + if (res != ERR_OK) { ZLOGW("unregister account event fail res:%d", res); } } @@ -285,6 +285,16 @@ bool AccountDelegateNormalImpl::Init() return true; } +bool AccountDelegateNormalImpl::IsUserForeground(int32_t userId) +{ + bool isForeground = false; + if (AccountSA::OsAccountManager::IsOsAccountForeground(userId, isForeground) != 0) { + ZLOGE("check foreground user error, userId:%{public}d", userId); + return false; + } + return isForeground; +} + bool AccountDelegateNormalImpl::IsDeactivating(int userId) { auto [success, res] = userDeactivating_.Find(userId); diff --git a/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.h b/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.h index 1fe64a14..7d16b388 100644 --- a/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/account/src/account_delegate_normal_impl.h @@ -49,6 +49,7 @@ public: void BindExecutor(std::shared_ptr executors) override; std::string GetUnencryptedAccountId(int32_t userId = 0) const override; bool QueryForegroundUserId(int &foregroundUserId) override; + bool IsUserForeground(int32_t userId) override; static bool Init(); private: @@ -58,7 +59,7 @@ private: void UpdateUserStatus(const AccountEventInfo &account); static constexpr int MAX_RETRY_TIME = 300; static constexpr int RETRY_WAIT_TIME_S = 1; - std::shared_ptr AccountSubscriber_{}; + std::shared_ptr accountSubscriber_{}; std::shared_ptr executors_; ConcurrentMap userStatus_{}; ConcurrentMap userDeactivating_{}; diff --git a/datamgr_service/services/distributeddataservice/adapter/account/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/account/test/BUILD.gn index 32ea5f92..39c09ee5 100644 --- a/datamgr_service/services/distributeddataservice/adapter/account/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/account/test/BUILD.gn @@ -12,7 +12,7 @@ # limitations under the License. import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### ohos_unittest("AccountDelegateTest") { @@ -27,7 +27,6 @@ ohos_unittest("AccountDelegateTest") { "${data_service_path}/adapter/include/autils", "${data_service_path}/adapter/include/utils", "${data_service_path}/framework/include", - "${data_service_path}/adapter/account/src", ] cflags = [ diff --git a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.cpp b/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.cpp deleted file mode 100644 index 4304be2d..00000000 --- a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender_impl.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "BroadcastSenderImpl" - -#include "broadcast_sender_impl.h" -#include "common_event_manager.h" -#include "common_event_support.h" -#include "log_print.h" -#include "string_wrapper.h" - -namespace OHOS::DistributedKv { -using namespace OHOS::EventFwk; -using namespace OHOS::AAFwk; -class CommonEventSubscriberListener : public CommonEventSubscriber { -public: - explicit CommonEventSubscriberListener(const CommonEventSubscribeInfo &subscriberInfo); - virtual ~CommonEventSubscriberListener() {} - virtual void OnReceiveEvent(const CommonEventData &data); -}; - -CommonEventSubscriberListener::CommonEventSubscriberListener(const CommonEventSubscribeInfo &subscriberInfo) - : CommonEventSubscriber(subscriberInfo) -{} - -void CommonEventSubscriberListener::OnReceiveEvent(const CommonEventData &data) -{ - (void) data; - ZLOGI("receive event."); -} - -void BroadcastSenderImpl::SendEvent(const EventParams ¶ms) -{ - ZLOGI("SendEvent code."); - bool result = false; - WantParams parameters; - parameters.SetParam(PKG_NAME, String::Box(params.appId)); - Want want; - want.SetAction(ACTION_NAME).SetParams(parameters); - CommonEventData commonEventData(want); - MatchingSkills matchingSkills; - matchingSkills.AddEvent(ACTION_NAME); - CommonEventSubscribeInfo subscribeInfo(matchingSkills); - auto subscriberPtr = std::make_shared(subscribeInfo); - if (CommonEventManager::SubscribeCommonEvent(subscriberPtr)) { - result = CommonEventManager::PublishCommonEvent(commonEventData); - } - CommonEventManager::UnSubscribeCommonEvent(subscriberPtr); - ZLOGI("SendEvent result:%{public}d.", result); -} -} // namespace OHOS::DistributedKv diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/communicator/BUILD.gn index ec2f3f16..522a20c2 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/BUILD.gn @@ -43,14 +43,16 @@ ohos_source_set("distributeddata_communicator") { include_dirs = [ "../include/communicator", - "../include/dfx", - "../include/log", - "../include/autils", "../include/utils", "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/framework/include/utils", ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] deps = [ "${data_service_path}/adapter/utils:distributeddata_utils", @@ -64,6 +66,7 @@ ohos_source_set("distributeddata_communicator") { "dsoftbus:softbus_client", "hilog:libhilog", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:datamgr_common", "kv_store:distributeddb", ] diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.cpp index 022f303d..40d6f8bb 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.cpp @@ -19,7 +19,7 @@ #include #include #include "log_print.h" -#include "reporter.h" +#include "dfx/reporter.h" #ifdef LOG_TAG #undef LOG_TAG diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.h b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.h index 9ea30448..6a9dc009 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.h +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_handler.h @@ -20,8 +20,8 @@ #include #include #include +#include "dfx/reporter.h" #include "log_print.h" -#include "reporter.h" #include "app_data_change_listener.h" #include "softbus_adapter.h" diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_mgr.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_mgr.cpp index e96fde31..fb62e8a4 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_mgr.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/src/app_pipe_mgr.cpp @@ -15,7 +15,7 @@ #include "app_pipe_mgr.h" #include "kvstore_utils.h" -#include "reporter.h" +#include "dfx/reporter.h" #undef LOG_TAG #define LOG_TAG "AppPipeMgr" diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/src/process_communicator_impl.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/src/process_communicator_impl.cpp index 07799948..5d7edbdd 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/src/process_communicator_impl.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/src/process_communicator_impl.cpp @@ -20,6 +20,7 @@ #include "log_print.h" #include "softbus_adapter.h" #include "process_communicator_impl.h" +#include "utils/anonymous.h" namespace OHOS { namespace AppDistributedKv { using namespace DistributedDB; @@ -250,10 +251,8 @@ std::shared_ptr ProcessCommunicatorImpl::GetExtendHeaderHand return {}; } -DBStatus ProcessCommunicatorImpl::CheckAndGetDataHeadInfo( - const uint8_t *data, uint32_t dataLen, uint32_t &headLen, std::vector &users) +DBStatus ProcessCommunicatorImpl::GetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength) { - ZLOGD("begin"); if (routeHeadHandlerCreator_ == nullptr) { ZLOGE("header handler creator not registered"); return DBStatus::DB_ERROR; @@ -263,16 +262,36 @@ DBStatus ProcessCommunicatorImpl::CheckAndGetDataHeadInfo( ZLOGE("failed to get header handler"); return DBStatus::DB_ERROR; } - auto ret = handler->ParseHeadData(data, dataLen, headLen, users); + auto ret = handler->ParseHeadDataLen(data, totalLen, headLength); if (!ret) { - ZLOGD("illegal head format"); + ZLOGE("illegal head format, dataLen:%{public}u, headLength:%{public}u", totalLen, headLength); return DBStatus::INVALID_FORMAT; } - if (users.empty()) { + return DBStatus::OK; +} + +DBStatus ProcessCommunicatorImpl::GetDataUserInfo(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) +{ + if (routeHeadHandlerCreator_ == nullptr) { + ZLOGE("header handler creator not registered"); + return DBStatus::DB_ERROR; + } + auto handler = routeHeadHandlerCreator_({}); + if (handler == nullptr) { + ZLOGE("failed to get header handler"); + return DBStatus::DB_ERROR; + } + auto ret = handler->ParseHeadDataUser(data, totalLen, label, userInfos); + if (!ret) { + ZLOGD("illegal head format, dataLen:%{public}u, label:%{public}s", totalLen, Anonymous::Change(label).c_str()); + return DBStatus::INVALID_FORMAT; + } + if (userInfos.empty()) { ZLOGW("no valid user"); return DBStatus::NO_PERMISSION; } - ZLOGD("ok, result:%{public}zu, user:%{public}s", users.size(), users.front().c_str()); + ZLOGD("ok, userInfos.size:%{public}zu", userInfos.size()); return DBStatus::OK; } diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/src/softbus_adapter_standard.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/src/softbus_adapter_standard.cpp index 3b240afa..6fbbcb80 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/src/softbus_adapter_standard.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/src/softbus_adapter_standard.cpp @@ -16,14 +16,14 @@ #include #include -#include "communicator_context.h" #include "communication/connect_manager.h" +#include "communicator_context.h" #include "data_level.h" #include "device_manager_adapter.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" +#include "dfx/reporter.h" #include "kvstore_utils.h" #include "log_print.h" -#include "reporter.h" #include "securec.h" #include "session.h" #include "softbus_adapter.h" @@ -69,7 +69,7 @@ SoftBusAdapter *AppDataListenerWrap::softBusAdapter_; std::shared_ptr SoftBusAdapter::instance_; namespace { -void OnDataLevelChanged(const char* networkId, const DataLevel dataLevel) +void OnDataLevelChanged(const char *networkId, const DataLevel dataLevel) { if (networkId == nullptr) { return; @@ -207,22 +207,23 @@ std::shared_ptr SoftBusAdapter::GetConnect(const PipeInfo &pipeIn uint32_t qosType) { std::shared_ptr conn; - connects_.Compute(deviceId.deviceId, [&pipeInfo, &deviceId, &conn, qosType](const auto &key, - std::vector> &connects) -> bool { - for (auto &connect : connects) { - if (connect == nullptr) { - continue; - } - if (connect->GetQoSType() == qosType) { - conn = connect; - return true; + connects_.Compute(deviceId.deviceId, + [&pipeInfo, &deviceId, &conn, qosType](const auto &key, + std::vector> &connects) -> bool { + for (auto &connect : connects) { + if (connect == nullptr) { + continue; + } + if (connect->GetQoSType() == qosType) { + conn = connect; + return true; + } } - } - auto connect = std::make_shared(pipeInfo, deviceId, qosType); - connects.emplace_back(connect); - conn = connect; - return true; - }); + auto connect = std::make_shared(pipeInfo, deviceId, qosType); + connects.emplace_back(connect); + conn = connect; + return true; + }); return conn; } @@ -314,8 +315,9 @@ SoftBusAdapter::Task SoftBusAdapter::GetCloseSessionTask() taskId_ = ExecutorPool::INVALID_TASK_ID; return; } - taskId_ = Context::GetInstance().GetThreadPool()->Schedule( - next > now ? next - now : ExecutorPool::INVALID_DELAY, GetCloseSessionTask()); + taskId_ = + Context::GetInstance().GetThreadPool()->Schedule(next > now ? next - now : ExecutorPool::INVALID_DELAY, + GetCloseSessionTask()); next_ = next; }; } @@ -343,23 +345,7 @@ uint32_t SoftBusAdapter::GetMtuSize(const DeviceId &deviceId) uint32_t SoftBusAdapter::GetTimeout(const DeviceId &deviceId) { - uint32_t interval = DEFAULT_TIMEOUT; - connects_.ComputeIfPresent(deviceId.deviceId, [&interval](auto, auto &connects) { - uint32_t time = 0; - for (auto conn : connects) { - if (conn == nullptr) { - continue; - } - if (time < conn->GetTimeout()) { - time = conn->GetTimeout(); - } - } - if (time != 0) { - interval = time; - } - return true; - }); - return interval; + return DEFAULT_TIMEOUT; } std::string SoftBusAdapter::DelConnect(int32_t socket, bool isForce) @@ -614,19 +600,19 @@ Status SoftBusAdapter::ReuseConnect(const PipeInfo &pipeInfo, const DeviceId &de return status; } // Avoid being cleared by scheduled tasks - connects_.Compute(deviceId.deviceId, [&conn, qosType](const auto &key, - std::vector> &connects) -> bool { - for (auto &connect : connects) { - if (connect == nullptr) { - continue; - } - if (connect->GetQoSType() == qosType) { - return true; + connects_.Compute(deviceId.deviceId, + [&conn, qosType](const auto &key, std::vector> &connects) -> bool { + for (auto &connect : connects) { + if (connect == nullptr) { + continue; + } + if (connect->GetQoSType() == qosType) { + return true; + } } - } - connects.emplace_back(conn); - return true; - }); + connects.emplace_back(conn); + return true; + }); StartCloseSessionTask(deviceId.deviceId); return Status::SUCCESS; } diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/communicator/test/BUILD.gn index ecd594ab..5e5a3489 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/BUILD.gn @@ -14,7 +14,7 @@ import("//build/ohos.gni") import("//build/ohos_var.gni") import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### ohos_unittest("CommunicationProviderTest") { @@ -22,9 +22,7 @@ ohos_unittest("CommunicationProviderTest") { sources = [ "./unittest/communication_provider_impl_test.cpp" ] include_dirs = [ - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/autils", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/communicator", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/dfx", + "${data_service_path}/adapter/include/communicator", "../src", ] external_deps = [ @@ -40,14 +38,70 @@ ohos_unittest("CommunicationProviderTest") { defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } +ohos_unittest("CommunicatorContextTest") { + module_out_path = module_output_path + + sources = [ "unittest/communicator_context_test.cpp" ] + + sanitize = { + cfi = true + cfi_cross_dso = true + debug = false + } + + include_dirs = [ + "../src", + "../../include/communicator", + ] + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "dsoftbus:softbus_client", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} + +ohos_unittest("CommunicatorDataBufferTest") { + module_out_path = module_output_path + + sources = [ "unittest/data_buffer_test.cpp" ] + include_dirs = [ "../src" ] + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "dsoftbus:softbus_client", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} + ohos_unittest("DeviceManagerAdapterTest") { module_out_path = module_output_path sources = [ "unittest/device_manager_adapter_test.cpp" ] include_dirs = [ - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/autils", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/communicator", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/dfx", + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/dfx", "../src", ] external_deps = [ @@ -71,15 +125,94 @@ ohos_unittest("DeviceManagerAdapterTest") { defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } +ohos_unittest("ProcessCommunicatorImplTest") { + module_out_path = module_output_path + + sources = [ + "${data_service_path}/service/test/mock/device_manager_adapter_mock.cpp", + "../src/app_pipe_handler.cpp", + "../src/app_pipe_handler.h", + "../src/app_pipe_mgr.cpp", + "../src/app_pipe_mgr.h", + "../src/communication_provider_impl.cpp", + "../src/communication_provider_impl.h", + "../src/communicator_context.cpp", + "../src/data_buffer.cpp", + "../src/process_communicator_impl.cpp", + "../src/softbus_adapter.h", + "../src/softbus_adapter_standard.cpp", + "../src/softbus_client.cpp", + "../src/softbus_client.h", + "unittest/process_communicator_impl_test.cpp", + ] + + include_dirs = [ + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/service/test/mock", + "../src", + "../../include/communicator", + "../../include/utils", + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/framework/include/utils", + ] + + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "dsoftbus:softbus_client", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "ipc:ipc_core", + "kv_store:datamgr_common", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + deps = [ + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + "${data_service_path}/service:distributeddatasvc", + "../../dfx:distributeddata_dfx", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} + ohos_unittest("SoftbusAdapterStandardTest") { module_out_path = module_output_path - sources = [ "unittest/softbus_adapter_standard_test.cpp" ] + sources = [ + "../src/app_pipe_handler.cpp", + "../src/app_pipe_handler.h", + "../src/app_pipe_mgr.cpp", + "../src/app_pipe_mgr.h", + "../src/ark_communication_provider.cpp", + "../src/communication_provider.cpp", + "../src/communication_provider_impl.cpp", + "../src/communicator_context.cpp", + "../src/data_buffer.cpp", + "../src/device_manager_adapter.cpp", + "../src/process_communicator_impl.cpp", + "../src/softbus_adapter.h", + "../src/softbus_client.cpp", + "../src/softbus_client.h", + "unittest/softbus_adapter_standard_test.cpp", + ] + include_dirs = [ - "${data_service_path}/adapter/include/autils", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", + "${data_service_path}/framework/include/dfx", "../src", + "../../include/communicator", + "../../include/utils", + "${data_service_path}/framework/include/utils", ] external_deps = [ "access_token:libaccesstoken_sdk", @@ -90,31 +223,80 @@ ohos_unittest("SoftbusAdapterStandardTest") { "dsoftbus:softbus_client", "hilog:libhilog", "ipc:ipc_core", + "json:nlohmann_json_static", + "kv_store:datamgr_common", "kv_store:distributeddata_inner", + "kv_store:distributeddb", ] cflags = [ "-Dprivate=public", "-Dprotected=public", ] deps = [ - "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + "../../dfx:distributeddata_dfx", ] defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } -############################################################################### -config("module_comm_config") { - visibility = [ ":*" ] +ohos_unittest("SoftbusClientTest") { + module_out_path = module_output_path + + sources = [ "unittest/softbus_client_test.cpp" ] + include_dirs = [ + "${data_service_path}/adapter/include/autils", + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/adapter/include/dfx", + "../src", + ] + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "dsoftbus:softbus_client", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} +ohos_unittest("AppPipeMgrServiceTest") { + module_out_path = module_output_path include_dirs = [ - "./unittest/communicator", - "./unittest/communicator/include", - "//commonlibrary/c_utils/base/include", - "//foundation/distributeddatamgr/kv_store/interfaces/innerkits/distributeddata/include", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/communicator", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/dfx", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/include/autils", + "${data_service_path}/adapter/include/autils", + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/adapter/include/dfx", + "${data_service_path}/adapter/include/utils", + "${data_service_path}/framework/include/dfx", + "../src", ] + sources = [ "unittest/app_pipe_mgr_service_test.cpp" ] + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "dsoftbus:softbus_client", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } ############################################################################### @@ -124,9 +306,14 @@ group("unittest") { deps = [] deps += [ + ":AppPipeMgrServiceTest", ":CommunicationProviderTest", + ":CommunicatorContextTest", + ":CommunicatorDataBufferTest", ":DeviceManagerAdapterTest", + ":ProcessCommunicatorImplTest", ":SoftbusAdapterStandardTest", + ":SoftbusClientTest", ] } ############################################################################### diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/fuzztest/softbusadapter_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/communicator/test/fuzztest/softbusadapter_fuzzer/BUILD.gn index 8f9e579b..cc55d56d 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/test/fuzztest/softbusadapter_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/fuzztest/softbusadapter_fuzzer/BUILD.gn @@ -21,9 +21,6 @@ ohos_fuzztest("SoftBusAdapterFuzzTest") { include_dirs = [ "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", - "${data_service_path}/adapter/include/log", - "${data_service_path}/adapter/include/autils", "${data_service_path}/adapter/include/utils", "${data_service_path}/adapter/communicator/src", "${data_service_path}/framework/include", @@ -40,13 +37,9 @@ ohos_fuzztest("SoftBusAdapterFuzzTest") { sources = [ "softbusadapter_fuzzer.cpp" ] - deps = [ - "${data_service_path}/adapter/dfx:distributeddata_dfx", - "${data_service_path}/adapter/utils:distributeddata_utils", - ] - external_deps = [ "c_utils:utils", + "datamgr_service:distributeddatasvcfwk", "device_manager:devicemanagersdk", "dsoftbus:softbus_client", "hilog:libhilog", diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/app_pipe_mgr_service_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/app_pipe_mgr_service_test.cpp new file mode 100644 index 00000000..f9ebce1a --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/app_pipe_mgr_service_test.cpp @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "AppPipeMgrServiceTest" + +#include "app_pipe_mgr.h" +#include "kvstore_utils.h" +#include "log_print.h" +#include "dfx/reporter.h" +#include "types.h" +#include + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::AppDistributedKv; +class AppDataChangeListenerMock : public AppDataChangeListener { + void OnMessage(const DeviceInfo &info, const uint8_t *ptr, const int size, + const struct PipeInfo &id) const override; +}; + +void AppDataChangeListenerMock::OnMessage(const DeviceInfo &info, + const uint8_t *ptr, const int size, const struct PipeInfo &id) const +{ + return; +} + +class AppPipeMgrServiceTest : public testing::Test { +public: + static void SetUpTestCase(void) {}; + static void TearDownTestCase(void) {}; + void SetUp() {}; + void TearDown() {}; +}; + +/** +* @tc.name: StopWatchDataChange001 +* @tc.desc: StopWatchDataChange test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, StopWatchDataChange001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + const AppDataChangeListener* observer = nullptr; + auto status = appPipeMgr.StopWatchDataChange(observer, pipeInfo); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); + + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + status = appPipeMgr.StopWatchDataChange(observer, pipeInfo); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); + + PipeInfo pipeInfoNull; + const AppDataChangeListener* observerListener = new AppDataChangeListenerMock(); + status = appPipeMgr.StopWatchDataChange(observerListener, pipeInfoNull); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); +} + +/** +* @tc.name: StopWatchDataChange002 +* @tc.desc: StopWatchDataChange test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, StopWatchDataChange002, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + const AppDataChangeListener* observer = new AppDataChangeListenerMock(); + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + auto status = appPipeMgr.StopWatchDataChange(observer, pipeInfo); + EXPECT_EQ(Status::ERROR, status); + + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + status = appPipeMgr.StopWatchDataChange(observer, pipeInfo); + EXPECT_EQ(Status::ERROR, status); +} + +/** +* @tc.name: SendData001 +* @tc.desc: SendData test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, SendData001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + DeviceId deviceId = {"deviceId"}; + std::string content = "Helloworlds"; + const uint8_t *t = reinterpret_cast(content.c_str()); + DataInfo dataInfo = { const_cast(t), static_cast(content.length())}; + uint32_t totalLength = 0; + MessageInfo info; + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + auto status = appPipeMgr.SendData(pipeInfo, deviceId, dataInfo, totalLength, info); + EXPECT_EQ(std::make_pair(Status::RATE_LIMIT, 0), status); +} + +/** +* @tc.name: Start001 +* @tc.desc: Start test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, Start001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + auto status = appPipeMgr.Start(pipeInfo); + EXPECT_EQ(Status::SUCCESS, status); +} + +/** +* @tc.name: Stop001 +* @tc.desc: Stop test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, Stop001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + auto status = appPipeMgr.Stop(pipeInfo); + EXPECT_EQ(Status::KEY_NOT_FOUND, status); + + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + status = appPipeMgr.Stop(pipeInfo); + EXPECT_EQ(Status::SUCCESS, status); +} + +/** +* @tc.name: IsSameStartedOnPeer001 +* @tc.desc: IsSameStartedOnPeer test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, IsSameStartedOnPeer001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + DeviceId deviceId; + auto status = appPipeMgr.IsSameStartedOnPeer(pipeInfo, deviceId); + EXPECT_EQ(false, status); + + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + status = appPipeMgr.IsSameStartedOnPeer(pipeInfo, deviceId); + EXPECT_EQ(false, status); + + PipeInfo pipeInfoNull; + deviceId.deviceId = "deviceId"; + status = appPipeMgr.IsSameStartedOnPeer(pipeInfoNull, deviceId); + EXPECT_EQ(false, status); +} + +/** +* @tc.name: IsSameStartedOnPeer002 +* @tc.desc: IsSameStartedOnPeer test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, IsSameStartedOnPeer002, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + DeviceId deviceId; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + deviceId.deviceId = "deviceId"; + auto status = appPipeMgr.IsSameStartedOnPeer(pipeInfo, deviceId); + EXPECT_EQ(false, status); + + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + status = appPipeMgr.IsSameStartedOnPeer(pipeInfo, deviceId); + EXPECT_EQ(true, status); +} + +/** +* @tc.name: SetMessageTransFlag001 +* @tc.desc: SetMessageTransFlag test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, SetMessageTransFlag001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + bool flag = true; + EXPECT_NO_FATAL_FAILURE( + appPipeMgr.SetMessageTransFlag(pipeInfo, flag)); + + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + EXPECT_NO_FATAL_FAILURE( + appPipeMgr.SetMessageTransFlag(pipeInfo, flag)); +} + +/** +* @tc.name: ReuseConnect001 +* @tc.desc: ReuseConnect test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, ReuseConnect001, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + DeviceId deviceId; + auto status = appPipeMgr.ReuseConnect(pipeInfo, deviceId); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); + + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + status = appPipeMgr.ReuseConnect(pipeInfo, deviceId); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); + + PipeInfo pipeInfoNull; + deviceId.deviceId = "deviceId"; + status = appPipeMgr.ReuseConnect(pipeInfoNull, deviceId); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); +} + +/** +* @tc.name: ReuseConnect002 +* @tc.desc: ReuseConnect test +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(AppPipeMgrServiceTest, ReuseConnect002, TestSize.Level0) +{ + AppPipeMgr appPipeMgr; + PipeInfo pipeInfo; + DeviceId deviceId; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + deviceId.deviceId = "deviceId"; + auto status = appPipeMgr.ReuseConnect(pipeInfo, deviceId); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); + + auto handler = std::make_shared(pipeInfo); + std::map> dataBusMap = {{"pipeId", handler}}; + appPipeMgr.dataBusMap_ = dataBusMap; + status = appPipeMgr.ReuseConnect(pipeInfo, deviceId); + EXPECT_EQ(Status::NOT_SUPPORT, status); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/communicator_context_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/communicator_context_test.cpp new file mode 100644 index 00000000..aeb43212 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/communicator_context_test.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include "communicator_context.h" + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::AppDistributedKv; +using namespace OHOS::DistributedData; +constexpr int32_t SOFTBUS_OK = 0; +using OnCloseAble = std::function; + +class DevChangeListener final : public AppDeviceChangeListener { +public: + void OnDeviceChanged(const DeviceInfo &info, const DeviceChangeType &type) const override + { + } + ChangeLevelType GetChangeLevelType() const override + { + return ChangeLevelType::MIN; + } +}; + +class CommunicatorContextTest : public testing::Test { +public: + static void SetUpTestCase(void) {} + static void TearDownTestCase(void) {} + void SetUp() {} + void TearDown() {} +}; + +/** +* @tc.name: RegSessionListener +* @tc.desc: RegSessionListener test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorContextTest, RegSessionListener, TestSize.Level0) +{ + DevChangeListener *devChangeListener = nullptr; + auto status = CommunicatorContext::GetInstance().RegSessionListener(devChangeListener); + EXPECT_EQ(status, Status::INVALID_ARGUMENT); + DevChangeListener listener; + devChangeListener = &listener; + CommunicatorContext::GetInstance().observers_.emplace_back(devChangeListener); + status = CommunicatorContext::GetInstance().RegSessionListener(devChangeListener); + EXPECT_EQ(status, Status::SUCCESS); +} + +/** +* @tc.name: UnRegSessionListener +* @tc.desc: UnRegSessionListener test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorContextTest, UnRegSessionListener, TestSize.Level0) +{ + DevChangeListener *devChangeListener = nullptr; + auto status = CommunicatorContext::GetInstance().UnRegSessionListener(devChangeListener); + EXPECT_EQ(status, Status::INVALID_ARGUMENT); + DevChangeListener listener; + devChangeListener = &listener; + CommunicatorContext::GetInstance().observers_.clear(); + EXPECT_EQ(CommunicatorContext::GetInstance().observers_.empty(), true); + status = CommunicatorContext::GetInstance().UnRegSessionListener(devChangeListener); + EXPECT_EQ(status, Status::SUCCESS); + CommunicatorContext::GetInstance().observers_.emplace_back(devChangeListener); + EXPECT_EQ(CommunicatorContext::GetInstance().observers_.empty(), false); + status = CommunicatorContext::GetInstance().UnRegSessionListener(devChangeListener); + EXPECT_EQ(status, Status::SUCCESS); +} + +/** +* @tc.name: NotifySessionReady +* @tc.desc: NotifySessionReady test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorContextTest, NotifySessionReady, TestSize.Level0) +{ + std::string deviceId[2] = {"CommunicatorContextTest", "CommunicatorContextTest"}; + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Empty(), true); + + DevChangeListener *devChangeListener = nullptr; + DevChangeListener *devChangeListener1 = nullptr; + DevChangeListener listener; + devChangeListener1 = &listener; + CommunicatorContext::GetInstance().observers_.clear(); + CommunicatorContext::GetInstance().observers_.emplace_back(devChangeListener); + CommunicatorContext::GetInstance().observers_.emplace_back(devChangeListener1); + + CommunicatorContext::GetInstance().closeListener_ = nullptr; + CommunicatorContext::GetInstance().NotifySessionReady(deviceId[0], 1); + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Empty(), true); + CommunicatorContext::GetInstance().NotifySessionReady(deviceId[0], SOFTBUS_OK); + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Contains(deviceId[0]), true); + auto myCloseHandler = [](const std::string &deviceId) { + const_cast(deviceId) = "CommunicatorContextTest01"; + }; + OnCloseAble closeAbleCallback; + closeAbleCallback = myCloseHandler; + CommunicatorContext::GetInstance().SetSessionListener(closeAbleCallback); + CommunicatorContext::GetInstance().NotifySessionReady(deviceId[0], SOFTBUS_OK); + ASSERT_NE(deviceId[0], deviceId[1]); +} + +/** +* @tc.name: NotifySessionClose +* @tc.desc: NotifySessionClose test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorContextTest, NotifySessionClose, TestSize.Level0) +{ + const std::string deviceId = "NotifySessionCloseTest"; + CommunicatorContext::GetInstance().devices_.Clear(); + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Empty(), true); + CommunicatorContext::GetInstance().devices_.Insert(deviceId, deviceId); + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Empty(), false); + CommunicatorContext::GetInstance().NotifySessionClose(deviceId); + EXPECT_EQ(CommunicatorContext::GetInstance().devices_.Empty(), true); +} + +/** +* @tc.name: IsSessionReady +* @tc.desc: IsSessionReady test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorContextTest, IsSessionReady, TestSize.Level0) +{ + const std::string deviceId = "NotifySessionCloseTest"; + const std::string deviceId1 = ""; + auto result = CommunicatorContext::GetInstance().IsSessionReady(deviceId1); + EXPECT_EQ(result, false); + result = CommunicatorContext::GetInstance().IsSessionReady(deviceId); + EXPECT_EQ(result, false); + CommunicatorContext::GetInstance().devices_.Insert(deviceId, deviceId); + result = CommunicatorContext::GetInstance().IsSessionReady(deviceId); + EXPECT_EQ(result, true); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/data_buffer_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/data_buffer_test.cpp new file mode 100644 index 00000000..71b76f40 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/data_buffer_test.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include "data_buffer.h" + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::AppDistributedKv; + +class CommunicatorDataBufferTest : public testing::Test { +public: + static void SetUpTestCase(void) {} + static void TearDownTestCase(void) {} + void SetUp() {} + void TearDown() {} +}; + +/** +* @tc.name: Init +* @tc.desc: Init Test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(CommunicatorDataBufferTest, Init, TestSize.Level0) +{ + DataBuffer buffer; + auto result = buffer.Init(0); + ASSERT_EQ(result, false); + char buff[] = "DataBuffer Init test"; + buffer.buf_ = buff; + result = buffer.Init(1); + ASSERT_EQ(result, false); + result = buffer.Init(0); + ASSERT_EQ(result, false); + buffer.buf_ = nullptr; + result = buffer.Init(1); + ASSERT_EQ(result, true); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/device_manager_adapter_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/device_manager_adapter_test.cpp index d9005d26..a813fb8a 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/device_manager_adapter_test.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/device_manager_adapter_test.cpp @@ -394,6 +394,62 @@ HWTEST_F(DeviceManagerAdapterTest, GetDeviceInfo, TestSize.Level0) EXPECT_TRUE(dvInfo.networkId.empty()); } +/** +* @tc.name: GetDeviceInfo +* @tc.desc: get device info +* @tc.type: FUNC +* @tc.author: nhj + */ +HWTEST_F(DeviceManagerAdapterTest, GetDeviceInfo01, TestSize.Level0) +{ + auto executors = std::make_shared(0, 0); + DeviceManagerAdapter::GetInstance().Init(executors); + OHOS::DistributedHardware::DmDeviceInfo deviceInfo = { + .deviceId = "123", + .deviceName = "asda", + .deviceTypeId = 1, + .networkId = "", + }; + DeviceInfo info; + info.uuid = "ohos.test.uuid"; + info.udid = "ohos.test.udid"; + info.networkId = "ohos.test.networkId"; + info.deviceName = "ohos.test.deviceName"; + info.deviceType = 1; + info.osType = 1; + info.authForm = 1; + auto dvInfo = DeviceManagerAdapter::GetInstance().GetDeviceInfo(deviceInfo, info); + EXPECT_EQ(dvInfo, false); +} + +/** +* @tc.name: GetDeviceInfo +* @tc.desc: get device info +* @tc.type: FUNC +* @tc.author: nhj + */ +HWTEST_F(DeviceManagerAdapterTest, GetDeviceInfo02, TestSize.Level0) +{ + auto executors = std::make_shared(0, 0); + DeviceManagerAdapter::GetInstance().Init(executors); + OHOS::DistributedHardware::DmDeviceInfo deviceInfo = { + .deviceId = "123", + .deviceName = "asda", + .deviceTypeId = 1, + .networkId = "14569", + }; + DeviceInfo info; + info.uuid = ""; + info.udid = ""; + info.networkId = "ohos.test.networkId"; + info.deviceName = "ohos.test.deviceName"; + info.deviceType = 1; + info.osType = 1; + info.authForm = 1; + auto dvInfo = DeviceManagerAdapter::GetInstance().GetDeviceInfo(deviceInfo, info); + EXPECT_EQ(dvInfo, false); +} + /** * @tc.name: GetDeviceInfo * @tc.desc: get device info, the id is invalid @@ -418,4 +474,152 @@ HWTEST_F(DeviceManagerAdapterTest, GetOnlineDevices, TestSize.Level0) EXPECT_TRUE(onInfos.empty()); } +/** +* @tc.name: Online +* @tc.desc: test OnDeviceOnline function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, Online, TestSize.Level0) +{ + DeviceManagerAdapter DeviceManagerAdapterTest; + OHOS::DistributedHardware::DmDeviceInfo deviceInfo({ "cloudDeviceId", + "cloudDeviceName", 0, "cloudNetworkId", 0 }); + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.Online(deviceInfo)); +} + +/** +* @tc.name: Offline +* @tc.desc: test Offline function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, Offline, TestSize.Level0) +{ + DeviceManagerAdapter DeviceManagerAdapterTest; + OHOS::DistributedHardware::DmDeviceInfo info01 = { + .deviceId = "123", + .deviceName = "asda", + .deviceTypeId = 1, + .networkId = "14569", + }; + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.Offline(info01)); +} + +/** +* @tc.name: OnChanged +* @tc.desc: test OnChanged function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, OnChanged, TestSize.Level0) +{ + DeviceManagerAdapter DeviceManagerAdapterTest; + OHOS::DistributedHardware::DmDeviceInfo deviceInfo({ "cloudDeviceId", + "cloudDeviceName", 1, "cloudNetworkId", 1 }); + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.OnChanged(deviceInfo)); +} + +/** +* @tc.name: OnReady +* @tc.desc: test OnReady function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, OnReady, TestSize.Level0) +{ + DeviceManagerAdapter DeviceManagerAdapterTest; + OHOS::DistributedHardware::DmDeviceInfo info = { + .deviceId = "111", + .deviceName = "device", + .deviceTypeId = 1, + .networkId = "12345", + }; + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.OnReady(info)); +} + +/** +* @tc.name: SaveDeviceInfo +* @tc.desc: test SaveDeviceInfo function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, SaveDeviceInfo, TestSize.Level0) +{ + DeviceManagerAdapter DeviceManagerAdapterTest; + DeviceInfo info1; + info1.uuid = ""; + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.SaveDeviceInfo(info1, DeviceChangeType::DEVICE_OFFLINE)); + + DeviceInfo info; + info.uuid = "ohos.test.uuid"; + info.udid = "ohos.test.udid"; + info.networkId = "ohos.test.networkId"; + info.deviceName = "ohos.test.deviceName"; + info.deviceType = 1; + info.osType = 1; + info.authForm = 1; + EXPECT_NO_FATAL_FAILURE(DeviceManagerAdapterTest.SaveDeviceInfo(info, DeviceChangeType::DEVICE_ONLINE)); +} + +/** +* @tc.name: GetEncryptedUuidByNetworkId +* @tc.desc: get uuid by networkId, the networkId is invalid +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj + */ +HWTEST_F(DeviceManagerAdapterTest, GetEncryptedUuidByNetworkId, TestSize.Level0) +{ + auto uuid = DeviceManagerAdapter::GetInstance().GetEncryptedUuidByNetworkId(EMPTY_DEVICE_ID); + EXPECT_TRUE(uuid.empty()); + uuid = DeviceManagerAdapter::GetInstance().GetEncryptedUuidByNetworkId(INVALID_DEVICE_ID); + EXPECT_TRUE(uuid.empty()); +} + +/** +* @tc.name: GetEncryptedUuidByNetworkId +* @tc.desc: get uuid by networkId +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj + */ +HWTEST_F(DeviceManagerAdapterTest, GetEncryptedUuidByNetworkId01, TestSize.Level0) +{ + auto result = DeviceManagerAdapter::GetInstance().GetEncryptedUuidByNetworkId(""); + EXPECT_EQ(result, ""); +} + +/** +* @tc.name: IsSameAccount +* @tc.desc: test IsSameAccount function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, IsSameAccount, TestSize.Level0) +{ + std::string networkId = ""; + auto status = DeviceManagerAdapter::GetInstance().IsSameAccount(networkId); + EXPECT_EQ(status, false); +} + +/** +* @tc.name: IsSameAccount +* @tc.desc: test IsSameAccount function +* @tc.type: FUNC +* @tc.require: +* @tc.author: nhj +*/ +HWTEST_F(DeviceManagerAdapterTest, IsSameAccount01, TestSize.Level0) +{ + std::string networkId = "test_network_id"; + auto status = DeviceManagerAdapter::GetInstance().IsSameAccount(networkId); + EXPECT_EQ(status, false); +} } // namespace \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/process_communicator_impl_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/process_communicator_impl_test.cpp new file mode 100644 index 00000000..f5c4cc30 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/process_communicator_impl_test.cpp @@ -0,0 +1,506 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "communication_provider.h" +#include "device_manager_adapter_mock.h" +#include "process_communicator_impl.h" +#include "softbus_adapter.h" +#include "softbus_error_code.h" + +using namespace testing; +using namespace testing::ext; +using namespace OHOS::AppDistributedKv; +using namespace OHOS::DistributedData; +using OnDeviceChange = DistributedDB::OnDeviceChange; +using OnDataReceive = DistributedDB::OnDataReceive; +using OnSendAble = DistributedDB::OnSendAble; +using DeviceInfos = DistributedDB::DeviceInfos; +using DeviceInfoo = OHOS::AppDistributedKv::DeviceInfo; +using UserInfo = DistributedDB::UserInfo; + +namespace OHOS::AppDistributedKv { +class MockCommunicationProvider : public CommunicationProvider { +public: + ~MockCommunicationProvider() = default; + static MockCommunicationProvider& Init() + { + static MockCommunicationProvider instance; + return instance; + } + MOCK_METHOD(Status, StartWatchDataChange, (const AppDataChangeListener *, const PipeInfo &), (override)); + MOCK_METHOD(Status, StopWatchDataChange, (const AppDataChangeListener *, const PipeInfo &), (override)); + MOCK_METHOD(Status, Start, (const PipeInfo&), (override)); + MOCK_METHOD(Status, Stop, (const PipeInfo&), (override)); + MOCK_METHOD((std::pair), SendData, (const PipeInfo&, const DeviceId&, + const DataInfo&, uint32_t, const MessageInfo &), (override)); + MOCK_METHOD(bool, IsSameStartedOnPeer, (const PipeInfo &, const DeviceId &), (const)); + MOCK_METHOD(Status, ReuseConnect, (const PipeInfo&, const DeviceId&), (override)); + MOCK_METHOD(void, SetDeviceQuery, (std::shared_ptr), (override)); + MOCK_METHOD(void, SetMessageTransFlag, (const PipeInfo &, bool), (override)); + MOCK_METHOD(Status, Broadcast, (const PipeInfo &, const LevelInfo &), (override)); + MOCK_METHOD(int32_t, ListenBroadcastMsg, (const PipeInfo &, + std::function), (override)); +}; + +CommunicationProvider& CommunicationProvider::GetInstance() +{ + return MockCommunicationProvider::Init(); +} + +std::shared_ptr CommunicationProvider::MakeCommunicationProvider() +{ + static std::shared_ptr instance( + &MockCommunicationProvider::Init(), + [](void*) { + } + ); + return instance; +} +} + +class ProcessCommunicatorImplTest : public testing::Test { +public: + static inline std::shared_ptr deviceManagerAdapterMock = nullptr; + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + + ProcessCommunicatorImpl* communicator_; + MockCommunicationProvider* mockProvider; +}; + +namespace OHOS::DistributedData { +class ConcreteRouteHeadHandler : public RouteHeadHandler { + public: + bool ParseHeadDataLen(const uint8_t *data, uint32_t totalLen, uint32_t &headSize) + { + if (totalLen == 0) { + return true; + } + return false; + } + + bool ParseHeadDataUser(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) + { + if (totalLen == 0) { + return true; + } + return false; + } +}; +} + +void ProcessCommunicatorImplTest::SetUp(void) +{ + communicator_ = ProcessCommunicatorImpl::GetInstance(); + communicator_->Stop(); + mockProvider = &MockCommunicationProvider::Init(); +} + +void ProcessCommunicatorImplTest::TearDown(void) +{ + mockProvider = nullptr; +} + +void ProcessCommunicatorImplTest::SetUpTestCase(void) +{ + deviceManagerAdapterMock = std::make_shared(); + BDeviceManagerAdapter::deviceManagerAdapter = deviceManagerAdapterMock; +} + +void ProcessCommunicatorImplTest::TearDownTestCase() +{ + deviceManagerAdapterMock = nullptr; + BDeviceManagerAdapter::deviceManagerAdapter = nullptr; +} + +/** +* @tc.name: StartTest01 +* @tc.desc: StartTest01 test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, StartTest01, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + EXPECT_CALL(*mockProvider, Start(_)).WillOnce(Return(Status::SUCCESS)); + auto status = communicator_->Start("test_process"); + EXPECT_EQ(status, DistributedDB::OK); + EXPECT_CALL(*mockProvider, Start(_)).WillOnce(Return(Status::INVALID_ARGUMENT)); + status = communicator_->Start("test_process"); + EXPECT_EQ(status, DistributedDB::DB_ERROR); +} + +/** +* @tc.name: StopTest01 +* @tc.desc: StopTest01 test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, StopTest01, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + EXPECT_CALL(*mockProvider, Stop(_)).WillOnce(Return(Status::SUCCESS)); + auto status = communicator_->Stop(); + EXPECT_EQ(status, DistributedDB::OK); + EXPECT_CALL(*mockProvider, Stop(_)).WillRepeatedly(Return(Status::INVALID_ARGUMENT)); + status = communicator_->Stop(); + EXPECT_EQ(status, DistributedDB::DB_ERROR); +} + +/** +* @tc.name: RegOnDeviceChange +* @tc.desc: RegOnDeviceChange test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, RegOnDeviceChange, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + EXPECT_CALL(*deviceManagerAdapterMock, StartWatchDeviceChange(_, _)).WillOnce(Return(Status::SUCCESS)) + .WillRepeatedly(Return(Status::SUCCESS)); + EXPECT_CALL(*deviceManagerAdapterMock, StopWatchDeviceChange(_, _)).WillOnce(Return(Status::SUCCESS)) + .WillRepeatedly(Return(Status::SUCCESS)); + auto myOnDeviceChange = [](const DeviceInfos &devInfo, bool isOnline) {}; + OnDeviceChange callback; + callback = myOnDeviceChange; + auto status = communicator_->RegOnDeviceChange(callback); + EXPECT_EQ(status, DistributedDB::OK); + callback = nullptr; + status = communicator_->RegOnDeviceChange(callback); + EXPECT_EQ(status, DistributedDB::OK); + + EXPECT_CALL(*deviceManagerAdapterMock, StartWatchDeviceChange(_, _)).WillOnce(Return(Status::INVALID_ARGUMENT)) + .WillRepeatedly(Return(Status::SUCCESS)); + EXPECT_CALL(*deviceManagerAdapterMock, StopWatchDeviceChange(_, _)).WillOnce(Return(Status::INVALID_ARGUMENT)) + .WillRepeatedly(Return(Status::SUCCESS)); + status = communicator_->RegOnDeviceChange(callback); + EXPECT_EQ(status, DistributedDB::DB_ERROR); + callback = myOnDeviceChange; + status = communicator_->RegOnDeviceChange(callback); + EXPECT_EQ(status, DistributedDB::DB_ERROR); +} + +/** +* @tc.name: RegOnDataReceive +* @tc.desc: RegOnDataReceive test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, RegOnDataReceive, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + auto myOnDeviceChange = [](const DeviceInfos &devInfo, const uint8_t *data, uint32_t length) {}; + OnDataReceive callback; + callback = myOnDeviceChange; + EXPECT_CALL(*mockProvider, StartWatchDataChange(_, _)).WillRepeatedly(Return(Status::SUCCESS)); + auto status = communicator_->RegOnDataReceive(callback); + EXPECT_EQ(status, DistributedDB::OK); + EXPECT_CALL(*mockProvider, StartWatchDataChange(_, _)).WillRepeatedly(Return(Status::ERROR)); + status = communicator_->RegOnDataReceive(callback); + EXPECT_EQ(status, DistributedDB::DB_ERROR); + + callback = nullptr; + EXPECT_CALL(*mockProvider, StopWatchDataChange(_, _)).WillRepeatedly(Return(Status::SUCCESS)); + status = communicator_->RegOnDataReceive(callback); + EXPECT_EQ(status, DistributedDB::OK); + EXPECT_CALL(*mockProvider, StopWatchDataChange(_, _)).WillRepeatedly(Return(Status::ERROR)); + status = communicator_->RegOnDataReceive(callback); + EXPECT_EQ(status, DistributedDB::DB_ERROR); +} + +/** +* @tc.name: RegOnSendAble +* @tc.desc: RegOnSendAble test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, RegOnSendAble, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + OnSendAble sendAbleCallback = nullptr; + EXPECT_NO_FATAL_FAILURE(communicator_->RegOnSendAble(sendAbleCallback)); +} + +/** +* @tc.name: SendData +* @tc.desc: SendData test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, SendData, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + DeviceInfos deviceInfos = {"SendDataTest"}; + const uint8_t *data; + uint8_t exampleData[] = {0x01, 0x02, 0x03, 0x04}; + data = exampleData; + uint32_t length = 1; + uint32_t totalLength = 1; + std::pair myPair = std::make_pair(Status::RATE_LIMIT, 1); + EXPECT_CALL(*mockProvider, SendData(_, _, _, _, _)).WillRepeatedly(Return(myPair)); + auto status = communicator_->SendData(deviceInfos, data, length, totalLength); + EXPECT_EQ(status, DistributedDB::RATE_LIMIT); + myPair = {Status::SUCCESS, 1}; + EXPECT_CALL(*mockProvider, SendData(_, _, _, _, _)).WillRepeatedly(Return(myPair)); + status = communicator_->SendData(deviceInfos, data, length, totalLength); + EXPECT_EQ(status, DistributedDB::OK); + myPair = {Status::ERROR, static_cast(DistributedDB::BUSY)}; + EXPECT_CALL(*mockProvider, SendData(_, _, _, _, _)).WillRepeatedly(Return(myPair)); + status = communicator_->SendData(deviceInfos, data, length, totalLength); + EXPECT_EQ(status, DistributedDB::BUSY); + myPair = {Status::ERROR, 0}; + EXPECT_CALL(*mockProvider, SendData(_, _, _, _, _)).WillRepeatedly(Return(myPair)); + status = communicator_->SendData(deviceInfos, data, length, totalLength); + EXPECT_EQ(status, DistributedDB::DB_ERROR); +} + +/** +* @tc.name: GetRemoteOnlineDeviceInfosList +* @tc.desc: GetRemoteOnlineDeviceInfosList test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, GetRemoteOnlineDeviceInfosList, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + auto remoteDevInfos = communicator_->GetRemoteOnlineDeviceInfosList(); + EXPECT_EQ(remoteDevInfos.empty(), true); + + DeviceInfoo deviceInfo; + deviceInfo.uuid = "GetRemoteOnlineDeviceInfosList"; + std::vector devInfos; + devInfos.push_back(deviceInfo); + EXPECT_CALL(*deviceManagerAdapterMock, GetRemoteDevices()).WillRepeatedly(Return(devInfos)); + remoteDevInfos = communicator_->GetRemoteOnlineDeviceInfosList(); + EXPECT_EQ(remoteDevInfos[0].identifier, deviceInfo.uuid); +} + +/** +* @tc.name: OnMessage +* @tc.desc: OnMessage test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, OnMessage, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + DeviceInfoo deviceInfo; + deviceInfo.uuid = "OnMessageTest"; + uint8_t data[] = {0x10, 0x20, 0x30, 0x40, 0x50}; + uint8_t *ptr = data; + int size = 1; + PipeInfo pipeInfo; + pipeInfo.pipeId = "OnMessageTest01"; + pipeInfo.userId = "OnMessageTest02"; + auto myOnDeviceChange = [](const DeviceInfos &devInfo, const uint8_t *data, uint32_t length) {}; + communicator_->onDataReceiveHandler_ = nullptr; + EXPECT_NO_FATAL_FAILURE(communicator_->OnMessage(deviceInfo, ptr, size, pipeInfo)); + communicator_->onDataReceiveHandler_ = myOnDeviceChange; + EXPECT_NO_FATAL_FAILURE(communicator_->OnMessage(deviceInfo, ptr, size, pipeInfo)); +} + +/** +* @tc.name: OnDeviceChanged +* @tc.desc: OnDeviceChanged test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, OnDeviceChanged, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + DeviceInfoo deviceInfo; + deviceInfo.uuid = "cloudDeviceUuid"; + EXPECT_NO_FATAL_FAILURE(communicator_->OnDeviceChanged(deviceInfo, DeviceChangeType::DEVICE_ONREADY)); + EXPECT_NO_FATAL_FAILURE(communicator_->OnDeviceChanged(deviceInfo, DeviceChangeType::DEVICE_OFFLINE)); + deviceInfo.uuid = "OnDeviceChangedTest"; + communicator_->onDeviceChangeHandler_ = nullptr; + EXPECT_NO_FATAL_FAILURE(communicator_->OnDeviceChanged(deviceInfo, DeviceChangeType::DEVICE_OFFLINE)); + auto myOnDeviceChange = [](const DeviceInfos &devInfo, bool isOnline) {}; + communicator_->onDeviceChangeHandler_ = myOnDeviceChange; + EXPECT_NO_FATAL_FAILURE(communicator_->OnDeviceChanged(deviceInfo, DeviceChangeType::DEVICE_OFFLINE)); +} + +/** +* @tc.name: OnSessionReady +* @tc.desc: OnSessionReady test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, OnSessionReady, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + DeviceInfoo deviceInfo; + deviceInfo.uuid = "OnSessionReadyTest"; + communicator_->sessionListener_ = nullptr; + EXPECT_NO_FATAL_FAILURE(communicator_->OnSessionReady(deviceInfo, 1)); + auto myOnSendAble = [](const DeviceInfos &deviceInfo, int deviceCommErrCode) {}; + communicator_->sessionListener_ = myOnSendAble; + EXPECT_NO_FATAL_FAILURE(communicator_->OnSessionReady(deviceInfo, 1)); +} + +/** +* @tc.name: GetExtendHeaderHandle +* @tc.desc: GetExtendHeaderHandle test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, GetExtendHeaderHandle, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + communicator_->routeHeadHandlerCreator_ = nullptr; + DistributedDB::ExtendInfo info; + auto handle = communicator_->GetExtendHeaderHandle(info); + EXPECT_EQ(handle, nullptr); + communicator_->routeHeadHandlerCreator_ = [](const DistributedDB::ExtendInfo &info) -> + std::shared_ptr { + return std::make_shared(); + }; + handle = communicator_->GetExtendHeaderHandle(info); + EXPECT_NE(handle, nullptr); +} + +/** +* @tc.name: GetDataHeadInfo +* @tc.desc: GetDataHeadInfo test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, GetDataHeadInfo, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + uint8_t data[] = {0x10, 0x20, 0x30, 0x40, 0x50}; + uint8_t *ptr = data; + uint32_t totalLen = 1; + uint32_t headLength = 1; + communicator_->routeHeadHandlerCreator_ = nullptr; + auto status = communicator_->GetDataHeadInfo(ptr, totalLen, headLength); + EXPECT_EQ(status, DistributedDB::DB_ERROR); + + communicator_->routeHeadHandlerCreator_ = [](const DistributedDB::ExtendInfo &info) -> + std::shared_ptr { + return std::make_shared(); + }; + status = communicator_->GetDataHeadInfo(ptr, totalLen, headLength); + EXPECT_EQ(status, DistributedDB::INVALID_FORMAT); + totalLen = 0; + status = communicator_->GetDataHeadInfo(ptr, totalLen, headLength); + EXPECT_EQ(status, DistributedDB::OK); +} + +/** +* @tc.name: GetDataUserInfo +* @tc.desc: GetDataUserInfo test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, GetDataUserInfo, TestSize.Level0) +{ + ASSERT_NE(communicator_, nullptr); + uint8_t data[] = {0x10, 0x20, 0x30, 0x40, 0x50}; + uint8_t *ptr = data; + uint32_t totalLen = 1; + std::string label = "GetDataUserInfoTest"; + std::vector userInfos; + UserInfo user1{"GetDataUserInfo01"}; + UserInfo user2{"GetDataUserInfo02"}; + UserInfo user3{"GetDataUserInfo03"}; + communicator_->routeHeadHandlerCreator_ = nullptr; + auto status = communicator_->GetDataUserInfo(ptr, totalLen, label, userInfos); + EXPECT_EQ(status, DistributedDB::DB_ERROR); + + communicator_->routeHeadHandlerCreator_ = [](const DistributedDB::ExtendInfo &info) -> + std::shared_ptr { + return std::make_shared(); + }; + status = communicator_->GetDataUserInfo(ptr, totalLen, label, userInfos); + EXPECT_EQ(status, DistributedDB::INVALID_FORMAT); + totalLen = 0; + EXPECT_EQ(userInfos.empty(), true); + status = communicator_->GetDataUserInfo(ptr, totalLen, label, userInfos); + EXPECT_EQ(status, DistributedDB::NO_PERMISSION); + userInfos.push_back(user1); + userInfos.push_back(user2); + userInfos.push_back(user3); + status = communicator_->GetDataUserInfo(ptr, totalLen, label, userInfos); + EXPECT_EQ(status, DistributedDB::OK); +} + +/** +* @tc.name: ReuseConnect01 +* @tc.desc: reuse connect +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(ProcessCommunicatorImplTest, ReuseConnect01, TestSize.Level0) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PipeInfo pipe; + pipe.pipeId = "appId"; + pipe.userId = "groupId"; + DeviceId device = {"DeviceId"}; + auto status = SoftBusAdapter::GetInstance()->ReuseConnect(pipe, device); + EXPECT_EQ(status, Status::NOT_SUPPORT); + EXPECT_CALL(*deviceManagerAdapterMock, IsOHOSType(testing::_)).WillOnce(testing::Return(true)) + .WillRepeatedly(testing::Return(true)); + status = SoftBusAdapter::GetInstance()->ReuseConnect(pipe, device); + EXPECT_EQ(status, Status::NETWORK_ERROR); +} + +/** +* @tc.name: CloseSession +* @tc.desc: close session +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(ProcessCommunicatorImplTest, CloseSession, TestSize.Level1) +{ + std::string networkId = "networkId"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + std::string uuid = "CloseSessionTest"; + EXPECT_CALL(*deviceManagerAdapterMock, GetUuidByNetworkId(testing::_)).WillOnce(testing::Return(uuid)) + .WillRepeatedly(testing::Return(uuid)); + std::shared_ptr conn = nullptr; + std::vector> clients; + clients.emplace_back(conn); + auto result = SoftBusAdapter::GetInstance()->connects_.Insert(uuid, clients); + EXPECT_EQ(result, true); + auto status = SoftBusAdapter::GetInstance()->CloseSession(networkId); + SoftBusAdapter::GetInstance()->connects_.Clear(); + EXPECT_EQ(status, true); +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_adapter_standard_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_adapter_standard_test.cpp index 41239891..4666457e 100644 --- a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_adapter_standard_test.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_adapter_standard_test.cpp @@ -12,14 +12,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define LOG_TAG "SoftbusAdapterStandardTest" - #include "app_device_change_listener.h" #include #include "gtest/gtest.h" #include -#include "log_print.h" #include "softbus_adapter.h" +#include "softbus_adapter_standard.cpp" +#include "softbus_error_code.h" #include "types.h" #include #include @@ -27,6 +26,7 @@ namespace OHOS::Test { using namespace testing::ext; using namespace OHOS::AppDistributedKv; +using namespace OHOS::DistributedData; using DeviceInfo = OHOS::AppDistributedKv::DeviceInfo; class AppDataChangeListenerImpl : public AppDataChangeListener { struct ServerSocketInfo { @@ -41,9 +41,7 @@ class AppDataChangeListenerImpl : public AppDataChangeListener { void AppDataChangeListenerImpl::OnMessage(const OHOS::AppDistributedKv::DeviceInfo &info, const uint8_t *ptr, const int size, const struct PipeInfo &id) const -{ - ZLOGI("data %{public}s %s", info.deviceName.c_str(), ptr); -} +{} class SoftbusAdapterStandardTest : public testing::Test { public: @@ -65,6 +63,7 @@ protected: */ HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange, TestSize.Level0) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); auto status = SoftBusAdapter::GetInstance()->StartWatchDataChange(nullptr, {}); EXPECT_EQ(status, Status::INVALID_ARGUMENT); } @@ -78,12 +77,16 @@ HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange, TestSize.Level0) */ HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange01, TestSize.Level0) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); PipeInfo appId; appId.pipeId = "appId"; appId.userId = "groupId"; const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); auto status = SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, appId); EXPECT_EQ(status, Status::SUCCESS); + status = SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, appId); + delete dataListener; + EXPECT_EQ(status, Status::ERROR); } /** @@ -95,14 +98,34 @@ HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange01, TestSize.Level0) */ HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange02, TestSize.Level0) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); PipeInfo appId; appId.pipeId = ""; appId.userId = "groupId"; const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); auto status = SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, appId); + delete dataListener; EXPECT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: StartWatchDeviceChange03 +* @tc.desc:the observer is nullptr +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, StartWatchDeviceChange03, TestSize.Level1) +{ + PipeInfo appId; + appId.pipeId = "appId06"; + appId.userId = "groupId06"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + SoftBusAdapter::GetInstance()->StartWatchDataChange(nullptr, appId); + auto status = SoftBusAdapter::GetInstance()->StartWatchDataChange(nullptr, appId); + EXPECT_EQ(Status::INVALID_ARGUMENT, status); +} + /** * @tc.name: StopWatchDataChange * @tc.desc: stop watch data change @@ -115,9 +138,14 @@ HWTEST_F(SoftbusAdapterStandardTest, StopWatchDataChange, TestSize.Level0) PipeInfo appId; appId.pipeId = "appId"; appId.userId = "groupId"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); auto status = SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, appId); EXPECT_EQ(status, Status::SUCCESS); + status = SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, appId); + delete dataListener; + EXPECT_EQ(status, Status::ERROR); } /** @@ -129,14 +157,34 @@ HWTEST_F(SoftbusAdapterStandardTest, StopWatchDataChange, TestSize.Level0) */ HWTEST_F(SoftbusAdapterStandardTest, StopWatchDataChange01, TestSize.Level0) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); PipeInfo appId; appId.pipeId = ""; appId.userId = "groupId"; const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); auto status = SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, appId); + delete dataListener; EXPECT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: GetExpireTime +* @tc.desc: GetExpireTime Test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusAdapterStandardTest, GetExpireTime, TestSize.Level0) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PipeInfo appId; + appId.pipeId = "appId"; + appId.userId = "groupId"; + DeviceId id = {"DeviceId"}; + std::shared_ptr conn = std::make_shared(appId, id, SoftBusClient::QoSType::QOS_HML); + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->GetExpireTime(conn)); +} + /** * @tc.name: SendData * @tc.desc: parse sent data @@ -145,6 +193,7 @@ HWTEST_F(SoftbusAdapterStandardTest, StopWatchDataChange01, TestSize.Level0) */ HWTEST_F(SoftbusAdapterStandardTest, SendData, TestSize.Level1) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); PipeInfo id; id.pipeId = "appId"; @@ -161,6 +210,215 @@ HWTEST_F(SoftbusAdapterStandardTest, SendData, TestSize.Level1) delete dataListener; } +/** +* @tc.name: SendData01 +* @tc.desc: parse sent data +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, SendData01, TestSize.Level1) +{ + const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); + PipeInfo pipe01; + pipe01.pipeId = "appId"; + pipe01.userId = "groupId"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto secRegister = SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, pipe01); + EXPECT_EQ(Status::SUCCESS, secRegister); + std::string content = ""; + const uint8_t *t = reinterpret_cast(content.c_str()); + DeviceId di = {"DeviceId"}; + DataInfo data = { const_cast(t), static_cast(content.length())}; + auto status = SoftBusAdapter::GetInstance()->SendData(pipe01, di, data, 10, { MessageType::FILE }); + EXPECT_NE(status.first, Status::ERROR); + EXPECT_EQ(status.first, Status::RATE_LIMIT); + SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, pipe01); + delete dataListener; +} + +/** +* @tc.name: StartCloseSessionTask +* @tc.desc: StartCloseSessionTask tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, StartCloseSessionTask, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + std::shared_ptr conn = nullptr; + std::vector> clients; + clients.emplace_back(conn); + auto status = SoftBusAdapter::GetInstance()->connects_.Insert("deviceId01", clients); + EXPECT_EQ(status, true); + SoftBusAdapter::GetInstance()->connects_.Clear(); + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->StartCloseSessionTask("deviceId02")); +} + +/** +* @tc.name: OnClientShutdown +* @tc.desc: DelConnect tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, OnClientShutdown, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PipeInfo appId; + appId.pipeId = "appId"; + appId.userId = "groupId"; + DeviceId id = {"DeviceId"}; + std::shared_ptr conn1 = std::make_shared(appId, id, SoftBusClient::QoSType::QOS_HML); + std::shared_ptr conn2 = nullptr; + std::vector> clients; + clients.emplace_back(conn1); + clients.emplace_back(conn2); + auto status = SoftBusAdapter::GetInstance()->connects_.Insert("deviceId01", clients); + EXPECT_EQ(status, true); + status = SoftBusAdapter::GetInstance()->connects_.Insert("deviceId02", {}); + EXPECT_EQ(status, true); + auto name = SoftBusAdapter::GetInstance()->OnClientShutdown(-1, true); + EXPECT_EQ(name, "deviceId01 "); + name = SoftBusAdapter::GetInstance()->OnClientShutdown(-1, false); + EXPECT_EQ(name, ""); + name = SoftBusAdapter::GetInstance()->OnClientShutdown(1, true); + SoftBusAdapter::GetInstance()->connects_.Clear(); + EXPECT_EQ(name, ""); +} + +/** +* @tc.name: NotifyDataListeners +* @tc.desc: NotifyDataListeners tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, NotifyDataListeners, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PipeInfo appId; + appId.pipeId = "appId"; + appId.userId = "groupId"; + std::string content = "Helloworlds"; + const uint8_t *t = reinterpret_cast(content.c_str()); + SoftBusAdapter::GetInstance()->dataChangeListeners_.Clear(); + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->NotifyDataListeners(t, 1, "deviceId", appId)); + const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); + SoftBusAdapter::GetInstance()->dataChangeListeners_.Insert(appId.pipeId, dataListener); + delete dataListener; + SoftBusAdapter::GetInstance()->dataChangeListeners_.Clear(); + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->NotifyDataListeners(t, 1, "deviceId", appId)); +} + +/** +* @tc.name: ListenBroadcastMsg +* @tc.desc: ListenBroadcastMsg tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, ListenBroadcastMsg, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + SoftBusAdapter::GetInstance()->onBroadcast_= nullptr; + PipeInfo appId; + appId.pipeId = "appId"; + appId.userId = "groupId"; + auto result = SoftBusAdapter::GetInstance()->ListenBroadcastMsg(appId, nullptr); + EXPECT_EQ(result, SoftBusErrNo::SOFTBUS_INVALID_PARAM); + + auto listener = [](const std::string &message, const LevelInfo &info) {}; + result = SoftBusAdapter::GetInstance()->ListenBroadcastMsg(appId, listener); + EXPECT_EQ(result, SoftBusErrNo::SOFTBUS_INVALID_PARAM); + result = SoftBusAdapter::GetInstance()->ListenBroadcastMsg(appId, listener); + EXPECT_EQ(result, SoftBusErrNo::SOFTBUS_ALREADY_EXISTED); +} + +/** +* @tc.name: OnBroadcast +* @tc.desc: OnBroadcast tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, OnBroadcast, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + DeviceId id = {"DeviceId"}; + LevelInfo level; + level.dynamic = 1; + level.statics = 1; + level.switches = 1; + level.switchesLen = 1; + EXPECT_NE(SoftBusAdapter::GetInstance()->onBroadcast_, nullptr); + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->OnBroadcast(id, level)); + SoftBusAdapter::GetInstance()->onBroadcast_ = nullptr; + EXPECT_NO_FATAL_FAILURE(SoftBusAdapter::GetInstance()->OnBroadcast(id, level)); +} + +/** +* @tc.name: OnClientSocketChanged +* @tc.desc: OnClientSocketChanged tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, OnClientSocketChanged, TestSize.Level1) +{ + QosTV qosTv; + qosTv.qos = QosType::QOS_TYPE_MIN_BW; + qosTv.value = 1; + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnClientSocketChanged(1, QoSEvent::QOS_SATISFIED, &qosTv, 1)); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnClientSocketChanged(1, QoSEvent::QOS_SATISFIED, &qosTv, 0)); + qosTv.qos = QosType::QOS_TYPE_MAX_WAIT_TIMEOUT; + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnClientSocketChanged(1, QoSEvent::QOS_SATISFIED, &qosTv, 0)); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnClientSocketChanged(1, QoSEvent::QOS_SATISFIED, nullptr, 0)); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnClientSocketChanged(1, QoSEvent::QOS_NOT_SATISFIED, nullptr, 0)); +} + +/** +* @tc.name: OnServerBytesReceived +* @tc.desc: OnServerBytesReceived tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, OnServerBytesReceived, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PeerSocketInfo info; + info.name = strdup(""); + info.networkId = strdup("peertest01"); + info.pkgName = strdup("ohos.kv.test"); + info.dataType = TransDataType::DATA_TYPE_MESSAGE; + AppDistributedKv::SoftBusAdapter::ServerSocketInfo serinfo; + auto result = SoftBusAdapter::GetInstance()->GetPeerSocketInfo(1, serinfo); + EXPECT_EQ(result, false); + char str[] = "Hello"; + const void* data = static_cast(str); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnServerBytesReceived(1, data, 10)); + SoftBusAdapter::GetInstance()->OnBind(1, info); + result = SoftBusAdapter::GetInstance()->GetPeerSocketInfo(1, serinfo); + EXPECT_EQ(result, true); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnServerBytesReceived(1, data, 10)); + info.name = strdup("name"); + SoftBusAdapter::GetInstance()->OnBind(2, info); + result = SoftBusAdapter::GetInstance()->GetPeerSocketInfo(2, serinfo); + EXPECT_EQ(result, true); + EXPECT_NO_FATAL_FAILURE(AppDataListenerWrap::OnServerBytesReceived(2, data, 10)); +} + +/** +* @tc.name: GetPipeId +* @tc.desc: GetPipeId tests +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(SoftbusAdapterStandardTest, GetPipeId, TestSize.Level1) +{ + std::string names = "GetPipeId"; + auto name = AppDataListenerWrap::GetPipeId(names); + EXPECT_EQ(name, names); + names = "test01_GetPipeId"; + name = AppDataListenerWrap::GetPipeId(names); + EXPECT_EQ(name, "test01"); +} + /** * @tc.name: GetMtuSize * @tc.desc: get size @@ -169,6 +427,7 @@ HWTEST_F(SoftbusAdapterStandardTest, SendData, TestSize.Level1) */ HWTEST_F(SoftbusAdapterStandardTest, GetMtuSize, TestSize.Level1) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); PipeInfo id; id.pipeId = "appId"; @@ -190,6 +449,7 @@ HWTEST_F(SoftbusAdapterStandardTest, GetMtuSize, TestSize.Level1) */ HWTEST_F(SoftbusAdapterStandardTest, GetTimeout, TestSize.Level1) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); PipeInfo id; id.pipeId = "appId01"; @@ -210,6 +470,7 @@ HWTEST_F(SoftbusAdapterStandardTest, GetTimeout, TestSize.Level1) */ HWTEST_F(SoftbusAdapterStandardTest, IsSameStartedOnPeer, TestSize.Level1) { + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); PipeInfo id; id.pipeId = "appId01"; id.userId = "groupId01"; @@ -218,4 +479,173 @@ HWTEST_F(SoftbusAdapterStandardTest, IsSameStartedOnPeer, TestSize.Level1) auto status = SoftBusAdapter::GetInstance()->IsSameStartedOnPeer(id, di); EXPECT_EQ(status, true); } + +/** +* @tc.name: ReuseConnect +* @tc.desc: reuse connect +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, ReuseConnect, TestSize.Level1) +{ + const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); + PipeInfo pipe; + pipe.pipeId = "appId"; + pipe.userId = "groupId"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, pipe); + DeviceId device = {"DeviceId"}; + auto reuse = SoftBusAdapter::GetInstance()->ReuseConnect(pipe, device); + EXPECT_EQ(reuse, Status::NOT_SUPPORT); + SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, pipe); + delete dataListener; +} + +/** +* @tc.name: ReuseConnect01 +* @tc.desc: reuse connect +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(SoftbusAdapterStandardTest, ReuseConnect01, TestSize.Level1) +{ + ASSERT_NE(SoftBusAdapter::GetInstance(), nullptr); + PipeInfo pipe; + pipe.pipeId = "appId"; + pipe.userId = "groupId"; + DeviceId device = {"DeviceId"}; + auto status = SoftBusAdapter::GetInstance()->ReuseConnect(pipe, device); + EXPECT_EQ(status, Status::NOT_SUPPORT); +} + +/** +* @tc.name: GetConnect +* @tc.desc: get connect +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, GetConnect, TestSize.Level1) +{ + const AppDataChangeListenerImpl *dataListener = new AppDataChangeListenerImpl(); + PipeInfo pipe; + pipe.pipeId = "appId01"; + pipe.userId = "groupId01"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + SoftBusAdapter::GetInstance()->StartWatchDataChange(dataListener, pipe); + DeviceId device = {"DeviceId01"}; + std::shared_ptr conn = nullptr; + auto reuse = SoftBusAdapter::GetInstance()->GetConnect(pipe, device, 1); + EXPECT_NE(reuse, nullptr); + SoftBusAdapter::GetInstance()->StopWatchDataChange(dataListener, pipe); + delete dataListener; +} + +/** +* @tc.name: Broadcast +* @tc.desc: broadcast +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, Broadcast, TestSize.Level1) +{ + PipeInfo id; + id.pipeId = "appId"; + id.userId = "groupId"; + LevelInfo level; + level.dynamic = 1; + level.statics = 1; + level.switches = 1; + level.switchesLen = 1; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + SoftBusAdapter::GetInstance()->SetMessageTransFlag(id, true); + auto status = SoftBusAdapter::GetInstance()->Broadcast(id, level); + EXPECT_EQ(status, Status::ERROR); +} + +/** +* @tc.name: OpenConnect +* @tc.desc: open connect +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, OpenConnect, TestSize.Level1) +{ + DeviceId device = {"DeviceId"}; + std::shared_ptr conn = nullptr; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto status = SoftBusAdapter::GetInstance()->OpenConnect(conn, device); + EXPECT_NE(status.first, Status::SUCCESS); + EXPECT_EQ(status.first, Status::RATE_LIMIT); + EXPECT_EQ(status.second, 0); +} + +/** +* @tc.name: CloseSession +* @tc.desc: close session +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, CloseSession, TestSize.Level1) +{ + std::string networkId = "networkId"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto status = SoftBusAdapter::GetInstance()->CloseSession(networkId); + EXPECT_EQ(status, false); +} + +/** +* @tc.name: CloseSession01 +* @tc.desc: close session +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, CloseSession01, TestSize.Level1) +{ + std::string networkId = ""; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto status = SoftBusAdapter::GetInstance()->CloseSession(networkId); + EXPECT_EQ(status, false); +} + +/** +* @tc.name: GetPeerSocketInfo +* @tc.desc: get socket info +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, GetPeerSocketInfo, TestSize.Level1) +{ + AppDistributedKv::SoftBusAdapter::ServerSocketInfo info; + info.name = "kv"; + info.networkId= "192.168.1.1"; + info.pkgName = "test"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto status = SoftBusAdapter::GetInstance()->GetPeerSocketInfo(-1, info); + EXPECT_EQ(status, false); +} + +/** +* @tc.name: GetPeerSocketInfo +* @tc.desc: get socket info +* @tc.type: FUNC +* @tc.author: nhj +*/ +HWTEST_F(SoftbusAdapterStandardTest, GetPeerSocketInfo01, TestSize.Level1) +{ + AppDistributedKv::SoftBusAdapter::ServerSocketInfo info; + info.name = "service"; + info.networkId= "192.168.1.1"; + info.pkgName = "test"; + auto flag = SoftBusAdapter::GetInstance(); + ASSERT_NE(flag, nullptr); + auto status = SoftBusAdapter::GetInstance()->GetPeerSocketInfo(1, info); + EXPECT_EQ(status, true); +} } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_client_test.cpp b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_client_test.cpp new file mode 100644 index 00000000..3ba19844 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest/softbus_client_test.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include "softbus_client.h" +#include "types.h" + + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::AppDistributedKv; +using PipeInfo = OHOS::AppDistributedKv::PipeInfo; +constexpr int32_t SOFTBUS_OK = 0; + +class SoftbusClientTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp() {} + void TearDown() {} + + static std::shared_ptr client; +}; + +std::shared_ptr SoftbusClientTest::client = nullptr; + +void SoftbusClientTest::SetUpTestCase(void) +{ + PipeInfo pipeInfo; + pipeInfo.pipeId = "pipeId"; + pipeInfo.userId = "userId"; + DeviceId id = {"DeviceId"}; + client = std::make_shared(pipeInfo, id); +} + +void SoftbusClientTest::TearDownTestCase(void) +{ + client = nullptr; +} + +/** +* @tc.name: SendData +* @tc.desc: SendData test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, SendData, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + std::string content = "Helloworlds"; + const uint8_t *data = reinterpret_cast(content.c_str()); + DataInfo info = { const_cast(data), static_cast(content.length())}; + ISocketListener *listener = nullptr; + auto status = client->SendData(info, listener); + EXPECT_EQ(status, Status::ERROR); + client->bindState_ = 0; + status = client->SendData(info, listener); + EXPECT_EQ(status, Status::ERROR); +} + +/** +* @tc.name: OpenConnect +* @tc.desc: OpenConnect test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, OpenConnect, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + ISocketListener *listener = nullptr; + client->bindState_ = 0; + auto status = client->OpenConnect(listener); + EXPECT_EQ(status, Status::SUCCESS); + client->bindState_ = 1; + client->isOpening_.store(true); + status = client->OpenConnect(listener); + EXPECT_EQ(status, Status::RATE_LIMIT); + client->isOpening_.store(false); + status = client->OpenConnect(listener); + EXPECT_EQ(status, Status::NETWORK_ERROR); +} + +/** +* @tc.name: CheckStatus +* @tc.desc: CheckStatus test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, CheckStatus, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + client->bindState_ = 0; + auto status = client->CheckStatus(); + EXPECT_EQ(status, Status::SUCCESS); + client->bindState_ = -1; + client->isOpening_.store(true); + status = client->CheckStatus(); + EXPECT_EQ(status, Status::RATE_LIMIT); + client->isOpening_.store(false); + status = client->CheckStatus(); + EXPECT_EQ(status, Status::ERROR); +} + +/** +* @tc.name: Open +* @tc.desc: Open test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, Open, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + ISocketListener *listener = nullptr; + auto status = client->Open(1, AppDistributedKv::SoftBusClient::QOS_BR, listener, false); + EXPECT_NE(status, SOFTBUS_OK); +} + +/** +* @tc.name: UpdateExpireTime +* @tc.desc: UpdateExpireTime test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, UpdateExpireTime, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + client->type_ = AppDistributedKv::SoftBusClient::QOS_BR; + auto expireTime = client->CalcExpireTime(); + EXPECT_NO_FATAL_FAILURE(client->UpdateExpireTime(true)); + EXPECT_LT(expireTime, client->expireTime_); + EXPECT_NO_FATAL_FAILURE(client->UpdateExpireTime(false)); + EXPECT_LT(expireTime, client->expireTime_); +} + +/** +* @tc.name: UpdateBindInfo +* @tc.desc: UpdateBindInfo test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, UpdateBindInfo, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + EXPECT_NO_FATAL_FAILURE(client->UpdateBindInfo(1, 1, AppDistributedKv::SoftBusClient::QOS_BR, false)); + EXPECT_NO_FATAL_FAILURE(client->UpdateBindInfo(1, 1, AppDistributedKv::SoftBusClient::QOS_BR, true)); +} + +/** +* @tc.name: ReuseConnect +* @tc.desc: ReuseConnect test +* @tc.type: FUNC +* @tc.require: +* @tc.author: wangbin + */ +HWTEST_F(SoftbusClientTest, ReuseConnect, TestSize.Level0) +{ + ASSERT_NE(client, nullptr); + ISocketListener *listener = nullptr; + client->bindState_ = 0; + auto status = client->ReuseConnect(listener); + EXPECT_EQ(status, Status::SUCCESS); + client->bindState_ = -1; + client->isOpening_.store(true); + status = client->ReuseConnect(listener); + EXPECT_EQ(status, Status::NETWORK_ERROR); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/dfx/BUILD.gn index b150f471..483c4544 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/BUILD.gn @@ -13,10 +13,6 @@ import("//build/ohos.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -config("dfx_public_config") { - include_dirs = [ "${data_service_path}/adapter/include/dfx" ] -} - ohos_source_set("distributeddata_dfx") { branch_protector_ret = "pac_ret" sanitize = { @@ -35,7 +31,7 @@ ohos_source_set("distributeddata_dfx") { "src/fault/service_fault_impl.cpp", "src/hiview_adapter.cpp", "src/radar_reporter.cpp", - "src/reporter.cpp", + "src/reporter_impl.cpp", "src/statistic/api_performance_statistic_impl.cpp", "src/statistic/database_statistic_impl.cpp", "src/statistic/traffic_statistic_impl.cpp", @@ -50,73 +46,23 @@ ohos_source_set("distributeddata_dfx") { "../include/log", "../include/utils", "../include/communicator", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/framework/include/utils", ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] external_deps = [ "c_utils:utils", "device_manager:devicemanagersdk", "hilog:libhilog", "hisysevent:libhisysevent", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", "kv_store:datamgr_common", "openssl:libcrypto_shared", ] - public_configs = [ ":dfx_public_config" ] - subsystem_name = "distributeddatamgr" - part_name = "datamgr_service" - defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] -} - -ohos_static_library("distributeddata_dfx_fault_static") { - branch_protector_ret = "pac_ret" - sanitize = { - cfi = true - cfi_cross_dso = true - debug = false - boundary_sanitize = true - ubsan = true - } - sources = [ - "src/behaviour/behaviour_reporter_impl.cpp", - "src/fault/cloud_sync_fault_impl.cpp", - "src/fault/communication_fault_impl.cpp", - "src/fault/database_fault_impl.cpp", - "src/fault/runtime_fault_impl.cpp", - "src/fault/service_fault_impl.cpp", - "src/hiview_adapter.cpp", - "src/reporter.cpp", - "src/statistic/api_performance_statistic_impl.cpp", - "src/statistic/database_statistic_impl.cpp", - "src/statistic/traffic_statistic_impl.cpp", - "src/statistic/visit_statistic_impl.cpp", - ] - - include_dirs = [ - "./src", - "./src/fault", - "./src/statistic", - "../include/dfx", - "../include/utils", - ] - - cflags_cc = [ "-fvisibility=hidden" ] - - deps = [ "${data_service_path}/adapter/utils:distributeddata_utils" ] - - external_deps = [ - "c_utils:utils", - "hilog:libhilog", - "hisysevent:libhisysevent", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", - "kv_store:distributeddata_inner", - "openssl:libcrypto_shared", - ] - - public_configs = [ ":dfx_public_config" ] subsystem_name = "distributeddatamgr" part_name = "datamgr_service" defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/behaviour/behaviour_reporter_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/behaviour/behaviour_reporter_impl.h index 5fff69e4..b460d27e 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/behaviour/behaviour_reporter_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/behaviour/behaviour_reporter_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_BEHAVIOUR_REPORTER_IMPL_H #define DISTRIBUTEDDATAMGR_BEHAVIOUR_REPORTER_IMPL_H -#include "behaviour_reporter.h" +#include "dfx/behaviour_reporter.h" #include "hiview_adapter.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/cloud_sync_fault_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/cloud_sync_fault_impl.h index e4e45bc9..89ee3d5b 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/cloud_sync_fault_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/cloud_sync_fault_impl.h @@ -16,9 +16,9 @@ #ifndef DISTRIBUTEDDATAMGR_DATAMGR_SERVICE_CLOUD_SYNC_FAULT_IMPL_H #define DISTRIBUTEDDATAMGR_DATAMGR_SERVICE_CLOUD_SYNC_FAULT_IMPL_H -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" #include "hiview_adapter.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/communication_fault_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/communication_fault_impl.h index b5ec17f5..bf25665a 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/communication_fault_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/communication_fault_impl.h @@ -16,9 +16,9 @@ #ifndef DISTRIBUTEDDATAMGR_COMMUNICATION_FAULT_IMPL_H #define DISTRIBUTEDDATAMGR_COMMUNICATION_FAULT_IMPL_H -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" #include "hiview_adapter.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/database_fault_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/database_fault_impl.h index 66c5b39b..755d0318 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/database_fault_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/database_fault_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_DATABASE_FAULT_IMPL_H #define DISTRIBUTEDDATAMGR_DATABASE_FAULT_IMPL_H -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" #include "hiview_adapter.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/fault_reporter.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/fault_reporter.cpp index 9e714ee7..1a7a35fd 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/fault_reporter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/fault_reporter.cpp @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/runtime_fault_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/runtime_fault_impl.h index f4aea1b0..ce2fa7c6 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/runtime_fault_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/runtime_fault_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_RUNTIME_FAULT_IMPL_H #define DISTRIBUTEDDATAMGR_RUNTIME_FAULT_IMPL_H #include "hiview_adapter.h" -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/service_fault_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/service_fault_impl.h index fb5eed6a..c627f53e 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/service_fault_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/fault/service_fault_impl.h @@ -17,7 +17,7 @@ #define DISTRIBUTEDDATAMGR_SERVICE_FAULT_IMPL_H #include "hiview_adapter.h" -#include "fault_reporter.h" +#include "dfx/fault_reporter.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.cpp index 01c883fa..c27789c7 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.cpp @@ -21,7 +21,7 @@ #include #include "log_print.h" -#include "time_utils.h" +#include "utils/time_utils.h" namespace OHOS { namespace DistributedDataDfx { @@ -212,7 +212,7 @@ void HiViewAdapter::ReportArkDataFault(int dfxCode, const ArkDataFaultMsg &msg, return; } ExecutorPool::Task task([dfxCode, msg]() { - std::string occurTime = TimeUtils::GetCurSysTimeWithMs(); + std::string occurTime = DistributedData::TimeUtils::GetCurSysTimeWithMs(); std::string bundleName = msg.bundleName; std::string moduleName = msg.moduleName; std::string storeName = msg.storeName; diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.h index a4c5f665..f5470386 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/hiview_adapter.h @@ -20,7 +20,7 @@ #include #include "dfx_code_constant.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" #include "executor_pool.h" #include "hisysevent_c.h" #include "value_hash.h" diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/radar_reporter.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/src/radar_reporter.cpp index cec370a5..87aafd4b 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/radar_reporter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/radar_reporter.cpp @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "radar_reporter.h" +#include "dfx/radar_reporter.h" #include "device_manager_adapter.h" #include "hisysevent.h" diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter_impl.cpp similarity index 64% rename from datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter.cpp rename to datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter_impl.cpp index 35ff346f..a03d4761 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/reporter_impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,96 +13,117 @@ * limitations under the License. */ -#include "reporter.h" +#include "dfx/reporter_impl.h" + +#include "behaviour/behaviour_reporter_impl.h" +#include "fault/cloud_sync_fault_impl.h" #include "fault/communication_fault_impl.h" #include "fault/database_fault_impl.h" #include "fault/runtime_fault_impl.h" #include "fault/service_fault_impl.h" -#include "fault/cloud_sync_fault_impl.h" - +#include "statistic/api_performance_statistic_impl.h" +#include "statistic/database_statistic_impl.h" #include "statistic/traffic_statistic_impl.h" #include "statistic/visit_statistic_impl.h" -#include "statistic/database_statistic_impl.h" -#include "statistic/api_performance_statistic_impl.h" - -#include "behaviour/behaviour_reporter_impl.h" namespace OHOS { namespace DistributedDataDfx { -Reporter* Reporter::GetInstance() +__attribute__((used)) static bool g_isInit = ReporterImpl::Init(); +bool ReporterImpl::Init() { - static Reporter reporter; - return &reporter; + static ReporterImpl reporterImpl; + static std::once_flag onceFlag; + std::call_once(onceFlag, [&]() { + Reporter::RegisterReporterInstance(&reporterImpl); + }); + return true; } -FaultReporter* Reporter::CommunicationFault() +FaultReporter* ReporterImpl::CommunicationFault() { static CommunicationFaultImpl communicationFault; communicationFault.SetThreadPool(executors_); return &communicationFault; } -FaultReporter* Reporter::DatabaseFault() +FaultReporter* ReporterImpl::DatabaseFault() { static DatabaseFaultImpl databaseFault; databaseFault.SetThreadPool(executors_); return &databaseFault; } -FaultReporter* Reporter::RuntimeFault() +FaultReporter* ReporterImpl::RuntimeFault() { static RuntimeFaultImpl runtimeFault; runtimeFault.SetThreadPool(executors_); return &runtimeFault; } -FaultReporter* Reporter::CloudSyncFault() +FaultReporter* ReporterImpl::CloudSyncFault() { static CloudSyncFaultImpl cloudSyncFault; cloudSyncFault.SetThreadPool(executors_); return &cloudSyncFault; } -FaultReporter* Reporter::ServiceFault() +FaultReporter* ReporterImpl::ServiceFault() { static ServiceFaultImpl serviceFault; serviceFault.SetThreadPool(executors_); return &serviceFault; } -StatisticReporter* Reporter::TrafficStatistic() +StatisticReporter* ReporterImpl::TrafficStatistic() { static TrafficStatisticImpl trafficStatistic; trafficStatistic.SetThreadPool(executors_); return &trafficStatistic; } -StatisticReporter* Reporter::VisitStatistic() +StatisticReporter* ReporterImpl::VisitStatistic() { static VisitStatisticImpl visitStatistic; visitStatistic.SetThreadPool(executors_); return &visitStatistic; } -StatisticReporter* Reporter::DatabaseStatistic() +StatisticReporter* ReporterImpl::DatabaseStatistic() { static DatabaseStatisticImpl databaseStatistic; databaseStatistic.SetThreadPool(executors_); return &databaseStatistic; } -StatisticReporter* Reporter::ApiPerformanceStatistic() +StatisticReporter* ReporterImpl::ApiPerformanceStatistic() { static ApiPerformanceStatisticImpl apiPerformanceStat; apiPerformanceStat.SetThreadPool(executors_); return &apiPerformanceStat; } -BehaviourReporter* Reporter::BehaviourReporter() +BehaviourReporter* ReporterImpl::GetBehaviourReporter() { static BehaviourReporterImpl behaviourReporterImpl; behaviourReporterImpl.SetThreadPool(executors_); return &behaviourReporterImpl; } + +void ReporterImpl::SetThreadPool(std::shared_ptr executors) +{ + executors_ = executors; + if (executors != nullptr) { + ServiceFault(); + RuntimeFault(); + DatabaseFault(); + CloudSyncFault(); + CommunicationFault(); + DatabaseStatistic(); + VisitStatistic(); + TrafficStatistic(); + ApiPerformanceStatistic(); + GetBehaviourReporter(); + } +} } // namespace DistributedDataDfx } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/api_performance_statistic_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/api_performance_statistic_impl.h index f253ee7b..8c44d269 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/api_performance_statistic_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/api_performance_statistic_impl.h @@ -16,9 +16,9 @@ #ifndef DISTRIBUTEDDATAMGR_API_PERFORMANCE_STATISTIC_IMPL_H #define DISTRIBUTEDDATAMGR_API_PERFORMANCE_STATISTIC_IMPL_H -#include "statistic_reporter.h" +#include "dfx/statistic_reporter.h" #include "hiview_adapter.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/database_statistic_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/database_statistic_impl.h index c3aa6774..453cd3dd 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/database_statistic_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/database_statistic_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_DATABASE_STATISTIC_IMPL_H #define DISTRIBUTEDDATAMGR_DATABASE_STATISTIC_IMPL_H -#include "statistic_reporter.h" +#include "dfx/statistic_reporter.h" #include "hiview_adapter.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/statistic_reporter.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/statistic_reporter.cpp index 017ea12e..e611e441 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/statistic_reporter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/statistic_reporter.cpp @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "statistic_reporter.h" +#include "dfx/statistic_reporter.h" namespace OHOS { namespace DistributedDataDfx { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/traffic_statistic_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/traffic_statistic_impl.h index c65e8888..b7cac17f 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/traffic_statistic_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/traffic_statistic_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_TRAFFIC_STATISTIC_IMPL_H #define DISTRIBUTEDDATAMGR_TRAFFIC_STATISTIC_IMPL_H -#include "statistic_reporter.h" +#include "dfx/statistic_reporter.h" #include "hiview_adapter.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/visit_statistic_impl.h b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/visit_statistic_impl.h index fb5b933e..c193a659 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/visit_statistic_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/src/statistic/visit_statistic_impl.h @@ -16,7 +16,7 @@ #ifndef DISTRIBUTEDDATAMGR_VISIT_STATISTIC_IMPL_H #define DISTRIBUTEDDATAMGR_VISIT_STATISTIC_IMPL_H -#include "statistic_reporter.h" +#include "dfx/statistic_reporter.h" #include "hiview_adapter.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/dfx/test/BUILD.gn index e355fb9f..78a3fa45 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/test/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### config("module_dfx_mst_config") { @@ -23,9 +23,6 @@ config("module_dfx_mst_config") { "../src", "../src/fault", "../src/statistic", - "../../include/log", - "../../include/utils", - "../../include/dfx", ] } @@ -38,6 +35,7 @@ ohos_unittest("DistributeddataDfxMSTTest") { external_deps = [ "c_utils:utils", + "datamgr_service:distributeddatasvcfwk", "hilog:libhilog", "hisysevent:libhisysevent", "hitrace:hitrace_meter", @@ -46,8 +44,7 @@ ohos_unittest("DistributeddataDfxMSTTest") { ] ldflags = [ "-Wl,--exclude-libs,ALL" ] deps = [ - "${data_service_path}/adapter/utils:distributeddata_utils", - "../../dfx:distributeddata_dfx", + "${data_service_path}/adapter/dfx:distributeddata_dfx", "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_shared", ] @@ -64,9 +61,6 @@ config("module_dfx_config") { "../src", "../src/fault", "../src/statistic", - "../../include/log", - "../../include/utils", - "../../include/dfx", ] } @@ -74,20 +68,6 @@ ohos_unittest("DistributeddataDfxUTTest") { module_out_path = module_output_path sources = [ - "../src/behaviour/behaviour_reporter_impl.cpp", - "../src/fault/cloud_sync_fault_impl.cpp", - "../src/fault/communication_fault_impl.cpp", - "../src/fault/database_fault_impl.cpp", - "../src/fault/fault_reporter.cpp", - "../src/fault/runtime_fault_impl.cpp", - "../src/fault/service_fault_impl.cpp", - "../src/hiview_adapter.cpp", - "../src/reporter.cpp", - "../src/statistic/api_performance_statistic_impl.cpp", - "../src/statistic/database_statistic_impl.cpp", - "../src/statistic/statistic_reporter.cpp", - "../src/statistic/traffic_statistic_impl.cpp", - "../src/statistic/visit_statistic_impl.cpp", "./unittest/distributeddata_dfx_ut_test.cpp", "./unittest/fake/hiview/fake_hiview.cpp", ] @@ -96,6 +76,7 @@ ohos_unittest("DistributeddataDfxUTTest") { external_deps = [ "c_utils:utils", + "datamgr_service:distributeddatasvcfwk", "hilog:libhilog", "hisysevent:libhisysevent", "hitrace:hitrace_meter", @@ -104,6 +85,7 @@ ohos_unittest("DistributeddataDfxUTTest") { ] ldflags = [ "-Wl,--exclude-libs,ALL" ] deps = [ + "${data_service_path}/adapter/dfx:distributeddata_dfx", "${data_service_path}/adapter/utils:distributeddata_utils", "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_shared", @@ -111,6 +93,37 @@ ohos_unittest("DistributeddataDfxUTTest") { defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } +ohos_unittest("HiViewAdapterDfxTest") { + module_out_path = module_output_path + include_dirs = [ "${data_service_path}/adapter/include/communicator" ] + sources = [ "./unittest/hiview_adapter_dfx_test.cpp" ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + configs = [ ":module_dfx_config" ] + + external_deps = [ + "c_utils:utils", + "datamgr_service:distributeddatasvcfwk", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + "kv_store:distributeddata_inner", + "kv_store:distributeddata_mgr", + "openssl:libcrypto_shared", + ] + ldflags = [ "-Wl,--exclude-libs,ALL" ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/adapter/dfx:distributeddata_dfx", + "${data_service_path}/adapter/utils:distributeddata_utils", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} + ############################################################################### group("unittest") { testonly = true @@ -119,6 +132,7 @@ group("unittest") { deps += [ ":DistributeddataDfxMSTTest", ":DistributeddataDfxUTTest", + ":HiViewAdapterDfxTest", ] } ############################################################################### diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_mst_test.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_mst_test.cpp index 5945937f..c5a67afe 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_mst_test.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_mst_test.cpp @@ -29,18 +29,23 @@ public: void SetUp(); void TearDown(); + static Reporter* reporter_; }; +Reporter* DistributedataDfxMSTTest::reporter_ = nullptr; void DistributedataDfxMSTTest::SetUpTestCase() { size_t max = 12; size_t min = 5; - Reporter::GetInstance()->SetThreadPool(std::make_shared(max, min)); + reporter_ = Reporter::GetInstance(); + ASSERT_NE(nullptr, reporter_); + reporter_->SetThreadPool(std::make_shared(max, min)); } void DistributedataDfxMSTTest::TearDownTestCase() { - Reporter::GetInstance()->SetThreadPool(nullptr); + reporter_->SetThreadPool(nullptr); + reporter_ = nullptr; } void DistributedataDfxMSTTest::SetUp() {} @@ -60,7 +65,7 @@ HWTEST_F(DistributedataDfxMSTTest, Dfx001, TestSize.Level0) * @tc.steps: step1. getcommunicationFault instance * @tc.expected: step1. Expect instance is not null. */ - auto comFault = Reporter::GetInstance()->CommunicationFault(); + auto comFault = reporter_->CommunicationFault(); EXPECT_NE(nullptr, comFault); struct CommFaultMsg msg{.userId = "user001", .appId = "myApp", .storeId = "storeTest"}; msg.deviceId.push_back("device001"); diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_ut_test.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_ut_test.cpp index d1b20ccd..d7fed644 100644 --- a/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_ut_test.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/distributeddata_dfx_ut_test.cpp @@ -31,19 +31,25 @@ public: void SetUp(); void TearDown(); + static Reporter* reporter_; }; +Reporter* DistributeddataDfxUTTest::reporter_ = nullptr; + void DistributeddataDfxUTTest::SetUpTestCase() { FakeHivew::Clear(); size_t max = 12; size_t min = 5; - Reporter::GetInstance()->SetThreadPool(std::make_shared(max, min)); + reporter_ = Reporter::GetInstance(); + ASSERT_NE(nullptr, reporter_); + reporter_->SetThreadPool(std::make_shared(max, min)); } void DistributeddataDfxUTTest::TearDownTestCase() { - Reporter::GetInstance()->SetThreadPool(nullptr); + reporter_->SetThreadPool(nullptr); + reporter_ = nullptr; FakeHivew::Clear(); } @@ -64,7 +70,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx001, TestSize.Level0) * @tc.steps: step1. getcommunicationFault instance * @tc.expected: step1. Expect instance is not null. */ - auto comFault = Reporter::GetInstance()->CommunicationFault(); + auto comFault = reporter_->CommunicationFault(); EXPECT_NE(nullptr, comFault); struct CommFaultMsg msg{.userId = "user001", .appId = "myApp", .storeId = "storeTest"}; msg.deviceId.push_back("device001"); @@ -107,7 +113,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx002, TestSize.Level0) * @tc.steps: step1. get database fault report instance * @tc.expected: step1. Expect get instance success. */ - auto dbFault = Reporter::GetInstance()->DatabaseFault(); + auto dbFault = reporter_->DatabaseFault(); EXPECT_NE(nullptr, dbFault); struct DBFaultMsg msg {.appId = "MyApp", .storeId = "MyDatabase", .moduleName = "database", .errorType = Fault::DF_DB_DAMAGE}; @@ -142,7 +148,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx003, TestSize.Level0) * @tc.steps: step1. get database reporter instance. * @tc.expected: step1. Expect get success. */ - auto dbs = Reporter::GetInstance()->DatabaseStatistic(); + auto dbs = reporter_->DatabaseStatistic(); EXPECT_NE(nullptr, dbs); DbStat ds = {"uid", "appid", "storeId001", 100}; auto dbsRet = dbs->Report(ds); @@ -167,8 +173,8 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx004, TestSize.Level0) * @tc.steps: step1. Get runtime fault instance. * @tc.expected: step1. Expect get runtime fault instance success. */ - auto rtFault = Reporter::GetInstance()->RuntimeFault(); - auto rtFault2 = Reporter::GetInstance()->RuntimeFault(); + auto rtFault = reporter_->RuntimeFault(); + auto rtFault2 = reporter_->RuntimeFault(); EXPECT_NE(nullptr, rtFault); EXPECT_EQ(rtFault, rtFault2); @@ -204,7 +210,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx005, TestSize.Level0) * @tc.steps:step1. send data to 1 device * @tc.expected: step1. Expect put success. */ - auto ts = Reporter::GetInstance()->TrafficStatistic(); + auto ts = reporter_->TrafficStatistic(); EXPECT_NE(nullptr, ts); struct TrafficStat tss = {"appId001", "deviceId001", 100, 200}; auto tsRet = ts->Report(tss); @@ -230,7 +236,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx006, TestSize.Level0) * @tc.steps:step1. create call interface statistic instance * @tc.expected: step1. Expect get instance success. */ - auto vs = Reporter::GetInstance()->VisitStatistic(); + auto vs = reporter_->VisitStatistic(); EXPECT_NE(nullptr, vs); struct VisitStat vss = {"appid001", "Put"}; auto vsRet = vs->Report(vss); @@ -260,7 +266,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx007, TestSize.Level0) * @tc.steps:step1. create call api perforamnce statistic instance * @tc.expected: step1. Expect get instance success. */ - auto ap = Reporter::GetInstance()->ApiPerformanceStatistic(); + auto ap = reporter_->ApiPerformanceStatistic(); EXPECT_NE(nullptr, ap); struct ApiPerformanceStat aps = { "interface", 10000, 5000, 20000 }; auto apRet = ap->Report(aps); @@ -285,7 +291,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx008, TestSize.Level0) * @tc.steps: step1. get database fault report instance * @tc.expected: step1. Expect get instance success. */ - auto behavior = Reporter::GetInstance()->BehaviourReporter(); + auto behavior = reporter_->GetBehaviourReporter(); EXPECT_NE(nullptr, behavior); struct BehaviourMsg msg{.userId = "user008", .appId = "myApp08", .storeId = "storeTest08", .behaviourType = BehaviourType::DATABASE_BACKUP, .extensionInfo="test111"}; @@ -323,8 +329,8 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx009, TestSize.Level0) * @tc.steps: step1. Get runtime fault instance. * @tc.expected: step1. Expect get runtime fault instance success. */ - auto svFault = Reporter::GetInstance()->ServiceFault(); - auto svFault2 = Reporter::GetInstance()->ServiceFault(); + auto svFault = reporter_->ServiceFault(); + auto svFault2 = reporter_->ServiceFault(); EXPECT_NE(nullptr, svFault); EXPECT_EQ(svFault, svFault2); @@ -359,7 +365,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx010, TestSize.Level0) * @tc.steps: step1. get database reporter instance. * @tc.expected: step1. Expect get success. */ - auto dbs = Reporter::GetInstance()->DatabaseStatistic(); + auto dbs = reporter_->DatabaseStatistic(); EXPECT_NE(nullptr, dbs); DbStat ds = {"uid", "appid", "storeId002", 100}; auto dbsRet = dbs->Report(ds); @@ -391,7 +397,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx011, TestSize.Level0) * @tc.steps: step1. get database fault report instance * @tc.expected: step1. Expect get instance success. */ - auto UdmfBehavior = Reporter::GetInstance()->BehaviourReporter(); + auto UdmfBehavior = reporter_->GetBehaviourReporter(); EXPECT_NE(nullptr, UdmfBehavior); struct UdmfBehaviourMsg UdMsg{"myApp", "channel", 200, "dataType", "operation", "result"}; auto repStatus = UdmfBehavior->UDMFReport(UdMsg); @@ -435,7 +441,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx012, TestSize.Level0) * @tc.steps:step1. send data to 1 device * @tc.expected: step1. Expect put success. */ - auto ts = Reporter::GetInstance()->TrafficStatistic(); + auto ts = reporter_->TrafficStatistic(); EXPECT_NE(nullptr, ts); struct TrafficStat tss = {"appId001", "deviceId001", 100, 200}; auto tsRet = ts->Report(tss); @@ -464,7 +470,7 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx013, TestSize.Level0) * @tc.steps:step1. create call api perforamnce statistic instance * @tc.expected: step1. Expect get instance success. */ - auto ap = Reporter::GetInstance()->ApiPerformanceStatistic(); + auto ap = reporter_->ApiPerformanceStatistic(); EXPECT_NE(nullptr, ap); struct ApiPerformanceStat aps = { "interface", 2000, 500, 1000 }; auto apRet = ap->Report(aps); @@ -478,4 +484,61 @@ HWTEST_F(DistributeddataDfxUTTest, Dfx013, TestSize.Level0) EXPECT_STREQ(string("interface").c_str(), val.c_str()); } FakeHivew::Clear(); +} + +/** + * @tc.name: Dfx014 + * @tc.desc: test msg.errorType equals to Fault::DF_DB_CORRUPTED. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(DistributeddataDfxUTTest, Dfx014, TestSize.Level0) +{ + FakeHivew::Clear(); + auto dbFault = reporter_->DatabaseFault(); + ASSERT_NE(nullptr, dbFault); + struct DBFaultMsg msg {.appId = "mYApp", .storeId = "mYDatabase", + .moduleName = "dataBase", .errorType = Fault::DF_DB_CORRUPTED}; + auto repStatus = dbFault->Report(msg); + EXPECT_TRUE(repStatus == ReportStatus::SUCCESS); + FakeHivew::Clear(); +} + +/** + * @tc.name: Dfx015 + * @tc.desc: test msg.errorType equals to Fault::DF_DB_REKEY_FAILED. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(DistributeddataDfxUTTest, Dfx015, TestSize.Level0) +{ + FakeHivew::Clear(); + auto dbFault = reporter_->DatabaseFault(); + ASSERT_NE(nullptr, dbFault); + struct DBFaultMsg msg {.appId = "MyApp", .storeId = "mYDatabase", + .moduleName = "DataBase", .errorType = Fault::DF_DB_REKEY_FAILED}; + auto repStatus = dbFault->Report(msg); + EXPECT_TRUE(repStatus == ReportStatus::SUCCESS); + FakeHivew::Clear(); +} + +/** + * @tc.name: Dfx016 + * @tc.desc: test other error scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(DistributeddataDfxUTTest, Dfx016, TestSize.Level0) +{ + FakeHivew::Clear(); + auto dbFault = reporter_->DatabaseFault(); + ASSERT_NE(nullptr, dbFault); + struct DBFaultMsg msg {.appId = "MyApp", .storeId = "mYDatabase", + .moduleName = "DataBase", .errorType = Fault::SF_SERVICE_PUBLISH}; + auto repStatus = dbFault->Report(msg); + EXPECT_TRUE(repStatus == ReportStatus::ERROR); + FakeHivew::Clear(); } \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/hiview_adapter_dfx_test.cpp b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/hiview_adapter_dfx_test.cpp new file mode 100644 index 00000000..f26e70e9 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/dfx/test/unittest/hiview_adapter_dfx_test.cpp @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "HiViewAdapterDfxTest" + +#include "device_manager_adapter.h" +#include "dfx_types.h" +#include "gtest/gtest.h" +#include "hiview_adapter.h" +#include "log_print.h" +#include "radar_reporter.h" + +using namespace OHOS; +using namespace testing; +using namespace testing::ext; +using namespace OHOS::DistributedData; +namespace OHOS::Test { +class HiViewAdapterDfxTest : public testing::Test { +public: + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +class RadarReporterDfxTest : public testing::Test { +public: + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +/** +* @tc.name: ReportFault001 +* @tc.desc: ReportFault function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportFault001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::FaultMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportFault(dfxCode, msg, executors)); +} + +/** +* @tc.name: ReportDBFault001 +* @tc.desc: ReportDBFault function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportDBFault001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::DBFaultMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportDBFault(dfxCode, msg, executors)); +} + +/** +* @tc.name: ReportCommFault001 +* @tc.desc: ReportCommFault function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportCommFault001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::CommFaultMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportCommFault(dfxCode, msg, executors)); +} + +/** +* @tc.name: ReportArkDataFault001 +* @tc.desc: ReportArkDataFault function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportArkDataFault001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::ArkDataFaultMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportArkDataFault(dfxCode, msg, executors)); +} + +/** +* @tc.name: ReportBehaviour001 +* @tc.desc: ReportBehaviour function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportBehaviour001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::BehaviourMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportBehaviour(dfxCode, msg, executors)); +} + +/** +* @tc.name: ReportDatabaseStatistic001 +* @tc.desc: ReportDatabaseStatistic function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportDatabaseStatistic001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::DbStat stat; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportDatabaseStatistic(dfxCode, stat, executors)); +} + +/** +* @tc.name: ReportTrafficStatistic001 +* @tc.desc: ReportTrafficStatistic function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportTrafficStatistic001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::TrafficStat stat; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportTrafficStatistic(dfxCode, stat, executors)); +} + +/** +* @tc.name: ReportVisitStatistic001 +* @tc.desc: ReportVisitStatistic function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportVisitStatistic001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::VisitStat stat; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportVisitStatistic(dfxCode, stat, executors)); +} + +/** +* @tc.name: ReportApiPerformanceStatistic001 +* @tc.desc: ReportApiPerformanceStatistic function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportApiPerformanceStatistic001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::ApiPerformanceStat stat; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportApiPerformanceStatistic(dfxCode, stat, executors)); +} + +/** +* @tc.name: ReportUdmfBehaviour001 +* @tc.desc: ReportUdmfBehaviour function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, ReportUdmfBehaviour001, TestSize.Level0) +{ + int dfxCode = 0; + DistributedDataDfx::UdmfBehaviourMsg msg; + std::shared_ptr executors = nullptr; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::HiViewAdapter::ReportUdmfBehaviour(dfxCode, msg, executors)); +} + +/** +* @tc.name: CoverEventID001 +* @tc.desc: CoverEventID function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(HiViewAdapterDfxTest, CoverEventID001, TestSize.Level0) +{ + int dfxCode = 0; + std::string str = DistributedDataDfx::HiViewAdapter::CoverEventID(dfxCode); + EXPECT_EQ(str, ""); +} + +/** +* @tc.name: AnonymousUuid001 +* @tc.desc: AnonymousUuid function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(RadarReporterDfxTest, AnonymousUuid001, TestSize.Level0) +{ + std::string uuid = "uuid"; + std::string str = DistributedDataDfx::RadarReporter::AnonymousUuid(uuid); + EXPECT_EQ(str, DistributedDataDfx::RadarReporter::DEFAULT_ANONYMOUS); +} + +/** +* @tc.name: Report001 +* @tc.desc: Report function error test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: SQL +*/ +HWTEST_F(RadarReporterDfxTest, Report001, TestSize.Level0) +{ + DistributedDataDfx::RadarParam param; + const char *funcName = "funcName"; + int32_t state = 0; + const char *eventName = "eventName"; + EXPECT_NO_FATAL_FAILURE( + DistributedDataDfx::RadarReporter::Report(param, funcName, state, eventName)); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/include/communicator/process_communicator_impl.h b/datamgr_service/services/distributeddataservice/adapter/include/communicator/process_communicator_impl.h index a196c69b..41df2779 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/communicator/process_communicator_impl.h +++ b/datamgr_service/services/distributeddataservice/adapter/include/communicator/process_communicator_impl.h @@ -33,6 +33,7 @@ public: using OnDataReceive = DistributedDB::OnDataReceive; using OnSendAble = DistributedDB::OnSendAble; using DeviceInfos = DistributedDB::DeviceInfos; + using UserInfo = DistributedDB::UserInfo; using RouteHeadHandlerCreator = std::function(const DistributedDB::ExtendInfo &info)>; @@ -61,8 +62,9 @@ public: API_EXPORT std::shared_ptr GetExtendHeaderHandle( const DistributedDB::ExtendInfo &info) override; - API_EXPORT DBStatus CheckAndGetDataHeadInfo( - const uint8_t *data, uint32_t dataLen, uint32_t &headLen, std::vector &users) override; + API_EXPORT DBStatus GetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength) override; + API_EXPORT DBStatus GetDataUserInfo(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) override; Status ReuseConnect(const DeviceId &deviceId); diff --git a/datamgr_service/services/distributeddataservice/adapter/include/communicator/route_head_handler.h b/datamgr_service/services/distributeddataservice/adapter/include/communicator/route_head_handler.h index f31eea2d..47596818 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/communicator/route_head_handler.h +++ b/datamgr_service/services/distributeddataservice/adapter/include/communicator/route_head_handler.h @@ -24,9 +24,11 @@ class RouteHeadHandler : public DistributedDB::ExtendHeaderHandle { public: using ExtendInfo = DistributedDB::ExtendInfo; using DBStatus = DistributedDB::DBStatus; + using UserInfo = DistributedDB::UserInfo; - virtual bool ParseHeadData( - const uint8_t *data, uint32_t totalLen, uint32_t &headSize, std::vector &users) = 0; + virtual bool ParseHeadDataLen(const uint8_t *data, uint32_t totalLen, uint32_t &headSize) = 0; + virtual bool ParseHeadDataUser(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) = 0; }; } // namespace OHOS::DistributedData #endif // DISTRIBUTEDDATAMGR_EXTEND_HEAD_HANDLER_H diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter_impl.h b/datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter_impl.h new file mode 100644 index 00000000..c5e9f1f2 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter_impl.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DISTRIBUTEDDATAMGR_REPORTER_IMPL_H +#define DISTRIBUTEDDATAMGR_REPORTER_IMPL_H + +#include +#include + +#include "dfx/behaviour_reporter.h" +#include "dfx/dfx_types.h" +#include "executor_pool.h" +#include "dfx/fault_reporter.h" +#include "dfx/reporter.h" +#include "dfx/statistic_reporter.h" + +namespace OHOS { +namespace DistributedDataDfx { +class ReporterImpl : public Reporter { +public: + static bool Init(); + FaultReporter* ServiceFault() override; + FaultReporter* RuntimeFault() override; + FaultReporter* DatabaseFault() override; + FaultReporter* CommunicationFault() override; + FaultReporter* CloudSyncFault() override; + + StatisticReporter* DatabaseStatistic() override; + StatisticReporter* VisitStatistic() override; + StatisticReporter* TrafficStatistic() override; + StatisticReporter* ApiPerformanceStatistic() override; + + BehaviourReporter* GetBehaviourReporter() override; + void SetThreadPool(std::shared_ptr executors) override; + +private: + ~ReporterImpl() override = default; + std::shared_ptr executors_; +}; +} // namespace DistributedDataDfx +} // namespace OHOS +#endif // DISTRIBUTEDDATAMGR_REPORTER_IMPL_H diff --git a/datamgr_service/services/distributeddataservice/adapter/include/schema_helper/get_schema_helper.h b/datamgr_service/services/distributeddataservice/adapter/include/schema_helper/get_schema_helper.h new file mode 100644 index 00000000..ab937d35 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/include/schema_helper/get_schema_helper.h @@ -0,0 +1,68 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#ifndef OHOS_DISTRIBUTED_DATA_SERVICES_ADAPTER_GET_SCHEMA_HELPER_H +#define OHOS_DISTRIBUTED_DATA_SERVICES_ADAPTER_GET_SCHEMA_HELPER_H + +#include +#include +#include + +#include "iremote_object.h" + +namespace OHOS { +namespace AppExecFwk { +class IBundleMgr; +} +} // namespace OHOS + +namespace OHOS::DistributedData { +struct AppInfo { + std::string bundleName; + int32_t userId; + int32_t appIndex; +}; + +class GetSchemaHelper final : public std::enable_shared_from_this { +public: + static GetSchemaHelper &GetInstance(); + std::vector GetSchemaFromHap(const std::string &schemaPath, const AppInfo &info); + +private: + GetSchemaHelper() = default; + ~GetSchemaHelper(); + class ServiceDeathRecipient : public IRemoteObject::DeathRecipient { + public: + explicit ServiceDeathRecipient(std::weak_ptr owner) : owner_(owner) + { + } + void OnRemoteDied(const wptr &object) override + { + auto owner = owner_.lock(); + if (owner != nullptr) { + owner->OnRemoteDied(); + } + } + + private: + std::weak_ptr owner_; + }; + void OnRemoteDied(); + sptr GetBundleMgr(); + std::mutex mutex_; + sptr object_; + sptr deathRecipient_; +}; +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_ADAPTER_GET_SCHEMA_HELPER_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/network/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/network/BUILD.gn similarity index 64% rename from datamgr_service/services/distributeddataservice/service/network/BUILD.gn rename to datamgr_service/services/distributeddataservice/adapter/network/BUILD.gn index 39bf8025..f120004b 100644 --- a/datamgr_service/services/distributeddataservice/service/network/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/network/BUILD.gn @@ -16,9 +16,12 @@ import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") config("network_public_config") { visibility = [ ":*" ] include_dirs = [ - "${data_service_path}/service/network", - "${data_service_path}/adapter/include/communicator", + "${data_service_path}/adapter/network/src", + "${data_service_path}/framework/include/network", ] + if (datamgr_service_cloud) { + include_dirs += [ "${data_service_path}/adapter/include/communicator" ] + } } ohos_source_set("distributeddata_network") { @@ -30,10 +33,11 @@ ohos_source_set("distributeddata_network") { boundary_sanitize = true ubsan = true } - sources = [ "network_adapter.cpp" ] + sources = [] cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] configs = [ ":network_public_config" ] @@ -45,18 +49,24 @@ ohos_source_set("distributeddata_network") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] - deps = [ - "${data_service_path}/adapter/communicator:distributeddata_communicator", - ] + external_deps = [ "kv_store:datamgr_common" ] - external_deps = [ - "device_manager:devicemanagersdk", - "hilog:libhilog", - "kv_store:datamgr_common", - "netmanager_base:net_conn_manager_if", - ] + if (datamgr_service_cloud) { + sources += [ "src/network_delegate_normal_impl.cpp" ] + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + ] + external_deps += [ + "device_manager:devicemanagersdk", + "hilog:libhilog", + "netmanager_base:net_conn_manager_if", + ] + } else { + sources += [ "src/network_delegate_default_impl.cpp" ] + } subsystem_name = "distributeddatamgr" part_name = "datamgr_service" diff --git a/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.cpp b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.cpp new file mode 100644 index 00000000..0372dc5b --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "NetworkDelegateDefaultImpl" +#include "network_delegate_default_impl.h" + +namespace OHOS::DistributedData { +__attribute__((used)) static bool g_isInit = NetworkDelegateDefaultImpl::Init(); +NetworkDelegateDefaultImpl::NetworkDelegateDefaultImpl() +{ +} + +NetworkDelegateDefaultImpl::~NetworkDelegateDefaultImpl() +{ +} + +bool NetworkDelegateDefaultImpl::IsNetworkAvailable() +{ + return false; +} + +void NetworkDelegateDefaultImpl::RegOnNetworkChange() +{ + return; +} + +NetworkDelegate::NetworkType NetworkDelegateDefaultImpl::GetNetworkType(bool retrieve) +{ + return NetworkType::NONE; +} + +bool NetworkDelegateDefaultImpl::Init() +{ + static NetworkDelegateDefaultImpl delegate; + static std::once_flag onceFlag; + std::call_once(onceFlag, [&]() { NetworkDelegate::RegisterNetworkInstance(&delegate); }); + return true; +} +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/include/broadcaster/broadcast_sender.h b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.h similarity index 46% rename from datamgr_service/services/distributeddataservice/adapter/include/broadcaster/broadcast_sender.h rename to datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.h index 656ad912..5c7583f2 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/broadcaster/broadcast_sender.h +++ b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_default_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,29 +13,22 @@ * limitations under the License. */ -#ifndef BROADCAST_SENDER_H -#define BROADCAST_SENDER_H +#ifndef OHOS_DISTRIBUTED_DATA_NETWORK_DELEGATE_DEFAULT_IMPL_H +#define OHOS_DISTRIBUTED_DATA_NETWORK_DELEGATE_DEFAULT_IMPL_H -#include -#include -#include -#include "visibility.h" +#include "network/network_delegate.h" -namespace OHOS::DistributedKv { -struct EventParams { - std::string userId; - std::string appId; - std::string storeId; -}; - -class BroadcastSender { +namespace OHOS::DistributedData { +class NetworkDelegateDefaultImpl : public NetworkDelegate { public: - KVSTORE_API virtual void SendEvent(const EventParams ¶ms) = 0; - KVSTORE_API virtual ~BroadcastSender() {} - KVSTORE_API static std::shared_ptr GetInstance(); + static bool Init(); + bool IsNetworkAvailable() override; + void RegOnNetworkChange() override; + NetworkType GetNetworkType(bool retrieve = false) override; + private: - static std::mutex mutex_; - static std::shared_ptr instance_; + NetworkDelegateDefaultImpl(); + ~NetworkDelegateDefaultImpl(); }; -} // namespace OHOS::DistributedKv -#endif // BROADCAST_SENDER_H +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_NETWORK_DELEGATE_DEFAULT_IMPL_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/network/network_adapter.cpp b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.cpp similarity index 71% rename from datamgr_service/services/distributeddataservice/service/network/network_adapter.cpp rename to datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.cpp index 7092184a..9389f8dc 100644 --- a/datamgr_service/services/distributeddataservice/service/network/network_adapter.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.cpp @@ -12,8 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define LOG_TAG "NetworkAdapter" -#include "network_adapter.h" +#define LOG_TAG "NetworkDelegateNormalImpl" + +#include "network_delegate_normal_impl.h" #include "device_manager_adapter.h" #include "log_print.h" @@ -22,23 +23,26 @@ #include "net_handle.h" namespace OHOS::DistributedData { using namespace OHOS::NetManagerStandard; -static NetworkAdapter::NetworkType Convert(NetManagerStandard::NetBearType bearType) +__attribute__((used)) static bool g_isInit = NetworkDelegateNormalImpl::Init(); +static NetworkDelegateNormalImpl::NetworkType Convert(NetManagerStandard::NetBearType bearType) { switch (bearType) { case NetManagerStandard::BEARER_WIFI: - return NetworkAdapter::WIFI; + return NetworkDelegate::WIFI; case NetManagerStandard::BEARER_CELLULAR: - return NetworkAdapter::CELLULAR; + return NetworkDelegate::CELLULAR; case NetManagerStandard::BEARER_ETHERNET: - return NetworkAdapter::ETHERNET; + return NetworkDelegate::ETHERNET; default: - return NetworkAdapter::OTHER; + return NetworkDelegate::OTHER; } } class NetConnCallbackObserver : public NetConnCallbackStub { public: - explicit NetConnCallbackObserver(NetworkAdapter &netAdapter) : netAdapter_(netAdapter) {} + explicit NetConnCallbackObserver(NetworkDelegateNormalImpl &delegate) : delegate_(delegate) + { + } ~NetConnCallbackObserver() override = default; int32_t NetAvailable(sptr &netHandle) override; int32_t NetCapabilitiesChange(sptr &netHandle, const sptr &netAllCap) override; @@ -48,7 +52,7 @@ public: int32_t NetBlockStatusChange(sptr &netHandle, bool blocked) override; private: - NetworkAdapter &netAdapter_; + NetworkDelegateNormalImpl &delegate_; }; int32_t NetConnCallbackObserver::NetAvailable(sptr &netHandle) @@ -60,7 +64,7 @@ int32_t NetConnCallbackObserver::NetAvailable(sptr &netHandl return 0; } if (netAllCap->netCaps_.count(NetManagerStandard::NET_CAPABILITY_VALIDATED) && !netAllCap->bearerTypes_.empty()) { - netAdapter_.SetNet(Convert(*netAllCap->bearerTypes_.begin())); + delegate_.SetNet(Convert(*netAllCap->bearerTypes_.begin())); } else { - netAdapter_.SetNet(NetworkAdapter::NONE); + delegate_.SetNet(NetworkDelegateNormalImpl::NONE); } return 0; } @@ -89,7 +93,7 @@ int32_t NetConnCallbackObserver::NetConnectionPropertiesChange(sptr & int32_t NetConnCallbackObserver::NetLost(sptr &netHandle) { ZLOGI("OnNetLost"); - netAdapter_.SetNet(NetworkAdapter::NONE); + delegate_.SetNet(NetworkDelegateNormalImpl::NONE); return 0; } @@ -99,22 +103,16 @@ int32_t NetConnCallbackObserver::NetBlockStatusChange(sptr &netHandle return 0; } -NetworkAdapter::NetworkAdapter() - : cloudDmInfo({ "cloudDeviceId", "cloudDeviceName", 0, "cloudNetworkId", 0 }) +NetworkDelegateNormalImpl::NetworkDelegateNormalImpl() + : cloudDmInfo_({ "cloudDeviceId", "cloudDeviceName", 0, "cloudNetworkId", 0 }) { } -NetworkAdapter::~NetworkAdapter() +NetworkDelegateNormalImpl::~NetworkDelegateNormalImpl() { } - NetworkAdapter &NetworkAdapter::GetInstance() -{ - static NetworkAdapter adapter; - return adapter; -} - -void NetworkAdapter::RegOnNetworkChange() +void NetworkDelegateNormalImpl::RegOnNetworkChange() { static std::atomic_bool flag = false; if (flag.exchange(true)) { @@ -134,7 +132,7 @@ void NetworkAdapter::RegOnNetworkChange() } } -bool NetworkAdapter::IsNetworkAvailable() +bool NetworkDelegateNormalImpl::IsNetworkAvailable() { if (defaultNetwork_ != NONE || expireTime_ > GetTimeStamp()) { return defaultNetwork_ != NONE; @@ -142,7 +140,7 @@ bool NetworkAdapter::IsNetworkAvailable() return RefreshNet() != NONE; } -NetworkAdapter::NetworkType NetworkAdapter::SetNet(NetworkType netWorkType) +NetworkDelegateNormalImpl::NetworkType NetworkDelegateNormalImpl::SetNet(NetworkType netWorkType) { auto oldNet = defaultNetwork_; bool ready = oldNet == NONE && netWorkType != NONE && (GetTimeStamp() - netLostTime_) > NET_LOST_DURATION; @@ -153,15 +151,15 @@ NetworkAdapter::NetworkType NetworkAdapter::SetNet(NetworkType netWorkType) defaultNetwork_ = netWorkType; expireTime_ = GetTimeStamp() + EFFECTIVE_DURATION; if (ready) { - DeviceManagerAdapter::GetInstance().OnReady(cloudDmInfo); + DeviceManagerAdapter::GetInstance().OnReady(cloudDmInfo_); } if (offline) { - DeviceManagerAdapter::GetInstance().Offline(cloudDmInfo); + DeviceManagerAdapter::GetInstance().Offline(cloudDmInfo_); } return netWorkType; } -NetworkAdapter::NetworkType NetworkAdapter::GetNetworkType(bool retrieve) +NetworkDelegate::NetworkType NetworkDelegateNormalImpl::GetNetworkType(bool retrieve) { if (!retrieve) { return defaultNetwork_; @@ -169,7 +167,7 @@ NetworkAdapter::NetworkType NetworkAdapter::GetNetworkType(bool retrieve) return RefreshNet(); } -NetworkAdapter::NetworkType NetworkAdapter::RefreshNet() +NetworkDelegateNormalImpl::NetworkType NetworkDelegateNormalImpl::RefreshNet() { NetHandle handle; auto status = NetConnClient::GetInstance().GetDefaultNet(handle); @@ -184,4 +182,12 @@ NetworkAdapter::NetworkType NetworkAdapter::RefreshNet() } return SetNet(Convert(*capabilities.bearerTypes_.begin())); } + +bool NetworkDelegateNormalImpl::Init() +{ + static NetworkDelegateNormalImpl delegate; + static std::once_flag onceFlag; + std::call_once(onceFlag, [&]() { NetworkDelegate::RegisterNetworkInstance(&delegate); }); + return true; } +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/network/network_adapter.h b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.h similarity index 62% rename from datamgr_service/services/distributeddataservice/service/network/network_adapter.h rename to datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.h index 63a9da6b..4000ef97 100644 --- a/datamgr_service/services/distributeddataservice/service/network/network_adapter.h +++ b/datamgr_service/services/distributeddataservice/adapter/network/src/network_delegate_normal_impl.h @@ -12,47 +12,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifndef OHOS_DISTRIBUTED_DATA_NETWORK_ADAPTER_H -#define OHOS_DISTRIBUTED_DATA_NETWORK_ADAPTER_H + +#ifndef OHOS_DISTRIBUTED_DATA_NETWORK_NORMAL_DELEGATE_IMPL_H +#define OHOS_DISTRIBUTED_DATA_NETWORK_NORMAL_DELEGATE_IMPL_H #include #include + #include "dm_device_info.h" +#include "network/network_delegate.h" namespace OHOS::DistributedData { -class NetworkAdapter { +class NetworkDelegateNormalImpl : public NetworkDelegate { public: - enum NetworkType { - NONE, - CELLULAR, - WIFI, - ETHERNET, - OTHER - }; - using DmDeviceInfo = OHOS::DistributedHardware::DmDeviceInfo; - static NetworkAdapter &GetInstance(); - bool IsNetworkAvailable(); - NetworkType GetNetworkType(bool retrieve = false); - void RegOnNetworkChange(); + using DmDeviceInfo = OHOS::DistributedHardware::DmDeviceInfo; + static bool Init(); + bool IsNetworkAvailable() override; + NetworkType GetNetworkType(bool retrieve = false) override; + void RegOnNetworkChange() override; friend class NetConnCallbackObserver; +private: + NetworkDelegateNormalImpl(); + ~NetworkDelegateNormalImpl(); + const DmDeviceInfo cloudDmInfo_; + NetworkType SetNet(NetworkType netWorkType); + NetworkType RefreshNet(); static inline uint64_t GetTimeStamp() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); } -private: - NetworkAdapter(); - ~NetworkAdapter(); - const DmDeviceInfo cloudDmInfo; - NetworkType SetNet(NetworkType netWorkType); - NetworkType RefreshNet(); static constexpr int32_t EFFECTIVE_DURATION = 30 * 1000; // ms - static constexpr int32_t NET_LOST_DURATION = 10 * 1000; // ms + static constexpr int32_t NET_LOST_DURATION = 10 * 1000; // ms NetworkType defaultNetwork_ = NONE; uint64_t expireTime_ = 0; uint64_t netLostTime_ = 0; }; -} -#endif // OHOS_DISTRIBUTED_DATA_NETWORK_ADAPTER_H +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_NETWORK_NORMAL_DELEGATE_IMPL_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/permission/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/network/test/BUILD.gn similarity index 31% rename from datamgr_service/services/distributeddataservice/adapter/permission/test/BUILD.gn rename to datamgr_service/services/distributeddataservice/adapter/network/test/BUILD.gn index 509f49b3..c1b564cc 100644 --- a/datamgr_service/services/distributeddataservice/adapter/permission/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/network/test/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Huawei Device Co., Ltd. +# Copyright (c) 2025 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,46 +11,88 @@ # See the License for the specific language governing permissions and # limitations under the License. import("//build/test.gni") - -module_output_path = "datamgr_service/distributeddatafwk" +import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### -config("module_private_config") { - visibility = [ ":*" ] +ohos_unittest("NetworkDelegateTest") { + module_out_path = module_output_path + sources = [ + "${data_service_path}/framework/network/network_delegate.cpp", + "network_delegate_test.cpp", + ] include_dirs = [ - "../../include/permission/", - "../../include/utils/", - "//commonlibrary/c_utils/base/include/", - "//foundation/distributeddatamgr/kv_store/interfaces/innerkits/distributeddata/include", - "//foundation/distributeddatamgr/kv_store/frameworks/common", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/framework/include", + "${data_service_path}/framework/include/network", + "${data_service_path}/adapter/include/utils", + "${data_service_path}/framework/include", + "${data_service_path}/adapter/network/src", + "${data_service_path}/adapter/include", + ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", ] + + deps = [ "../:distributeddata_network" ] + + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:datamgr_common", + "netmanager_base:net_conn_manager_if", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } -ohos_unittest("PermissionValidatorTest") { +ohos_unittest("NetworkDelegateNormalImplTest") { module_out_path = module_output_path - sources = [ "unittest/permission_validator_test.cpp" ] + sources = [ + "${data_service_path}/framework/network/network_delegate.cpp", + "network_delegate_normal_impl_test.cpp", + ] + include_dirs = [ + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/network", + "${data_service_path}/adapter/include/utils", + "${data_service_path}/framework/include", + "${data_service_path}/adapter/network/src", + "${data_service_path}/adapter/include", + ] - configs = [ ":module_private_config" ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] deps = [ - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/permission:distributeddata_permission_static", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/adapter/utils:distributeddata_utils_static", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/framework:distributeddatasvcfwk", - "//third_party/googletest:gtest_main", + "${data_service_path}/adapter/communicator:distributeddata_communicator", ] - external_deps = [ "hilog:libhilog" ] + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "ipc:ipc_core", + "kv_store:datamgr_common", + "netmanager_base:net_conn_manager_if", + ] defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } +############################################################################### group("unittest") { testonly = true deps = [] - deps += [ ":PermissionValidatorTest" ] + deps += [ + ":NetworkDelegateNormalImplTest", + ":NetworkDelegateTest", + ] } ############################################################################### diff --git a/datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_normal_impl_test.cpp b/datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_normal_impl_test.cpp new file mode 100644 index 00000000..9d4842a9 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_normal_impl_test.cpp @@ -0,0 +1,217 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "network/network_delegate.h" +#include "network_delegate_normal_impl.h" +#include "network_delegate_normal_impl.cpp" +#include +#include + +using namespace testing::ext; +using namespace std; +using namespace OHOS::DistributedData; +using namespace OHOS::NetManagerStandard; +using DmDeviceInfo = OHOS::DistributedHardware::DmDeviceInfo; +namespace OHOS::Test { +namespace DistributedDataTest { +class NetworkDelegateNormalImplTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void NetworkDelegateNormalImplTest::SetUpTestCase(void) +{ +} + +void NetworkDelegateNormalImplTest::TearDownTestCase() +{ +} + +void NetworkDelegateNormalImplTest::SetUp() +{ +} + +void NetworkDelegateNormalImplTest::TearDown() +{ +} + +/** +* @tc.name: GetNetworkType001 +* @tc.desc: GetNetworkType testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, GetNetworkType001, TestSize.Level1) +{ + NetworkDelegateNormalImpl delegate; + bool retrieve = false; + EXPECT_NO_FATAL_FAILURE(delegate.RegOnNetworkChange()); + NetworkDelegate::NetworkType status = delegate.GetNetworkType(retrieve); + EXPECT_EQ(status, NetworkDelegate::NetworkType::NONE); +} + +/** +* @tc.name: GetNetworkType002 +* @tc.desc: GetNetworkType testing normal branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, GetNetworkType002, TestSize.Level1) +{ + NetworkDelegateNormalImpl delegate; + bool retrieve = true; + EXPECT_NO_FATAL_FAILURE(delegate.RegOnNetworkChange()); + NetworkDelegate::NetworkType status = delegate.GetNetworkType(retrieve); + EXPECT_EQ(status, NetworkDelegate::NetworkType::NONE); +} + +/** +* @tc.name: IsNetworkAvailable +* @tc.desc: IsNetworkAvailable testing different branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, IsNetworkAvailable, TestSize.Level1) +{ + NetworkDelegateNormalImpl delegate; + bool ret = delegate.IsNetworkAvailable(); // false false + EXPECT_FALSE(ret); + + DmDeviceInfo& info = const_cast(delegate.cloudDmInfo_); + std::fill(info.networkId, info.networkId + sizeof(info.networkId), '\0'); + NetworkDelegateNormalImpl::NetworkType netWorkType = NetworkDelegate::NetworkType::NONE; + NetworkDelegateNormalImpl::NetworkType status = delegate.SetNet(netWorkType); + EXPECT_EQ(status, NetworkDelegate::NONE); + ret = delegate.IsNetworkAvailable(); // false true + EXPECT_FALSE(ret); + + netWorkType = NetworkDelegate::NetworkType::WIFI; + status = delegate.SetNet(netWorkType); + EXPECT_EQ(status, NetworkDelegate::WIFI); + ret = delegate.IsNetworkAvailable(); // true true + EXPECT_TRUE(ret); + + netWorkType = NetworkDelegate::NetworkType::NONE; + status = delegate.SetNet(netWorkType); + EXPECT_EQ(status, NetworkDelegate::NONE); + ret = delegate.IsNetworkAvailable(); // false true + EXPECT_FALSE(ret); + + netWorkType = NetworkDelegate::NetworkType::WIFI; + status = delegate.SetNet(netWorkType); + EXPECT_EQ(status, NetworkDelegate::WIFI); + ret = delegate.IsNetworkAvailable(); // false true + EXPECT_TRUE(ret); +} + +/** +* @tc.name: NetCapabilitiesChange001 +* @tc.desc: NetCapabilitiesChange testing different branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, NetCapabilitiesChange001, TestSize.Level1) +{ + NetworkDelegateNormalImpl delegate; + sptr observer = new (std::nothrow) NetConnCallbackObserver(delegate); + sptr netHandle = nullptr; + sptr netAllCap = nullptr; + int32_t status = observer->NetCapabilitiesChange(netHandle, netAllCap); + EXPECT_EQ(status, 0); + + netHandle = new (std::nothrow) NetHandle(); + status = observer->NetCapabilitiesChange(netHandle, netAllCap); + EXPECT_EQ(status, 0); + + netHandle = nullptr; + netAllCap = new (std::nothrow) NetAllCapabilities(); + status = observer->NetCapabilitiesChange(netHandle, netAllCap); + EXPECT_EQ(status, 0); +} + +/** +* @tc.name: NetCapabilitiesChange002 +* @tc.desc: NetCapabilitiesChange testing different branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, NetCapabilitiesChange002, TestSize.Level1) +{ + NetworkDelegateNormalImpl delegate; + DmDeviceInfo& info = const_cast(delegate.cloudDmInfo_); + std::fill(info.networkId, info.networkId + sizeof(info.networkId), '\0'); + sptr observer = new (std::nothrow) NetConnCallbackObserver(delegate); + sptr netHandle = new (std::nothrow) NetHandle(); + sptr netAllCap = new (std::nothrow) NetAllCapabilities(); + EXPECT_FALSE(netAllCap->netCaps_.count(NetManagerStandard::NET_CAPABILITY_VALIDATED)); + EXPECT_FALSE(!netAllCap->bearerTypes_.empty()); + int32_t status = observer->NetCapabilitiesChange(netHandle, netAllCap); + EXPECT_EQ(status, 0); + + netAllCap->netCaps_.insert(NetManagerStandard::NET_CAPABILITY_VALIDATED); + EXPECT_TRUE(netAllCap->netCaps_.count(NetManagerStandard::NET_CAPABILITY_VALIDATED)); + EXPECT_FALSE(!netAllCap->bearerTypes_.empty()); + status = observer->NetCapabilitiesChange(netHandle, netAllCap); + EXPECT_EQ(status, 0); + + sptr netAllCaps = new (std::nothrow) NetAllCapabilities(); + netAllCaps->bearerTypes_.insert(NetManagerStandard::BEARER_WIFI); + EXPECT_FALSE(netAllCaps->netCaps_.count(NetManagerStandard::NET_CAPABILITY_VALIDATED)); + EXPECT_TRUE(!netAllCaps->bearerTypes_.empty()); + status = observer->NetCapabilitiesChange(netHandle, netAllCaps); + EXPECT_EQ(status, 0); + + netAllCaps->netCaps_.insert(NetManagerStandard::NET_CAPABILITY_VALIDATED); + EXPECT_TRUE(netAllCaps->netCaps_.count(NetManagerStandard::NET_CAPABILITY_VALIDATED)); + EXPECT_TRUE(!netAllCaps->bearerTypes_.empty()); + status = observer->NetCapabilitiesChange(netHandle, netAllCaps); + EXPECT_EQ(status, 0); +} + +/** +* @tc.name: Convert +* @tc.desc: Convert testing different branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(NetworkDelegateNormalImplTest, Convert, TestSize.Level1) +{ + NetManagerStandard::NetBearType bearType = NetManagerStandard::BEARER_WIFI; + NetworkDelegateNormalImpl::NetworkType status = Convert(bearType); + EXPECT_EQ(status, NetworkDelegate::WIFI); + + bearType = NetManagerStandard::BEARER_CELLULAR; + status = Convert(bearType); + EXPECT_EQ(status, NetworkDelegate::CELLULAR); + + bearType = NetManagerStandard::BEARER_ETHERNET; + status = Convert(bearType); + EXPECT_EQ(status, NetworkDelegate::ETHERNET); + + bearType = NetManagerStandard::BEARER_VPN; + status = Convert(bearType); + EXPECT_EQ(status, NetworkDelegate::OTHER); +} +} // namespace DistributedDataTest +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/network_adapter_test.cpp b/datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_test.cpp similarity index 36% rename from datamgr_service/services/distributeddataservice/service/test/network_adapter_test.cpp rename to datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_test.cpp index e8c1db11..391aa008 100644 --- a/datamgr_service/services/distributeddataservice/service/test/network_adapter_test.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/network/test/network_delegate_test.cpp @@ -12,22 +12,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define LOG_TAG "NetworkAdapterTest" +#define LOG_TAG "NetworkDelegateTest" +#include "network/network_delegate.h" + #include #include -#include - -#include "communicator/device_manager_adapter.h" -#include "log_print.h" -#include "network_adapter.h" using namespace testing::ext; using namespace OHOS::DistributedData; -using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; namespace OHOS::Test { namespace DistributedDataTest { -class NetworkAdapterTest : public testing::Test { +class NetworkDelegateTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); @@ -35,68 +31,53 @@ public: void TearDown(); }; -void NetworkAdapterTest::SetUpTestCase(void) +void NetworkDelegateTest::SetUpTestCase(void) { - NetworkAdapter::GetInstance().RegOnNetworkChange(); + auto delegate = NetworkDelegate::GetInstance(); + if (delegate == nullptr) { + return; + } + delegate->RegOnNetworkChange(); } -void NetworkAdapterTest::TearDownTestCase() +void NetworkDelegateTest::TearDownTestCase() { } -void NetworkAdapterTest::SetUp() +void NetworkDelegateTest::SetUp() { } -void NetworkAdapterTest::TearDown() +void NetworkDelegateTest::TearDown() { } /** -* @tc.name: GetInstanceTest -* @tc.desc: GetInstance test. +* @tc.name: IsNetworkAvailableTest +* @tc.desc: IsNetworkAvailable test. * @tc.type: FUNC * @tc.require: * @tc.author: */ -HWTEST_F(NetworkAdapterTest, GetInstanceTest, TestSize.Level0) +HWTEST_F(NetworkDelegateTest, IsNetworkAvailableTest, TestSize.Level0) { - NetworkAdapter &instance = NetworkAdapter::GetInstance(); - EXPECT_NE(&instance, nullptr); + NetworkDelegate *instance = NetworkDelegate::GetInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_FALSE(instance->IsNetworkAvailable()); } /** -* @tc.name: SetNetTest -* @tc.desc: SetNet test. +* @tc.name: GetNetworkTypeTest +* @tc.desc: GetNetworkType test. * @tc.type: FUNC * @tc.require: * @tc.author: */ -HWTEST_F(NetworkAdapterTest, SetNetTest, TestSize.Level0) +HWTEST_F(NetworkDelegateTest, GetNetworkTypeTest, TestSize.Level0) { - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NetworkType::NONE); - auto ret = NetworkAdapter::GetInstance().IsNetworkAvailable(); - EXPECT_EQ(ret, false); - auto type = NetworkAdapter::GetInstance().GetNetworkType(true); - EXPECT_EQ(type, NetworkAdapter::NONE); - type = NetworkAdapter::GetInstance().GetNetworkType(); - EXPECT_EQ(type, NetworkAdapter::NONE); - - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NetworkType::WIFI); - type = NetworkAdapter::GetInstance().GetNetworkType(); - EXPECT_EQ(type, NetworkAdapter::WIFI); - - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NetworkType::CELLULAR); - type = NetworkAdapter::GetInstance().GetNetworkType(); - EXPECT_EQ(type, NetworkAdapter::CELLULAR); - - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NetworkType::ETHERNET); - type = NetworkAdapter::GetInstance().GetNetworkType(); - EXPECT_EQ(type, NetworkAdapter::ETHERNET); - - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NetworkType::OTHER); - type = NetworkAdapter::GetInstance().GetNetworkType(); - EXPECT_EQ(type, NetworkAdapter::OTHER); + NetworkDelegate *instance = NetworkDelegate::GetInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_EQ(instance->GetNetworkType(), NetworkDelegate::NetworkType::NONE); } } // namespace DistributedDataTest } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/broadcaster/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/schema_helper/BUILD.gn similarity index 65% rename from datamgr_service/services/distributeddataservice/adapter/broadcaster/BUILD.gn rename to datamgr_service/services/distributeddataservice/adapter/schema_helper/BUILD.gn index 45b8b189..4bfbe3be 100644 --- a/datamgr_service/services/distributeddataservice/adapter/broadcaster/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/schema_helper/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Huawei Device Co., Ltd. +# Copyright (c) 2025 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,7 +13,12 @@ import("//build/ohos.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -ohos_source_set("distributeddata_broadcaster") { +config("module_public_config") { + visibility = [ ":*" ] + include_dirs = [ "../include/schema_helper" ] +} + +ohos_source_set("distributeddata_schema_helper") { branch_protector_ret = "pac_ret" sanitize = { cfi = true @@ -22,28 +27,24 @@ ohos_source_set("distributeddata_broadcaster") { boundary_sanitize = true ubsan = true } - sources = [ - "src/broadcast_sender.cpp", - "src/broadcast_sender_impl.cpp", - ] + sources = [ "src/get_schema_helper.cpp" ] - include_dirs = [ - "../include/broadcaster", - "../include/log", - "./src", + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", ] - cflags_cc = [ "-fvisibility=hidden" ] + include_dirs = [ "../include/schema_helper" ] + public_configs = [ ":module_public_config" ] external_deps = [ - "ability_base:base", - "ability_base:want", "bundle_framework:appexecfwk_base", - "c_utils:utils", - "common_event_service:cesfwk_innerkits", + "bundle_framework:appexecfwk_core", "hilog:libhilog", - "ipc:ipc_core", + "ipc:ipc_single", "kv_store:datamgr_common", + "resource_management:global_resmgr", + "samgr:samgr_proxy", ] subsystem_name = "distributeddatamgr" part_name = "datamgr_service" diff --git a/datamgr_service/services/distributeddataservice/adapter/schema_helper/src/get_schema_helper.cpp b/datamgr_service/services/distributeddataservice/adapter/schema_helper/src/get_schema_helper.cpp new file mode 100644 index 00000000..b5c847b3 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/adapter/schema_helper/src/get_schema_helper.cpp @@ -0,0 +1,121 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#define LOG_TAG "SchemaHelper" +#include "schema_helper/get_schema_helper.h" + +#include "bundle_mgr_interface.h" +#include "if_system_ability_manager.h" +#include "iservice_registry.h" +#include "log_print.h" +#include "resource_manager.h" +#include "system_ability_definition.h" + +namespace OHOS { +namespace DistributedData { +using namespace OHOS::Global::Resource; + +sptr GetSchemaHelper::GetBundleMgr() +{ + std::lock_guard lock(mutex_); + if (object_ != nullptr) { + return iface_cast(object_); + } + sptr systemAbilityManager = + SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager(); + if (systemAbilityManager == nullptr) { + ZLOGE("Failed to get system ability mgr."); + return nullptr; + } + object_ = systemAbilityManager->GetSystemAbility(BUNDLE_MGR_SERVICE_SYS_ABILITY_ID); + if (object_ == nullptr) { + ZLOGE("BMS service not ready to complete."); + return nullptr; + } + deathRecipient_ = new (std::nothrow) GetSchemaHelper::ServiceDeathRecipient(weak_from_this()); + if (deathRecipient_ == nullptr) { + ZLOGE("deathRecipient alloc failed."); + object_ = nullptr; + return nullptr; + } + if (!object_->AddDeathRecipient(deathRecipient_)) { + ZLOGE("add death recipient failed."); + object_ = nullptr; + deathRecipient_ = nullptr; + return nullptr; + } + return iface_cast(object_); +} +std::vector GetSchemaHelper::GetSchemaFromHap(const std::string &schemaPath, const AppInfo &info) +{ + std::vector schemas; + auto bmsClient = GetBundleMgr(); + if (bmsClient == nullptr) { + ZLOGE("GetBundleMgr is nullptr!"); + return schemas; + } + OHOS::AppExecFwk::BundleInfo bundleInfo; + int32_t flag = static_cast(AppExecFwk::GetBundleInfoFlag::GET_BUNDLE_INFO_WITH_HAP_MODULE); + auto ret = bmsClient->GetCloneBundleInfo(info.bundleName, flag, info.appIndex, bundleInfo, info.userId); + if (ret != ERR_OK) { + ZLOGE("GetCloneBundleInfo failed. errCode:%{public}d", ret); + return schemas; + } + + std::shared_ptr resMgr(CreateResourceManager()); + if (resMgr == nullptr) { + ZLOGE("resMgr is nullptr."); + return schemas; + } + for (auto &hapModuleInfo : bundleInfo.hapModuleInfos) { + resMgr->AddResource(hapModuleInfo.hapPath.c_str()); + size_t length = 0; + std::unique_ptr fileContent; + auto ret = resMgr->GetRawFileFromHap(schemaPath, length, fileContent); + if (ret != ERR_OK) { + ZLOGD("GetRawFileFromHap failed. bundleName:%{public}s ret:%{public}d", info.bundleName.c_str(), ret); + continue; + } + std::string schema(fileContent.get(), fileContent.get() + length); + schemas.emplace_back(std::move(schema)); + } + return schemas; +} +void GetSchemaHelper::OnRemoteDied() +{ + std::lock_guard lock(mutex_); + ZLOGE("remote object died, object=null ? %{public}s.", object_ == nullptr ? "true" : "false"); + if (object_ != nullptr) { + object_->RemoveDeathRecipient(deathRecipient_); + } + object_ = nullptr; + deathRecipient_ = nullptr; +} + +GetSchemaHelper::~GetSchemaHelper() +{ + std::lock_guard lock(mutex_); + if (object_ != nullptr) { + object_->RemoveDeathRecipient(deathRecipient_); + } +} + +GetSchemaHelper &GetSchemaHelper::GetInstance() +{ + static GetSchemaHelper helper; + return helper; +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/screenlock/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/screenlock/BUILD.gn index ed1fd391..28f3b0b7 100644 --- a/datamgr_service/services/distributeddataservice/adapter/screenlock/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/screenlock/BUILD.gn @@ -24,7 +24,10 @@ ohos_source_set("distributeddata_screenlock") { } sources = [ "src/screen_lock.cpp" ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] include_dirs = [ "../include/screenlock", diff --git a/datamgr_service/services/distributeddataservice/adapter/screenlock/src/screen_lock.cpp b/datamgr_service/services/distributeddataservice/adapter/screenlock/src/screen_lock.cpp index 7cac20cb..36c5d601 100644 --- a/datamgr_service/services/distributeddataservice/adapter/screenlock/src/screen_lock.cpp +++ b/datamgr_service/services/distributeddataservice/adapter/screenlock/src/screen_lock.cpp @@ -14,7 +14,7 @@ */ #define LOG_TAG "ScreenLock" -#include "screen_lock.h" +#include "screenlock/screen_lock.h" #include "account/account_delegate.h" #include "log_print.h" diff --git a/datamgr_service/services/distributeddataservice/adapter/screenlock/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/screenlock/test/BUILD.gn index 8a1c3084..f1f16b53 100644 --- a/datamgr_service/services/distributeddataservice/adapter/screenlock/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/screenlock/test/BUILD.gn @@ -12,7 +12,7 @@ # limitations under the License. import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### ohos_unittest("ScreenLockTest") { diff --git a/datamgr_service/services/distributeddataservice/adapter/test/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/test/BUILD.gn index 4ae23205..6f6f3b43 100644 --- a/datamgr_service/services/distributeddataservice/adapter/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/test/BUILD.gn @@ -12,6 +12,7 @@ # limitations under the License. import("//build/ohos.gni") import("//build/ohos_var.gni") +import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") group("unittest") { testonly = true @@ -23,6 +24,10 @@ group("unittest") { "../dfx/test:unittest", "../screenlock/test:unittest", ] + + if (datamgr_service_cloud) { + deps += [ "../network/test:unittest" ] + } } group("fuzztest") { diff --git a/datamgr_service/services/distributeddataservice/adapter/utils/BUILD.gn b/datamgr_service/services/distributeddataservice/adapter/utils/BUILD.gn index 7ae119fd..8325bf05 100644 --- a/datamgr_service/services/distributeddataservice/adapter/utils/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/adapter/utils/BUILD.gn @@ -22,25 +22,19 @@ ohos_source_set("distributeddata_utils") { boundary_sanitize = true ubsan = true } - sources = [ - "src/kvstore_utils.cpp", - "src/time_utils.cpp", - ] + sources = [ "src/kvstore_utils.cpp" ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] if (build_public_version) { cflags_cc += [ "-DCONFIG_PUBLIC_VERSION" ] } - include_dirs = [ - "../include/permission", - "../include/utils", - "../include/log", - "../include/communicator", - "../include/dfx", - ] - ldflags = [ "-Wl,--exclude-libs,ALL" ] + include_dirs = [ "../include/utils" ] + ldflags = [ "-Wl,-z,relro,-z,now,--exclude-libs,ALL" ] external_deps = [ "c_utils:utils", diff --git a/datamgr_service/services/distributeddataservice/app/BUILD.gn b/datamgr_service/services/distributeddataservice/app/BUILD.gn index 03daee00..b66d0134 100644 --- a/datamgr_service/services/distributeddataservice/app/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/app/BUILD.gn @@ -47,11 +47,9 @@ config("module_private_config") { "${data_service_path}/adapter/include/installer", "${data_service_path}/adapter/include/broadcaster", "${data_service_path}/adapter/include/utils", - "${data_service_path}/adapter/include/dfx", "${data_service_path}/adapter/include", "${data_service_path}/app/src/session_manager", "${data_service_path}/framework/include", - "${data_service_path}/service/bootstrap/include", "${data_service_path}/service/common", "${data_service_path}/service/config/include", "${data_service_path}/service/crypto/include", @@ -76,12 +74,18 @@ config("module_private_config") { "-Wno-multichar", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fdata-sections", + "-ffunction-sections", + "-fstack-protector-strong", ] cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] + + ldflags = [ "-Wl,-z,relro,-z,now,--gc-sections" ] } ohos_shared_library("distributeddataservice") { diff --git a/datamgr_service/services/distributeddataservice/app/CMakeLists.txt b/datamgr_service/services/distributeddataservice/app/CMakeLists.txt index 47ba4989..0d383c14 100644 --- a/datamgr_service/services/distributeddataservice/app/CMakeLists.txt +++ b/datamgr_service/services/distributeddataservice/app/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10.2) project(app) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE -pthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -fpermissive -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") diff --git a/datamgr_service/services/distributeddataservice/app/src/checker/BUILD.gn b/datamgr_service/services/distributeddataservice/app/src/checker/BUILD.gn index f0fa8dd2..6695b0d6 100644 --- a/datamgr_service/services/distributeddataservice/app/src/checker/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/app/src/checker/BUILD.gn @@ -30,6 +30,7 @@ ohos_source_set("distributeddata_checker") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "${data_service_path}/framework/include" ] @@ -37,9 +38,12 @@ ohos_source_set("distributeddata_checker") { if (build_public_version) { cflags_cc += [ "-DCONFIG_PUBLIC_VERSION" ] } - ldflags = [ "-Wl,--exclude-libs,ALL" ] + ldflags = [ "-Wl,-z,relro,-z,now,--exclude-libs,ALL" ] deps = [ "${data_service_path}/adapter/utils:distributeddata_utils" ] - cflags = [ "-Oz" ] + cflags = [ + "-Oz", + "-fstack-protector-strong", + ] external_deps = [ "ability_base:base", "ability_base:want", diff --git a/datamgr_service/services/distributeddataservice/app/src/clone/clone_backup_info.cpp b/datamgr_service/services/distributeddataservice/app/src/clone/clone_backup_info.cpp index 83bfa59c..65408d43 100644 --- a/datamgr_service/services/distributeddataservice/app/src/clone/clone_backup_info.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/clone/clone_backup_info.cpp @@ -16,19 +16,11 @@ namespace OHOS { namespace DistributedData { -constexpr const char* CLONE_INFO_DETAIL = "detail"; -constexpr const char* ENCRYPTION_INFO = "encryption_info"; -constexpr const char* APPLICATION_SELECTION = "application_selection"; -constexpr const char* USER_ID = "userId"; -constexpr const char* ENCRYPTION_SYMKEY = "encryption_symkey"; -constexpr const char* ENCRYPTION_ALGORITHM = "encryption_algname"; -constexpr const char* GCM_PARAMS_IV = "gcmParams_iv"; - bool CloneEncryptionInfo::Unmarshal(const json &node) { - bool res = GetValue(node, ENCRYPTION_SYMKEY, symkey); - res = GetValue(node, ENCRYPTION_ALGORITHM, algName) && res; - res = GetValue(node, GCM_PARAMS_IV, iv) && res; + bool res = GetValue(node, "encryption_symkey", symkey); + res = GetValue(node, "encryption_algname", algName) && res; + res = GetValue(node, "gcmParams_iv", iv) && res; return res; } @@ -58,22 +50,23 @@ bool CloneBundleInfo::Marshal(json &node) const bool CloneBackupInfo::Unmarshal(const json &node) { - if (!node.is_array()) { + auto items = DistributedData::Serializable::ToJson(node); + if (!items.is_array()) { return false; } std::string type; - auto size = node.size(); + auto size = items.size(); for (size_t i = 0; i < size; i++) { - bool result = GetValue(node[i], GET_NAME(type), type); + bool result = GetValue(items[i], GET_NAME(type), type); if (!result || type.empty()) { continue; } - if (type == ENCRYPTION_INFO) { - GetValue(node[i], CLONE_INFO_DETAIL, encryptionInfo); - } else if (type == APPLICATION_SELECTION) { - GetValue(node[i], CLONE_INFO_DETAIL, bundleInfos); - } else if (type == USER_ID) { - GetValue(node[i], CLONE_INFO_DETAIL, userId); + if (type == "encryption_info") { + GetValue(items[i], "detail", encryptionInfo); + } else if (type == "application_selection") { + GetValue(items[i], "detail", bundleInfos); + } else if (type == "userId") { + GetValue(items[i], "detail", userId); } } return true; diff --git a/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.cpp b/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.cpp index 95a5ada4..2a28c6d8 100644 --- a/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.cpp @@ -53,6 +53,7 @@ bool SecretKeyBackupData::BackupItem::Marshal(json &node) const SetValue(node[GET_NAME(time)], time); SetValue(node[GET_NAME(sKey)], sKey); SetValue(node[GET_NAME(storeType)], storeType); + SetValue(node[GET_NAME(area)], area); return true; } @@ -65,6 +66,7 @@ bool SecretKeyBackupData::BackupItem::Unmarshal(const json &node) ret = GetValue(node, GET_NAME(time), time) && ret; ret = GetValue(node, GET_NAME(sKey), sKey) && ret; ret = GetValue(node, GET_NAME(storeType), storeType) && ret; + ret = GetValue(node, GET_NAME(area), area) && ret; return ret; } diff --git a/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.h b/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.h index 5cfc2cdc..ef503708 100644 --- a/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.h +++ b/datamgr_service/services/distributeddataservice/app/src/clone/secret_key_backup_data.h @@ -27,6 +27,7 @@ public: std::string sKey; int32_t instanceId; int32_t storeType; + int32_t area = -1; std::vector time; API_EXPORT BackupItem(); diff --git a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.cpp b/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.cpp deleted file mode 100644 index f023df18..00000000 --- a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define LOG_TAG "KvStoreFlowCtrlManager" - -#include "kvstore_flowctrl_manager.h" -#include -#include - -namespace OHOS { -namespace DistributedKv { -const int SECOND_TO_MICROSECOND = 1000; -uint64_t CurrentTimeMicros() -{ - struct timeval tv = { 0, 0 }; - gettimeofday(&tv, nullptr); - return (tv.tv_sec * SECOND_TO_MICROSECOND + tv.tv_usec / SECOND_TO_MICROSECOND); -} - -KvStoreFlowCtrlManager::KvStoreFlowCtrlManager(const int burstCapacity, const int sustainedCapacity) -{ - burstTokenBucket_.maxCapacity = burstCapacity; - burstTokenBucket_.refreshTimeGap = BURST_REFRESH_TIME; - sustainedTokenBucket_.maxCapacity = sustainedCapacity; - sustainedTokenBucket_.refreshTimeGap = SUSTAINED_REFRESH_TIME; -} - -void KvStoreFlowCtrlManager::RefreshTokenBucket(TokenBucket &tokenBucket, uint64_t timestamp) -{ - tokenBucket.leftNumInTokenBucket = tokenBucket.maxCapacity; - tokenBucket.tokenBucketRefreshTime = timestamp; -} - -bool KvStoreFlowCtrlManager::IsTokenEnough() -{ - uint64_t curTime = CurrentTimeMicros(); - if (IsTokenEnoughSlice(burstTokenBucket_, curTime) && IsTokenEnoughSlice(sustainedTokenBucket_, curTime)) { - burstTokenBucket_.lastAccessTime = curTime; - burstTokenBucket_.leftNumInTokenBucket--; - - sustainedTokenBucket_.lastAccessTime = curTime; - sustainedTokenBucket_.leftNumInTokenBucket--; - return true; - } - return false; -} - -bool KvStoreFlowCtrlManager::IsTokenEnoughSlice(TokenBucket &tokenBucket, uint64_t timestamp) -{ - // the first time to get token will be allowed; - // if the gap between this time to get token and the least time to fill the bucket - // to the full is larger than 10ms, this operation will be allowed; - if (tokenBucket.tokenBucketRefreshTime == 0 || - timestamp - tokenBucket.tokenBucketRefreshTime > tokenBucket.refreshTimeGap) { - RefreshTokenBucket(tokenBucket, timestamp); - return true; - } else { - return tokenBucket.leftNumInTokenBucket >= 1; - } -} -} // namespace DistributedKv -} // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.h b/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.h deleted file mode 100644 index 838cb3bd..00000000 --- a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/kvstore_flowctrl_manager.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FOUNDATION_KVSTORE_FLOW_CTRL_MANAGER_H -#define FOUNDATION_KVSTORE_FLOW_CTRL_MANAGER_H - -#include - -namespace OHOS { -namespace DistributedKv { -struct TokenBucket { - uint64_t tokenBucketRefreshTime = 0; // last time to refresh the bucket - - uint64_t lastAccessTime = 0; // last time to access - - volatile std::atomic leftNumInTokenBucket {0}; // rest numbers of token in the bucket - - int maxCapacity; // max capacity - - uint64_t refreshTimeGap; // time gap between refreshing -}; - -class KvStoreFlowCtrlManager { -public: - KvStoreFlowCtrlManager() = delete; - - KvStoreFlowCtrlManager(const int burstCapacity, const int sustainedCapacity); - - ~KvStoreFlowCtrlManager() = default; - - bool IsTokenEnough(); - - static const int BURST_REFRESH_TIME = 1000; - - static const int SUSTAINED_REFRESH_TIME = 60000; - -private: - void RefreshTokenBucket(TokenBucket &tokenBucket, uint64_t timestamp); - - bool IsTokenEnoughSlice(TokenBucket &tokenBucket, uint64_t timestamp); - - TokenBucket burstTokenBucket_; // token bucket to deal with events in a burst - - TokenBucket sustainedTokenBucket_; // token bucket to deal with sustained events. -}; -} // namespace DistributedKv -} // namespace OHOS - -#endif // FOUNDATION_KVSTORE_FLOW_CTRL_MANAGER_H diff --git a/datamgr_service/services/distributeddataservice/app/src/installer/BUILD.gn b/datamgr_service/services/distributeddataservice/app/src/installer/BUILD.gn index 4dcc6a77..49eb62d2 100644 --- a/datamgr_service/services/distributeddataservice/app/src/installer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/app/src/installer/BUILD.gn @@ -35,15 +35,18 @@ ohos_source_set("distributeddata_installer") { "${data_service_path}/service/kvdb", "${data_service_path}/service/permission/include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] - cflags = [ "-Oz" ] + cflags = [ + "-Oz", + "-fstack-protector-strong", + ] deps = [ "${data_service_path}/framework:distributeddatasvcfwk", diff --git a/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.cpp b/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.cpp index 72feb1e8..6977b3ab 100644 --- a/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.cpp @@ -58,7 +58,14 @@ void InstallEventSubscriber::OnReceiveEvent(const CommonEventData &event) std::string bundleName = want.GetElement().GetBundleName(); int32_t userId = want.GetIntParam(USER_ID, -1); int32_t appIndex = want.GetIntParam(SANDBOX_APP_INDEX, 0); - ZLOGI("bundleName:%{public}s, user:%{public}d, appIndex:%{public}d", bundleName.c_str(), userId, appIndex); + int32_t newAppIndex = want.GetIntParam(APP_INDEX, 0); + ZLOGI("bundleName:%{public}s, user:%{public}d, appIndex:%{public}d, newAppIndex:%{public}d", + bundleName.c_str(), userId, appIndex, newAppIndex); + // appIndex's key in want is "appIndex", the value of the elder key "sandbox_app_index" is unsure, + // to avoid effecting historical function, passing non-zero value to the function + if (appIndex == 0 && newAppIndex != 0) { + appIndex = newAppIndex; + } (this->*(it->second))(bundleName, userId, appIndex); } } @@ -87,6 +94,7 @@ void InstallEventSubscriber::OnUninstall(const std::string &bundleName, int32_t MetaDataManager::GetInstance().DelMeta(meta.GetBackupSecretKey(), true); MetaDataManager::GetInstance().DelMeta(meta.GetAutoLaunchKey(), true); MetaDataManager::GetInstance().DelMeta(meta.GetDebugInfoKey(), true); + MetaDataManager::GetInstance().DelMeta(meta.GetDfxInfoKey(), true); MetaDataManager::GetInstance().DelMeta(meta.GetCloneSecretKey(), true); PermitDelegate::GetInstance().DelCache(meta.GetKey()); } @@ -96,21 +104,6 @@ void InstallEventSubscriber::OnUninstall(const std::string &bundleName, int32_t void InstallEventSubscriber::OnUpdate(const std::string &bundleName, int32_t userId, int32_t appIndex) { kvStoreDataService_->OnUpdate(bundleName, userId, appIndex); - std::string prefix = StoreMetaData::GetPrefix( - { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid, std::to_string(userId), "default", bundleName }); - std::vector storeMetaData; - if (!MetaDataManager::GetInstance().LoadMeta(prefix, storeMetaData, true)) { - ZLOGE("load meta failed! bundleName:%{public}s, userId:%{public}d, appIndex:%{public}d", bundleName.c_str(), - userId, appIndex); - return; - } - for (auto &meta : storeMetaData) { - if (meta.instanceId == appIndex && !meta.appId.empty() && !meta.storeId.empty()) { - ZLOGI("updated bundleName:%{public}s, storeId:%{public}s, userId:%{public}d, appIndex:%{public}d", - bundleName.c_str(), Anonymous::Change(meta.storeId).c_str(), userId, appIndex); - MetaDataManager::GetInstance().DelMeta(CloudInfo::GetSchemaKey(meta), true); - } - } } void InstallEventSubscriber::OnInstall(const std::string &bundleName, int32_t userId, int32_t appIndex) diff --git a/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.h b/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.h index 51441cdc..29d56b94 100644 --- a/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.h +++ b/datamgr_service/services/distributeddataservice/app/src/installer/installer_impl.h @@ -33,6 +33,7 @@ using InstallEventCallback = void (InstallEventSubscriber::*) private: static constexpr const char *USER_ID = "userId"; static constexpr const char *SANDBOX_APP_INDEX = "sandbox_app_index"; + static constexpr const char *APP_INDEX = "appIndex"; void OnUninstall(const std::string &bundleName, int32_t userId, int32_t appIndex); void OnUpdate(const std::string &bundleName, int32_t userId, int32_t appIndex); void OnInstall(const std::string &bundleName, int32_t userId, int32_t appIndex); diff --git a/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service.cpp b/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service.cpp index ba292c0c..ac38e8a7 100644 --- a/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service.cpp @@ -37,6 +37,7 @@ #include "db_info_handle_impl.h" #include "device_manager_adapter.h" #include "device_matrix.h" +#include "dfx/reporter.h" #include "dump/dump_manager.h" #include "dump_helper.h" #include "eventcenter/event_center.h" @@ -50,11 +51,9 @@ #include "mem_mgr_proxy.h" #include "metadata/appid_meta_data.h" #include "metadata/meta_data_manager.h" -#include "metadata/secret_key_meta_data.h" #include "permission_validator.h" #include "permit_delegate.h" #include "process_communicator_impl.h" -#include "reporter.h" #include "route_head_handler_impl.h" #include "runtime_config.h" #include "store/auto_cache.h" @@ -79,11 +78,9 @@ using namespace OHOS::Security::AccessToken; using KvStoreDelegateManager = DistributedDB::KvStoreDelegateManager; using SecretKeyMeta = DistributedData::SecretKeyMetaData; using DmAdapter = DistributedData::DeviceManagerAdapter; -constexpr const char* EXTENSION_BACKUP = "backup"; -constexpr const char* EXTENSION_RESTORE = "restore"; -constexpr const char* SECRET_KEY_BACKUP_PATH = - "/data/service/el1/public/database/distributeddata/" - "secret_key_backup.conf"; +constexpr const char *EXTENSION_BACKUP = "backup"; +constexpr const char *EXTENSION_RESTORE = "restore"; +constexpr const char *CLONE_KEY_ALIAS = "distributed_db_backup_key"; REGISTER_SYSTEM_ABILITY_BY_ID(KvStoreDataService, DISTRIBUTED_KV_DATA_SERVICE_ABILITY_ID, true); @@ -93,8 +90,7 @@ constexpr int MAX_DOWNLOAD_TASK = 5; constexpr int KEY_SIZE = 32; constexpr int AES_256_NONCE_SIZE = 32; -KvStoreDataService::KvStoreDataService(bool runOnCreate) - : SystemAbility(runOnCreate), clients_() +KvStoreDataService::KvStoreDataService(bool runOnCreate) : SystemAbility(runOnCreate), clients_() { ZLOGI("begin."); } @@ -164,7 +160,7 @@ sptr KvStoreDataService::GetFeatureInterface(const std::string &n { sptr feature; bool isFirstCreate = false; - features_.Compute(name, [&feature, &isFirstCreate](const auto &key, auto &value) ->bool { + features_.Compute(name, [&feature, &isFirstCreate](const auto &key, auto &value) -> bool { if (value != nullptr) { feature = value; return true; @@ -196,6 +192,13 @@ void KvStoreDataService::LoadFeatures() for (auto const &feature : features) { GetFeatureInterface(feature); } + auto staticActs = FeatureSystem::GetInstance().GetStaticActs(); + staticActs.ForEach([exec = executors_](const auto &name, std::shared_ptr acts) { + if (acts != nullptr) { + acts->SetThreadPool(exec); + } + return false; + }); } /* RegisterClientDeathObserver */ @@ -218,8 +221,8 @@ Status KvStoreDataService::RegisterClientDeathObserver(const AppId &appId, sptr< return Status::PERMISSION_DENIED; } auto pid = IPCSkeleton::GetCallingPid(); - clients_.Compute( - info.tokenId, [&appId, &info, pid, this, obs = std::move(observer)](const auto tokenId, auto &clients) { + clients_.Compute(info.tokenId, + [&appId, &info, pid, this, obs = std::move(observer)](const auto tokenId, auto &clients) { auto res = clients.try_emplace(pid, appId, *this, std::move(obs)); ZLOGI("bundleName:%{public}s, uid:%{public}d, pid:%{public}d, inserted:%{public}s.", appId.appId.c_str(), info.uid, pid, res.second ? "success" : "failed"); @@ -316,16 +319,16 @@ void KvStoreDataService::OnStart() AddSystemAbilityListener(COMMON_EVENT_SERVICE_ID); AddSystemAbilityListener(MEMORY_MANAGER_SA_ID); RegisterStoreInfo(); - Handler handlerStoreInfo = std::bind(&KvStoreDataService::DumpStoreInfo, this, std::placeholders::_1, - std::placeholders::_2); + Handler handlerStoreInfo = + std::bind(&KvStoreDataService::DumpStoreInfo, this, std::placeholders::_1, std::placeholders::_2); DumpManager::GetInstance().AddHandler("STORE_INFO", uintptr_t(this), handlerStoreInfo); RegisterUserInfo(); - Handler handlerUserInfo = std::bind(&KvStoreDataService::DumpUserInfo, this, std::placeholders::_1, - std::placeholders::_2); + Handler handlerUserInfo = + std::bind(&KvStoreDataService::DumpUserInfo, this, std::placeholders::_1, std::placeholders::_2); DumpManager::GetInstance().AddHandler("USER_INFO", uintptr_t(this), handlerUserInfo); RegisterBundleInfo(); - Handler handlerBundleInfo = std::bind(&KvStoreDataService::DumpBundleInfo, this, std::placeholders::_1, - std::placeholders::_2); + Handler handlerBundleInfo = + std::bind(&KvStoreDataService::DumpBundleInfo, this, std::placeholders::_1, std::placeholders::_2); DumpManager::GetInstance().AddHandler("BUNDLE_INFO", uintptr_t(this), handlerBundleInfo); StartService(); } @@ -353,8 +356,7 @@ void KvStoreDataService::OnAddSystemAbility(int32_t systemAbilityId, const std:: Installer::GetInstance().Init(this, executors_); ScreenManager::GetInstance()->SubscribeScreenEvent(); } else if (systemAbilityId == MEMORY_MANAGER_SA_ID) { - Memory::MemMgrClient::GetInstance().NotifyProcessStatus(getpid(), 1, 1, - DISTRIBUTED_KV_DATA_SERVICE_ABILITY_ID); + Memory::MemMgrClient::GetInstance().NotifyProcessStatus(getpid(), 1, 1, DISTRIBUTED_KV_DATA_SERVICE_ABILITY_ID); } return; } @@ -382,7 +384,7 @@ int32_t KvStoreDataService::OnExtension(const std::string &extension, MessagePar return 0; } -std::string GetBackupReplyCode(int replyCode, const std::string &info = "") +std::string KvStoreDataService::GetBackupReplyCode(int replyCode, const std::string &info) { CloneReplyCode reply; CloneReplyResult result; @@ -392,80 +394,52 @@ std::string GetBackupReplyCode(int replyCode, const std::string &info = "") return Serializable::Marshall(reply); } -bool ParseBackupInfo(MessageParcel &data, CloneBackupInfo &backupInfo) -{ - std::string info = data.ReadString(); - bool success = backupInfo.Unmarshal(DistributedData::Serializable::ToJson(info)); - if (!success) { - ZLOGE("parse backup json failed."); - return false; - } - return true; -} - -bool CheckBackupInfo(CloneBackupInfo &backupInfo) -{ - if (backupInfo.bundleInfos.empty()) { - ZLOGE("bundle empty."); - return false; - } - if (backupInfo.userId.empty()) { - ZLOGE("userId invalid"); - return false; - } - return true; -} - std::vector ConvertDecStrToVec(const std::string &inData) { std::vector outData; auto splitedToken = Constant::Split(inData, ","); outData.reserve(splitedToken.size()); for (auto &token : splitedToken) { - uint8_t num = atoi(token.c_str()); - outData.push_back(num); + outData.push_back(static_cast(atoi(token.c_str()))); } return outData; } -bool ImportCloneKey(const std::string &keyStr, const std::string &ivStr) +bool KvStoreDataService::ImportCloneKey(const std::string &keyStr) { auto key = ConvertDecStrToVec(keyStr); if (key.size() != KEY_SIZE) { - ZLOGE("ImportKey failed, key length not correct."); + ZLOGE("ImportKey failed, key size not correct, key size:%{public}zu", key.size()); key.assign(key.size(), 0); return false; } - auto iv = ConvertDecStrToVec(ivStr); - if (iv.size() != AES_256_NONCE_SIZE) { - ZLOGE("ImportKey failed, iv length not correct."); - key.assign(key.size(), 0); - iv.assign(iv.size(), 0); - return false; - } - if (!CryptoManager::GetInstance().ImportCloneKey(key, iv)) { - ZLOGE("ImportCloneKey failed."); + auto cloneKeyAlias = std::vector(CLONE_KEY_ALIAS, CLONE_KEY_ALIAS + strlen(CLONE_KEY_ALIAS)); + if (!CryptoManager::GetInstance().ImportKey(key, cloneKeyAlias)) { key.assign(key.size(), 0); - iv.assign(iv.size(), 0); return false; } key.assign(key.size(), 0); - iv.assign(iv.size(), 0); return true; } -bool WriteBackupInfo(const std::string &content) +void KvStoreDataService::DeleteCloneKey() { - FILE *fp = fopen(SECRET_KEY_BACKUP_PATH, "w"); + auto cloneKeyAlias = std::vector(CLONE_KEY_ALIAS, CLONE_KEY_ALIAS + strlen(CLONE_KEY_ALIAS)); + CryptoManager::GetInstance().DeleteKey(cloneKeyAlias); +} + +bool KvStoreDataService::WriteBackupInfo(const std::string &content, const std::string &backupPath) +{ + FILE *fp = fopen(backupPath.c_str(), "w"); if (!fp) { - ZLOGE("Secret key backup file open failed"); + ZLOGE("Secret key backup file fopen failed, path: %{public}s, errno: %{public}d", backupPath.c_str(), errno); return false; } size_t ret = fwrite(content.c_str(), 1, content.length(), fp); if (ret != content.length()) { - ZLOGE("Save config file fwrite() failed!"); + ZLOGE("Secret key backup file fwrite failed, path: %{public}s, errno: %{public}d", backupPath.c_str(), errno); (void)fclose(fp); return false; } @@ -479,29 +453,42 @@ bool WriteBackupInfo(const std::string &content) int32_t KvStoreDataService::OnBackup(MessageParcel &data, MessageParcel &reply) { CloneBackupInfo backupInfo; - if (!ParseBackupInfo(data, backupInfo)) { + if (!backupInfo.Unmarshal(data.ReadString()) || backupInfo.bundleInfos.empty() || backupInfo.userId.empty()) { + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); return -1; } - if (!CheckBackupInfo(backupInfo)) { + if (!ImportCloneKey(backupInfo.encryptionInfo.symkey)) { + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); return -1; } + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); - if (!ImportCloneKey(backupInfo.encryptionInfo.symkey, backupInfo.encryptionInfo.iv)) { + auto iv = ConvertDecStrToVec(backupInfo.encryptionInfo.iv); + if (iv.size() != AES_256_NONCE_SIZE) { + ZLOGE("Iv size not correct, iv size:%{public}zu", iv.size()); + iv.assign(iv.size(), 0); return -1; } std::string content; - if (!GetSecretKeyBackup(backupInfo.bundleInfos, backupInfo.userId, content)) { + if (!GetSecretKeyBackup(backupInfo.bundleInfos, backupInfo.userId, iv, content)) { + DeleteCloneKey(); return -1; }; + DeleteCloneKey(); - if (!WriteBackupInfo(content)) { + std::string backupPath = DirectoryManager::GetInstance().GetClonePath(backupInfo.userId); + if (backupPath.empty()) { + ZLOGE("GetClonePath failed, userId:%{public}s errno: %{public}d", backupInfo.userId.c_str(), errno); + return -1; + } + if (!WriteBackupInfo(content, backupPath)) { return -1; } UniqueFd fd(-1); - fd = UniqueFd(open(SECRET_KEY_BACKUP_PATH, O_RDONLY)); + fd = UniqueFd(open(backupPath.c_str(), O_RDONLY)); std::string replyCode = GetBackupReplyCode(0); if (!reply.WriteFileDescriptor(fd) || !reply.WriteString(replyCode)) { close(fd.Release()); @@ -513,36 +500,36 @@ int32_t KvStoreDataService::OnBackup(MessageParcel &data, MessageParcel &reply) return 0; } -std::vector ReEncryptKey(const std::string &key, SecretKeyMetaData &secretKeyMeta) +std::vector KvStoreDataService::ReEncryptKey(const std::string &key, SecretKeyMetaData &secretKeyMeta, + const StoreMetaData &metaData, const std::vector &iv) { if (!MetaDataManager::GetInstance().LoadMeta(key, secretKeyMeta, true)) { - ZLOGE("Secret key meta load failed."); return {}; }; std::vector password; - if (!CryptoManager::GetInstance().Decrypt(secretKeyMeta.sKey, password)) { - ZLOGE("Secret key decrypt failed."); + if (!CryptoManager::GetInstance().Decrypt(metaData, secretKeyMeta, password)) { return {}; }; - auto reEncryptedKey = CryptoManager::GetInstance().EncryptCloneKey(password); + + auto cloneKeyAlias = std::vector(CLONE_KEY_ALIAS, CLONE_KEY_ALIAS + strlen(CLONE_KEY_ALIAS)); + CryptoManager::EncryptParams encryptParams = { .keyAlias = cloneKeyAlias, .nonce = iv }; + auto reEncryptedKey = CryptoManager::GetInstance().Encrypt(password, encryptParams); + password.assign(password.size(), 0); if (reEncryptedKey.size() == 0) { - ZLOGE("Secret key encrypt failed."); return {}; }; return reEncryptedKey; } -bool KvStoreDataService::GetSecretKeyBackup( - const std::vector &bundleInfos, - const std::string &userId, std::string &content) +bool KvStoreDataService::GetSecretKeyBackup(const std::vector &bundleInfos, + const std::string &userId, const std::vector &iv, std::string &content) { SecretKeyBackupData backupInfos; std::string deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; - for (const auto& bundleInfo : bundleInfos) { + for (const auto &bundleInfo : bundleInfos) { std::string metaPrefix = StoreMetaData::GetKey({ deviceId, userId, "default", bundleInfo.bundleName }); std::vector StoreMetas; if (!MetaDataManager::GetInstance().LoadMeta(metaPrefix, StoreMetas, true)) { - ZLOGW("Store meta load failed, bundleName: %{public}s", bundleInfo.bundleName.c_str()); continue; }; for (const auto &storeMeta : StoreMetas) { @@ -551,12 +538,8 @@ bool KvStoreDataService::GetSecretKeyBackup( }; auto key = storeMeta.GetSecretKey(); SecretKeyMetaData secretKeyMeta; - auto reEncryptedKey = ReEncryptKey(key, secretKeyMeta); + auto reEncryptedKey = ReEncryptKey(key, secretKeyMeta, storeMeta, iv); if (reEncryptedKey.size() == 0) { - ZLOGE("Secret key re-encrypt failed, user: %{public}s, bundleName: %{public}s, Db: " - "%{public}s, instanceId: %{public}d", userId.c_str(), - storeMeta.bundleName.c_str(), Anonymous::Change(storeMeta.storeId).c_str(), - storeMeta.instanceId); continue; }; SecretKeyBackupData::BackupItem item; @@ -567,6 +550,7 @@ bool KvStoreDataService::GetSecretKeyBackup( item.sKey = DistributedData::Base64::Encode(reEncryptedKey); item.storeType = secretKeyMeta.storeType; item.user = userId; + item.area = storeMeta.area; backupInfos.infos.push_back(std::move(item)); } } @@ -578,28 +562,35 @@ int32_t KvStoreDataService::OnRestore(MessageParcel &data, MessageParcel &reply) { SecretKeyBackupData backupData; if (!ParseSecretKeyFile(data, backupData) || backupData.infos.size() == 0) { - ZLOGE("Read backup file failed or infos is empty!"); return ReplyForRestore(reply, -1); } CloneBackupInfo backupInfo; - bool ret = backupInfo.Unmarshal(DistributedData::Serializable::ToJson(data.ReadString())); + bool ret = backupInfo.Unmarshal(data.ReadString()); if (!ret || backupInfo.userId.empty()) { - ZLOGE("Clone info invalid or userId is empty!"); + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); + return ReplyForRestore(reply, -1); + } + + auto iv = ConvertDecStrToVec(backupInfo.encryptionInfo.iv); + if (iv.size() != AES_256_NONCE_SIZE) { + ZLOGE("Iv size not correct, iv size:%{public}zu", iv.size()); + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); return ReplyForRestore(reply, -1); } - if (!ImportCloneKey(backupInfo.encryptionInfo.symkey, backupInfo.encryptionInfo.iv)) { + if (!ImportCloneKey(backupInfo.encryptionInfo.symkey)) { + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); + DeleteCloneKey(); return ReplyForRestore(reply, -1); } + std::fill(backupInfo.encryptionInfo.symkey.begin(), backupInfo.encryptionInfo.symkey.end(), '\0'); + for (const auto &item : backupData.infos) { - if (!item.IsValid() || !RestoreSecretKey(item, backupInfo.userId)) { - ZLOGW("Restore secret key failed! bundleName:%{public}s, dbName:%{public}s, instanceId:%{public}d, " - "storeType:%{public}d, time.size:%{public}zu, sKey:%{public}s, user:%{public}s", - item.bundleName.c_str(), Anonymous::Change(item.dbName).c_str(), item.instanceId, item.storeType, - item.time.size(), Anonymous::Change(item.sKey).c_str(), item.user.c_str()); + if (!item.IsValid() || !RestoreSecretKey(item, backupInfo.userId, iv)) { continue; } } + DeleteCloneKey(); return ReplyForRestore(reply, 0); } @@ -607,7 +598,6 @@ int32_t KvStoreDataService::ReplyForRestore(MessageParcel &reply, int32_t result { std::string replyCode = GetBackupReplyCode(result); if (!reply.WriteString(replyCode)) { - ZLOGE("Write reply failed"); return -1; } return result; @@ -634,7 +624,8 @@ bool KvStoreDataService::ParseSecretKeyFile(MessageParcel &data, SecretKeyBackup return true; } -bool KvStoreDataService::RestoreSecretKey(const SecretKeyBackupData::BackupItem &item, const std::string &userId) +bool KvStoreDataService::RestoreSecretKey(const SecretKeyBackupData::BackupItem &item, const std::string &userId, + const std::vector &iv) { StoreMetaData metaData; metaData.bundleName = item.bundleName; @@ -643,23 +634,28 @@ bool KvStoreDataService::RestoreSecretKey(const SecretKeyBackupData::BackupItem metaData.instanceId = item.instanceId; auto sKey = DistributedData::Base64::Decode(item.sKey); std::vector rawKey; - if (!CryptoManager::GetInstance().DecryptCloneKey(sKey, rawKey)) { - ZLOGE("Decrypt by clonekey failed."); + + auto cloneKeyAlias = std::vector(CLONE_KEY_ALIAS, CLONE_KEY_ALIAS + strlen(CLONE_KEY_ALIAS)); + CryptoManager::EncryptParams encryptParams = { .keyAlias = cloneKeyAlias, .nonce = iv }; + if (!CryptoManager::GetInstance().Decrypt(sKey, rawKey, encryptParams)) { + ZLOGE("Decrypt failed, bundleName:%{public}s, storeName:%{public}s, storeType:%{public}d", + item.bundleName.c_str(), Anonymous::Change(item.dbName).c_str(), item.storeType); sKey.assign(sKey.size(), 0); rawKey.assign(rawKey.size(), 0); return false; } SecretKeyMetaData secretKey; secretKey.storeType = item.storeType; - secretKey.sKey = CryptoManager::GetInstance().Encrypt(rawKey); + if (item.area < 0) { + secretKey.sKey = CryptoManager::GetInstance().Encrypt(rawKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); + } else { + secretKey.sKey = CryptoManager::GetInstance().Encrypt(rawKey, item.area, userId); + secretKey.area = item.area; + } secretKey.time = { item.time.begin(), item.time.end() }; sKey.assign(sKey.size(), 0); rawKey.assign(rawKey.size(), 0); - if (!MetaDataManager::GetInstance().SaveMeta(metaData.GetCloneSecretKey(), secretKey, true)) { - ZLOGE("Save meta failed."); - return false; - } - return true; + return MetaDataManager::GetInstance().SaveMeta(metaData.GetCloneSecretKey(), secretKey, true); } void KvStoreDataService::StartService() @@ -693,8 +689,8 @@ void KvStoreDataService::StartService() ZLOGI("Start distributedata Success, Publish ret: %{public}d", static_cast(ret)); } -void KvStoreDataService::OnStoreMetaChanged( - const std::vector &key, const std::vector &value, CHANGE_FLAG flag) +void KvStoreDataService::OnStoreMetaChanged(const std::vector &key, const std::vector &value, + CHANGE_FLAG flag) { if (flag != CHANGE_FLAG::UPDATE) { return; @@ -721,8 +717,8 @@ void KvStoreDataService::OnStop() Memory::MemMgrClient::GetInstance().NotifyProcessStatus(getpid(), 1, 0, DISTRIBUTED_KV_DATA_SERVICE_ABILITY_ID); } -KvStoreDataService::KvStoreClientDeathObserverImpl::KvStoreClientDeathObserverImpl( - const AppId &appId, KvStoreDataService &service, sptr observer) +KvStoreDataService::KvStoreClientDeathObserverImpl::KvStoreClientDeathObserverImpl(const AppId &appId, + KvStoreDataService &service, sptr observer) : appId_(appId), dataService_(service), observerProxy_(std::move(observer)), deathRecipient_(new KvStoreDeathRecipient(*this)) { @@ -812,7 +808,7 @@ KvStoreDataService::KvStoreClientDeathObserverImpl::KvStoreDeathRecipient::~KvSt void KvStoreDataService::KvStoreClientDeathObserverImpl::KvStoreDeathRecipient::OnRemoteDied( const wptr &remote) { - (void) remote; + (void)remote; ZLOGI("begin"); kvStoreClientDeathObserverImpl_.NotifyClientDie(); } @@ -826,7 +822,7 @@ void KvStoreDataService::AccountEventChanged(const AccountEventInfo &eventInfo) g_kvStoreAccountEventStatus = 1; // delete all kvstore meta belong to this user std::vector metaData; - MetaDataManager::GetInstance().LoadMeta(StoreMetaData::GetPrefix({""}), metaData, true); + MetaDataManager::GetInstance().LoadMeta(StoreMetaData::GetPrefix({ "" }), metaData, true); for (const auto &meta : metaData) { if (meta.user != eventInfo.userId) { continue; @@ -841,6 +837,7 @@ void KvStoreDataService::AccountEventChanged(const AccountEventInfo &eventInfo) MetaDataManager::GetInstance().DelMeta(meta.GetAutoLaunchKey(), true); MetaDataManager::GetInstance().DelMeta(meta.appId, true); MetaDataManager::GetInstance().DelMeta(meta.GetDebugInfoKey(), true); + MetaDataManager::GetInstance().DelMeta(meta.GetDfxInfoKey(), true); MetaDataManager::GetInstance().DelMeta(meta.GetCloneSecretKey(), true); PermitDelegate::GetInstance().DelCache(meta.GetKey()); } @@ -876,7 +873,7 @@ void KvStoreDataService::NotifyAccountEvent(const AccountEventInfo &eventInfo) AutoCache::GetInstance().CloseStore([&userIds](const StoreMetaData &meta) { if (userIds.count(atoi(meta.user.c_str())) == 0) { ZLOGW("Illegal use of database by user %{public}s, %{public}s:%{public}s", meta.user.c_str(), - meta.bundleName.c_str(), meta.GetStoreAlias().c_str()); + meta.bundleName.c_str(), meta.GetStoreAlias().c_str()); return true; } return false; @@ -907,15 +904,13 @@ void KvStoreDataService::InitSecurityAdapter(std::shared_ptr execu auto dbStatus = DistributedDB::RuntimeConfig::SetProcessSystemAPIAdapter(security_); ZLOGD("set distributed db system api adapter: %d.", static_cast(dbStatus)); - auto status = DmAdapter::GetInstance().StartWatchDeviceChange(security_.get(), {"security"}); + auto status = DmAdapter::GetInstance().StartWatchDeviceChange(security_.get(), { "security" }); if (status != Status::SUCCESS) { ZLOGD("security register device change failed, status:%d", static_cast(status)); } } -void KvStoreDataService::SetCompatibleIdentify(const AppDistributedKv::DeviceInfo &info) const -{ -} +void KvStoreDataService::SetCompatibleIdentify(const AppDistributedKv::DeviceInfo &info) const {} void KvStoreDataService::OnDeviceOnline(const AppDistributedKv::DeviceInfo &info) { @@ -964,7 +959,7 @@ void KvStoreDataService::OnSessionReady(const AppDistributedKv::DeviceInfo &info int32_t KvStoreDataService::OnUninstall(const std::string &bundleName, int32_t user, int32_t index) { auto staticActs = FeatureSystem::GetInstance().GetStaticActs(); - staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr& acts) { + staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr &acts) { acts->OnAppUninstall(bundleName, user, index); return false; }); @@ -974,7 +969,7 @@ int32_t KvStoreDataService::OnUninstall(const std::string &bundleName, int32_t u int32_t KvStoreDataService::OnUpdate(const std::string &bundleName, int32_t user, int32_t index) { auto staticActs = FeatureSystem::GetInstance().GetStaticActs(); - staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr& acts) { + staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr &acts) { acts->OnAppUpdate(bundleName, user, index); return false; }); @@ -984,7 +979,7 @@ int32_t KvStoreDataService::OnUpdate(const std::string &bundleName, int32_t user int32_t KvStoreDataService::OnInstall(const std::string &bundleName, int32_t user, int32_t index) { auto staticActs = FeatureSystem::GetInstance().GetStaticActs(); - staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr& acts) { + staticActs.ForEachCopies([bundleName, user, index](const auto &, const std::shared_ptr &acts) { acts->OnAppInstall(bundleName, user, index); return false; }); @@ -1040,6 +1035,7 @@ int32_t KvStoreDataService::ClearAppStorage(const std::string &bundleName, int32 MetaDataManager::GetInstance().DelMeta(meta.GetBackupSecretKey(), true); MetaDataManager::GetInstance().DelMeta(meta.appId, true); MetaDataManager::GetInstance().DelMeta(meta.GetDebugInfoKey(), true); + MetaDataManager::GetInstance().DelMeta(meta.GetDfxInfoKey(), true); MetaDataManager::GetInstance().DelMeta(meta.GetAutoLaunchKey(), true); PermitDelegate::GetInstance().DelCache(meta.GetKey()); } @@ -1120,25 +1116,58 @@ void KvStoreDataService::PrintfInfo(int fd, const std::vector &me } info.append(GetIndentation(indentation)) .append("--------------------------------------------------------------------------\n") - .append(GetIndentation(indentation)).append("StoreID : ").append(data.storeId).append("\n") - .append(GetIndentation(indentation)).append("UId : ").append(data.user).append("\n") - .append(GetIndentation(indentation)).append("BundleName : ").append(data.bundleName).append("\n") - .append(GetIndentation(indentation)).append("AppID : ").append(data.appId).append("\n") - .append(GetIndentation(indentation)).append("StorePath : ").append(data.dataDir).append("\n") - .append(GetIndentation(indentation)).append("StoreType : ") - .append(std::to_string(data.storeType)).append("\n") - .append(GetIndentation(indentation)).append("encrypt : ") - .append(std::to_string(data.isEncrypt)).append("\n") - .append(GetIndentation(indentation)).append("autoSync : ") - .append(std::to_string(data.isAutoSync)).append("\n") - .append(GetIndentation(indentation)).append("schema : ").append(data.schema).append("\n") - .append(GetIndentation(indentation)).append("securityLevel : ") - .append(std::to_string(data.securityLevel)).append("\n") - .append(GetIndentation(indentation)).append("area : ") - .append(std::to_string(data.area)).append("\n") - .append(GetIndentation(indentation)).append("instanceID : ") - .append(std::to_string(data.instanceId)).append("\n") - .append(GetIndentation(indentation)).append("version : ").append(version).append("\n"); + .append(GetIndentation(indentation)) + .append("StoreID : ") + .append(data.storeId) + .append("\n") + .append(GetIndentation(indentation)) + .append("UId : ") + .append(data.user) + .append("\n") + .append(GetIndentation(indentation)) + .append("BundleName : ") + .append(data.bundleName) + .append("\n") + .append(GetIndentation(indentation)) + .append("AppID : ") + .append(data.appId) + .append("\n") + .append(GetIndentation(indentation)) + .append("StorePath : ") + .append(data.dataDir) + .append("\n") + .append(GetIndentation(indentation)) + .append("StoreType : ") + .append(std::to_string(data.storeType)) + .append("\n") + .append(GetIndentation(indentation)) + .append("encrypt : ") + .append(std::to_string(data.isEncrypt)) + .append("\n") + .append(GetIndentation(indentation)) + .append("autoSync : ") + .append(std::to_string(data.isAutoSync)) + .append("\n") + .append(GetIndentation(indentation)) + .append("schema : ") + .append(data.schema) + .append("\n") + .append(GetIndentation(indentation)) + .append("securityLevel : ") + .append(std::to_string(data.securityLevel)) + .append("\n") + .append(GetIndentation(indentation)) + .append("area : ") + .append(std::to_string(data.area)) + .append("\n") + .append(GetIndentation(indentation)) + .append("instanceID : ") + .append(std::to_string(data.instanceId)) + .append("\n") + .append(GetIndentation(indentation)) + .append("version : ") + .append(version) + .append("\n"); } dprintf(fd, "--------------------------------------StoreInfo-------------------------------------\n%s\n", info.c_str()); @@ -1287,12 +1316,9 @@ void KvStoreDataService::PrintfInfo(int fd, const std::map &bundleInfos, - const std::string &userId, std::string &content); + bool WriteBackupInfo(const std::string &content, const std::string &backupPath); + bool GetSecretKeyBackup(const std::vector &bundleInfos, + const std::string &userId, const std::vector &iv, std::string &content); - private: +private: void NotifyAccountEvent(const AccountEventInfo &eventInfo); class KvStoreClientDeathObserverImpl { public: @@ -186,9 +187,17 @@ public: void InitExecutor(); + std::vector ReEncryptKey(const std::string &key, SecretKeyMetaData &secretKeyMeta, + const StoreMetaData &metaData, const std::vector &iv); + bool ParseSecretKeyFile(MessageParcel &data, SecretKeyBackupData &backupData); - bool RestoreSecretKey(const SecretKeyBackupData::BackupItem &item, const std::string &userId); + bool RestoreSecretKey(const SecretKeyBackupData::BackupItem &item, const std::string &userId, + const std::vector &iv); + bool ImportCloneKey(const std::string &keyStr); + void DeleteCloneKey(); + + std::string GetBackupReplyCode(int replyCode, const std::string &info = ""); int32_t ReplyForRestore(MessageParcel &reply, int32_t result); @@ -212,5 +221,5 @@ public: static constexpr pid_t INVALID_PID = -1; static constexpr uint32_t INVALID_TOKEN = 0; }; -} -#endif // KVSTORE_DATASERVICE_H \ No newline at end of file +} // namespace OHOS::DistributedKv +#endif // KVSTORE_DATASERVICE_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service_stub.cpp b/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service_stub.cpp index ca3deb44..4d0a55fa 100644 --- a/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service_stub.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/kvstore_data_service_stub.cpp @@ -31,7 +31,7 @@ constexpr KvStoreDataServiceStub::RequestHandler int32_t KvStoreDataServiceStub::RegisterClientDeathObserverOnRemote(MessageParcel &data, MessageParcel &reply) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); AppId appId = { data.ReadString() }; sptr kvStoreClientDeathObserverProxy = data.ReadRemoteObject(); if (kvStoreClientDeathObserverProxy == nullptr) { @@ -46,7 +46,7 @@ int32_t KvStoreDataServiceStub::RegisterClientDeathObserverOnRemote(MessageParce int32_t KvStoreDataServiceStub::GetFeatureInterfaceOnRemote(MessageParcel &data, MessageParcel &reply) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); std::string name; if (!ITypesUtil::Unmarshal(data, name)) { return -1; diff --git a/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.cpp b/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.cpp index f7e315ca..06b07a9c 100644 --- a/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.cpp @@ -33,8 +33,16 @@ #include "kv_radar_reporter.h" #include "log_print.h" #include "matrix_event.h" +#include "metadata/auto_launch_meta_data.h" +#include "metadata/capability_meta_data.h" +#include "metadata/device_meta_data.h" #include "metadata/meta_data_manager.h" +#include "metadata/matrix_meta_data.h" +#include "metadata/strategy_meta_data.h" #include "metadata/store_meta_data_local.h" +#include "metadata/strategy_meta_data.h" +#include "metadata/switches_meta_data.h" +#include "metadata/user_meta_data.h" #include "metadata/version_meta_data.h" #include "runtime_config.h" #include "safe_block_queue.h" @@ -46,6 +54,7 @@ #include "utils/crypto.h" #include "utils/ref_count.h" #include "utils/converter.h" +#include "utils/constant.h" namespace OHOS { namespace DistributedKv { @@ -134,9 +143,10 @@ void KvStoreMetaManager::InitMetaData() ZLOGI("get meta failed."); return; } + + CheckMetaDeviceId(); auto uid = getuid(); auto tokenId = IPCSkeleton::GetCallingTokenID(); - const std::string accountId = AccountDelegate::GetInstance()->GetCurrentAccountId(); auto userId = AccountDelegate::GetInstance()->GetUserByToken(tokenId); StoreMetaData data; data.appId = label_; @@ -153,7 +163,7 @@ void KvStoreMetaManager::InitMetaData() data.dataType = DataType::TYPE_DYNAMICAL; data.schema = ""; data.storeId = Bootstrap::GetInstance().GetMetaDBName(); - data.account = accountId; + data.account = AccountDelegate::GetInstance()->GetCurrentAccountId(); data.uid = static_cast(uid); data.version = META_STORE_VERSION; data.securityLevel = SecurityLevel::S1; @@ -554,5 +564,118 @@ AppDistributedKv::ChangeLevelType KvStoreMetaManager::DBInfoDeviceChangeListener { return AppDistributedKv::ChangeLevelType::MIN; } + +void KvStoreMetaManager::CheckMetaDeviceId() +{ + DeviceMetaData deviceMeta; + auto localUuid = DmAdapter::GetInstance().GetLocalDevice().uuid; + if (localUuid.empty()) { + ZLOGW("get uuid failed"); + return; + } + if (!MetaDataManager::GetInstance().LoadMeta(deviceMeta.GetKey(), deviceMeta, true)) { + deviceMeta.newUuid = localUuid; + MetaDataManager::GetInstance().SaveMeta(deviceMeta.GetKey(), deviceMeta, true); + return; + } + if (deviceMeta.newUuid != localUuid) { + UpdateStoreMetaData(localUuid, deviceMeta.newUuid); + UpdateMetaDatas(localUuid, deviceMeta.newUuid); + deviceMeta.oldUuid = deviceMeta.newUuid; + deviceMeta.newUuid = localUuid; + MetaDataManager::GetInstance().SaveMeta(deviceMeta.GetKey(), deviceMeta, true); + ZLOGI("meta changed! curruuid:%{public}s, olduuid:%{public}s", + deviceMeta.newUuid.c_str(), deviceMeta.oldUuid.c_str()); + } +} + +void KvStoreMetaManager::UpdateStoreMetaData(const std::string &newUuid, const std::string &oldUuid) +{ + std::vector storeMetas; + MetaDataManager::GetInstance().LoadMeta(StoreMetaData::GetPrefix({ oldUuid }), storeMetas, true); + for (auto &storeMeta : storeMetas) { + auto oldMeta = storeMeta; + storeMeta.isNeedUpdateDeviceId = true; + storeMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(storeMeta.GetKey(), storeMeta, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey(), true); + + StoreMetaData syncStoreMeta; + if (MetaDataManager::GetInstance().LoadMeta(oldMeta.GetKey(), syncStoreMeta)) { + syncStoreMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(storeMeta.GetKey(), syncStoreMeta); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey()); + } + + StrategyMeta strategyMeta; + if (MetaDataManager::GetInstance().LoadMeta(oldMeta.GetStrategyKey(), strategyMeta)) { + strategyMeta.devId = newUuid; + MetaDataManager::GetInstance().SaveMeta(storeMeta.GetStrategyKey(), strategyMeta); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetStrategyKey()); + } + + StoreMetaDataLocal metaDataLocal; + if (MetaDataManager::GetInstance().LoadMeta(oldMeta.GetKeyLocal(), metaDataLocal, true)) { + MetaDataManager::GetInstance().SaveMeta(storeMeta.GetKeyLocal(), metaDataLocal, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKeyLocal(), true); + } + + AutoLaunchMetaData autoLaunchMetaData; + bool isExist = MetaDataManager::GetInstance().LoadMeta(oldMeta.GetAutoLaunchKey(), autoLaunchMetaData, true); + if (!isExist) { + oldMeta.storeId = ""; + isExist = MetaDataManager::GetInstance().LoadMeta(oldMeta.GetAutoLaunchKey(), autoLaunchMetaData, true); + } + if (isExist) { + MetaDataManager::GetInstance().DelMeta(oldMeta.GetAutoLaunchKey(), true); + oldMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(oldMeta.GetAutoLaunchKey(), autoLaunchMetaData, true); + } + if (storeMeta.isEncrypt) { + MetaDataManager::GetInstance().DelMeta(storeMeta.GetSecretKey(), true); + MetaDataManager::GetInstance().DelMeta(storeMeta.GetCloneSecretKey(), true); + } + } +} + +void KvStoreMetaManager::UpdateMetaDatas(const std::string &newUuid, const std::string &oldUuid) +{ + MatrixMetaData matrixMeta; + if (MetaDataManager::GetInstance().LoadMeta(MatrixMetaData::GetPrefix({ oldUuid }), matrixMeta, true)) { + MetaDataManager::GetInstance().DelMeta(MatrixMetaData::GetPrefix({ oldUuid }), true); + matrixMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(MatrixMetaData::GetPrefix({ newUuid }), matrixMeta, true); + } + + if (MetaDataManager::GetInstance().LoadMeta(MatrixMetaData::GetPrefix({ oldUuid }), matrixMeta)) { + MetaDataManager::GetInstance().DelMeta(MatrixMetaData::GetPrefix({ oldUuid })); + matrixMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(MatrixMetaData::GetPrefix({ newUuid }), matrixMeta); + } + + SwitchesMetaData switchesMetaData; + if (MetaDataManager::GetInstance().LoadMeta(SwitchesMetaData::GetPrefix({ oldUuid }), + switchesMetaData, true)) { + MetaDataManager::GetInstance().DelMeta(SwitchesMetaData::GetPrefix({ oldUuid }), true); + switchesMetaData.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(SwitchesMetaData::GetPrefix({ newUuid }), + switchesMetaData, true); + } + + UserMetaData userMeta; + if (MetaDataManager::GetInstance().LoadMeta(UserMetaRow::GetKeyFor(oldUuid), userMeta)) { + MetaDataManager::GetInstance().DelMeta(UserMetaRow::GetKeyFor(oldUuid)); + userMeta.deviceId = newUuid; + MetaDataManager::GetInstance().SaveMeta(UserMetaRow::GetKeyFor(newUuid), userMeta); + } + + CapMetaData capMetaData; + auto capKey = CapMetaRow::GetKeyFor(oldUuid); + if (MetaDataManager::GetInstance().LoadMeta(std::string(capKey.begin(), capKey.end()), capMetaData)) { + auto newCapKey = CapMetaRow::GetKeyFor(newUuid); + MetaDataManager::GetInstance().DelMeta(std::string(capKey.begin(), capKey.end())); + MetaDataManager::GetInstance().SaveMeta(std::string(newCapKey.begin(), newCapKey.end()), capMetaData); + } +} } // namespace DistributedKv } // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.h b/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.h index 87bcc716..f7b28e1c 100644 --- a/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.h +++ b/datamgr_service/services/distributeddataservice/app/src/kvstore_meta_manager.h @@ -95,6 +95,12 @@ private: ExecutorPool::Task GetTask(uint32_t retry); DistributedDB::KvStoreNbDelegate::Option InitDBOption(); + + void CheckMetaDeviceId(); + + void UpdateStoreMetaData(const std::string &newUuid, const std::string &oldUuid); + + void UpdateMetaDatas(const std::string &newUuid, const std::string &oldUuid); static ExecutorPool::Task GetBackupTask( TaskQueue queue, std::shared_ptr executors, const NbDelegate store); diff --git a/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.cpp b/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.cpp index d602b206..f52cde72 100644 --- a/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "RouteHeadHandler" #include #include +#include "account/account_delegate.h" #include "auth_delegate.h" #include "device_manager_adapter.h" #include "kvstore_meta_manager.h" @@ -33,6 +34,7 @@ namespace OHOS::DistributedData { using namespace OHOS::DistributedKv; using namespace std::chrono; using DmAdapter = DistributedData::DeviceManagerAdapter; +using DBManager = DistributedDB::KvStoreDelegateManager; constexpr const int ALIGN_WIDTH = 8; constexpr const char *DEFAULT_USERID = "0"; std::shared_ptr RouteHeadHandlerImpl::Create(const ExtendInfo &info) @@ -59,8 +61,19 @@ void RouteHeadHandlerImpl::Init() if (deviceId_.empty()) { return; } - if (!DmAdapter::GetInstance().IsOHOSType(deviceId_) && userId_ != DEFAULT_USERID) { - userId_ = DEFAULT_USERID; + if (userId_ != DEFAULT_USERID) { + if (!DmAdapter::GetInstance().IsOHOSType(deviceId_)) { + userId_ = DEFAULT_USERID; + } else { + StoreMetaData metaData; + metaData.deviceId = deviceId_; + metaData.user = DEFAULT_USERID; + metaData.bundleName = appId_; + metaData.storeId = storeId_; + if (MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData)) { + userId_ = DEFAULT_USERID; + } + } } SessionPoint localPoint { DmAdapter::GetInstance().GetLocalDevice().uuid, static_cast(atoi(userId_.c_str())), appId_, storeId_ }; @@ -194,17 +207,65 @@ bool RouteHeadHandlerImpl::PackDataBody(uint8_t *data, uint32_t totalLen) return true; } -bool RouteHeadHandlerImpl::ParseHeadData( - const uint8_t *data, uint32_t len, uint32_t &headSize, std::vector &users) +bool RouteHeadHandlerImpl::ParseHeadDataLen(const uint8_t *data, uint32_t totalLen, uint32_t &headSize) +{ + if (data == nullptr) { + ZLOGE("invalid input data, totalLen:%{public}d", totalLen); + return false; + } + RouteHead head = { 0 }; + auto ret = UnPackDataHead(data, totalLen, head); + headSize = ret ? sizeof(RouteHead) + head.dataLen : 0; + ZLOGI("unpacked data size:%{public}u, ret:%{public}d", headSize, ret); + return ret; +} + +std::string RouteHeadHandlerImpl::ParseStoreId(const std::string &deviceId, const std::string &label) +{ + std::vector metaData; + auto prefix = StoreMetaData::GetPrefix({ deviceId }); + if (!MetaDataManager::GetInstance().LoadMeta(prefix, metaData)) { + return ""; + } + for (const auto &storeMeta : metaData) { + auto labelTag = DBManager::GetKvStoreIdentifier("", storeMeta.appId, storeMeta.storeId, true); + if (labelTag != label) { + continue; + } + return storeMeta.storeId; + } + return ""; +} + +bool RouteHeadHandlerImpl::ParseHeadDataUser(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) { - auto ret = UnPackData(data, len, headSize); + uint32_t headSize = 0; + auto ret = UnPackData(data, totalLen, headSize); if (!ret) { - headSize = 0; return false; } - auto time = - static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); + auto time = static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); ZLOGI("unpacked size:%{public}u times %{public}" PRIu64 ".", headSize, time); + + if (DmAdapter::GetInstance().IsOHOSType(session_.sourceDeviceId)) { + auto storeId = ParseStoreId(session_.targetDeviceId, label); + if (!storeId.empty() && std::to_string(session_.sourceUserId) == DEFAULT_USERID) { + StoreMetaData metaData; + metaData.deviceId = session_.targetDeviceId; + metaData.user = DEFAULT_USERID; + metaData.bundleName = session_.appId; + metaData.storeId = storeId; + if (!MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData)) { + int foregroundUserId = 0; + AccountDelegate::GetInstance()->QueryForegroundUserId(foregroundUserId); + UserInfo userInfo = { .receiveUser = std::to_string(foregroundUserId) }; + userInfos.emplace_back(userInfo); + return true; + } + } + } + // flip the local and peer ends SessionPoint local { .deviceId = session_.targetDeviceId, .appId = session_.appId }; SessionPoint peer { .deviceId = session_.sourceDeviceId, .userId = session_.sourceUserId, .appId = session_.appId }; @@ -214,7 +275,8 @@ bool RouteHeadHandlerImpl::ParseHeadData( for (const auto &item : session_.targetUserIds) { local.userId = item; if (SessionManager::GetInstance().CheckSession(local, peer)) { - users.emplace_back(std::to_string(item)); + UserInfo userInfo = { .receiveUser = std::to_string(item) }; + userInfos.emplace_back(userInfo); } } return true; diff --git a/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.h b/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.h index 4cc31add..f23360dd 100644 --- a/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.h +++ b/datamgr_service/services/distributeddataservice/app/src/session_manager/route_head_handler_impl.h @@ -61,7 +61,9 @@ public: explicit RouteHeadHandlerImpl(const ExtendInfo &info); DBStatus GetHeadDataSize(uint32_t &headSize) override; DBStatus FillHeadData(uint8_t *data, uint32_t headSize, uint32_t totalLen) override; - bool ParseHeadData(const uint8_t *data, uint32_t len, uint32_t &headSize, std::vector &users) override; + bool ParseHeadDataLen(const uint8_t *data, uint32_t totalLen, uint32_t &headSize) override; + bool ParseHeadDataUser(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) override; private: void Init(); @@ -71,6 +73,7 @@ private: bool UnPackData(const uint8_t *data, uint32_t totalLen, uint32_t &unpackedSize); bool UnPackDataHead(const uint8_t *data, uint32_t totalLen, RouteHead &routeHead); bool UnPackDataBody(const uint8_t *data, uint32_t totalLen); + std::string ParseStoreId(const std::string &deviceId, const std::string &label); std::string userId_; std::string appId_; diff --git a/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.cpp b/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.cpp index 3fd3f6a5..e13e4357 100644 --- a/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.cpp +++ b/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.cpp @@ -32,72 +32,68 @@ namespace OHOS::DistributedData { using namespace OHOS::DistributedKv; using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; +using Account = AccountDelegate; SessionManager &SessionManager::GetInstance() { static SessionManager instance; return instance; } -Session SessionManager::GetSession(const SessionPoint &from, const std::string &targetDeviceId) const +Session SessionManager::GetSession(const SessionPoint &local, const std::string &targetDeviceId) const { ZLOGD("begin. peer device:%{public}s", Anonymous::Change(targetDeviceId).c_str()); Session session; - session.appId = from.appId; - session.sourceUserId = from.userId; - session.sourceDeviceId = from.deviceId; + session.appId = local.appId; + session.sourceUserId = local.userId; + session.sourceDeviceId = local.deviceId; session.targetDeviceId = targetDeviceId; auto users = UserDelegate::GetInstance().GetRemoteUserStatus(targetDeviceId); // system service - if (from.userId == UserDelegate::SYSTEM_USER) { - StoreMetaData metaData; - metaData.deviceId = from.deviceId; - metaData.user = std::to_string(from.userId); - metaData.bundleName = from.appId; - metaData.storeId = from.storeId; - if (MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData) && - CheckerManager::GetInstance().GetAppId(Converter::ConvertToStoreInfo(metaData)) == from.appId) { - session.targetUserIds.push_back(UserDelegate::SYSTEM_USER); - } + if (local.userId == UserDelegate::SYSTEM_USER) { + session.targetUserIds.push_back(UserDelegate::SYSTEM_USER); } AclParams aclParams; - if (!GetSendAuthParams(from, targetDeviceId, aclParams)) { + if (!GetSendAuthParams(local, targetDeviceId, aclParams)) { ZLOGE("get send auth params failed:%{public}s", Anonymous::Change(targetDeviceId).c_str()); return session; } + + std::vector targetUsers {}; for (const auto &user : users) { aclParams.accCallee.userId = user.id; - auto [isPermitted, isSameAccount] = AuthDelegate::GetInstance()->CheckAccess(from.userId, user.id, + auto [isPermitted, isSameAccount] = AuthDelegate::GetInstance()->CheckAccess(local.userId, user.id, targetDeviceId, aclParams); if (isPermitted) { auto it = std::find(session.targetUserIds.begin(), session.targetUserIds.end(), user.id); if (it == session.targetUserIds.end() && isSameAccount) { - session.targetUserIds.insert(session.targetUserIds.begin(), user.id); + session.targetUserIds.push_back(user.id); } if (it == session.targetUserIds.end() && !isSameAccount) { - session.targetUserIds.push_back(user.id); + targetUsers.push_back(user.id); } } } + session.targetUserIds.insert(session.targetUserIds.end(), targetUsers.begin(), targetUsers.end()); ZLOGD("access to peer users:%{public}s", DistributedData::Serializable::Marshall(session.targetUserIds).c_str()); return session; } -bool SessionManager::GetSendAuthParams(const SessionPoint &from, const std::string &targetDeviceId, +bool SessionManager::GetSendAuthParams(const SessionPoint &local, const std::string &targetDeviceId, AclParams &aclParams) const { std::vector metaData; - if (!MetaDataManager::GetInstance().LoadMeta(StoreMetaData::GetPrefix({ from.deviceId }), metaData)) { - ZLOGE("load meta failed, deviceId:%{public}s, user:%{public}d", Anonymous::Change(from.deviceId).c_str(), - from.userId); + if (!MetaDataManager::GetInstance().LoadMeta(StoreMetaData::GetPrefix({ local.deviceId }), metaData)) { + ZLOGE("load meta failed, deviceId:%{public}s, user:%{public}d", Anonymous::Change(local.deviceId).c_str(), + local.userId); return false; } for (const auto &storeMeta : metaData) { - if (storeMeta.appId == from.appId && storeMeta.storeId == from.storeId) { + if (storeMeta.appId == local.appId && storeMeta.storeId == local.storeId) { aclParams.accCaller.bundleName = storeMeta.bundleName; aclParams.accCaller.accountId = AccountDelegate::GetInstance()->GetCurrentAccountId(); - aclParams.accCaller.userId = from.userId; - aclParams.accCaller.networkId = DmAdapter::GetInstance().ToNetworkID(from.deviceId); + aclParams.accCaller.userId = local.userId; + aclParams.accCaller.networkId = DmAdapter::GetInstance().ToNetworkID(local.deviceId); aclParams.accCallee.networkId = DmAdapter::GetInstance().ToNetworkID(targetDeviceId); aclParams.authType = storeMeta.authType; @@ -105,12 +101,12 @@ bool SessionManager::GetSendAuthParams(const SessionPoint &from, const std::stri } } ZLOGE("get params failed,appId:%{public}s,localDevId:%{public}s,tarDevid:%{public}s,user:%{public}d,", - from.appId.c_str(), Anonymous::Change(from.deviceId).c_str(), - Anonymous::Change(targetDeviceId).c_str(), from.userId); + local.appId.c_str(), Anonymous::Change(local.deviceId).c_str(), + Anonymous::Change(targetDeviceId).c_str(), local.userId); return false; } -bool SessionManager::GetRecvAuthParams(const SessionPoint &from, const std::string &targetDeviceId, +bool SessionManager::GetRecvAuthParams(const SessionPoint &local, const std::string &targetDeviceId, AclParams &aclParams, int32_t peerUser) const { std::vector metaData; @@ -120,12 +116,12 @@ bool SessionManager::GetRecvAuthParams(const SessionPoint &from, const std::stri return false; } for (const auto &storeMeta : metaData) { - if (storeMeta.appId == from.appId) { + if (storeMeta.appId == local.appId) { auto accountId = AccountDelegate::GetInstance()->GetCurrentAccountId(); aclParams.accCaller.bundleName = storeMeta.bundleName; aclParams.accCaller.accountId = accountId; - aclParams.accCaller.userId = from.userId; - aclParams.accCaller.networkId = DmAdapter::GetInstance().ToNetworkID(from.deviceId); + aclParams.accCaller.userId = local.userId; + aclParams.accCaller.networkId = DmAdapter::GetInstance().ToNetworkID(local.deviceId); aclParams.accCallee.accountId = accountId; aclParams.accCallee.userId = peerUser; @@ -136,19 +132,22 @@ bool SessionManager::GetRecvAuthParams(const SessionPoint &from, const std::stri } ZLOGE("get params failed,appId:%{public}s,tarDevid:%{public}s,user:%{public}d,peer:%{public}d", - from.appId.c_str(), Anonymous::Change(targetDeviceId).c_str(), from.userId, peerUser); + local.appId.c_str(), Anonymous::Change(targetDeviceId).c_str(), local.userId, peerUser); return false; } -bool SessionManager::CheckSession(const SessionPoint &from, const SessionPoint &to) const +bool SessionManager::CheckSession(const SessionPoint &local, const SessionPoint &peer) const { AclParams aclParams; - if (!GetRecvAuthParams(from, to.deviceId, aclParams, to.userId)) { - ZLOGE("get recv auth params failed:%{public}s", Anonymous::Change(to.deviceId).c_str()); + if (!GetRecvAuthParams(local, peer.deviceId, aclParams, peer.userId)) { + ZLOGE("get recv auth params failed:%{public}s", Anonymous::Change(peer.deviceId).c_str()); return false; } - auto [isPermitted, isSameAccount] = AuthDelegate::GetInstance()->CheckAccess(from.userId, - to.userId, to.deviceId, aclParams); + auto [isPermitted, isSameAccount] = AuthDelegate::GetInstance()->CheckAccess(local.userId, + peer.userId, peer.deviceId, aclParams); + if (isPermitted && local.userId != UserDelegate::SYSTEM_USER) { + isPermitted = Account::GetInstance()->IsUserForeground(local.userId); + } return isPermitted; } diff --git a/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.h b/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.h index df16d859..ca83970b 100644 --- a/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.h +++ b/datamgr_service/services/distributeddataservice/app/src/session_manager/session_manager.h @@ -50,12 +50,12 @@ public: class SessionManager { public: static SessionManager &GetInstance(); - Session GetSession(const SessionPoint &from, const std::string &targetDeviceId) const; - bool CheckSession(const SessionPoint &from, const SessionPoint &to) const; + Session GetSession(const SessionPoint &local, const std::string &targetDeviceId) const; + bool CheckSession(const SessionPoint &local, const SessionPoint &peer) const; private: - bool GetSendAuthParams(const SessionPoint &from, const std::string &targetDeviceId, + bool GetSendAuthParams(const SessionPoint &local, const std::string &targetDeviceId, AclParams &aclParams) const; - bool GetRecvAuthParams(const SessionPoint &from, const std::string &targetDeviceId, + bool GetRecvAuthParams(const SessionPoint &local, const std::string &targetDeviceId, AclParams &aclParams, int peerUser) const; }; } // namespace OHOS::DistributedData diff --git a/datamgr_service/services/distributeddataservice/app/test/BUILD.gn b/datamgr_service/services/distributeddataservice/app/test/BUILD.gn index 010a6546..f3f91289 100644 --- a/datamgr_service/services/distributeddataservice/app/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/app/test/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddataservice" +module_output_path = "datamgr_service/datamgr_service/distributeddataservice" ############################################################################### config("module_private_config") { @@ -28,8 +28,6 @@ config("module_private_config") { "${data_service_path}/adapter/include/permission", "${data_service_path}/adapter/include/account", "${data_service_path}/adapter/include", - "${data_service_path}/adapter/include/dfx", - "${data_service_path}/adapter/include/broadcaster", "${data_service_path}/adapter/include/utils", "${data_service_path}/framework/include", "${data_service_path}/service/bootstrap/include", @@ -51,7 +49,6 @@ config("module_private_config") { "../src/security", "unittest", "../src/installer", - "../src/flowctrl_manager", "../../service/backup/include", "../../../../interfaces/innerkits/distributeddata", "../../service/dumper/include", @@ -76,18 +73,16 @@ config("module_private_config") { } ohos_unittest("KvStoreDataServiceTest") { - sanitize = { - cfi = true - cfi_cross_dso = true - debug = false - } - module_out_path = module_output_path sources = [ + "${data_service_path}/app/src/checker/bundle_checker.cpp", + "${data_service_path}/app/src/checker/system_checker.cpp", "${data_service_path}/app/src/clone/clone_backup_info.cpp", "${data_service_path}/app/src/clone/secret_key_backup_data.cpp", "${data_service_path}/app/src/db_info_handle_impl.cpp", "${data_service_path}/app/src/feature_stub_impl.cpp", + "${data_service_path}/app/src/installer/installer.cpp", + "${data_service_path}/app/src/installer/installer_impl.cpp", "${data_service_path}/app/src/kvstore_account_observer.cpp", "${data_service_path}/app/src/kvstore_data_service.cpp", "${data_service_path}/app/src/kvstore_data_service_stub.cpp", @@ -107,8 +102,11 @@ ohos_unittest("KvStoreDataServiceTest") { configs = [ ":module_private_config" ] external_deps = [ + "ability_base:want", "access_token:libaccesstoken_sdk", + "bundle_framework:appexecfwk_core", "c_utils:utils", + "common_event_service:cesfwk_innerkits", "dataclassification:data_transit_mgr", "device_auth:deviceauth_sdk", "file_api:securitylabel", @@ -133,11 +131,6 @@ ohos_unittest("KvStoreDataServiceTest") { } deps = [ - "${data_service_path}/adapter/broadcaster:distributeddata_broadcaster", - "${data_service_path}/adapter/utils:distributeddata_utils", - "${data_service_path}/app/src/checker:distributeddata_checker", - "${data_service_path}/app/src/flowctrl_manager:distributeddata_flowctrl", - "${data_service_path}/app/src/installer:distributeddata_installer", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", "//third_party/googletest:gtest_main", @@ -188,7 +181,6 @@ ohos_unittest("SessionManagerTest") { } deps = [ - "${data_service_path}/adapter/broadcaster:distributeddata_broadcaster", "${data_service_path}/adapter/utils:distributeddata_utils", "${data_service_path}/app/src/checker:distributeddata_checker", "${kv_store_path}/interfaces/innerkits/distributeddatamgr:distributeddata_mgr", @@ -202,43 +194,6 @@ ohos_unittest("SessionManagerTest") { part_name = "datamgr_service" } -ohos_unittest("KvStoreFlowCtrlManagerTest") { - module_out_path = module_output_path - sources = [ "unittest/kvstore_flowctrl_manager_test.cpp" ] - - configs = [ ":module_private_config" ] - - external_deps = [ - "access_token:libaccesstoken_sdk", - "c_utils:utils", - "dataclassification:data_transit_mgr", - "device_auth:deviceauth_sdk", - "file_api:securitylabel", - "hilog:libhilog", - "hisysevent:libhisysevent", - "ipc:ipc_core", - "safwk:system_ability_fwk", - "samgr:samgr_proxy", - ] - - if (datamgr_service_power) { - external_deps += [ - "battery_manager:batterysrv_client", - "power_manager:powermgr_client", - ] - } - - deps = [ - "${data_service_path}/app/src/flowctrl_manager:distributeddata_flowctrl", - "${kv_store_path}/interfaces/innerkits/distributeddatamgr:distributeddata_mgr", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/framework:distributeddatasvcfwk", - "//foundation/distributeddatamgr/datamgr_service/services/distributeddataservice/service:distributeddatasvc", - "//foundation/distributeddatamgr/kv_store/interfaces/innerkits/distributeddata:distributeddata_inner", - "//third_party/googletest:gtest_main", - ] - part_name = "datamgr_service" -} - ohos_unittest("KvStoreDataServiceClearTest") { module_out_path = module_output_path sources = [ @@ -301,10 +256,8 @@ ohos_unittest("KvStoreDataServiceClearTest") { } deps = [ - "${data_service_path}/adapter/broadcaster:distributeddata_broadcaster", "${data_service_path}/adapter/utils:distributeddata_utils", "${data_service_path}/app/src/checker:distributeddata_checker", - "${data_service_path}/app/src/flowctrl_manager:distributeddata_flowctrl", "${data_service_path}/app/src/installer:distributeddata_installer", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", @@ -321,8 +274,6 @@ ohos_unittest("FeatureStubImplTest") { "${data_service_path}/adapter/include/permission", "${data_service_path}/adapter/include/account", "${data_service_path}/adapter/include", - "${data_service_path}/adapter/include/dfx", - "${data_service_path}/adapter/include/broadcaster", "${data_service_path}/adapter/include/utils", "${data_service_path}/framework/include", "${data_service_path}/service/bootstrap/include", @@ -372,7 +323,6 @@ ohos_unittest("FeatureStubImplTest") { } deps = [ - "${data_service_path}/app/src/flowctrl_manager:distributeddata_flowctrl", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", ] @@ -389,7 +339,6 @@ group("unittest") { ":FeatureStubImplTest", ":KvStoreDataServiceClearTest", ":KvStoreDataServiceTest", - ":KvStoreFlowCtrlManagerTest", ":SessionManagerTest", ] } @@ -405,7 +354,6 @@ group("moduletest") { deps += [ #":DistributedDataAccountEventModuleTest", - #":DistributedDataFlowCtrlManagerTest", ] } ############################################################################### diff --git a/datamgr_service/services/distributeddataservice/app/test/fuzztest/dataservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/app/test/fuzztest/dataservicestub_fuzzer/BUILD.gn index 4b3b0124..a04a1d62 100644 --- a/datamgr_service/services/distributeddataservice/app/test/fuzztest/dataservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/app/test/fuzztest/dataservicestub_fuzzer/BUILD.gn @@ -33,7 +33,6 @@ ohos_fuzztest("DataServiceStubFuzzTest") { "${data_service_path}/app/src", "${data_service_path}/app/src/backup_rule/include", "${data_service_path}/app/src/checker", - "${data_service_path}/app/src/flowctrl_manager", "${data_service_path}/app/src/security", "${data_service_path}/app/src/session_manager", "${data_service_path}/app/src/installer", @@ -45,9 +44,7 @@ ohos_fuzztest("DataServiceStubFuzzTest") { "${data_service_path}/adapter/include/account", "${data_service_path}/adapter/include/permission", "${data_service_path}/adapter/include/installer", - "${data_service_path}/adapter/include/broadcaster", "${data_service_path}/adapter/include/utils", - "${data_service_path}/adapter/include/dfx", "${data_service_path}/adapter/include", "${device_manager_path}/interfaces/inner_kits/native_cpp/include", "//third_party/json/single_include", @@ -87,10 +84,8 @@ ohos_fuzztest("DataServiceStubFuzzTest") { deps = [ "${data_service_path}/adapter/account:distributeddata_account", - "${data_service_path}/adapter/broadcaster:distributeddata_broadcaster", "${data_service_path}/adapter/utils:distributeddata_utils", "${data_service_path}/app/src/checker:distributeddata_checker", - "${data_service_path}/app/src/flowctrl_manager:distributeddata_flowctrl", "${data_service_path}/app/src/installer:distributeddata_installer", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", diff --git a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_clear_test.cpp b/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_clear_test.cpp index 26881120..ebd8c46d 100644 --- a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_clear_test.cpp +++ b/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_clear_test.cpp @@ -79,31 +79,9 @@ void KvStoreDataServiceClearTest::TearDown(void) { } -void KvStoreDataServiceClearTest::InitMetaData() -{ - metaData_.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; - metaData_.user = TEST_USER; - metaData_.bundleName = TEST_BUNDLE; - metaData_.storeId = TEST_STORE; - metaData_.appId = TEST_BUNDLE; - metaData_.tokenId = AccessTokenKit::GetHapTokenID(TEST_USERID, TEST_BUNDLE, 0); - metaData_.uid = TEST_UID; - metaData_.storeType = 1; - metaData_.area = EL1; - metaData_.instanceId = 0; - metaData_.isAutoSync = true; - metaData_.version = 1; - metaData_.appType = "default"; - metaData_.dataDir = "/data/service/el1/public/database/kvstore_data_service_clear_test"; - - DistributedData::PolicyValue value; - value.type = OHOS::DistributedKv::PolicyType::IMMEDIATE_SYNC_ON_ONLINE; - localMeta_.policies = { std::move(value) }; -} - /** * @tc.name: ClearAppStorage001 - * @tc.desc: The parameters are valid but have no metaData + * @tc.desc: Test for abnormal situations in ClearAppStore * @tc.type: FUNC * @tc.require: * @tc.author: suoqilong @@ -111,56 +89,13 @@ void KvStoreDataServiceClearTest::InitMetaData() HWTEST_F(KvStoreDataServiceClearTest, ClearAppStorage001, TestSize.Level1) { auto kvDataService = OHOS::DistributedKv::KvStoreDataService(); - auto tokenIdOk = AccessTokenKit::GetNativeTokenId("foundation"); - SetSelfTokenID(tokenIdOk); - auto ret = kvDataService.ClearAppStorage(BUNDLE_NAME, USER_ID, APP_INDEX, tokenIdOk); + // tokenId mismatch + auto ret = kvDataService.ClearAppStorage(BUNDLE_NAME, TEST_USERID, APP_INDEX, 0); EXPECT_EQ(ret, Status::ERROR); -} - -/** - * @tc.name: ClearAppStorage002 - * @tc.desc: Test that the cleanup is implemented - * @tc.type: FUNC - * @tc.require: - * @tc.author: suoqilong - */ -HWTEST_F(KvStoreDataServiceClearTest, ClearAppStorage002, TestSize.Level1) -{ - auto executors = std::make_shared(2, 1); - // Create an object of the ExecutorPool class and pass 2 and 1 as arguments to the constructor of the class - KvStoreMetaManager::GetInstance().BindExecutor(executors); - KvStoreMetaManager::GetInstance().InitMetaParameter(); - DmAdapter::GetInstance().Init(executors); - KvStoreMetaManager::GetInstance().InitMetaListener(); - - InitMetaData(); - - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetKey(), metaData_)); - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetKey(), metaData_, true)); - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetSecretKey(), metaData_, true)); - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetStrategyKey(), metaData_)); - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.appId, metaData_, true)); - EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetKeyLocal(), localMeta_, true)); - - EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKey(), metaData_)); - EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetSecretKey(), metaData_, true)); - EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetStrategyKey(), metaData_)); - EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData_.appId, metaData_, true)); - EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKeyLocal(), localMeta_, true)); - - auto tokenIdOk = AccessTokenKit::GetNativeTokenId("foundation"); + // no meta data + auto tokenIdOk = Security::AccessToken::AccessTokenKit::GetNativeTokenId("foundation"); SetSelfTokenID(tokenIdOk); - auto kvDataService = OHOS::DistributedKv::KvStoreDataService(); - auto ret = kvDataService.ClearAppStorage(BUNDLE_NAME, USER_ID, APP_INDEX, tokenIdOk); - EXPECT_EQ(ret, Status::SUCCESS); - - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKey(), metaData_)); - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetSecretKey(), metaData_, true)); - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetStrategyKey(), metaData_)); - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.appId, metaData_, true)); - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKeyLocal(), localMeta_, true)); - - MetaDataManager::GetInstance().DelMeta(metaData_.GetKey()); - EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKey(), metaData_)); + ret = kvDataService.ClearAppStorage(BUNDLE_NAME, TEST_USERID, APP_INDEX, tokenIdOk); + EXPECT_EQ(ret, Status::ERROR); } } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_test.cpp b/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_test.cpp index b6b05a97..7b9aeb3d 100644 --- a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_test.cpp +++ b/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_data_service_test.cpp @@ -994,6 +994,7 @@ HWTEST_F(KvStoreDataServiceTest, OnExtensionBackup008, TestSize.Level0) { testMeta.bundleName = "com.example.restore_test"; testMeta.storeId = "Source"; testMeta.user = "100"; + testMeta.area = DEFAULT_ENCRYPTION_LEVEL; testMeta.instanceId = 0; testMeta.deviceId = DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid; @@ -1004,15 +1005,11 @@ HWTEST_F(KvStoreDataServiceTest, OnExtensionBackup008, TestSize.Level0) { 131, 104, 141, 43, 96, 119, 214, 34, 177, 129, 233, 96, 98, 164, 87, 115, 187, 170}; SecretKeyMetaData testSecret; - testSecret.sKey = CryptoManager::GetInstance().Encrypt(sKey); + testSecret.sKey = CryptoManager::GetInstance().Encrypt(sKey, testMeta.area, testMeta.user); testSecret.storeType = 10; testSecret.time = std::vector{233, 39, 137, 103, 0, 0, 0, 0}; - EXPECT_EQ(MetaDataManager::GetInstance().SaveMeta(testMeta.GetKey(), - testMeta, true), - true); - EXPECT_EQ(MetaDataManager::GetInstance().SaveMeta(testMeta.GetSecretKey(), - testSecret, true), - true); + EXPECT_EQ(MetaDataManager::GetInstance().SaveMeta(testMeta.GetKey(), testMeta, true), true); + EXPECT_EQ(MetaDataManager::GetInstance().SaveMeta(testMeta.GetSecretKey(), testSecret, true), true); std::string cloneInfoStr = "[{\"type\":\"encryption_info\",\"detail\":{\"encryption_symkey\":\"27," diff --git a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_flowctrl_manager_test.cpp b/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_flowctrl_manager_test.cpp deleted file mode 100644 index 506f32c1..00000000 --- a/datamgr_service/services/distributeddataservice/app/test/unittest/kvstore_flowctrl_manager_test.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "flowctrl_manager/kvstore_flowctrl_manager.h" -#include -#include -#include -using namespace testing::ext; -using namespace OHOS::DistributedKv; -using namespace OHOS; -using namespace std::chrono; - -class KvStoreFlowCtrlManagerTest : public testing::Test { -public: - static inline const int MANAGER_BURST_CAPACITY = 50; - static inline const int MANAGER_SUSTAINED_CAPACITY = 500; - static inline const int OPERATION_BURST_CAPACITY = 1000; - static inline const int OPERATION_SUSTAINED_CAPACITY = 10000; - static void SetUpTestCase(void); - static void TearDownTestCase(void); - void SetUp(); - void TearDown(); -}; - -void KvStoreFlowCtrlManagerTest::SetUpTestCase(void) -{} - -void KvStoreFlowCtrlManagerTest::TearDownTestCase(void) -{} - -void KvStoreFlowCtrlManagerTest::SetUp(void) -{} - -void KvStoreFlowCtrlManagerTest::TearDown(void) -{} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest001 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest001, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 1001; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(1, arr[0]); - EXPECT_EQ(1000, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest002 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest002, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 1000; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(1000, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest003 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest003, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 999; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(999, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest004 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: SR000F3H5U AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest004, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 9999; i++) { - arr[ptr->IsTokenEnough()]++; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(9999, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest005 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest005, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 10000; i++) { - arr[ptr->IsTokenEnough()]++; - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(10000, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest006 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest006, TestSize.Level1) -{ - auto ptr = std::make_shared(OPERATION_BURST_CAPACITY, OPERATION_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - uint64_t curTime = 0; - uint64_t lastTime = duration_cast(steady_clock::now().time_since_epoch()).count(); - for (int i = 0; i < 10001; i++) { - arr[ptr->IsTokenEnough()]++; - while (true) { - curTime = duration_cast(steady_clock::now().time_since_epoch()).count(); - if ((curTime - lastTime) > 1000) { - lastTime = curTime; - break; - } - } - } - EXPECT_EQ(1, arr[0]); - EXPECT_EQ(10000, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest007 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest007, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 51; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(1, arr[0]); - EXPECT_EQ(50, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest008 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest008, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 50; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(50, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest009 -* @tc.desc: burst flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP7 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest009, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 49; i++) { - arr[ptr->IsTokenEnough()]++; - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(49, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest010 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest010, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 499; i++) { - arr[ptr->IsTokenEnough()]++; - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(499, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest011 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest011, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 500; i++) { - arr[ptr->IsTokenEnough()]++; - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } - EXPECT_EQ(0, arr[0]); - EXPECT_EQ(500, arr[1]); -} - -/** -* @tc.name: KvStoreFlowCtrlManagerTest012 -* @tc.desc: sustained flow control -* @tc.type: FUNC -* @tc.require: AR000F3OP8 -* @tc.author: jishengwu -*/ -HWTEST_F(KvStoreFlowCtrlManagerTest, KvStoreFlowCtrlManagerTest012, TestSize.Level1) -{ - auto ptr = std::make_shared(MANAGER_BURST_CAPACITY, MANAGER_SUSTAINED_CAPACITY); - int arr[2] = {0, 0}; - for (int i = 0; i < 501; i++) { - arr[ptr->IsTokenEnough()]++; - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } - EXPECT_EQ(1, arr[0]); - EXPECT_EQ(500, arr[1]); -} - diff --git a/datamgr_service/services/distributeddataservice/app/test/unittest/session_manager_test.cpp b/datamgr_service/services/distributeddataservice/app/test/unittest/session_manager_test.cpp index 96fa8ca7..8031dc6c 100644 --- a/datamgr_service/services/distributeddataservice/app/test/unittest/session_manager_test.cpp +++ b/datamgr_service/services/distributeddataservice/app/test/unittest/session_manager_test.cpp @@ -148,12 +148,13 @@ HWTEST_F(SessionManagerTest, PackAndUnPack01, TestSize.Level2) std::unique_ptr data = std::make_unique(routeHeadSize); sendHandler->FillHeadData(data.get(), routeHeadSize, routeHeadSize); - std::vector users; + std::vector users; auto recvHandler = RouteHeadHandlerImpl::Create({}); ASSERT_NE(recvHandler, nullptr); uint32_t parseSize = 1; - recvHandler->ParseHeadData(data.get(), routeHeadSize, parseSize, users); + recvHandler->ParseHeadDataLen(data.get(), routeHeadSize, parseSize); EXPECT_EQ(routeHeadSize, parseSize); + recvHandler->ParseHeadDataUser(data.get(), routeHeadSize, "", users); ASSERT_EQ(users.size(), 0); } } // namespace \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/BUILD.gn b/datamgr_service/services/distributeddataservice/framework/BUILD.gn index 9f86825e..96970573 100644 --- a/datamgr_service/services/distributeddataservice/framework/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/framework/BUILD.gn @@ -21,6 +21,7 @@ config("module_config") { visibility = [ ":*" ] include_dirs = [ "include", + "include/utils", "${data_service_path}/adapter/include", "${kv_store_path}/interfaces/innerkits/distributeddata/include", "${kv_store_common_path}", @@ -34,6 +35,8 @@ config("module_public_config") { include_dirs = [ "include", "${data_service_path}/framework/include/account", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/framework/include/network", "${kv_store_common_path}", ] } @@ -59,7 +62,9 @@ ohos_shared_library("distributeddatasvcfwk") { "cloud/cloud_event.cpp", "cloud/cloud_extra_data.cpp", "cloud/cloud_info.cpp", + "cloud/cloud_last_sync_info.cpp", "cloud/cloud_lock_event.cpp", + "cloud/cloud_mark.cpp", "cloud/cloud_report.cpp", "cloud/cloud_server.cpp", "cloud/cloud_share_event.cpp", @@ -71,6 +76,7 @@ ohos_shared_library("distributeddatasvcfwk") { "cloud/sync_event.cpp", "cloud/sync_strategy.cpp", "communication/connect_manager.cpp", + "dfx/reporter.cpp", "directory/directory_manager.cpp", "dump/dump_manager.cpp", "eventcenter/event.cpp", @@ -82,6 +88,7 @@ ohos_shared_library("distributeddatasvcfwk") { "metadata/capability_meta_data.cpp", "metadata/capability_range.cpp", "metadata/corrupted_meta_data.cpp", + "metadata/device_meta_data.cpp", "metadata/matrix_meta_data.cpp", "metadata/meta_data.cpp", "metadata/meta_data_manager.cpp", @@ -93,6 +100,7 @@ ohos_shared_library("distributeddatasvcfwk") { "metadata/switches_meta_data.cpp", "metadata/user_meta_data.cpp", "metadata/version_meta_data.cpp", + "network/network_delegate.cpp", "screen/screen_manager.cpp", "serializable/serializable.cpp", "snapshot/bind_event.cpp", @@ -107,14 +115,24 @@ ohos_shared_library("distributeddatasvcfwk") { "utils/corrupt_reporter.cpp", "utils/crypto.cpp", "utils/ref_count.cpp", + "utils/time_utils.cpp", ] + cflags = [ "-Werror", "-Wno-multichar", "-D_LIBCPP_HAS_COND_CLOCKWAIT", + "-fdata-sections", + "-ffunction-sections", + "-fstack-protector-strong", ] - cflags_cc = [ "-fvisibility=hidden" ] + ldflags = [ "-Wl,-z,relro,-z,now,--gc-sections" ] + + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] configs = [ ":module_config" ] diff --git a/datamgr_service/services/distributeddataservice/framework/CMakeLists.txt b/datamgr_service/services/distributeddataservice/framework/CMakeLists.txt index a83b5cdb..6b9b30cf 100644 --- a/datamgr_service/services/distributeddataservice/framework/CMakeLists.txt +++ b/datamgr_service/services/distributeddataservice/framework/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10.2) project(svcFwk) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") @@ -18,10 +18,12 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/checker svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/cloud svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/communication svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/directory svcFwkSrc) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dfx svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/dump svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/eventcenter svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/feature svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/metadata svcFwkSrc) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/network svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/screen svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/serializable svcFwkSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/snapshot svcFwkSrc) diff --git a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_db.cpp b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_db.cpp index ab0a7b50..86347588 100644 --- a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_db.cpp +++ b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_db.cpp @@ -104,4 +104,4 @@ void CloudDB::SetPrepareTraceId(const std::string &prepareTraceId) { (void)prepareTraceId; } -} // namespace OHOS::DistributedData +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_info.cpp b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_info.cpp index 191c6f88..0d80c15d 100644 --- a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_info.cpp +++ b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_info.cpp @@ -148,4 +148,16 @@ std::string CloudInfo::GetKey(const std::string &prefix, const std::initializer_ { return Constant::Join(prefix, Constant::KEY_SEPARATOR, fields); } + +std::optional CloudInfo::GetAppInfo(const std::string &bundleName) const +{ + if (bundleName.empty()) { + return std::nullopt; + } + auto it = apps.find(bundleName); + if (it == apps.end()) { + return std::nullopt; + } + return it->second; +} } // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_last_sync_info.cpp b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_last_sync_info.cpp new file mode 100644 index 00000000..463ea514 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_last_sync_info.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "CloudLastSyncInfo" +#include "cloud/cloud_last_sync_info.h" + +#include "utils/constant.h" +namespace OHOS::DistributedData { +bool CloudLastSyncInfo::Marshal(Serializable::json &node) const +{ + SetValue(node[GET_NAME(id)], id); + SetValue(node[GET_NAME(storeId)], storeId); + SetValue(node[GET_NAME(startTime)], startTime); + SetValue(node[GET_NAME(finishTime)], finishTime); + SetValue(node[GET_NAME(code)], code); + SetValue(node[GET_NAME(syncStatus)], syncStatus); + SetValue(node[GET_NAME(instanceId)], instanceId); + return true; +} + +bool CloudLastSyncInfo::Unmarshal(const Serializable::json &node) +{ + GetValue(node, GET_NAME(id), id); + GetValue(node, GET_NAME(storeId), storeId); + GetValue(node, GET_NAME(startTime), startTime); + GetValue(node, GET_NAME(finishTime), finishTime); + GetValue(node, GET_NAME(code), code); + GetValue(node, GET_NAME(syncStatus), syncStatus); + GetValue(node, GET_NAME(instanceId), instanceId); + return true; +} + +std::string CloudLastSyncInfo::GetKey(const int32_t user, const std::string &bundleName, + const std::string &storeId, int32_t instanceId) +{ + return GetKey(INFO_PREFIX, { std::to_string(user), bundleName, std::to_string(instanceId), storeId }); +} + +std::string CloudLastSyncInfo::GetKey(const std::string &prefix, const std::initializer_list &fields) +{ + return Constant::Join(prefix, Constant::KEY_SEPARATOR, fields); +} +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/cloud/cloud_mark.cpp b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_mark.cpp new file mode 100644 index 00000000..37e69ad2 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/cloud/cloud_mark.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cloud/cloud_mark.h" + +namespace OHOS::DistributedData { + +static constexpr const char *KEY_PREFIX = "CloudMark"; +static constexpr const char *KEY_SEPARATOR = "###"; +bool CloudMark::Marshal(Serializable::json &node) const +{ + SetValue(node[GET_NAME(isClearWaterMark)], isClearWaterMark); + return true; +} + +bool CloudMark::Unmarshal(const Serializable::json &node) +{ + GetValue(node, GET_NAME(isClearWaterMark), isClearWaterMark); + return true; +} + +std::string CloudMark::GetKey() +{ + return GetKey({ deviceId, std::to_string(userId), "default", bundleName, storeId, std::to_string(index) }); +} + +std::string CloudMark::GetKey(const std::initializer_list &fields) +{ + std::string prefix = KEY_PREFIX; + for (const auto &field : fields) { + prefix.append(KEY_SEPARATOR).append(field); + } + return prefix; +} +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/cloud/schema_meta.cpp b/datamgr_service/services/distributeddataservice/framework/cloud/schema_meta.cpp index 7ced1856..7844058a 100644 --- a/datamgr_service/services/distributeddataservice/framework/cloud/schema_meta.cpp +++ b/datamgr_service/services/distributeddataservice/framework/cloud/schema_meta.cpp @@ -136,6 +136,7 @@ bool Field::Unmarshal(const Serializable::json &node) GetValue(node, GET_NAME(alias), alias); GetValue(node, GET_NAME(type), type); GetValue(node, GET_NAME(primary), primary); + GetValue(node, GET_NAME(primaryKey), primary); GetValue(node, GET_NAME(nullable), nullable); GetValue(node, GET_NAME(columnName), colName); GetValue(node, GET_NAME(notNull), nullable); @@ -156,4 +157,13 @@ bool SchemaMeta::IsValid() const { return !bundleName.empty() && !databases.empty(); } + +std::vector SchemaMeta::GetStores() +{ + std::vector stores; + for (const auto &it : databases) { + stores.push_back(it.name); + } + return stores; +} } // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender.cpp b/datamgr_service/services/distributeddataservice/framework/dfx/reporter.cpp similarity index 52% rename from datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender.cpp rename to datamgr_service/services/distributeddataservice/framework/dfx/reporter.cpp index 095ba4b3..14ba18c8 100644 --- a/datamgr_service/services/distributeddataservice/adapter/broadcaster/src/broadcast_sender.cpp +++ b/datamgr_service/services/distributeddataservice/framework/dfx/reporter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,21 +13,23 @@ * limitations under the License. */ -#include "broadcast_sender.h" -#include -#include "broadcast_sender_impl.h" +#include "dfx/reporter.h" -namespace OHOS::DistributedKv { -std::shared_ptr BroadcastSender::instance_; -std::mutex BroadcastSender::mutex_; -std::shared_ptr BroadcastSender::GetInstance() +namespace OHOS { +namespace DistributedDataDfx { +Reporter *Reporter::instance_ = nullptr; +bool Reporter::RegisterReporterInstance(Reporter *instance) { - if (instance_ == nullptr) { - std::lock_guard lock(mutex_); - if (instance_ == nullptr) { - instance_ = std::make_shared(); - } + if (instance_ != nullptr) { + return false; } + instance_ = instance; + return true; +} + +Reporter* Reporter::GetInstance() +{ return instance_; } -} // namespace OHOS::DistributedKv +} // namespace DistributedDataDfx +} // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/framework/directory/directory_manager.cpp b/datamgr_service/services/distributeddataservice/framework/directory/directory_manager.cpp index 4cc56b21..9deb434e 100644 --- a/datamgr_service/services/distributeddataservice/framework/directory/directory_manager.cpp +++ b/datamgr_service/services/distributeddataservice/framework/directory/directory_manager.cpp @@ -85,6 +85,28 @@ std::string DirectoryManager::GetMetaBackupPath(uint32_t version) return path; } +std::string DirectoryManager::GetClonePath(const std::string &userId, uint32_t version) +{ + int32_t index = GetVersionIndex(version); + if (index < 0) { + return ""; + } + + auto &strategy = strategies_[index]; + auto path = strategy.clonePath; + std::string pattern = "{userId}"; + size_t pos = path.find(pattern); + if (pos != std::string::npos) { + path.replace(pos, pattern.length(), userId); + if (CreateDirectory(path.substr(0, path.rfind('/')))) { + return path; + } else { + return ""; + }; + } + return ""; +} + void DirectoryManager::Initialize(const std::vector &strategies) { strategies_.resize(strategies.size()); @@ -95,6 +117,7 @@ void DirectoryManager::Initialize(const std::vector &strategies) impl.version = strategy.version; impl.metaPath = strategy.metaPath; impl.path = Split(strategy.pattern, "/"); + impl.clonePath = strategy.clonePath; impl.pipes.clear(); for (auto &value : impl.path) { auto it = actions_.find(value); diff --git a/datamgr_service/services/distributeddataservice/framework/feature/static_acts.cpp b/datamgr_service/services/distributeddataservice/framework/feature/static_acts.cpp index 28b4d0c4..f023d8da 100644 --- a/datamgr_service/services/distributeddataservice/framework/feature/static_acts.cpp +++ b/datamgr_service/services/distributeddataservice/framework/feature/static_acts.cpp @@ -37,4 +37,18 @@ int32_t StaticActs::OnClearAppStorage(const std::string &bundleName, int32_t use { return E_OK; } + +void StaticActs::SetThreadPool(std::shared_ptr executors) +{ + executors_ = executors; +} + +void StaticActs::Execute(Task task) +{ + auto executor = executors_; + if (executor == nullptr) { + return; + } + executor->Execute(std::move(task)); +} } diff --git a/datamgr_service/services/distributeddataservice/framework/include/account/account_delegate.h b/datamgr_service/services/distributeddataservice/framework/include/account/account_delegate.h index 92584aac..5636131f 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/account/account_delegate.h +++ b/datamgr_service/services/distributeddataservice/framework/include/account/account_delegate.h @@ -76,6 +76,7 @@ public: API_EXPORT virtual std::string GetUnencryptedAccountId(int32_t userId = 0) const = 0; API_EXPORT static AccountDelegate *GetInstance(); API_EXPORT static bool RegisterAccountInstance(AccountDelegate *instance); + API_EXPORT virtual bool IsUserForeground(int32_t userId) = 0; private: static AccountDelegate *instance_; diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/asset_loader.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/asset_loader.h index a7caf47b..1e9a7ca0 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/asset_loader.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/asset_loader.h @@ -31,4 +31,4 @@ public: virtual int32_t CancelDownload(); }; } // namespace OHOS::DistributedData -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_ASSET_LOADER_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_ASSET_LOADER_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_db.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_db.h index a6818dbb..fcf7cfca 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_db.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_db.h @@ -67,4 +67,4 @@ public: virtual void SetPrepareTraceId(const std::string &prepareTraceId); }; } // namespace OHOS::DistributedData -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_DB_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_DB_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_event.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_event.h index abb96c1d..7c347470 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_event.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_event.h @@ -39,6 +39,7 @@ public: LOCK_CLOUD_CONTAINER, UNLOCK_CLOUD_CONTAINER, SET_SEARCH_TRIGGER, + UPGRADE_SCHEMA, CLOUD_BUTT }; diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_extra_data.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_extra_data.h index 1a45499c..f6beb5b8 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_extra_data.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_extra_data.h @@ -57,4 +57,4 @@ public: static constexpr const char *SHARED_TABLE = "shared"; }; } // namespace OHOS::DistributedData -#endif // DISTRIBUTEDDATAMGR_DATAMGR_SERVICE5_CLOUD_EXTRA_DATA_H +#endif // DISTRIBUTEDDATAMGR_DATAMGR_SERVICE5_CLOUD_EXTRA_DATA_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_info.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_info.h index b8ef793b..c565fcd2 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_info.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_info.h @@ -15,6 +15,7 @@ #ifndef OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_INFO_H #define OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_INFO_H +#include #include "metadata/store_meta_data.h" #include "serializable/serializable.h" namespace OHOS::DistributedData { @@ -52,6 +53,7 @@ public: bool IsOn(const std::string &bundleName, int32_t instanceId = 0) const; bool IsAllSwitchOff() const; static std::string GetPrefix(const std::initializer_list &field); + std::optional GetAppInfo(const std::string &bundleName) const; bool Marshal(json &node) const override; bool Unmarshal(const json &node) override; diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_last_sync_info.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_last_sync_info.h new file mode 100644 index 00000000..c362dfb0 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_last_sync_info.h @@ -0,0 +1,41 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_LAST_SYNC_INFO_H +#define OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_LAST_SYNC_INFO_H + +#include "serializable/serializable.h" +#include "visibility.h" + +namespace OHOS::DistributedData { +class API_EXPORT CloudLastSyncInfo final : public Serializable { +public: + std::string id = ""; + std::string storeId = ""; + int64_t startTime = 0; + int64_t finishTime = 0; + int32_t code = -1; + int32_t syncStatus = 0; + int32_t instanceId = 0; + bool Marshal(json &node) const override; + bool Unmarshal(const json &node) override; + static std::string GetKey(const int32_t user, const std::string &bundleName, + const std::string &storeId, int32_t instanceId = 0); +private: + static constexpr const char *INFO_PREFIX = "CLOUD_LAST_SYNC_INFO"; + static std::string GetKey(const std::string &prefix, const std::initializer_list &fields); +}; +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_CLOUD_LAST_SYNC_INFO_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_mark.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_mark.h new file mode 100644 index 00000000..47c5783d --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/cloud_mark.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_MARK_H +#define OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_MARK_H +#include "serializable/serializable.h" +#include "store/store_info.h" +namespace OHOS::DistributedData { +class API_EXPORT CloudMark final : public Serializable { +public: + bool isClearWaterMark = false; + std::string bundleName; + std::string deviceId; + std::string storeId; + int32_t userId = 0; + int32_t index = 0; + CloudMark() = default; + CloudMark(const StoreInfo &info) + : bundleName(info.bundleName), deviceId(info.deviceId), storeId(info.storeName), userId(info.user), + index(info.instanceId) + { + } + bool Marshal(json &node) const override; + bool Unmarshal(const json &node) override; + std::string GetKey(); + std::string GetKey(const std::initializer_list &fields); +}; + +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_MARK_H diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/schema_meta.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/schema_meta.h index bfa6f14f..bc74d210 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/schema_meta.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/schema_meta.h @@ -92,6 +92,7 @@ public: bool Unmarshal(const json &node) override; bool IsValid() const; Database GetDataBase(const std::string &storeId); + std::vector GetStores(); }; // Table mode of device data sync time diff --git a/datamgr_service/services/distributeddataservice/framework/include/cloud/sync_event.h b/datamgr_service/services/distributeddataservice/framework/include/cloud/sync_event.h index 9396a6e8..d81b387e 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/cloud/sync_event.h +++ b/datamgr_service/services/distributeddataservice/framework/include/cloud/sync_event.h @@ -61,4 +61,4 @@ private: EventInfo info_; }; } // namespace OHOS::DistributedData -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_SYNC_EVENT_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_CLOUD_SYNC_EVENT_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/behaviour_reporter.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/behaviour_reporter.h similarity index 81% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/behaviour_reporter.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/behaviour_reporter.h index 82431c96..709d9be6 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/behaviour_reporter.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/behaviour_reporter.h @@ -22,9 +22,9 @@ namespace OHOS { namespace DistributedDataDfx { class BehaviourReporter { public: - KVSTORE_API virtual ReportStatus Report(const BehaviourMsg &msg) = 0; - KVSTORE_API virtual ReportStatus UDMFReport(const UdmfBehaviourMsg &msg) = 0; - KVSTORE_API virtual ~BehaviourReporter() {} + API_EXPORT virtual ReportStatus Report(const BehaviourMsg &msg) = 0; + API_EXPORT virtual ReportStatus UDMFReport(const UdmfBehaviourMsg &msg) = 0; + API_EXPORT virtual ~BehaviourReporter() {} }; } // namespace DistributedDataDfx } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/db_meta_callback_delegate.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/db_meta_callback_delegate.h similarity index 82% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/db_meta_callback_delegate.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/db_meta_callback_delegate.h index 8f4f6622..ca644737 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/db_meta_callback_delegate.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/db_meta_callback_delegate.h @@ -32,9 +32,9 @@ struct StoreInfo { }; class DbMetaCallbackDelegate { public: - KVSTORE_API virtual ~DbMetaCallbackDelegate() {} - KVSTORE_API virtual bool GetKvStoreDiskSize(const std::string &storeId, uint64_t &size) = 0; - KVSTORE_API virtual void GetKvStoreKeys(std::vector &entries) = 0; + API_EXPORT virtual ~DbMetaCallbackDelegate() {} + API_EXPORT virtual bool GetKvStoreDiskSize(const std::string &storeId, uint64_t &size) = 0; + API_EXPORT virtual void GetKvStoreKeys(std::vector &entries) = 0; }; } // namespace DistributedKv } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/dfx_types.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/dfx_types.h similarity index 97% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/dfx_types.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/dfx_types.h index 8fd03bb7..bb1a0e91 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/dfx_types.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/dfx_types.h @@ -176,7 +176,7 @@ struct UdmfBehaviourMsg { struct VisitStat { std::string appId; std::string interfaceName; - KVSTORE_API std::string GetKey() const + API_EXPORT std::string GetKey() const { return appId + interfaceName; } @@ -187,7 +187,7 @@ struct TrafficStat { std::string deviceId; int sendSize; int receivedSize; - KVSTORE_API std::string GetKey() const + API_EXPORT std::string GetKey() const { return appId + deviceId; } @@ -200,7 +200,7 @@ struct DbStat { int dbSize; std::shared_ptr delegate; - KVSTORE_API std::string GetKey() const + API_EXPORT std::string GetKey() const { return userId + appId + storeId; } @@ -221,7 +221,7 @@ struct ApiPerformanceStat { uint64_t costTime; uint64_t averageTime; uint64_t worstTime; - KVSTORE_API std::string GetKey() const + API_EXPORT std::string GetKey() const { return interfaceName; } diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/fault_reporter.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/fault_reporter.h similarity index 71% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/fault_reporter.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/fault_reporter.h index 07386302..a9dcdfb9 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/fault_reporter.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/fault_reporter.h @@ -22,11 +22,11 @@ namespace OHOS { namespace DistributedDataDfx { class FaultReporter { public: - KVSTORE_API virtual ReportStatus Report(const FaultMsg &msg) = 0; - KVSTORE_API virtual ReportStatus Report(const CommFaultMsg &msg) = 0; - KVSTORE_API virtual ReportStatus Report(const DBFaultMsg &ms) = 0; - KVSTORE_API virtual ReportStatus Report(const ArkDataFaultMsg &msg){ return ReportStatus::SUCCESS; }; - KVSTORE_API virtual ~FaultReporter() {} + API_EXPORT virtual ReportStatus Report(const FaultMsg &msg) = 0; + API_EXPORT virtual ReportStatus Report(const CommFaultMsg &msg) = 0; + API_EXPORT virtual ReportStatus Report(const DBFaultMsg &ms) = 0; + API_EXPORT virtual ReportStatus Report(const ArkDataFaultMsg &msg){ return ReportStatus::SUCCESS; }; + API_EXPORT virtual ~FaultReporter() {} }; } // namespace DistributedDataDfx } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/radar_reporter.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/radar_reporter.h similarity index 91% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/radar_reporter.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/radar_reporter.h index 450bcdb8..966758dd 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/radar_reporter.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/radar_reporter.h @@ -78,10 +78,10 @@ struct RadarParam { class RadarReporter { public: - KVSTORE_API RadarReporter(const char *eventName, int32_t scene, const char *bundleName, const char *funcName); - KVSTORE_API ~RadarReporter(); - KVSTORE_API RadarReporter &operator=(int32_t errCode); - KVSTORE_API static void Report(const RadarParam ¶m, const char *funcName, int32_t state = 0, + API_EXPORT RadarReporter(const char *eventName, int32_t scene, const char *bundleName, const char *funcName); + API_EXPORT ~RadarReporter(); + API_EXPORT RadarReporter &operator=(int32_t errCode); + API_EXPORT static void Report(const RadarParam ¶m, const char *funcName, int32_t state = 0, const char *eventName = EventName::CLOUD_SYNC_BEHAVIOR); private: diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/reporter.h similarity index 45% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/reporter.h index 0ecf7c93..3288980d 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/reporter.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/reporter.h @@ -16,51 +16,35 @@ #ifndef DISTRIBUTEDDATAMGR_REPORTER_H #define DISTRIBUTEDDATAMGR_REPORTER_H -#include -#include -#include "dfx_types.h" -#include "statistic_reporter.h" -#include "fault_reporter.h" #include "behaviour_reporter.h" +#include "dfx_types.h" #include "executor_pool.h" +#include "fault_reporter.h" +#include "statistic_reporter.h" namespace OHOS { namespace DistributedDataDfx { class Reporter { public: - KVSTORE_API static Reporter* GetInstance(); - KVSTORE_API FaultReporter* ServiceFault(); - KVSTORE_API FaultReporter* RuntimeFault(); - KVSTORE_API FaultReporter* DatabaseFault(); - KVSTORE_API FaultReporter* CommunicationFault(); - KVSTORE_API FaultReporter* CloudSyncFault(); - - KVSTORE_API StatisticReporter* DatabaseStatistic(); - KVSTORE_API StatisticReporter* VisitStatistic(); - KVSTORE_API StatisticReporter* TrafficStatistic(); - KVSTORE_API StatisticReporter* ApiPerformanceStatistic(); - - KVSTORE_API BehaviourReporter* BehaviourReporter(); - void SetThreadPool(std::shared_ptr executors) - { - executors_ = executors; - if (executors != nullptr) { - ServiceFault(); - RuntimeFault(); - DatabaseFault(); - CloudSyncFault(); - CommunicationFault(); - DatabaseStatistic(); - VisitStatistic(); - TrafficStatistic(); - ApiPerformanceStatistic(); - BehaviourReporter(); - } - }; + API_EXPORT virtual ~Reporter() = default; + API_EXPORT static bool RegisterReporterInstance(Reporter *reporter); + API_EXPORT static Reporter* GetInstance(); + API_EXPORT virtual FaultReporter* ServiceFault() = 0; + API_EXPORT virtual FaultReporter* RuntimeFault() = 0; + API_EXPORT virtual FaultReporter* DatabaseFault() = 0; + API_EXPORT virtual FaultReporter* CommunicationFault() = 0; + API_EXPORT virtual FaultReporter* CloudSyncFault() = 0; + API_EXPORT virtual StatisticReporter* DatabaseStatistic() = 0; + API_EXPORT virtual StatisticReporter* VisitStatistic() = 0; + API_EXPORT virtual StatisticReporter* TrafficStatistic() = 0; + API_EXPORT virtual StatisticReporter* ApiPerformanceStatistic() = 0; + API_EXPORT virtual BehaviourReporter* GetBehaviourReporter() = 0; + API_EXPORT virtual void SetThreadPool(std::shared_ptr executors) = 0; private: std::shared_ptr executors_; + static Reporter *instance_; }; -} // namespace DistributedDataDfx -} // namespace OHOS +} // namespace DistributedDataDfx +} // namespace OHOS #endif // DISTRIBUTEDDATAMGR_REPORTER_H diff --git a/datamgr_service/services/distributeddataservice/adapter/include/dfx/statistic_reporter.h b/datamgr_service/services/distributeddataservice/framework/include/dfx/statistic_reporter.h similarity index 89% rename from datamgr_service/services/distributeddataservice/adapter/include/dfx/statistic_reporter.h rename to datamgr_service/services/distributeddataservice/framework/include/dfx/statistic_reporter.h index 6667be48..a53fe572 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/dfx/statistic_reporter.h +++ b/datamgr_service/services/distributeddataservice/framework/include/dfx/statistic_reporter.h @@ -23,8 +23,8 @@ namespace DistributedDataDfx { template class StatisticReporter { public: - KVSTORE_API virtual ReportStatus Report(const T &stat) = 0; - KVSTORE_API virtual ~StatisticReporter() {} + API_EXPORT virtual ReportStatus Report(const T &stat) = 0; + API_EXPORT virtual ~StatisticReporter() {} }; } // namespace DistributedDataDfx } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/framework/include/directory/directory_manager.h b/datamgr_service/services/distributeddataservice/framework/include/directory/directory_manager.h index e2642ec8..0570bcd6 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/directory/directory_manager.h +++ b/datamgr_service/services/distributeddataservice/framework/include/directory/directory_manager.h @@ -30,11 +30,13 @@ public: uint32_t version = 0; std::string pattern; std::string metaPath; + std::string clonePath; }; API_EXPORT static DirectoryManager &GetInstance(); API_EXPORT std::string GetStorePath(const StoreMetaData &metaData, uint32_t version = INVALID_VERSION); API_EXPORT std::string GetSecretKeyPath(const StoreMetaData &metaData, uint32_t version = INVALID_VERSION); API_EXPORT std::string GetStoreBackupPath(const StoreMetaData &metaData, uint32_t version = INVALID_VERSION); + API_EXPORT std::string GetClonePath(const std::string &userId, uint32_t version = INVALID_VERSION); API_EXPORT std::string GetMetaStorePath(uint32_t version = INVALID_VERSION); API_EXPORT std::string GetMetaBackupPath(uint32_t version = INVALID_VERSION); API_EXPORT std::vector GetVersions(); @@ -48,6 +50,7 @@ private: bool autoCreate = false; uint32_t version; std::string metaPath; + std::string clonePath; std::vector path; std::vector pipes; }; diff --git a/datamgr_service/services/distributeddataservice/framework/include/error/general_error.h b/datamgr_service/services/distributeddataservice/framework/include/error/general_error.h index 1359397f..7049f353 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/error/general_error.h +++ b/datamgr_service/services/distributeddataservice/framework/include/error/general_error.h @@ -65,4 +65,4 @@ enum GeneralError : int32_t { E_BUTT, }; } -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_STORE_GENERAL_ERROR_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_STORE_GENERAL_ERROR_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/feature/feature_system.h b/datamgr_service/services/distributeddataservice/framework/include/feature/feature_system.h index ecadb34d..8f0d1424 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/feature/feature_system.h +++ b/datamgr_service/services/distributeddataservice/framework/include/feature/feature_system.h @@ -78,4 +78,4 @@ private: }; } // namespace DistributedData } -#endif // OHOS_DISTRIBUTED_DATA_FRAMEWORK_SYSTEM_SYSTEM_H +#endif // OHOS_DISTRIBUTED_DATA_FRAMEWORK_SYSTEM_SYSTEM_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/feature/static_acts.h b/datamgr_service/services/distributeddataservice/framework/include/feature/static_acts.h index 92a19772..beda3e42 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/feature/static_acts.h +++ b/datamgr_service/services/distributeddataservice/framework/include/feature/static_acts.h @@ -17,16 +17,25 @@ #define OHOS_DISTRIBUTED_DATA_FRAMEWORK_STATIC_ACTS_H #include #include + #include "error/general_error.h" +#include "executor_pool.h" #include "visibility.h" + namespace OHOS::DistributedData { class API_EXPORT StaticActs { public: + using Task = ExecutorPool::Task; virtual ~StaticActs(); virtual int32_t OnAppUninstall(const std::string &bundleName, int32_t user, int32_t index); virtual int32_t OnAppUpdate(const std::string &bundleName, int32_t user, int32_t index); virtual int32_t OnAppInstall(const std::string &bundleName, int32_t user, int32_t index); virtual int32_t OnClearAppStorage(const std::string &bundleName, int32_t user, int32_t index, int32_t tokenId); + void SetThreadPool(std::shared_ptr executors); + void Execute(Task task); + +private: + std::shared_ptr executors_; }; } // namespace OHOS::DistributedData #endif // OHOS_DISTRIBUTED_DATA_FRAMEWORK_STATIC_ACTS_H diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/device_meta_data.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/device_meta_data.h new file mode 100644 index 00000000..7df8c556 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/device_meta_data.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +#ifndef OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_METADATA_DEVICE_META_DATA_H +#define OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_METADATA_DEVICE_META_DATA_H +#include + +#include "serializable/serializable.h" +namespace OHOS::DistributedData { +class API_EXPORT DeviceMetaData final : public Serializable { +public: + API_EXPORT DeviceMetaData(); + API_EXPORT ~DeviceMetaData(); + API_EXPORT bool Marshal(json &node) const override; + API_EXPORT bool Unmarshal(const json &node) override; + API_EXPORT std::string GetKey() const; + + std::string newUuid = ""; + std::string oldUuid = ""; +}; +} // namespace OHOS::DistributedData +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_METADATA_DEVICE_META_DATA_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/meta_data_manager.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/meta_data_manager.h index 91d1d244..08a2465c 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/metadata/meta_data_manager.h +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/meta_data_manager.h @@ -52,7 +52,7 @@ public: using Bytes = std::vector; using OnComplete = std::function &)>; API_EXPORT static MetaDataManager &GetInstance(); - API_EXPORT void Initialize(std::shared_ptr metaStore, const Backup &backup, const std::string storeId); + API_EXPORT void Initialize(std::shared_ptr metaStore, const Backup &backup, const std::string &storeId); API_EXPORT void SetSyncer(const Syncer &syncer); API_EXPORT void SetCloudSyncer(const CloudSyncer &cloudSyncer); API_EXPORT bool SaveMeta(const std::string &key, const Serializable &value, bool isLocal = false); @@ -78,7 +78,7 @@ public: API_EXPORT bool Subscribe(std::shared_ptr filter, Observer observer); API_EXPORT bool Subscribe(std::string prefix, Observer observer, bool isLocal = false); API_EXPORT bool Unsubscribe(std::string filter); - API_EXPORT bool Sync(const std::vector &devices, OnComplete complete); + API_EXPORT bool Sync(const std::vector &devices, OnComplete complete, bool wait = false); private: MetaDataManager(); diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/secret_key_meta_data.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/secret_key_meta_data.h index a97b04e3..ed7537cd 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/metadata/secret_key_meta_data.h +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/secret_key_meta_data.h @@ -22,6 +22,7 @@ struct API_EXPORT SecretKeyMetaData final : public Serializable { std::vector time {}; std::vector sKey {}; int32_t storeType = 0; + int32_t area = -1; API_EXPORT SecretKeyMetaData(); API_EXPORT ~SecretKeyMetaData(); API_EXPORT bool Marshal(json &node) const override; diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data.h index e80e14b4..2d177634 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data.h +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data.h @@ -60,6 +60,7 @@ struct API_EXPORT StoreMetaData final : public Serializable { std::string account = ""; int32_t authType = 0; bool asyncDownloadAsset = false; + bool isNeedUpdateDeviceId = false; enum StoreType { STORE_KV_BEGIN = 0, @@ -86,6 +87,7 @@ struct API_EXPORT StoreMetaData final : public Serializable { API_EXPORT std::string GetBackupSecretKey() const; API_EXPORT std::string GetAutoLaunchKey() const; API_EXPORT std::string GetDebugInfoKey() const; + API_EXPORT std::string GetDfxInfoKey() const; API_EXPORT std::string GetStoreAlias() const; API_EXPORT StoreInfo GetStoreInfo() const; API_EXPORT static std::string GetKey(const std::initializer_list &fields); diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data_local.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data_local.h index 2ac940d2..717b495f 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data_local.h +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/store_meta_data_local.h @@ -69,5 +69,17 @@ struct API_EXPORT StoreMetaDataLocal final : public Serializable { private: static constexpr const char *KEY_PREFIX = "KvStoreMetaDataLocal"; }; + +struct API_EXPORT StoreDfxInfo final : public Serializable { + std::string lastOpenTime; + API_EXPORT StoreDfxInfo() = default; + API_EXPORT ~StoreDfxInfo() = default; + API_EXPORT static std::string GetPrefix(const std::initializer_list &fields); + bool Marshal(Serializable::json &node) const override; + bool Unmarshal(const Serializable::json &node) override; + +private: + static constexpr const char *KEY_PREFIX = "StoreDfxInfo"; +}; } // namespace OHOS::DistributedData #endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_METADATA_STORE_META_DATA_LOCAL_H diff --git a/datamgr_service/services/distributeddataservice/framework/include/metadata/strategy_meta_data.h b/datamgr_service/services/distributeddataservice/framework/include/metadata/strategy_meta_data.h index a3cb9275..25feaede 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/metadata/strategy_meta_data.h +++ b/datamgr_service/services/distributeddataservice/framework/include/metadata/strategy_meta_data.h @@ -29,6 +29,7 @@ struct API_EXPORT StrategyMeta final : public Serializable { API_EXPORT StrategyMeta(const std::string &devId, const std::string &userId, const std::string &bundleName, const std::string &storeId); + API_EXPORT StrategyMeta() {}; API_EXPORT ~StrategyMeta() {}; API_EXPORT bool Marshal(json &node) const override; API_EXPORT bool Unmarshal(const json &node) override; diff --git a/datamgr_service/services/distributeddataservice/adapter/include/permission/permission_validator.h b/datamgr_service/services/distributeddataservice/framework/include/network/network_delegate.h similarity index 46% rename from datamgr_service/services/distributeddataservice/adapter/include/permission/permission_validator.h rename to datamgr_service/services/distributeddataservice/framework/include/network/network_delegate.h index 322d8b00..ae7e9f94 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/permission/permission_validator.h +++ b/datamgr_service/services/distributeddataservice/framework/include/network/network_delegate.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,26 +13,35 @@ * limitations under the License. */ -#ifndef PERMISSION_VALIDATOR_H -#define PERMISSION_VALIDATOR_H -#include +#ifndef DISTRIBUTEDDATAMGR_NETWORK_DELEGATE_H +#define DISTRIBUTEDDATAMGR_NETWORK_DELEGATE_H + +#include +#include + #include "types.h" -#include "visibility.h" namespace OHOS { -namespace DistributedKv { -class PermissionValidator { +namespace DistributedData { +class NetworkDelegate { public: - API_EXPORT static PermissionValidator &GetInstance(); - // check whether the client process have enough privilege to share data with the other devices. - // tokenId: client process tokenId - API_EXPORT bool CheckSyncPermission(uint32_t tokenId); - API_EXPORT bool IsCloudConfigPermit(uint32_t tokenId); + enum NetworkType { + NONE, + CELLULAR, + WIFI, + ETHERNET, + OTHER + }; + API_EXPORT virtual ~NetworkDelegate() = default; + API_EXPORT static NetworkDelegate *GetInstance(); + API_EXPORT static bool RegisterNetworkInstance(NetworkDelegate *instance); + virtual bool IsNetworkAvailable() = 0; + virtual void RegOnNetworkChange() = 0; + virtual NetworkType GetNetworkType(bool retrieve = false) = 0; private: - static constexpr const char *DISTRIBUTED_DATASYNC = "ohos.permission.DISTRIBUTED_DATASYNC"; - static constexpr const char *CLOUD_DATA_CONFIG = "ohos.permission.CLOUDDATA_CONFIG"; + static NetworkDelegate *instance_; }; } // namespace DistributedKv } // namespace OHOS -#endif // PERMISSION_VALIDATOR_H +#endif // DISTRIBUTEDDATAMGR_NETWORK_DELEGATE_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/include/store/auto_cache.h b/datamgr_service/services/distributeddataservice/framework/include/store/auto_cache.h index cc9e6c79..1549d0f1 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/store/auto_cache.h +++ b/datamgr_service/services/distributeddataservice/framework/include/store/auto_cache.h @@ -52,21 +52,24 @@ public: API_EXPORT std::pair GetDBStore(const StoreMetaData &meta, const Watchers &watchers); - API_EXPORT Stores GetStoresIfPresent(uint32_t tokenId, const std::string &storeName = ""); + API_EXPORT Stores GetStoresIfPresent(uint32_t tokenId, const std::string &storeName = "", + const std::string &userId = ""); - API_EXPORT void CloseStore(uint32_t tokenId, const std::string &storeId = ""); + API_EXPORT void CloseStore(uint32_t tokenId, const std::string &storeId = "", const std::string &userId = ""); API_EXPORT void CloseStore(const Filter &filter); - API_EXPORT void SetObserver(uint32_t tokenId, const std::string &storeId, const Watchers &watchers); + API_EXPORT void SetObserver(uint32_t tokenId, const std::string &storeId, const Watchers &watchers, + const std::string &userId = ""); - API_EXPORT void Enable(uint32_t tokenId, const std::string &storeId = ""); + API_EXPORT void Enable(uint32_t tokenId, const std::string &storeId = "", const std::string &userId = ""); - API_EXPORT void Disable(uint32_t tokenId, const std::string &storeId); + API_EXPORT void Disable(uint32_t tokenId, const std::string &storeId, const std::string &userId = ""); private: AutoCache(); ~AutoCache(); + std::string GenerateKey(const std::string &userId, const std::string &storeId) const; void GarbageCollect(bool isForce); void StartTimer(); struct Delegate : public GeneralWatcher { diff --git a/datamgr_service/services/distributeddataservice/framework/include/store/general_store.h b/datamgr_service/services/distributeddataservice/framework/include/store/general_store.h index ee5e8504..d50b1ab8 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/store/general_store.h +++ b/datamgr_service/services/distributeddataservice/framework/include/store/general_store.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "executor_pool.h" @@ -115,8 +116,14 @@ public: int32_t maxRetryConflictTimes = 3; // default max retry 3 times when version conflict }; + enum class DistributedTableMode : int { + COLLABORATION = 0, // Save all devices data in user table + SPLIT_BY_DEVICE // Save device data in each table split by device + }; + struct StoreConfig { bool enableCloud_ = false; + std::optional tableMode; }; enum ErrOffset { @@ -191,6 +198,11 @@ public: virtual std::pair LockCloudDB() = 0; virtual int32_t UnLockCloudDB() = 0; + + virtual int32_t UpdateDBStatus() + { + return 0; + } }; } // namespace OHOS::DistributedData #endif // OHOS_DISTRIBUTED_DATA_SERVICES_FRAMEWORK_STORE_GENERAL_STORE_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/include/utils/time_utils.h b/datamgr_service/services/distributeddataservice/framework/include/utils/time_utils.h similarity index 79% rename from datamgr_service/services/distributeddataservice/adapter/include/utils/time_utils.h rename to datamgr_service/services/distributeddataservice/framework/include/utils/time_utils.h index b7ca6181..d202f7a8 100644 --- a/datamgr_service/services/distributeddataservice/adapter/include/utils/time_utils.h +++ b/datamgr_service/services/distributeddataservice/framework/include/utils/time_utils.h @@ -19,14 +19,16 @@ #include #include +#include "visibility.h" + namespace OHOS { -namespace DistributedKv { +namespace DistributedData { class TimeUtils { public: - static std::string GetCurSysTimeWithMs(); - static std::string GetTimeWithMs(time_t sec, int64_t nsec); + API_EXPORT static std::string GetCurSysTimeWithMs(); + API_EXPORT static std::string GetTimeWithMs(time_t sec, int64_t nsec); }; -} // namespace DistributedKv +} // namespace DistributedData } // namespace OHOS #endif \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/metadata/device_meta_data.cpp b/datamgr_service/services/distributeddataservice/framework/metadata/device_meta_data.cpp new file mode 100644 index 00000000..c16c6400 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/metadata/device_meta_data.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "metadata/device_meta_data.h" + +namespace OHOS { +namespace DistributedData { +static constexpr const char *KEY_PREFIX = "DeviceMeta"; +bool DeviceMetaData::Marshal(json &node) const +{ + SetValue(node[GET_NAME(newUuid)], newUuid); + SetValue(node[GET_NAME(oldUuid)], oldUuid); + return true; +} + +bool DeviceMetaData::Unmarshal(const json &node) +{ + GetValue(node, GET_NAME(newUuid), newUuid); + GetValue(node, GET_NAME(oldUuid), oldUuid); + return true; +} + +DeviceMetaData::DeviceMetaData() +{ +} + +DeviceMetaData::~DeviceMetaData() +{ +} + +std::string DeviceMetaData::GetKey() const +{ + return KEY_PREFIX; +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/metadata/meta_data_manager.cpp b/datamgr_service/services/distributeddataservice/framework/metadata/meta_data_manager.cpp index ed12b363..e7ade4b8 100644 --- a/datamgr_service/services/distributeddataservice/framework/metadata/meta_data_manager.cpp +++ b/datamgr_service/services/distributeddataservice/framework/metadata/meta_data_manager.cpp @@ -155,7 +155,7 @@ MetaDataManager::~MetaDataManager() metaObservers_.Clear(); } -void MetaDataManager::Initialize(std::shared_ptr metaStore, const Backup &backup, const std::string storeId) +void MetaDataManager::Initialize(std::shared_ptr metaStore, const Backup &backup, const std::string &storeId) { if (metaStore == nullptr) { return; @@ -167,7 +167,7 @@ void MetaDataManager::Initialize(std::shared_ptr metaStore, const Bac } metaStore_ = std::move(metaStore); backup_ = backup; - storeId_ = std::move(storeId); + storeId_ = storeId; inited_ = true; } @@ -290,7 +290,7 @@ bool MetaDataManager::DelMeta(const std::string &key, bool isLocal) return ((status == DistributedDB::DBStatus::OK) || (status == DistributedDB::DBStatus::NOT_FOUND)); } -bool MetaDataManager::Sync(const std::vector &devices, OnComplete complete) +bool MetaDataManager::Sync(const std::vector &devices, OnComplete complete, bool wait) { if (!inited_ || devices.empty()) { return false; @@ -301,7 +301,7 @@ bool MetaDataManager::Sync(const std::vector &devices, OnComplete c results.insert_or_assign(uuid, static_cast(status)); } complete(results); - }); + }, wait); if (status == DistributedDB::DBStatus::INVALID_PASSWD_OR_CORRUPTED_DB) { ZLOGE("db corrupted! status:%{public}d", status); CorruptReporter::CreateCorruptedFlag(DirectoryManager::GetInstance().GetMetaStorePath(), storeId_); diff --git a/datamgr_service/services/distributeddataservice/framework/metadata/secret_key_meta_data.cpp b/datamgr_service/services/distributeddataservice/framework/metadata/secret_key_meta_data.cpp index a28bec14..168844c4 100644 --- a/datamgr_service/services/distributeddataservice/framework/metadata/secret_key_meta_data.cpp +++ b/datamgr_service/services/distributeddataservice/framework/metadata/secret_key_meta_data.cpp @@ -31,6 +31,7 @@ bool SecretKeyMetaData::Marshal(json &node) const SetValue(node[GET_NAME(time)], time); SetValue(node[GET_NAME(sKey)], sKey); SetValue(node[GET_NAME(storeType)], storeType); + SetValue(node[GET_NAME(area)], area); return true; } @@ -39,6 +40,7 @@ bool SecretKeyMetaData::Unmarshal(const json &node) GetValue(node, GET_NAME(time), time); GetValue(node, GET_NAME(sKey), sKey); GetValue(node, GET_NAME(storeType), storeType); + GetValue(node, GET_NAME(area), area); return true; } diff --git a/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data.cpp b/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data.cpp index 8a950a14..61d67858 100644 --- a/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data.cpp +++ b/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data.cpp @@ -57,6 +57,7 @@ bool StoreMetaData::Marshal(json &node) const SetValue(node[GET_NAME(enableCloud)], enableCloud); SetValue(node[GET_NAME(cloudAutoSync)], cloudAutoSync); SetValue(node[GET_NAME(asyncDownloadAsset)], asyncDownloadAsset); + SetValue(node[GET_NAME(isNeedUpdateDeviceId)], isNeedUpdateDeviceId); // compatible with the versions which lower than VERSION_TAG_0000 SetValue(node[GET_NAME(kvStoreType)], storeType); SetValue(node[GET_NAME(deviceAccountID)], user); @@ -98,6 +99,7 @@ bool StoreMetaData::Unmarshal(const json &node) GetValue(node, GET_NAME(enableCloud), enableCloud); GetValue(node, GET_NAME(cloudAutoSync), cloudAutoSync); GetValue(node, GET_NAME(asyncDownloadAsset), asyncDownloadAsset); + GetValue(node, GET_NAME(isNeedUpdateDeviceId), isNeedUpdateDeviceId); // compatible with the older versions if (version < FIELD_CHANGED_TAG) { GetValue(node, GET_NAME(kvStoreType), storeType); @@ -138,7 +140,8 @@ bool StoreMetaData::operator==(const StoreMetaData &metaData) const Constant::NotEqual(isNeedCompress, metaData.isNeedCompress) || Constant::NotEqual(enableCloud, metaData.enableCloud) || Constant::NotEqual(cloudAutoSync, metaData.cloudAutoSync) || - Constant::NotEqual(isManualClean, metaData.isManualClean)) { + Constant::NotEqual(isManualClean, metaData.isManualClean) || + Constant::NotEqual(isNeedUpdateDeviceId, metaData.isNeedUpdateDeviceId)) { return false; } return (version == metaData.version && storeType == metaData.storeType && dataType == metaData.dataType && @@ -196,6 +199,11 @@ std::string StoreMetaData::GetDebugInfoKey() const return StoreDebugInfo::GetPrefix({ deviceId, user, "default", bundleName, storeId, std::to_string(instanceId) }); } +std::string StoreMetaData::GetDfxInfoKey() const +{ + return StoreDfxInfo::GetPrefix({ deviceId, user, "default", bundleName, storeId, std::to_string(instanceId) }); +} + std::string StoreMetaData::GetStoreAlias() const { return Anonymous::Change(storeId); diff --git a/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data_local.cpp b/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data_local.cpp index 28a040b5..19aea85c 100644 --- a/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data_local.cpp +++ b/datamgr_service/services/distributeddataservice/framework/metadata/store_meta_data_local.cpp @@ -136,5 +136,24 @@ std::string StoreMetaDataLocal::GetPrefix(const std::initializer_list &fields) +{ + auto prefix = Constant::Join(StoreDfxInfo::KEY_PREFIX, Constant::KEY_SEPARATOR, fields); + prefix.append(Constant::KEY_SEPARATOR); + return prefix; +} + +bool StoreDfxInfo::Marshal(json &node) const +{ + SetValue(node[GET_NAME(lastOpenTime)], lastOpenTime); + return true; +} + +bool StoreDfxInfo::Unmarshal(const json &node) +{ + GetValue(node, GET_NAME(lastOpenTime), lastOpenTime); + return true; +} } // namespace DistributedData } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/framework/network/network_delegate.cpp b/datamgr_service/services/distributeddataservice/framework/network/network_delegate.cpp new file mode 100644 index 00000000..37326525 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/framework/network/network_delegate.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "network/network_delegate.h" + +namespace OHOS { +namespace DistributedData { +NetworkDelegate *NetworkDelegate::instance_ = nullptr; +bool NetworkDelegate::RegisterNetworkInstance(NetworkDelegate *instance) +{ + if (instance_ != nullptr) { + return false; + } + instance_ = instance; + return true; +} + +NetworkDelegate *NetworkDelegate::GetInstance() +{ + return instance_; +} + +bool NetworkDelegate::IsNetworkAvailable() +{ + return true; +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/store/auto_cache.cpp b/datamgr_service/services/distributeddataservice/framework/store/auto_cache.cpp index ad799d6e..0b17f5bc 100644 --- a/datamgr_service/services/distributeddataservice/framework/store/auto_cache.cpp +++ b/datamgr_service/services/distributeddataservice/framework/store/auto_cache.cpp @@ -25,6 +25,7 @@ #include "utils/anonymous.h" namespace OHOS::DistributedData { using Account = AccountDelegate; +static constexpr const char *KEY_SEPARATOR = "###"; AutoCache &AutoCache::GetInstance() { static AutoCache cache; @@ -60,14 +61,23 @@ AutoCache::~AutoCache() } } +std::string AutoCache::GenerateKey(const std::string &userId, const std::string &storeId) const +{ + std::string key = ""; + if (userId.empty() || storeId.empty()) { + return key; + } + return key.append(userId).append(KEY_SEPARATOR).append(storeId); +} + std::pair AutoCache::GetDBStore(const StoreMetaData &meta, const Watchers &watchers) { Store store; + auto storeKey = GenerateKey(meta.user, meta.storeId); if (meta.storeType >= MAX_CREATOR_NUM || meta.storeType < 0 || !creators_[meta.storeType] || disables_.ContainIf(meta.tokenId, - [&meta](const std::set &stores) -> bool { return stores.count(meta.storeId) != 0; })) { - ZLOGW("storeType is invalid or store is disabled, user:%{public}s, bundleName:%{public}s, " - "storeName:%{public}s", + [&storeKey](const std::set &stores) -> bool { return stores.count(storeKey) != 0; })) { + ZLOGW("storeType is invalid or store is disabled,user:%{public}s,bundleName:%{public}s,storeName:%{public}s", meta.user.c_str(), meta.bundleName.c_str(), meta.GetStoreAlias().c_str()); return { E_ERROR, store }; } @@ -82,14 +92,13 @@ std::pair AutoCache::GetDBStore(const StoreMetaData & return { E_USER_DEACTIVATING, store }; } stores_.Compute(meta.tokenId, - [this, &meta, &watchers, &store](auto &, std::map &stores) -> bool { + [this, &meta, &watchers, &store, &storeKey](auto &, std::map &stores) -> bool { if (disableStores_.count(meta.dataDir) != 0) { - ZLOGW("store is closing, tokenId:0x%{public}x user:%{public}s" - "bundleName:%{public}s storeName:%{public}s", + ZLOGW("store is closing,tokenId:0x%{public}x,user:%{public}s,bundleName:%{public}s,storeId:%{public}s", meta.tokenId, meta.user.c_str(), meta.bundleName.c_str(), meta.GetStoreAlias().c_str()); return !stores.empty(); } - auto it = stores.find(meta.storeId); + auto it = stores.find(storeKey); if (it != stores.end()) { if (!watchers.empty()) { it->second.SetObservers(watchers); @@ -103,7 +112,7 @@ std::pair AutoCache::GetDBStore(const StoreMetaData & return !stores.empty(); } dbStore->SetExecutor(executor_); - auto result = stores.emplace(std::piecewise_construct, std::forward_as_tuple(meta.storeId), + auto result = stores.emplace(std::piecewise_construct, std::forward_as_tuple(storeKey), std::forward_as_tuple(dbStore, watchers, atoi(meta.user.c_str()), meta)); store = result.first->second; StartTimer(); @@ -117,17 +126,19 @@ AutoCache::Store AutoCache::GetStore(const StoreMetaData &meta, const Watchers & return GetDBStore(meta, watchers).second; } -AutoCache::Stores AutoCache::GetStoresIfPresent(uint32_t tokenId, const std::string &storeName) +AutoCache::Stores AutoCache::GetStoresIfPresent(uint32_t tokenId, const std::string &storeName, + const std::string &userId) { Stores stores; + auto storeKey = GenerateKey(userId, storeName); stores_.ComputeIfPresent( - tokenId, [&stores, &storeName](auto &, std::map &delegates) -> bool { - if (storeName.empty()) { + tokenId, [&stores, &storeKey](auto &, std::map &delegates) -> bool { + if (storeKey.empty()) { for (auto &[_, delegate] : delegates) { stores.push_back(delegate); } } else { - auto it = delegates.find(storeName); + auto it = delegates.find(storeKey); if (it != delegates.end()) { stores.push_back(it->second); } @@ -159,17 +170,18 @@ void AutoCache::StartTimer() ZLOGD("start timer,taskId: %{public}" PRIu64, taskId_); } -void AutoCache::CloseStore(uint32_t tokenId, const std::string &storeId) +void AutoCache::CloseStore(uint32_t tokenId, const std::string &storeId, const std::string &userId) { ZLOGD("close store start, store:%{public}s, token:%{public}u", Anonymous::Change(storeId).c_str(), tokenId); std::set storeIds; std::list closeStores; bool isScreenLocked = ScreenManager::GetInstance()->IsLocked(); + auto storeKey = GenerateKey(userId, storeId); stores_.ComputeIfPresent(tokenId, - [this, &storeId, isScreenLocked, &storeIds, &closeStores](auto &key, auto &delegates) { + [this, &storeKey, isScreenLocked, &storeIds, &closeStores](auto &, auto &delegates) { auto it = delegates.begin(); while (it != delegates.end()) { - if ((storeId == it->first || storeId.empty()) && + if ((it->first == storeKey || storeKey.empty()) && (!isScreenLocked || it->second.GetArea() != GeneralStore::EL4) && disableStores_.count(it->second.GetDataDir()) == 0) { disableStores_.insert(it->second.GetDataDir()); @@ -229,12 +241,14 @@ void AutoCache::CloseStore(const AutoCache::Filter &filter) }); } -void AutoCache::SetObserver(uint32_t tokenId, const std::string &storeId, const AutoCache::Watchers &watchers) +void AutoCache::SetObserver(uint32_t tokenId, const std::string &storeId, const AutoCache::Watchers &watchers, + const std::string &userId) { - stores_.ComputeIfPresent(tokenId, [&storeId, &watchers](auto &key, auto &stores) { + auto storeKey = GenerateKey(userId, storeId); + stores_.ComputeIfPresent(tokenId, [&storeKey, &watchers](auto &key, auto &stores) { ZLOGD("tokenId:0x%{public}x storeId:%{public}s observers:%{public}zu", key, - Anonymous::Change(storeId).c_str(), watchers.size()); - auto it = stores.find(storeId); + Anonymous::Change(storeKey).c_str(), watchers.size()); + auto it = stores.find(storeKey); if (it != stores.end()) { it->second.SetObservers(watchers); } @@ -260,21 +274,23 @@ void AutoCache::GarbageCollect(bool isForce) }); } -void AutoCache::Enable(uint32_t tokenId, const std::string &storeId) +void AutoCache::Enable(uint32_t tokenId, const std::string &storeId, const std::string &userId) { - disables_.ComputeIfPresent(tokenId, [&storeId](auto key, std::set &stores) { - stores.erase(storeId); - return !(stores.empty() || storeId.empty()); + auto storeKey = GenerateKey(userId, storeId); + disables_.ComputeIfPresent(tokenId, [&storeKey](auto key, std::set &stores) { + stores.erase(storeKey); + return !(stores.empty() || storeKey.empty()); }); } -void AutoCache::Disable(uint32_t tokenId, const std::string &storeId) +void AutoCache::Disable(uint32_t tokenId, const std::string &storeId, const std::string &userId) { - disables_.Compute(tokenId, [&storeId](auto key, std::set &stores) { - stores.insert(storeId); + auto storeKey = GenerateKey(userId, storeId); + disables_.Compute(tokenId, [&storeKey](auto key, std::set &stores) { + stores.insert(storeKey); return !stores.empty(); }); - CloseStore(tokenId, storeId); + CloseStore(tokenId, storeId, userId); } AutoCache::Delegate::Delegate(GeneralStore *delegate, const Watchers &watchers, int32_t user, const StoreMetaData &meta) diff --git a/datamgr_service/services/distributeddataservice/framework/test/BUILD.gn b/datamgr_service/services/distributeddataservice/framework/test/BUILD.gn index 4d644b36..24a5a8d6 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/framework/test/BUILD.gn @@ -14,7 +14,7 @@ import("//build/ohos_var.gni") import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### config("module_private_config") { @@ -146,13 +146,15 @@ ohos_unittest("ServiceUtilsTest") { ohos_unittest("StoreTest") { module_out_path = module_output_path - include_dirs = [ "${data_service_path}/service/test/mock" ] + include_dirs = [ + "${data_service_path}/service/test/mock", + "${data_service_path}/framework/include/screen", + ] sources = [ "${data_service_path}/framework/metadata/store_meta_data.cpp", "${data_service_path}/framework/store/auto_cache.cpp", "${data_service_path}/service/rdb/rdb_query.cpp", - "${data_service_path}/service/test/mock/general_store_mock.cpp", "store_test.cpp", ] @@ -176,9 +178,11 @@ ohos_unittest("StoreTest") { ] deps = [ + "${data_service_path}/adapter/account:distributeddata_account", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", "${data_service_path}/service/common:distributeddata_common", + "${data_service_path}/service/test/mock:distributeddata_mock_static", "//third_party/googletest:gtest_main", ] } @@ -297,6 +301,11 @@ ohos_unittest("ServiceMetaDataTest") { debug = false } + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + configs = [ ":module_private_config" ] include_dirs = [ @@ -326,7 +335,6 @@ ohos_unittest("ServiceMetaDataTest") { "${data_service_path}/service/kvdb", "//third_party/json/single_include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] external_deps = [ @@ -347,7 +355,6 @@ ohos_unittest("ServiceMetaDataTest") { deps = [ "${data_service_path}/adapter/account:distributeddata_account", - "${data_service_path}/adapter/broadcaster:distributeddata_broadcaster", "${data_service_path}/adapter/communicator:distributeddata_communicator", "${data_service_path}/adapter/utils:distributeddata_utils", "${data_service_path}/app/src/checker:distributeddata_checker", diff --git a/datamgr_service/services/distributeddataservice/framework/test/cloud_test.cpp b/datamgr_service/services/distributeddataservice/framework/test/cloud_test.cpp index 2688a1ec..1940f95f 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/cloud_test.cpp +++ b/datamgr_service/services/distributeddataservice/framework/test/cloud_test.cpp @@ -66,12 +66,17 @@ public: class ScreenManagerTest : public testing::Test { public: - static void SetUpTestCase(void){}; + static void SetUpTestCase(void); static void TearDownTestCase(void){}; void SetUp(){}; void TearDown(){}; }; +void ScreenManagerTest::SetUpTestCase() +{ + ScreenManager::GetInstance()->BindExecutor(nullptr); +} + class MockGeneralWatcher : public DistributedData::GeneralWatcher { public: int32_t OnChange(const Origin &origin, const PRIFields &primaryFields, diff --git a/datamgr_service/services/distributeddataservice/framework/test/meta_data_manager_test.cpp b/datamgr_service/services/distributeddataservice/framework/test/meta_data_manager_test.cpp index 6166af1c..a733b88c 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/meta_data_manager_test.cpp +++ b/datamgr_service/services/distributeddataservice/framework/test/meta_data_manager_test.cpp @@ -1,22 +1,22 @@ /* -* Copyright (c) 2024 Huawei Device Co., Ltd. -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #define LOG_TAG "MetaDataManagerTest" -#include #include "log_print.h" #include "metadata/meta_data_manager.h" +#include using namespace OHOS; using namespace testing::ext; using namespace OHOS::DistributedData; @@ -26,21 +26,21 @@ class MetaDataManagerTest : public testing::Test { public: static const std::string INVALID_DEVICE_ID; static const std::string EMPTY_DEVICE_ID; - static void SetUpTestCase(void){}; - static void TearDownTestCase(void){}; - void SetUp(){}; - void TearDown(){}; + static void SetUpTestCase(void) {}; + static void TearDownTestCase(void) {}; + void SetUp() {}; + void TearDown() {}; }; const std::string MetaDataManagerTest::INVALID_DEVICE_ID = "1234567890"; const std::string MetaDataManagerTest::EMPTY_DEVICE_ID = ""; /** -* @tc.name: FilterConstructorAndGetKeyTest -* @tc.desc: FilterConstructor and GetKey. -* @tc.type: FUNC -* @tc.require: -* @tc.author: MengYao -*/ + * @tc.name: FilterConstructorAndGetKeyTest + * @tc.desc: FilterConstructor and GetKey. + * @tc.type: FUNC + * @tc.require: + * @tc.author: MengYao + */ HWTEST_F(MetaDataManagerTest, FilterConstructorAndGetKeyTest, TestSize.Level1) { std::string pattern = "test"; @@ -51,12 +51,12 @@ HWTEST_F(MetaDataManagerTest, FilterConstructorAndGetKeyTest, TestSize.Level1) } /** -* @tc.name: FilterOperatorTest -* @tc.desc: FilterOperator. -* @tc.type: FUNC -* @tc.require: -* @tc.author: MengYao -*/ + * @tc.name: FilterOperatorTest + * @tc.desc: FilterOperator. + * @tc.type: FUNC + * @tc.require: + * @tc.author: MengYao + */ HWTEST_F(MetaDataManagerTest, FilterOperatorTest, TestSize.Level1) { std::string pattern = "test"; @@ -70,12 +70,12 @@ HWTEST_F(MetaDataManagerTest, FilterOperatorTest, TestSize.Level1) } /** -* @tc.name: SyncTest001 -* @tc.desc: devices is empty. -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: SyncTest001 + * @tc.desc: devices is empty. + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(MetaDataManagerTest, SyncTest001, TestSize.Level1) { std::vector devices; @@ -86,12 +86,12 @@ HWTEST_F(MetaDataManagerTest, SyncTest001, TestSize.Level1) } /** -* @tc.name: SyncTest002 -* @tc.desc: devices is invalid. -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: SyncTest002 + * @tc.desc: devices is invalid. + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(MetaDataManagerTest, SyncTest002, TestSize.Level1) { std::vector devices; diff --git a/datamgr_service/services/distributeddataservice/framework/test/meta_data_test.cpp b/datamgr_service/services/distributeddataservice/framework/test/meta_data_test.cpp index 2bf97f93..6ab4d6ce 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/meta_data_test.cpp +++ b/datamgr_service/services/distributeddataservice/framework/test/meta_data_test.cpp @@ -13,23 +13,28 @@ * limitations under the License. */ -#include "gtest/gtest.h" -#include "utils/constant.h" -#include #include "bootstrap.h" #include "kvstore_meta_manager.h" + #include "metadata/appid_meta_data.h" -#include "metadata/corrupted_meta_data.h" +#include "metadata/auto_launch_meta_data.h" +#include "metadata/appid_meta_data.h" +#include "metadata/capability_meta_data.h" #include "metadata/capability_range.h" +#include "metadata/corrupted_meta_data.h" +#include "metadata/matrix_meta_data.h" #include "metadata/meta_data.h" #include "metadata/meta_data_manager.h" #include "metadata/secret_key_meta_data.h" #include "metadata/store_meta_data.h" #include "metadata/store_meta_data_local.h" #include "metadata/strategy_meta_data.h" -#include "metadata/capability_meta_data.h" +#include "metadata/switches_meta_data.h" #include "metadata/user_meta_data.h" -#include "metadata/matrix_meta_data.h" +#include "metadata/device_meta_data.h" +#include "utils/constant.h" +#include "gtest/gtest.h" +#include using namespace testing::ext; using namespace OHOS; using namespace OHOS::DistributedKv; @@ -57,12 +62,12 @@ public: }; /** -* @tc.name: AppIDMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: AppIDMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, AppIDMetaData, TestSize.Level1) { AppIDMetaData appIdMetaData("appid", "ohos.test.demo"); @@ -92,12 +97,12 @@ HWTEST_F(ServiceMetaDataTest, AppIDMetaData, TestSize.Level1) } /** -* @tc.name: corruptedMeta -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: corruptedMeta + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, corruptedMeta, TestSize.Level1) { CorruptedMetaData corruptedMeta("appid", "ohos.test.demo", "test_store"); @@ -130,18 +135,18 @@ HWTEST_F(ServiceMetaDataTest, corruptedMeta, TestSize.Level1) } /** -* @tc.name: SecretKeyMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: SecretKeyMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, SecretKeyMetaData001, TestSize.Level1) { SecretKeyMetaData secretKeyMeta; SecretKeyMetaData secretKeyMetaData; secretKeyMeta.storeType = 1; - std::initializer_list fields = {"time", "skey"}; + std::initializer_list fields = { "time", "skey" }; std::string key = secretKeyMeta.GetKey(fields); EXPECT_EQ(key, "SecretKey###time###skey###SINGLE_KEY"); @@ -170,18 +175,18 @@ HWTEST_F(ServiceMetaDataTest, SecretKeyMetaData001, TestSize.Level1) } /** -* @tc.name: SecretKeyMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: SecretKeyMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, SecretKeyMetaData002, TestSize.Level1) { SecretKeyMetaData secretKeyMeta; SecretKeyMetaData secretKeyMetaData; secretKeyMeta.storeType = 1; - std::initializer_list fields = {"time", "skey"}; + std::initializer_list fields = { "time", "skey" }; std::string prefix = secretKeyMeta.GetPrefix(fields); EXPECT_EQ(prefix, "SecretKey###time###skey###"); @@ -210,12 +215,12 @@ HWTEST_F(ServiceMetaDataTest, SecretKeyMetaData002, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData001, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -225,7 +230,7 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData001, TestSize.Level1) EXPECT_EQ(key, "KvStoreMetaData######100###default######test_store"); std::string keylocal = storeMetaData.GetKeyLocal(); EXPECT_EQ(keylocal, "KvStoreMetaDataLocal######100###default######test_store"); - std::initializer_list fields = {"100", "appid", "test_store"}; + std::initializer_list fields = { "100", "appid", "test_store" }; std::string keyfields = storeMetaData.GetKey(fields); EXPECT_EQ(keyfields, "KvStoreMetaData###100###appid###test_store"); @@ -253,12 +258,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData001, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData002, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -291,12 +296,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData002, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData003, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -306,7 +311,7 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData003, TestSize.Level1) EXPECT_EQ(storealias, "test_store"); std::string strategykey = storeMetaData.GetStrategyKey(); EXPECT_EQ(strategykey, "StrategyMetaData######100###default######test_store"); - std::initializer_list fields = {"100", "appid", "test_store"}; + std::initializer_list fields = { "100", "appid", "test_store" }; std::string prefix = storeMetaData.GetPrefix(fields); EXPECT_EQ(prefix, "KvStoreMetaData###100###appid###test_store###"); @@ -334,12 +339,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData003, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData004, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -374,12 +379,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData004, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData005, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -418,12 +423,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData005, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData006, TestSize.Level1) { StoreMetaData storemetaData1("100", "appid", "test_store"); @@ -474,12 +479,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData006, TestSize.Level1) } /** -* @tc.name: StoreMetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StoreMetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StoreMetaData007, TestSize.Level1) { StoreMetaData storemetaData1("100", "appid", "test_store"); @@ -530,12 +535,12 @@ HWTEST_F(ServiceMetaDataTest, StoreMetaData007, TestSize.Level1) } /** -* @tc.name: GetStoreInfo -* @tc.desc: test StoreMetaData GetStoreInfo function -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: GetStoreInfo + * @tc.desc: test StoreMetaData GetStoreInfo function + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, GetStoreInfo, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -549,18 +554,18 @@ HWTEST_F(ServiceMetaDataTest, GetStoreInfo, TestSize.Level1) } /** -* @tc.name: StrategyMeta001 -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StrategyMeta001 + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StrategyMeta001, TestSize.Level1) { auto deviceId = "deviceId"; StrategyMeta strategyMeta(deviceId, "100", "ohos.test.demo", "test_store"); - std::vector local = {"local1"}; - std::vector remote = {"remote1"}; + std::vector local = { "local1" }; + std::vector remote = { "remote1" }; strategyMeta.capabilityRange.localLabel = local; strategyMeta.capabilityRange.remoteLabel = remote; strategyMeta.capabilityEnabled = true; @@ -596,18 +601,18 @@ HWTEST_F(ServiceMetaDataTest, StrategyMeta001, TestSize.Level1) } /** -* @tc.name: StrategyMeta -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: StrategyMeta + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, StrategyMeta002, TestSize.Level1) { auto deviceId = "deviceId"; StrategyMeta strategyMeta(deviceId, "100", "ohos.test.demo", "test_store"); - std::vector local = {"local1"}; - std::vector remote = {"remote1"}; + std::vector local = { "local1" }; + std::vector remote = { "remote1" }; strategyMeta.capabilityRange.localLabel = local; strategyMeta.capabilityRange.remoteLabel = remote; strategyMeta.capabilityEnabled = true; @@ -637,12 +642,12 @@ HWTEST_F(ServiceMetaDataTest, StrategyMeta002, TestSize.Level1) } /** -* @tc.name: MetaData -* @tc.desc: -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: MetaData + * @tc.desc: + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, MetaData, TestSize.Level1) { StoreMetaData storeMetaData("100", "appid", "test_store"); @@ -652,7 +657,7 @@ HWTEST_F(ServiceMetaDataTest, MetaData, TestSize.Level1) metaData.storeMetaData = storeMetaData; metaData.secretKeyMetaData = secretKeyMetaData; metaData.storeType = 1; - std::initializer_list fields = {"time", "skey"}; + std::initializer_list fields = { "time", "skey" }; std::string key = metaData.storeMetaData.GetKey(); std::string secretkey = metaData.secretKeyMetaData.GetKey(fields); @@ -676,12 +681,12 @@ HWTEST_F(ServiceMetaDataTest, MetaData, TestSize.Level1) } /** -* @tc.name: CapMetaData -* @tc.desc: test CapMetaData function -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: CapMetaData + * @tc.desc: test CapMetaData function + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, CapMetaData, TestSize.Level1) { CapMetaData capMetaData; @@ -702,12 +707,12 @@ HWTEST_F(ServiceMetaDataTest, CapMetaData, TestSize.Level1) } /** -* @tc.name: UserMetaData -* @tc.desc: test UserMetaData function -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: UserMetaData + * @tc.desc: test UserMetaData function + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, UserMetaData, TestSize.Level1) { UserMetaData userMetaData; @@ -744,17 +749,17 @@ HWTEST_F(ServiceMetaDataTest, UserMetaData, TestSize.Level1) } /** -* @tc.name: CapabilityRange -* @tc.desc: test CapabilityRange function -* @tc.type: FUNC -* @tc.require: -* @tc.author: SQL -*/ + * @tc.name: CapabilityRange + * @tc.desc: test CapabilityRange function + * @tc.type: FUNC + * @tc.require: + * @tc.author: SQL + */ HWTEST_F(ServiceMetaDataTest, CapabilityRange, TestSize.Level1) { CapabilityRange capabilityRange; - std::vector local = {"local1"}; - std::vector remote = {"remote1"}; + std::vector local = { "local1" }; + std::vector remote = { "remote1" }; capabilityRange.localLabel = local; capabilityRange.remoteLabel = remote; Serializable::json node1; @@ -769,12 +774,12 @@ HWTEST_F(ServiceMetaDataTest, CapabilityRange, TestSize.Level1) } /** -* @tc.name: MatrixMetaData -* @tc.desc: test MatrixMetaData operator!= function -* @tc.type: FUNC -* @tc.require: -* @tc.author: nhj -*/ + * @tc.name: MatrixMetaData + * @tc.desc: test MatrixMetaData operator!= function + * @tc.type: FUNC + * @tc.require: + * @tc.author: nhj + */ HWTEST_F(ServiceMetaDataTest, MatrixMetaData, TestSize.Level1) { MatrixMetaData matrixMetaData1; @@ -794,4 +799,199 @@ HWTEST_F(ServiceMetaDataTest, MatrixMetaData, TestSize.Level1) std::string key = matrixMetaData3.GetConsistentKey(); EXPECT_EQ(key, "MatrixMeta###DEVICE_ID###Consistent"); } + +/** + * @tc.name: DeviceMetaData + * @tc.desc: test DeviceMetaData function + * @tc.type: FUNC + * @tc.require: + * @tc.author: yl + */ +HWTEST_F(ServiceMetaDataTest, DeviceMetaData, TestSize.Level1) +{ + DeviceMetaData metaData; + std::string newUuid = "newuuid"; + std::string oldUuid = "olduuid"; + metaData.newUuid = newUuid; + metaData.oldUuid = oldUuid; + Serializable::json node1; + metaData.Marshal(node1); + EXPECT_EQ(node1["newUuid"], newUuid); + EXPECT_EQ(node1["oldUuid"], oldUuid); + + DeviceMetaData newMetaData; + newMetaData.Unmarshal(node1); + EXPECT_EQ(newMetaData.newUuid, newUuid); + EXPECT_EQ(newMetaData.oldUuid, oldUuid); +} + +/** +* @tc.name: InitMeta +* @tc.desc: test Init TestMeta +* @tc.type: FUNC +* @tc.require: +* @tc.author: yl +*/ +HWTEST_F(ServiceMetaDataTest, InitTestMeta, TestSize.Level1) +{ + StoreMetaData oldMeta; + oldMeta.deviceId = "mockOldUuid"; + oldMeta.user = "200"; + oldMeta.bundleName = "test_appid_001"; + oldMeta.storeId = "test_storeid_001"; + oldMeta.isEncrypt = true; + bool isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta, true); + EXPECT_TRUE(isSuccess); + StoreMetaDataLocal metaDataLocal; + isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKeyLocal(), metaDataLocal, true); + EXPECT_TRUE(isSuccess); + SwitchesMetaData switchesMetaData; + isSuccess = MetaDataManager::GetInstance().SaveMeta(SwitchesMetaData::GetPrefix({"mockOldUuid"}), + switchesMetaData, true); + EXPECT_TRUE(isSuccess); + AutoLaunchMetaData autoLaunchMetaData; + MetaDataManager::GetInstance().SaveMeta(AutoLaunchMetaData::GetPrefix({ oldMeta.deviceId, oldMeta.user, + "default", oldMeta.bundleName, "" }), autoLaunchMetaData, true); + EXPECT_TRUE(isSuccess); + MatrixMetaData matrixMeta0; + isSuccess = MetaDataManager::GetInstance().SaveMeta(MatrixMetaData::GetPrefix({"mockOldUuid"}), matrixMeta0, true); + EXPECT_TRUE(isSuccess); + + isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta); + EXPECT_TRUE(isSuccess); + MatrixMetaData matrixMeta; + isSuccess = MetaDataManager::GetInstance().SaveMeta(MatrixMetaData::GetPrefix({"mockOldUuid"}), matrixMeta); + EXPECT_TRUE(isSuccess); + UserMetaData userMeta; + isSuccess = MetaDataManager::GetInstance().SaveMeta(UserMetaRow::GetKeyFor("mockOldUuid"), userMeta); + EXPECT_TRUE(isSuccess); + CapMetaData capMetaData; + auto capKey = CapMetaRow::GetKeyFor("mockOldUuid"); + isSuccess = MetaDataManager::GetInstance().SaveMeta(std::string(capKey.begin(), capKey.end()), capMetaData); + EXPECT_TRUE(isSuccess); + StrategyMeta strategyMeta; + isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetStrategyKey(), strategyMeta); + EXPECT_TRUE(isSuccess); +} + +/** +* @tc.name: UpdateStoreMetaData +* @tc.desc: test UpdateStoreMetaData function +* @tc.type: FUNC +* @tc.require: +* @tc.author: yl +*/ +HWTEST_F(ServiceMetaDataTest, UpdateStoreMetaData, TestSize.Level1) +{ + std::string mockNewUuid = "mockNewUuid"; + std::string mockOldUuid = "mockOldUuid"; + StoreMetaData newMeta; + newMeta.deviceId = "mockNewUuid"; + newMeta.user = "200"; + newMeta.bundleName = "test_appid_001"; + newMeta.storeId = "test_storeid_001"; + KvStoreMetaManager::GetInstance().UpdateStoreMetaData(mockNewUuid, mockOldUuid); + bool isSuccess = MetaDataManager::GetInstance().LoadMeta(newMeta.GetKey(), newMeta, true); + EXPECT_TRUE(isSuccess); + EXPECT_TRUE(newMeta.isNeedUpdateDeviceId); + isSuccess = MetaDataManager::GetInstance().LoadMeta(newMeta.GetKey(), newMeta); + EXPECT_TRUE(isSuccess); + AutoLaunchMetaData autoLaunchMetaData; + isSuccess = MetaDataManager::GetInstance().LoadMeta(AutoLaunchMetaData::GetPrefix({ newMeta.deviceId, newMeta.user, + "default", newMeta.bundleName, "" }), autoLaunchMetaData, true); + EXPECT_TRUE(isSuccess); + StrategyMeta strategyMeta; + isSuccess = MetaDataManager::GetInstance().LoadMeta(newMeta.GetStrategyKey(), strategyMeta); + EXPECT_TRUE(isSuccess); +} + +/** +* @tc.name: UpdateMetaDatas +* @tc.desc: test UpdateMetaDatas function +* @tc.type: FUNC +* @tc.require: +* @tc.author: yl +*/ +HWTEST_F(ServiceMetaDataTest, UpdateMetaDatas, TestSize.Level1) +{ + std::string mockNewUuid = "mockNewUuid"; + std::string mockOldUuid = "mockOldUuid"; + StoreMetaData newMeta; + newMeta.deviceId = "mockNewUuid"; + newMeta.user = "200"; + newMeta.bundleName = "test_appid_001"; + newMeta.storeId = "test_storeid_001"; + KvStoreMetaManager::GetInstance().UpdateMetaDatas(mockNewUuid, mockOldUuid); + MatrixMetaData matrixMeta; + bool isSuccess = MetaDataManager::GetInstance().LoadMeta(MatrixMetaData::GetPrefix({ "mockNewUuid" }), + matrixMeta, true); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().LoadMeta(MatrixMetaData::GetPrefix({ "mockNewUuid" }), matrixMeta); + EXPECT_TRUE(isSuccess); + UserMetaData userMeta; + isSuccess = MetaDataManager::GetInstance().LoadMeta(MatrixMetaData::GetPrefix({ "mockNewUuid" }), userMeta); + EXPECT_TRUE(isSuccess); + CapMetaData capMetaData; + auto capKey = CapMetaRow::GetKeyFor("mockNewUuid"); + isSuccess = MetaDataManager::GetInstance().LoadMeta(std::string(capKey.begin(), capKey.end()), capMetaData); + EXPECT_TRUE(isSuccess); + SwitchesMetaData switchesMetaData; + isSuccess = MetaDataManager::GetInstance().LoadMeta(SwitchesMetaData::GetPrefix({ "mockNewUuid" }), + switchesMetaData, true); + EXPECT_TRUE(isSuccess); +} + +/** +* @tc.name: DelInitTestMeta +* @tc.desc: test Del TestMeta +* @tc.type: FUNC +* @tc.require: +* @tc.author: yl +*/ +HWTEST_F(ServiceMetaDataTest, DelTestMeta, TestSize.Level1) +{ + StoreMetaData newMeta; + newMeta.deviceId = "mockNewUuid"; + newMeta.user = "200"; + newMeta.bundleName = "test_appid_001"; + newMeta.storeId = "test_storeid_001"; + bool isSuccess = MetaDataManager::GetInstance().DelMeta(newMeta.GetKey(), true); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(newMeta.GetKeyLocal(), true); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(SwitchesMetaData::GetPrefix({ "mockNewUuid" }), true); + EXPECT_TRUE(isSuccess); + MetaDataManager::GetInstance().DelMeta(AutoLaunchMetaData::GetPrefix({ "mockNewUuid", newMeta.user, + "default", newMeta.bundleName, "" }), true); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(MatrixMetaData::GetPrefix({ "mockNewUuid" }), true); + EXPECT_TRUE(isSuccess); + + isSuccess = MetaDataManager::GetInstance().DelMeta(newMeta.GetKey()); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(MatrixMetaData::GetPrefix({"mockNewUuid"})); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(UserMetaRow::GetKeyFor("mockNewUuid")); + EXPECT_TRUE(isSuccess); + auto capKey = CapMetaRow::GetKeyFor("mockNewUuid"); + isSuccess = MetaDataManager::GetInstance().DelMeta(std::string(capKey.begin(), capKey.end())); + EXPECT_TRUE(isSuccess); + isSuccess = MetaDataManager::GetInstance().DelMeta(newMeta.GetStrategyKey()); + EXPECT_TRUE(isSuccess); +} + +/** +* @tc.name: GetKeyTest +* @tc.desc: GetKey +* @tc.type: FUNC +* @tc.require: +* @tc.author: yl +*/ +HWTEST_F(ServiceMetaDataTest, GetKey, TestSize.Level1) +{ + DeviceMetaData metaData; + std::string expectedPrefix = "DeviceMeta"; + std::string prefix = metaData.GetKey(); + EXPECT_EQ(prefix, expectedPrefix); +} } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/test/store_test.cpp b/datamgr_service/services/distributeddataservice/framework/test/store_test.cpp index 18db822c..7bc31f25 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/store_test.cpp +++ b/datamgr_service/services/distributeddataservice/framework/test/store_test.cpp @@ -25,6 +25,7 @@ #include "store/general_store.h" #include "store/general_value.h" #include "store/general_watcher.h" +#include "screen_lock_mock.h" using namespace testing::ext; using namespace OHOS::DistributedData; @@ -47,17 +48,26 @@ public: class AutoCacheTest : public testing::Test { public: - static void SetUpTestCase(void){}; + static void SetUpTestCase(void); static void TearDownTestCase(void){}; void SetUp(){}; void TearDown(){}; +protected: + static std::shared_ptr mock_; }; + +void AutoCacheTest::SetUpTestCase(void) +{ + ScreenManager::RegisterInstance(mock_); +} + +std::shared_ptr AutoCacheTest::mock_ = std::make_shared(); /** * @tc.name: SetQueryNodesTest * @tc.desc: Set and query nodes. * @tc.type: FUNC - * @tc.require: AR000F8N0 + * @tc.require: */ HWTEST_F(GeneralValueTest, SetQueryNodesTest, TestSize.Level2) { @@ -171,4 +181,83 @@ HWTEST_F(AutoCacheTest, operatorStore, TestSize.Level2) EXPECT_EQ(ret, GeneralError::E_OK); EXPECT_EQ(delegate.GetUser(), user); } + +/** +* @tc.name: GetMeta +* @tc.desc: AutoCache Delegate operator GetMeta() +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(AutoCacheTest, GetMeta, TestSize.Level2) +{ + GeneralStoreMock* store = new (std::nothrow) GeneralStoreMock(); + ASSERT_NE(store, nullptr); + AutoCache::Watchers watchers; + int32_t user = 0; + StoreMetaData meta; + meta.enableCloud = true; + AutoCache::Delegate delegate(store, watchers, user, meta); + auto newMate = delegate.GetMeta(); + ASSERT_TRUE(newMate.enableCloud); +} + +/** +* @tc.name: GetArea +* @tc.desc: AutoCache Delegate operator GetArea() +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(AutoCacheTest, GetArea, TestSize.Level2) +{ + AutoCache::Watchers watchers; + int32_t user = 0; + StoreMetaData meta; + meta.area = GeneralStore::EL1; + AutoCache::Delegate delegate(nullptr, watchers, user, meta); + auto newArea = delegate.GetArea(); + ASSERT_EQ(newArea, GeneralStore::EL1); +} + +/** +* @tc.name: GetDBStore +* @tc.desc: AutoCache GetDBStore +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(AutoCacheTest, GetDBStore, TestSize.Level2) +{ + auto creator = [](const StoreMetaData &metaData) -> GeneralStore* { + return new (std::nothrow) GeneralStoreMock(); + }; + AutoCache::GetInstance().RegCreator(DistributedRdb::RDB_DEVICE_COLLABORATION, creator); + AutoCache::Watchers watchers; + StoreMetaData meta; + meta.storeType = DistributedRdb::RDB_DEVICE_COLLABORATION; + mock_->isLocked_ = true; + meta.area = GeneralStore::EL4; + EXPECT_EQ(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL1; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL2; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL3; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL5; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + + mock_->isLocked_ = false; + meta.area = GeneralStore::EL4; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL1; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL2; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL3; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); + meta.area = GeneralStore::EL5; + EXPECT_NE(AutoCache::GetInstance().GetDBStore(meta, watchers).first, GeneralError::E_SCREEN_LOCKED); +} } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/framework/test/utils_test.cpp b/datamgr_service/services/distributeddataservice/framework/test/utils_test.cpp index 31e8ecf1..1893d0d1 100644 --- a/datamgr_service/services/distributeddataservice/framework/test/utils_test.cpp +++ b/datamgr_service/services/distributeddataservice/framework/test/utils_test.cpp @@ -259,10 +259,13 @@ HWTEST_F(ServiceUtilsTest, CorruptTest001, TestSize.Level1) HWTEST_F(ServiceUtilsTest, CorruptTest002, TestSize.Level1) { mode_t mode = S_IRWXU | S_IRWXG | S_IXOTH; // 0771 - mkdir(TEST_CORRUPT_PATH, mode); + auto ret = mkdir(TEST_CORRUPT_PATH, mode); + ASSERT_EQ(0, ret); + ASSERT_EQ(true, CorruptReporter::CreateCorruptedFlag(TEST_CORRUPT_PATH, TEST_CORRUPT_STOREID)); ASSERT_EQ(true, CorruptReporter::CreateCorruptedFlag(TEST_CORRUPT_PATH, TEST_CORRUPT_STOREID)); ASSERT_EQ(true, CorruptReporter::HasCorruptedFlag(TEST_CORRUPT_PATH, TEST_CORRUPT_STOREID)); ASSERT_EQ(true, CorruptReporter::DeleteCorruptedFlag(TEST_CORRUPT_PATH, TEST_CORRUPT_STOREID)); - rmdir(TEST_CORRUPT_PATH); + ret = rmdir(TEST_CORRUPT_PATH); + ASSERT_EQ(0, ret); } } // namespace OHOS::Test diff --git a/datamgr_service/services/distributeddataservice/framework/utils/corrupt_reporter.cpp b/datamgr_service/services/distributeddataservice/framework/utils/corrupt_reporter.cpp index 90c25a3b..405e2991 100644 --- a/datamgr_service/services/distributeddataservice/framework/utils/corrupt_reporter.cpp +++ b/datamgr_service/services/distributeddataservice/framework/utils/corrupt_reporter.cpp @@ -32,6 +32,9 @@ bool CorruptReporter::CreateCorruptedFlag(const std::string &path, const std::st return false; } std::string flagFileName = path + "/" + dbName + DB_CORRUPTED_POSTFIX; + if (access(flagFileName.c_str(), F_OK) == 0) { + return true; + } int fd = creat(flagFileName.c_str(), S_IRWXU | S_IRWXG); if (fd == -1) { ZLOGW("Create corrupted flag fail, flagFileName:%{public}s, errno:%{public}d", diff --git a/datamgr_service/services/distributeddataservice/adapter/utils/src/time_utils.cpp b/datamgr_service/services/distributeddataservice/framework/utils/time_utils.cpp similarity index 94% rename from datamgr_service/services/distributeddataservice/adapter/utils/src/time_utils.cpp rename to datamgr_service/services/distributeddataservice/framework/utils/time_utils.cpp index a5f0d9d8..e133f6f2 100644 --- a/datamgr_service/services/distributeddataservice/adapter/utils/src/time_utils.cpp +++ b/datamgr_service/services/distributeddataservice/framework/utils/time_utils.cpp @@ -14,14 +14,14 @@ */ #define LOG_TAG "TimeUtils" -#include "time_utils.h" +#include "utils/time_utils.h" #include #include #include namespace OHOS { -namespace DistributedKv { +namespace DistributedData { constexpr int MAX_TIME_BUF_LEN = 32; constexpr int MILLISECONDS_LEN = 3; constexpr int NANO_TO_MILLI = 1000000; @@ -45,5 +45,5 @@ std::string TimeUtils::GetTimeWithMs(time_t sec, int64_t nsec) oss << buffer << '.' << std::setfill('0') << std::setw(MILLISECONDS_LEN) << (nsec / NANO_TO_MILLI) % MILLI_PRE_SEC; return oss.str(); } -} // namespace DistributedKv +} // namespace DistributedData } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/rust/connect_adapter/BUILD.gn b/datamgr_service/services/distributeddataservice/rust/connect_adapter/BUILD.gn index 49162568..292dc200 100644 --- a/datamgr_service/services/distributeddataservice/rust/connect_adapter/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/rust/connect_adapter/BUILD.gn @@ -34,8 +34,14 @@ ohos_shared_library("conn_adapter") { "src/connect_wrapper.cpp", ] configs = [ ":export_connect" ] - cflags = [ "-Werror" ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags = [ + "-Werror", + "-fstack-protector-strong", + ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] external_deps = [ "ability_base:want", diff --git a/datamgr_service/services/distributeddataservice/rust/extension/BUILD.gn b/datamgr_service/services/distributeddataservice/rust/extension/BUILD.gn index d1c0fdc1..6730b990 100644 --- a/datamgr_service/services/distributeddataservice/rust/extension/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/rust/extension/BUILD.gn @@ -47,9 +47,13 @@ ohos_shared_library("opencloudextension") { cflags = [ "-Werror", "-Wno-multichar", + "-fstack-protector-strong", ] - cflags_cc = [ "-fvisibility=hidden" ] + cflags_cc = [ + "-fvisibility=hidden", + "-fstack-protector-strong", + ] configs = [ ":module_public_config" ] diff --git a/datamgr_service/services/distributeddataservice/service/BUILD.gn b/datamgr_service/services/distributeddataservice/service/BUILD.gn index 4f264fde..b07724a0 100644 --- a/datamgr_service/services/distributeddataservice/service/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/BUILD.gn @@ -42,6 +42,10 @@ config("module_public_config") { "${kv_store_distributeddb_path}/interfaces/include/", "${kv_store_distributeddb_path}/interfaces/include/relational", ] + cflags = [ + "-Wno-c99-designator", + "-fstack-protector-strong", + ] } ohos_shared_library("distributeddatasvc") { @@ -60,11 +64,17 @@ ohos_shared_library("distributeddatasvc") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fdata-sections", + "-ffunction-sections", + "-fstack-protector-strong", ] + ldflags = [ "-Wl,-z,relro,-z,now,--gc-sections" ] + cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] configs = [ ":module_public_config" ] diff --git a/datamgr_service/services/distributeddataservice/service/CMakeLists.txt b/datamgr_service/services/distributeddataservice/service/CMakeLists.txt index e3334218..c0bfb2f7 100644 --- a/datamgr_service/services/distributeddataservice/service/CMakeLists.txt +++ b/datamgr_service/services/distributeddataservice/service/CMakeLists.txt @@ -39,7 +39,6 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf/lifecycle serviceSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf/permission serviceSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf/preprocess serviceSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf/store serviceSrc) -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf/utd serviceSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/udmf serviceSrc) #aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/waterversion serviceSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/permission/src serviceSrc) @@ -93,4 +92,5 @@ target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/dumper/include) target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/network) target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/object/include) +target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/permission/include) #target_include_directories(service PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/waterversion) diff --git a/datamgr_service/services/distributeddataservice/service/backup/BUILD.gn b/datamgr_service/services/distributeddataservice/service/backup/BUILD.gn index 6baaacf3..6a492ef4 100644 --- a/datamgr_service/services/distributeddataservice/service/backup/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/backup/BUILD.gn @@ -32,6 +32,7 @@ ohos_source_set("distributeddata_backup") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ @@ -46,6 +47,7 @@ ohos_source_set("distributeddata_backup") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ @@ -56,6 +58,7 @@ ohos_source_set("distributeddata_backup") { external_deps = [ "device_manager:devicemanagersdk", "hilog:libhilog", + "json:nlohmann_json_static", "kv_store:datamgr_common", ] subsystem_name = "distributeddatamgr" diff --git a/datamgr_service/services/distributeddataservice/service/backup/src/backup_manager.cpp b/datamgr_service/services/distributeddataservice/service/backup/src/backup_manager.cpp index 204ac5b9..ddbb2319 100644 --- a/datamgr_service/services/distributeddataservice/service/backup/src/backup_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/backup/src/backup_manager.cpp @@ -134,7 +134,10 @@ void BackupManager::DoBackup(const StoreMetaData &meta) std::vector decryptKey; SecretKeyMetaData secretKey; if (MetaDataManager::GetInstance().LoadMeta(key, secretKey, true)) { - CryptoManager::GetInstance().Decrypt(secretKey.sKey, decryptKey); + CryptoManager::GetInstance().Decrypt(meta, secretKey, decryptKey); + if (secretKey.area < 0) { + MetaDataManager::GetInstance().LoadMeta(key, secretKey, true); + } } auto backupPath = DirectoryManager::GetInstance().GetStoreBackupPath(meta); std::string backupFullPath = backupPath + "/" + AUTO_BACKUP_NAME; @@ -297,10 +300,11 @@ void BackupManager::CopyFile(const std::string &oldPath, const std::string &newP bool BackupManager::GetPassWord(const StoreMetaData &meta, std::vector &password) { - std::string key = meta.GetBackupSecretKey(); SecretKeyMetaData secretKey; - MetaDataManager::GetInstance().LoadMeta(key, secretKey, true); - return CryptoManager::GetInstance().Decrypt(secretKey.sKey, password); + if (!MetaDataManager::GetInstance().LoadMeta(meta.GetBackupSecretKey(), secretKey, true)) { + return false; + } + return CryptoManager::GetInstance().Decrypt(meta, secretKey, password); } bool BackupManager::IsFileExist(const std::string &path) diff --git a/datamgr_service/services/distributeddataservice/service/bootstrap/BUILD.gn b/datamgr_service/services/distributeddataservice/service/bootstrap/BUILD.gn index ff48eaec..70b1c24c 100644 --- a/datamgr_service/services/distributeddataservice/service/bootstrap/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/bootstrap/BUILD.gn @@ -32,6 +32,7 @@ ohos_source_set("distributeddata_bootstrap") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "${data_service_path}/service/config/include" ] @@ -43,6 +44,7 @@ ohos_source_set("distributeddata_bootstrap") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ diff --git a/datamgr_service/services/distributeddataservice/service/cloud/BUILD.gn b/datamgr_service/services/distributeddataservice/service/cloud/BUILD.gn index d2f8c227..3c592276 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/cloud/BUILD.gn @@ -39,6 +39,7 @@ ohos_source_set("distributeddata_cloud") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ @@ -46,12 +47,11 @@ ohos_source_set("distributeddata_cloud") { "${data_service_path}/service/bootstrap/include", "${data_service_path}/service/kvdb", "${data_service_path}/service/matrix/include", - "${data_service_path}/service/network", "${data_service_path}/service/permission/include", "${data_service_path}/framework/include", "${data_service_path}/adapter/include/communicator", "sync_strategies", - "${data_service_path}/adapter/include/dfx", + "${data_service_path}/framework/include/dfx", ] configs = [ ":cloud_public_config" ] cflags = [ @@ -60,14 +60,19 @@ ohos_source_set("distributeddata_cloud") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] - deps = [ "${data_service_path}/service/network:distributeddata_network" ] + deps = [ + "${data_service_path}/adapter/network:distributeddata_network", + "${data_service_path}/adapter/schema_helper:distributeddata_schema_helper", + ] external_deps = [ "access_token:libtokenid_sdk", "device_manager:devicemanagersdk", "hicollie:libhicollie", + "json:nlohmann_json_static", "kv_store:datamgr_common", "kv_store:distributeddb", "relational_store:cloud_data_inner", diff --git a/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.cpp b/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.cpp index a2d43de0..88c7aa99 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.cpp +++ b/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.cpp @@ -23,6 +23,8 @@ #include "accesstoken_kit.h" #include "account/account_delegate.h" #include "checker/checker_manager.h" +#include "cloud/cloud_last_sync_info.h" +#include "cloud/cloud_mark.h" #include "cloud/cloud_server.h" #include "cloud/cloud_share_event.h" #include "cloud/make_query_event.h" @@ -30,17 +32,18 @@ #include "cloud_data_translate.h" #include "cloud_value_util.h" #include "device_manager_adapter.h" -#include "radar_reporter.h" +#include "dfx/radar_reporter.h" #include "eventcenter/event_center.h" #include "hap_token_info.h" #include "ipc_skeleton.h" #include "log_print.h" #include "metadata/meta_data_manager.h" -#include "network_adapter.h" +#include "network/network_delegate.h" #include "rdb_types.h" -#include "reporter.h" +#include "dfx/reporter.h" #include "relational_store_manager.h" #include "runtime_config.h" +#include "schema_helper/get_schema_helper.h" #include "store/auto_cache.h" #include "sync_manager.h" #include "sync_strategies/network_sync_strategy.h" @@ -48,6 +51,7 @@ #include "values_bucket.h" #include "xcollie.h" + namespace OHOS::CloudData { using namespace DistributedData; using namespace std::chrono; @@ -57,7 +61,7 @@ using namespace DistributedDataDfx; using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; using Account = AccountDelegate; using AccessTokenKit = Security::AccessToken::AccessTokenKit; -static constexpr uint32_t RESTART_SERVICE_TIME_THRESHOLD = 60; +static constexpr uint32_t RESTART_SERVICE_TIME_THRESHOLD = 120; static constexpr const char *FT_ENABLE_CLOUD = "ENABLE_CLOUD"; static constexpr const char *FT_DISABLE_CLOUD = "DISABLE_CLOUD"; static constexpr const char *FT_SWITCH_ON = "SWITCH_ON"; @@ -68,6 +72,7 @@ static constexpr const char *FT_USER_UNLOCK = "USER_UNLOCK"; static constexpr const char *FT_NETWORK_RECOVERY = "NETWORK_RECOVERY"; static constexpr const char *FT_SERVICE_INIT = "SERVICE_INIT"; static constexpr const char *FT_SYNC_TASK = "SYNC_TASK"; +static constexpr const char *CLOUD_SCHEMA = "arkdata/cloud/cloud_schema.json"; __attribute__((used)) CloudServiceImpl::Factory CloudServiceImpl::factory_; const CloudServiceImpl::SaveStrategy CloudServiceImpl::STRATEGY_SAVERS[Strategy::STRATEGY_BUTT] = { &CloudServiceImpl::SaveNetworkStrategy @@ -98,6 +103,9 @@ CloudServiceImpl::CloudServiceImpl() EventCenter::GetInstance().Subscribe(CloudEvent::CLOUD_SHARE, [this](const Event &event) { CloudShare(event); }); + EventCenter::GetInstance().Subscribe(CloudEvent::UPGRADE_SCHEMA, [this](const Event &event) { + DoSync(event); + }); MetaDataManager::GetInstance().Subscribe( Subscription::GetPrefix({ "" }), [this](const std::string &key, const std::string &value, int32_t flag) -> auto { @@ -114,7 +122,7 @@ CloudServiceImpl::CloudServiceImpl() int32_t CloudServiceImpl::EnableCloud(const std::string &id, const std::map &switches) { XCollie xcollie( - __FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY, RESTART_SERVICE_TIME_THRESHOLD); + __FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY, RESTART_SERVICE_TIME_THRESHOLD); auto tokenId = IPCSkeleton::GetCallingTokenID(); auto user = Account::GetInstance()->GetUserByToken(tokenId); auto [status, cloudInfo] = GetCloudInfo(user); @@ -152,7 +160,7 @@ void CloudServiceImpl::Report( int32_t CloudServiceImpl::DisableCloud(const std::string &id) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto tokenId = IPCSkeleton::GetCallingTokenID(); auto user = Account::GetInstance()->GetUserByToken(tokenId); ReleaseUserInfo(user, CloudSyncScene::DISABLE_CLOUD); @@ -178,7 +186,7 @@ int32_t CloudServiceImpl::DisableCloud(const std::string &id) int32_t CloudServiceImpl::ChangeAppSwitch(const std::string &id, const std::string &bundleName, int32_t appSwitch) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto tokenId = IPCSkeleton::GetCallingTokenID(); auto user = Account::GetInstance()->GetUserByToken(tokenId); CloudSyncScene scene; @@ -430,7 +438,7 @@ int32_t CloudServiceImpl::NotifyDataChange(const std::string &id, const std::str int32_t CloudServiceImpl::NotifyDataChange(const std::string &eventId, const std::string &extraData, int32_t userId) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); ExtraData exData; if (eventId != DATA_CHANGE_EVENT_ID || extraData.empty() || !exData.Unmarshall(extraData)) { ZLOGE("invalid args, eventId:%{public}s, user:%{public}d, extraData is %{public}s", eventId.c_str(), @@ -638,7 +646,7 @@ std::pair CloudServiceImpl::QueryLastSyncInfo(const s databases = schema.databases; for (const auto &database : schema.databases) { if (storeId.empty() || database.alias == storeId) { - queryKeys.push_back({ id, bundleName, database.name }); + queryKeys.push_back({ user, id, bundleName, database.name }); } } if (queryKeys.empty()) { @@ -647,26 +655,20 @@ std::pair CloudServiceImpl::QueryLastSyncInfo(const s } } - auto ret = syncManager_.QueryLastSyncInfo(queryKeys, results); - ZLOGI("code:%{public}d, accountId:%{public}s, bundleName:%{public}s, storeId:%{public}s", ret, - Anonymous::Change(id).c_str(), bundleName.c_str(), Anonymous::Change(storeId).c_str()); - if (results.empty()) { + auto [ret, lastSyncInfos] = syncManager_.QueryLastSyncInfo(queryKeys); + ZLOGI("code:%{public}d, id:%{public}s, bundleName:%{public}s, storeId:%{public}s, size:%{public}d", ret, + Anonymous::Change(id).c_str(), bundleName.c_str(), Anonymous::Change(storeId).c_str(), + static_cast(lastSyncInfos.size())); + if (lastSyncInfos.empty()) { return { ret, results }; } - for (const auto &database : databases) { - if (results.find(database.name) != results.end()) { - auto node = results.extract(database.name); - node.key() = database.alias; - results.insert(std::move(node)); - } - } - return { ret, results }; + return { ret, AssembleLastResults(databases, lastSyncInfos) }; } int32_t CloudServiceImpl::OnInitialize() { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); - NetworkAdapter::GetInstance().RegOnNetworkChange(); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); + NetworkDelegate::GetInstance()->RegOnNetworkChange(); DistributedDB::RuntimeConfig::SetCloudTranslate(std::make_shared()); Execute(GenTask(0, 0, CloudSyncScene::SERVICE_INIT, { WORK_CLOUD_INFO_UPDATE, WORK_SCHEMA_UPDATE, WORK_DO_CLOUD_SYNC, WORK_SUB })); @@ -703,7 +705,7 @@ int32_t CloudServiceImpl::OnBind(const BindInfo &info) int32_t CloudServiceImpl::OnUserChange(uint32_t code, const std::string &user, const std::string &account) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); int32_t userId = atoi(user.c_str()); ZLOGI("code:%{public}d, user:%{public}s, account:%{public}s", code, user.c_str(), Anonymous::Change(account).c_str()); @@ -729,7 +731,7 @@ int32_t CloudServiceImpl::OnUserChange(uint32_t code, const std::string &user, c int32_t CloudServiceImpl::OnReady(const std::string &device) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (device != DeviceManagerAdapter::CLOUD_DEVICE_UUID) { return SUCCESS; } @@ -738,7 +740,7 @@ int32_t CloudServiceImpl::OnReady(const std::string &device) if (users.empty()) { return SUCCESS; } - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { return NETWORK_ERROR; } for (auto user : users) { @@ -781,7 +783,7 @@ std::pair CloudServiceImpl::GetCloudInfoFromServer(int32_t u { CloudInfo cloudInfo; cloudInfo.user = userId; - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { ZLOGD("network is not available"); return { NETWORK_ERROR, cloudInfo }; } @@ -843,8 +845,18 @@ bool CloudServiceImpl::UpdateSchema(int32_t user, CloudSyncScene scene) } auto keys = cloudInfo.GetSchemaKey(); for (const auto &[bundle, key] : keys) { + HapInfo hapInfo{ .user = user, .instIndex = 0, .bundleName = bundle }; + auto appInfoOpt = cloudInfo.GetAppInfo(bundle); + if (appInfoOpt.has_value()) { + const CloudInfo::AppInfo &appInfo = appInfoOpt.value(); + hapInfo.instIndex = appInfo.instanceId; + } + SchemaMeta schemaMeta; - std::tie(status, schemaMeta) = GetAppSchemaFromServer(user, bundle); + std::tie(status, schemaMeta) = GetSchemaFromHap(hapInfo); + if (status != SUCCESS) { + std::tie(status, schemaMeta) = GetAppSchemaFromServer(user, bundle); + } if (status == NOT_SUPPORT) { ZLOGW("app not support, del cloudInfo! user:%{public}d, bundleName:%{public}s", user, bundle.c_str()); MetaDataManager::GetInstance().DelMeta(cloudInfo.GetKey(), true); @@ -856,6 +868,7 @@ bool CloudServiceImpl::UpdateSchema(int32_t user, CloudSyncScene scene) SchemaMeta oldMeta; if (MetaDataManager::GetInstance().LoadMeta(key, oldMeta, true)) { UpgradeSchemaMeta(user, oldMeta); + UpdateClearWaterMark(hapInfo, schemaMeta, oldMeta); } MetaDataManager::GetInstance().SaveMeta(key, schemaMeta, true); } @@ -865,7 +878,7 @@ bool CloudServiceImpl::UpdateSchema(int32_t user, CloudSyncScene scene) std::pair CloudServiceImpl::GetAppSchemaFromServer(int32_t user, const std::string &bundleName) { SchemaMeta schemaMeta; - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { ZLOGD("network is not available"); return { NETWORK_ERROR, schemaMeta }; } @@ -903,7 +916,7 @@ ExecutorPool::Task CloudServiceImpl::GenTask(int32_t retry, int32_t user, CloudS if (retry >= RETRY_TIMES || executor == nullptr || works.empty()) { return; } - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { ZLOGD("network is not available"); return; } @@ -953,7 +966,12 @@ std::pair CloudServiceImpl::GetSchemaMeta(int32_t userId, c return { SUCCESS, schemaMeta }; } UpgradeSchemaMeta(userId, schemaMeta); - + HapInfo hapInfo{ .user = userId, .instIndex = instanceId, .bundleName = bundleName }; + std::tie(status, schemaMeta) = GetSchemaFromHap(hapInfo); + if (status == SUCCESS) { + MetaDataManager::GetInstance().SaveMeta(schemaKey, schemaMeta, true); + return { status, schemaMeta }; + } if (!Account::GetInstance()->IsVerified(userId)) { ZLOGE("user:%{public}d is locked!", userId); return { ERROR, schemaMeta }; @@ -1013,7 +1031,92 @@ int32_t CloudServiceImpl::CloudStatic::OnAppUninstall(const std::string &bundleN int32_t CloudServiceImpl::CloudStatic::OnAppInstall(const std::string &bundleName, int32_t user, int32_t index) { - return UpdateCloudInfoFromServer(user); + ZLOGI("bundleName:%{public}s,user:%{public}d,instanceId:%{public}d", bundleName.c_str(), user, index); + auto ret = UpdateCloudInfoFromServer(user); + if (ret == E_OK) { + StoreInfo info{.bundleName = bundleName, .instanceId = index, .user = user}; + EventCenter::GetInstance().PostEvent(std::make_unique(CloudEvent::GET_SCHEMA, info)); + } + return ret; +} + +int32_t CloudServiceImpl::CloudStatic::OnAppUpdate(const std::string &bundleName, int32_t user, int32_t index) +{ + ZLOGI("bundleName:%{public}s,user:%{public}d,instanceId:%{public}d", bundleName.c_str(), user, index); + HapInfo hapInfo{ .user = user, .instIndex = index, .bundleName = bundleName }; + Execute([this, hapInfo]() { UpdateSchemaFromHap(hapInfo); }); + return SUCCESS; +} + +int32_t CloudServiceImpl::UpdateSchemaFromHap(const HapInfo &hapInfo) +{ + auto [status, cloudInfo] = GetCloudInfoFromMeta(hapInfo.user); + if (status != SUCCESS) { + return status; + } + if (!cloudInfo.Exist(hapInfo.bundleName, hapInfo.instIndex)) { + return ERROR; + } + + std::string schemaKey = CloudInfo::GetSchemaKey(hapInfo.user, hapInfo.bundleName, hapInfo.instIndex); + auto [ret, newSchemaMeta] = GetSchemaFromHap(hapInfo); + if (ret != SUCCESS) { + std::tie(ret, newSchemaMeta) = GetAppSchemaFromServer(hapInfo.user, hapInfo.bundleName); + } + if (ret != SUCCESS) { + MetaDataManager::GetInstance().DelMeta(schemaKey, true); + return ret; + } + + SchemaMeta schemaMeta; + if (MetaDataManager::GetInstance().LoadMeta(schemaKey, schemaMeta, true)) { + UpdateClearWaterMark(hapInfo, newSchemaMeta, schemaMeta); + } + MetaDataManager::GetInstance().SaveMeta(schemaKey, newSchemaMeta, true); + return SUCCESS; +} + +void CloudServiceImpl::UpdateClearWaterMark( + const HapInfo &hapInfo, const SchemaMeta &newSchemaMeta, const SchemaMeta &schemaMeta) +{ + if (newSchemaMeta.version == schemaMeta.version) { + return; + } + ZLOGI("update schemaMeta newVersion:%{public}d,oldVersion:%{public}d", newSchemaMeta.version, schemaMeta.version); + CloudMark metaData; + metaData.bundleName = hapInfo.bundleName; + metaData.userId = hapInfo.user; + metaData.index = hapInfo.instIndex; + metaData.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; + + std::unordered_map dbMap; + for (const auto &database : schemaMeta.databases) { + dbMap[database.name] = database.version; + } + + for (const auto &database : newSchemaMeta.databases) { + if (dbMap.find(database.name) != dbMap.end() && database.version != dbMap[database.name]) { + metaData.storeId = database.name; + metaData.isClearWaterMark = true; + MetaDataManager::GetInstance().SaveMeta(metaData.GetKey(), metaData, true); + ZLOGI("clear watermark, storeId:%{public}s, newVersion:%{public}d, oldVersion:%{public}d", + Anonymous::Change(metaData.storeId).c_str(), database.version, dbMap[database.name]); + } + } +} + +std::pair CloudServiceImpl::GetSchemaFromHap(const HapInfo &hapInfo) +{ + SchemaMeta schemaMeta; + AppInfo info{ .bundleName = hapInfo.bundleName, .userId = hapInfo.user, .appIndex = hapInfo.instIndex }; + auto schemas = GetSchemaHelper::GetInstance().GetSchemaFromHap(CLOUD_SCHEMA, info); + for (auto &schema : schemas) { + if (schemaMeta.Unmarshall(schema)) { + return { SUCCESS, schemaMeta }; + } + } + ZLOGD("get schema from hap failed, bundleName:%{public}s", hapInfo.bundleName.c_str()); + return { ERROR, schemaMeta }; } void CloudServiceImpl::GetSchema(const Event &event) @@ -1054,6 +1157,15 @@ void CloudServiceImpl::CloudShare(const Event &event) } } +void CloudServiceImpl::DoSync(const Event &event) +{ + auto &cloudEvent = static_cast(event); + auto &storeInfo = cloudEvent.GetStoreInfo(); + SyncManager::SyncInfo info(storeInfo.user, storeInfo.bundleName); + syncManager_.DoCloudSync(info); + return; +} + std::pair> CloudServiceImpl::PreShare( const StoreInfo &storeInfo, GenQuery &query) { @@ -1312,7 +1424,7 @@ std::string CloudServiceImpl::GetDfxFaultType(CloudSyncScene scene) int32_t CloudServiceImpl::Share(const std::string &sharingRes, const Participants &participants, Results &results) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto hapInfo = GetHapInfo(IPCSkeleton::GetCallingTokenID()); if (hapInfo.bundleName.empty()) { ZLOGE("bundleName is empty, sharingRes:%{public}s", Anonymous::Change(sharingRes).c_str()); @@ -1333,7 +1445,7 @@ int32_t CloudServiceImpl::Share(const std::string &sharingRes, const Participant int32_t CloudServiceImpl::Unshare(const std::string &sharingRes, const Participants &participants, Results &results) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto hapInfo = GetHapInfo(IPCSkeleton::GetCallingTokenID()); if (hapInfo.bundleName.empty()) { ZLOGE("bundleName is empty, sharingRes:%{public}s", Anonymous::Change(sharingRes).c_str()); @@ -1354,7 +1466,7 @@ int32_t CloudServiceImpl::Unshare(const std::string &sharingRes, const Participa int32_t CloudServiceImpl::Exit(const std::string &sharingRes, std::pair &result) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto hapInfo = GetHapInfo(IPCSkeleton::GetCallingTokenID()); if (hapInfo.bundleName.empty()) { ZLOGE("bundleName is empty, sharingRes:%{public}s", Anonymous::Change(sharingRes).c_str()); @@ -1377,7 +1489,7 @@ int32_t CloudServiceImpl::Exit(const std::string &sharingRes, std::pair &result) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto hapInfo = GetHapInfo(IPCSkeleton::GetCallingTokenID()); if (hapInfo.bundleName.empty()) { ZLOGE("bundleName is empty, invitation:%{public}s, confirmation:%{public}d", @@ -1462,7 +1574,7 @@ int32_t CloudServiceImpl::ConfirmInvitation(const std::string &invitation, int32 int32_t CloudServiceImpl::ChangeConfirmation(const std::string &sharingRes, int32_t confirmation, std::pair &result) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto hapInfo = GetHapInfo(IPCSkeleton::GetCallingTokenID()); if (hapInfo.bundleName.empty()) { ZLOGE("bundleName is empty, sharingRes:%{public}s", Anonymous::Change(sharingRes).c_str()); @@ -1578,4 +1690,19 @@ int32_t CloudServiceImpl::OnScreenUnlocked(int32_t user) syncManager_.OnScreenUnlocked(user); return E_OK; } + +QueryLastResults CloudServiceImpl::AssembleLastResults(const std::vector &databases, + const std::map &lastSyncInfos) +{ + QueryLastResults results; + for (const auto &database : databases) { + auto iter = lastSyncInfos.find(database.name); + if (iter != lastSyncInfos.end()) { + CloudSyncInfo syncInfo = { .startTime = iter->second.startTime, .finishTime = iter->second.finishTime, + .code = iter->second.code, .syncStatus = iter->second.syncStatus }; + results.insert({ database.alias, std::move(syncInfo) }); + } + } + return results; +} } // namespace OHOS::CloudData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.h b/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.h index 9b598599..fb777f13 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.h +++ b/datamgr_service/services/distributeddataservice/service/cloud/cloud_service_impl.h @@ -26,7 +26,7 @@ #include "cloud/sharing_center.h" #include "cloud/subscription.h" #include "cloud_service_stub.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" #include "feature/static_acts.h" #include "store/general_store.h" #include "sync_manager.h" @@ -35,6 +35,7 @@ namespace OHOS::CloudData { class CloudServiceImpl : public CloudServiceStub { public: + using CloudLastSyncInfo = DistributedData::CloudLastSyncInfo; using StoreMetaData = DistributedData::StoreMetaData; using StoreInfo = DistributedData::StoreInfo; CloudServiceImpl(); @@ -81,6 +82,7 @@ private: ~CloudStatic() override{}; int32_t OnAppUninstall(const std::string &bundleName, int32_t user, int32_t index) override; int32_t OnAppInstall(const std::string &bundleName, int32_t user, int32_t index) override; + int32_t OnAppUpdate(const std::string &bundleName, int32_t user, int32_t index) override; }; class Factory { public: @@ -105,6 +107,7 @@ private: ACCOUNT_STOP = 9, }; + using Database = DistributedData::Database; using CloudInfo = DistributedData::CloudInfo; using SchemaMeta = DistributedData::SchemaMeta; using Event = DistributedData::Event; @@ -147,9 +150,9 @@ private: static std::pair GetCloudInfoFromMeta(int32_t userId); static std::pair GetCloudInfoFromServer(int32_t userId); static int32_t UpdateCloudInfoFromServer(int32_t user); + static std::pair GetAppSchemaFromServer(int32_t user, const std::string &bundleName); std::pair GetSchemaMeta(int32_t userId, const std::string &bundleName, int32_t instanceId); - std::pair GetAppSchemaFromServer(int32_t user, const std::string &bundleName); void UpgradeSchemaMeta(int32_t user, const SchemaMeta &schemaMeta); std::map ExecuteStatistics( const std::string &storeId, const CloudInfo &cloudInfo, const SchemaMeta &schemaMeta); @@ -159,6 +162,7 @@ private: void GetSchema(const Event &event); void CloudShare(const Event &event); + void DoSync(const Event &event); Task GenTask(int32_t retry, int32_t user, CloudSyncScene scene, Handles handles = { WORK_SUB }); Task GenSubTask(Task task, int32_t user); @@ -183,6 +187,13 @@ private: void Report(const std::string &faultType, DistributedDataDfx::Fault errCode, const std::string &bundleName, const std::string &appendix); + static std::pair GetSchemaFromHap(const HapInfo &hapInfo); + static int32_t UpdateSchemaFromHap(const HapInfo &hapInfo); + static void UpdateClearWaterMark( + const HapInfo &hapInfo, const SchemaMeta &newSchemaMeta, const SchemaMeta &schemaMeta); + QueryLastResults AssembleLastResults(const std::vector &databases, + const std::map &lastSyncInfos); + std::shared_ptr executor_; SyncManager syncManager_; std::mutex mutex_; diff --git a/datamgr_service/services/distributeddataservice/service/cloud/cloud_types_util.cpp b/datamgr_service/services/distributeddataservice/service/cloud/cloud_types_util.cpp index f765a556..116d1b72 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/cloud_types_util.cpp +++ b/datamgr_service/services/distributeddataservice/service/cloud/cloud_types_util.cpp @@ -101,12 +101,12 @@ bool Unmarshalling(Strategy &output, MessageParcel &data) template<> bool Marshalling(const CloudSyncInfo &input, MessageParcel &data) { - return Marshal(data, input.startTime, input.finishTime, input.code); + return Marshal(data, input.startTime, input.finishTime, input.code, input.syncStatus); } template<> bool Unmarshalling(CloudSyncInfo &output, MessageParcel &data) { - return Unmarshal(data, output.startTime, output.finishTime, output.code); + return Unmarshal(data, output.startTime, output.finishTime, output.code, output.syncStatus); } } // namespace OHOS::ITypesUtil \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.cpp b/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.cpp index e914ce6a..aa290128 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.cpp @@ -24,16 +24,14 @@ #include "cloud/cloud_report.h" #include "cloud/cloud_server.h" #include "cloud/schema_meta.h" -#include "cloud/sync_event.h" #include "cloud_value_util.h" #include "device_manager_adapter.h" -#include "dfx_types.h" -#include "radar_reporter.h" +#include "dfx/dfx_types.h" +#include "dfx/reporter.h" #include "eventcenter/event_center.h" #include "log_print.h" #include "metadata/meta_data_manager.h" -#include "network_adapter.h" -#include "reporter.h" +#include "network/network_delegate.h" #include "screen/screen_manager.h" #include "sync_strategies/network_sync_strategy.h" #include "user_delegate.h" @@ -51,8 +49,8 @@ std::atomic SyncManager::genId_ = 0; constexpr int32_t SYSTEM_USER_ID = 0; static constexpr const char *FT_GET_STORE = "GET_STORE"; static constexpr const char *FT_CALLBACK = "CALLBACK"; -SyncManager::SyncInfo::SyncInfo( - int32_t user, const std::string &bundleName, const Store &store, const Tables &tables, int32_t triggerMode) +SyncManager::SyncInfo::SyncInfo(int32_t user, const std::string &bundleName, const Store &store, const Tables &tables, + int32_t triggerMode) : user_(user), bundleName_(bundleName), triggerMode_(triggerMode) { if (!store.empty()) { @@ -278,7 +276,7 @@ GeneralError SyncManager::IsValid(SyncInfo &info, CloudInfo &cloud) ZLOGD("enable:%{public}d, bundleName:%{public}s", cloud.enableCloud, info.bundleName_.c_str()); return E_CLOUD_DISABLED; } - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { info.SetError(E_NETWORK_ERROR); ZLOGD("network unavailable"); return E_NETWORK_ERROR; @@ -296,28 +294,29 @@ std::function SyncManager::GetPostEventTask(const std::vectorsecond, SyncStage::END, - E_ERROR }); + UpdateFinishSyncInfo({ cloud.user, cloud.id, schema.bundleName, "" }, syncId, E_ERROR); + Report( + { cloud.user, schema.bundleName, it == traceIds.end() ? "" : it->second, SyncStage::END, E_ERROR }); continue; } for (const auto &database : schema.databases) { if (!info.Contains(database.name)) { - UpdateFinishSyncInfo({ cloud.id, schema.bundleName, "" }, info.syncId_, E_ERROR); + UpdateFinishSyncInfo({ cloud.user, cloud.id, schema.bundleName, database.name }, syncId, E_ERROR); Report({ cloud.user, schema.bundleName, it == traceIds.end() ? "" : it->second, SyncStage::END, - E_ERROR }); + E_ERROR }); continue; } StoreInfo storeInfo = { 0, schema.bundleName, database.name, cloud.apps[schema.bundleName].instanceId, - info.user_, "", info.syncId_ }; + info.user_, "", syncId }; auto status = syncStrategy_->CheckSyncAction(storeInfo); if (status != SUCCESS) { ZLOGW("Verification strategy failed, status:%{public}d. %{public}d:%{public}s:%{public}s", status, storeInfo.user, storeInfo.bundleName.c_str(), Anonymous::Change(storeInfo.storeName).c_str()); - UpdateFinishSyncInfo({ cloud.id, schema.bundleName, "" }, info.syncId_, status); + UpdateFinishSyncInfo({ cloud.user, cloud.id, schema.bundleName, database.name }, syncId, status); Report({ cloud.user, schema.bundleName, it == traceIds.end() ? "" : it->second, SyncStage::END, status }); info.SetError(status); @@ -411,33 +410,55 @@ std::function SyncManager::GetSyncHandler(Retryer retryer) prepareTraceId.c_str()); return DoExceptionalCallback(async, details, storeInfo, prepareTraceId); } + if (!meta.enableCloud) { + ZLOGW("meta.enableCloud is false, storeId:%{public}s, prepareTraceId:%{public}s", + meta.GetStoreAlias().c_str(), prepareTraceId.c_str()); + return DoExceptionalCallback(async, details, storeInfo, prepareTraceId, E_CLOUD_DISABLED); + } ZLOGI("database:<%{public}d:%{public}s:%{public}s:%{public}s> sync start, asyncDownloadAsset?[%{public}d]", - storeInfo.user, storeInfo.bundleName.c_str(), meta.GetStoreAlias().c_str(), prepareTraceId.c_str(), - meta.asyncDownloadAsset); - RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, TRIGGER_SYNC, storeInfo.syncId, - evt.GetTriggerMode() }, "GetSyncHandler", BizState::BEGIN); - Report({ user, storeInfo.bundleName, prepareTraceId, SyncStage::START, E_OK }); + storeInfo.user, storeInfo.bundleName.c_str(), meta.GetStoreAlias().c_str(), prepareTraceId.c_str(), + meta.asyncDownloadAsset); + ReportSyncEvent(evt, BizState::BEGIN, E_OK); SyncParam syncParam = { evt.GetMode(), evt.GetWait(), evt.IsCompensation(), MODE_DEFAULT, prepareTraceId }; syncParam.asyncDownloadAsset = meta.asyncDownloadAsset; auto [status, dbCode] = store->Sync({ SyncInfo::DEFAULT_ID }, *(evt.GetQuery()), evt.AutoRetry() ? RetryCallback(storeInfo, retryer, evt.GetTriggerMode(), prepareTraceId, user) - : GetCallback(async, storeInfo, evt.GetTriggerMode(), prepareTraceId, user), syncParam); + : GetCallback(async, storeInfo, evt.GetTriggerMode(), prepareTraceId, user), + syncParam); if (status != E_OK) { if (async) { detail.code = status; async(std::move(details)); } - UpdateFinishSyncInfo({ GetAccountId(storeInfo.user), storeInfo.bundleName, "" }, storeInfo.syncId, E_ERROR); + UpdateFinishSyncInfo({ storeInfo.user, GetAccountId(storeInfo.user), storeInfo.bundleName, + storeInfo.storeName }, + storeInfo.syncId, E_ERROR); if (status != GeneralError::E_NOT_SUPPORT) { auto code = dbCode == 0 ? GenStore::CLOUD_ERR_OFFSET + status : dbCode; - RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, FINISH_SYNC, storeInfo.syncId, - evt.GetTriggerMode(), code }, "GetSyncHandler", BizState::END); - Report({ user, storeInfo.bundleName, prepareTraceId, SyncStage::END, code }); + ReportSyncEvent(evt, BizState::END, code); } } }; } +void SyncManager::ReportSyncEvent(const SyncEvent &evt, BizState bizState, int32_t code) +{ + SyncStage syncStage = SyncStage::START; + auto &storeInfo = evt.GetStoreInfo(); + if (bizState == BizState::BEGIN) { + syncStage = SyncStage::START; + RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, TRIGGER_SYNC, storeInfo.syncId, + evt.GetTriggerMode() }, + "GetSyncHandler", bizState); + } else { + syncStage = SyncStage::END; + RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, FINISH_SYNC, storeInfo.syncId, + evt.GetTriggerMode(), code }, + "GetSyncHandler", bizState); + } + Report({ evt.GetUser(), storeInfo.bundleName, evt.GetPrepareTraceId(), syncStage, code }); +} + std::function SyncManager::GetClientChangeHandler() { return [this](const Event &event) { @@ -455,8 +476,8 @@ std::function SyncManager::GetClientChangeHandler() }; } -void SyncManager::Report( - const std::string &faultType, const std::string &bundleName, int32_t errCode, const std::string &appendix) +void SyncManager::Report(const std::string &faultType, const std::string &bundleName, int32_t errCode, + const std::string &appendix) { ArkDataFaultMsg msg = { .faultType = faultType, .bundleName = bundleName, @@ -476,7 +497,7 @@ SyncManager::Retryer SyncManager::GetRetryer(int32_t times, const SyncInfo &sync } info.SetError(code); RadarReporter::Report({ info.bundleName_.c_str(), CLOUD_SYNC, FINISH_SYNC, info.syncId_, info.triggerMode_, - dbCode }, + dbCode }, "GetRetryer", BizState::END); Report({ user, info.bundleName_, prepareTraceId, SyncStage::END, dbCode == GenStore::DB_ERR_OFFSET ? 0 : dbCode }); @@ -493,12 +514,12 @@ SyncManager::Retryer SyncManager::GetRetryer(int32_t times, const SyncInfo &sync if (code == E_NO_SPACE_FOR_ASSET || code == E_RECODE_LIMIT_EXCEEDED) { info.SetError(code); RadarReporter::Report({ info.bundleName_.c_str(), CLOUD_SYNC, FINISH_SYNC, info.syncId_, info.triggerMode_, - dbCode }, + dbCode }, "GetRetryer", BizState::END); Report({ user, info.bundleName_, prepareTraceId, SyncStage::END, dbCode == GenStore::DB_ERR_OFFSET ? 0 : dbCode }); Report(FT_CALLBACK, info.bundleName_, static_cast(Fault::CSF_GS_CLOUD_SYNC), - "code=" + std::to_string(code) + ",dbCode=" + std::to_string(static_cast(dbCode))); + "code=" + std::to_string(code) + ",dbCode=" + std::to_string(static_cast(dbCode))); return true; } @@ -666,7 +687,7 @@ SyncManager::TraceIds SyncManager::GetPrepareTraceId(const SyncInfo &info, const bool SyncManager::NeedGetCloudInfo(CloudInfo &cloud) { return (!MetaDataManager::GetInstance().LoadMeta(cloud.GetKey(), cloud, true) || !cloud.enableCloud) && - NetworkAdapter::GetInstance().IsNetworkAvailable() && Account::GetInstance()->IsLoginAccount(); + NetworkDelegate::GetInstance()->IsNetworkAvailable() && Account::GetInstance()->IsLoginAccount(); } std::vector> SyncManager::GetCloudSyncInfo(const SyncInfo &info, CloudInfo &cloud) @@ -688,26 +709,27 @@ std::vector> SyncManager::GetCloudSyncInfo(const ZLOGW("save cloud info fail, user: %{public}d", cloud.user); } } - if (info.bundleName_.empty()) { - for (const auto &it : cloud.apps) { - QueryKey queryKey{ .accountId = cloud.id, .bundleName = it.first, .storeId = "" }; - cloudSyncInfos.emplace_back(std::make_tuple(queryKey, info.syncId_)); - } - } else { - QueryKey queryKey{ .accountId = cloud.id, .bundleName = info.bundleName_, .storeId = "" }; - cloudSyncInfos.emplace_back(std::make_tuple(queryKey, info.syncId_)); + auto schemaKey = CloudInfo::GetSchemaKey(cloud.user, info.bundleName_); + SchemaMeta schemaMeta; + if (!MetaDataManager::GetInstance().LoadMeta(schemaKey, schemaMeta, true)) { + ZLOGE("load schema fail, bundleName: %{public}s, user %{public}d", info.bundleName_.c_str(), info.user_); + return cloudSyncInfos; + } + auto stores = schemaMeta.GetStores(); + for (auto &storeId : stores) { + QueryKey queryKey{ cloud.user, cloud.id, info.bundleName_, std::move(storeId) }; + cloudSyncInfos.emplace_back(std::make_tuple(std::move(queryKey), info.syncId_)); } return cloudSyncInfos; } -void SyncManager::GetLastResults( - const std::string &storeId, std::map &infos, QueryLastResults &results) +std::pair SyncManager::GetLastResults(std::map &infos) { - for (auto &[key, info] : infos) { - if (info.code != -1) { - results.insert(std::pair(storeId, info)); - } + auto iter = infos.rbegin(); + if (iter != infos.rend() && iter->second.code != -1) { + return { SUCCESS, std::move(iter->second) }; } + return { E_ERROR, {} }; } bool SyncManager::NeedSaveSyncInfo(const QueryKey &queryKey) @@ -721,18 +743,30 @@ bool SyncManager::NeedSaveSyncInfo(const QueryKey &queryKey) return true; } -int32_t SyncManager::QueryLastSyncInfo(const std::vector &queryKeys, QueryLastResults &results) +std::pair> SyncManager::QueryLastSyncInfo( + const std::vector &queryKeys) { + std::map lastSyncInfoMap; for (const auto &queryKey : queryKeys) { std::string storeId = queryKey.storeId; - QueryKey key{ .accountId = queryKey.accountId, .bundleName = queryKey.bundleName, .storeId = "" }; - lastSyncInfos_.ComputeIfPresent( - key, [&storeId, &results](auto &key, std::map &vals) { - GetLastResults(storeId, vals, results); + QueryKey key{ queryKey.user, queryKey.accountId, queryKey.bundleName, queryKey.storeId }; + lastSyncInfos_.ComputeIfPresent(key, + [&storeId, &lastSyncInfoMap](auto &key, std::map &vals) { + auto [status, syncInfo] = GetLastResults(vals); + if (status == SUCCESS) { + lastSyncInfoMap.insert(std::make_pair(std::move(storeId), std::move(syncInfo))); + } return !vals.empty(); }); + if (lastSyncInfoMap.find(queryKey.storeId) != lastSyncInfoMap.end()) { + continue; + } + auto [status, syncInfo] = GetLastSyncInfoFromMeta(queryKey); + if (status == SUCCESS) { + lastSyncInfoMap.insert(std::make_pair(std::move(syncInfo.storeId), std::move(syncInfo))); + } } - return SUCCESS; + return { SUCCESS, lastSyncInfoMap }; } void SyncManager::UpdateStartSyncInfo(const std::vector> &cloudSyncInfos) @@ -742,8 +776,13 @@ void SyncManager::UpdateStartSyncInfo(const std::vector &val) { - val[id] = { .startTime = startTime }; + lastSyncInfos_.Compute(queryKey, [id = syncId, startTime](auto &key, std::map &val) { + CloudLastSyncInfo syncInfo; + syncInfo.id = key.accountId; + syncInfo.storeId = key.storeId; + syncInfo.startTime = startTime; + syncInfo.code = 0; + val[id] = std::move(syncInfo); return !val.empty(); }); } @@ -754,7 +793,7 @@ void SyncManager::UpdateFinishSyncInfo(const QueryKey &queryKey, uint64_t syncId if (!NeedSaveSyncInfo(queryKey)) { return; } - lastSyncInfos_.ComputeIfPresent(queryKey, [syncId, code](auto &key, std::map &val) { + lastSyncInfos_.ComputeIfPresent(queryKey, [syncId, code](auto &key, std::map &val) { auto now = duration_cast(system_clock::now().time_since_epoch()).count(); for (auto iter = val.begin(); iter != val.end();) { bool isExpired = ((now - iter->second.startTime) >= EXPIRATION_TIME) && iter->second.code == -1; @@ -763,7 +802,9 @@ void SyncManager::UpdateFinishSyncInfo(const QueryKey &queryKey, uint64_t syncId } else if (iter->first == syncId) { iter->second.finishTime = duration_cast(system_clock::now().time_since_epoch()).count(); iter->second.code = code; - iter++; + iter->second.syncStatus = SyncStatus::FINISHED; + SaveLastSyncInfo(key, std::move(iter->second)); + iter = val.erase(iter); } else { iter++; } @@ -791,7 +832,7 @@ std::function SyncManager::GetCallback(const Gen int32_t dbCode = (result.begin()->second.dbCode == GenStore::DB_ERR_OFFSET) ? 0 : result.begin()->second.dbCode; RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, FINISH_SYNC, storeInfo.syncId, triggerMode, - dbCode, result.begin()->second.changeCount }, + dbCode, result.begin()->second.changeCount }, "GetCallback", BizState::END); Report({ user, storeInfo.bundleName, prepareTraceId, SyncStage::END, dbCode }); if (dbCode != 0) { @@ -803,14 +844,8 @@ std::function SyncManager::GetCallback(const Gen ZLOGD("account id is empty"); return; } - QueryKey queryKey{ - .accountId = id, - .bundleName = storeInfo.bundleName, - .storeId = "" - }; - int32_t code = result.begin()->second.code; - UpdateFinishSyncInfo(queryKey, storeInfo.syncId, code); + UpdateFinishSyncInfo({ user, id, storeInfo.bundleName, storeInfo.storeName }, storeInfo.syncId, code); }; } @@ -846,15 +881,15 @@ std::vector SyncManager::GetSchemaMeta(const CloudInfo &cloud, const } void SyncManager::DoExceptionalCallback(const GenAsync &async, GenDetails &details, const StoreInfo &storeInfo, - const std::string &prepareTraceId) + const std::string &prepareTraceId, int32_t code) { if (async) { - details[SyncInfo::DEFAULT_ID].code = E_ERROR; + details[SyncInfo::DEFAULT_ID].code = code; async(details); } - QueryKey queryKey{ GetAccountId(storeInfo.user), storeInfo.bundleName, "" }; - UpdateFinishSyncInfo(queryKey, storeInfo.syncId, E_ERROR); - Report({ storeInfo.user, storeInfo.bundleName, prepareTraceId, SyncStage::END, E_ERROR }); + QueryKey queryKey{ storeInfo.user, GetAccountId(storeInfo.user), storeInfo.bundleName, storeInfo.storeName }; + UpdateFinishSyncInfo(queryKey, storeInfo.syncId, code); + Report({ storeInfo.user, storeInfo.bundleName, prepareTraceId, SyncStage::END, code }); } bool SyncManager::InitDefaultUser(int32_t &user) @@ -881,11 +916,12 @@ std::function SyncManager::Retr int32_t code = details.begin()->second.code; int32_t dbCode = details.begin()->second.dbCode; if (details.begin()->second.progress == GenProgress::SYNC_FINISH) { - QueryKey queryKey{ GetAccountId(storeInfo.user), storeInfo.bundleName, "" }; + auto user = storeInfo.user; + QueryKey queryKey{ user, GetAccountId(user), storeInfo.bundleName, storeInfo.storeName }; UpdateFinishSyncInfo(queryKey, storeInfo.syncId, code); if (code == E_OK) { RadarReporter::Report({ storeInfo.bundleName.c_str(), CLOUD_SYNC, FINISH_SYNC, storeInfo.syncId, - triggerMode, code, details.begin()->second.changeCount }, + triggerMode, code, details.begin()->second.changeCount }, "RetryCallback", BizState::END); Report({ user, storeInfo.bundleName, prepareTraceId, SyncStage::END, dbCode == GenStore::DB_ERR_OFFSET ? 0 : dbCode }); @@ -920,7 +956,7 @@ std::pair SyncManager::GetMetaData(const StoreInfo &storeIn if (!MetaDataManager::GetInstance().LoadMeta(meta.GetKeyLocal(), localMetaData, true) || !localMetaData.isPublic || !MetaDataManager::GetInstance().LoadMeta(meta.GetKey(), meta, true)) { ZLOGE("failed, no store meta. bundleName:%{public}s, storeId:%{public}s", meta.bundleName.c_str(), - meta.GetStoreAlias().c_str()); + meta.GetStoreAlias().c_str()); return { false, meta }; } } @@ -965,4 +1001,31 @@ void SyncManager::AddCompensateSync(const StoreMetaData &meta) return true; }); } + +std::pair SyncManager::GetLastSyncInfoFromMeta(const QueryKey &queryKey) +{ + CloudLastSyncInfo lastSyncInfo; + if (!MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(queryKey.user, queryKey.bundleName, + queryKey.storeId), + lastSyncInfo, true)) { + ZLOGE("load last sync info fail, bundleName: %{public}s, user:%{public}d", queryKey.bundleName.c_str(), + queryKey.user); + return { E_ERROR, lastSyncInfo }; + } + if (queryKey.accountId != lastSyncInfo.id || + (!queryKey.storeId.empty() && queryKey.storeId != lastSyncInfo.storeId)) { + return { E_ERROR, lastSyncInfo }; + } + return { SUCCESS, std::move(lastSyncInfo) }; +} + +void SyncManager::SaveLastSyncInfo(const QueryKey &queryKey, CloudLastSyncInfo &&info) +{ + if (!MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(queryKey.user, queryKey.bundleName, + queryKey.storeId), + info, true)) { + ZLOGE("save cloud last info fail, bundleName: %{public}s, user:%{public}d", queryKey.bundleName.c_str(), + queryKey.user); + } +} } // namespace OHOS::CloudData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.h b/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.h index e250bd6c..6214a046 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.h +++ b/datamgr_service/services/distributeddataservice/service/cloud/sync_manager.h @@ -18,9 +18,12 @@ #include "cloud/cloud_event.h" #include "cloud/cloud_info.h" +#include "cloud/cloud_last_sync_info.h" #include "cloud/sync_strategy.h" #include "cloud_types.h" +#include "cloud/sync_event.h" #include "concurrent_map.h" +#include "dfx/radar_reporter.h" #include "eventcenter/event.h" #include "executor_pool.h" #include "metadata/store_meta_data_local.h" @@ -32,6 +35,7 @@ namespace OHOS::CloudData { class SyncManager { public: + using CloudLastSyncInfo = DistributedData::CloudLastSyncInfo; using GenAsync = DistributedData::GenAsync; using GenStore = DistributedData::GeneralStore; using GenQuery = DistributedData::GenQuery; @@ -94,7 +98,8 @@ public: int32_t Bind(std::shared_ptr executor); int32_t DoCloudSync(SyncInfo syncInfo); int32_t StopCloudSync(int32_t user = 0); - int32_t QueryLastSyncInfo(const std::vector &queryKeys, QueryLastResults &results); + std::pair> QueryLastSyncInfo( + const std::vector &queryKeys); void Report(const ReportParam &reportParam); void OnScreenUnlocked(int32_t user); void CleanCompensateSync(int32_t userId); @@ -145,13 +150,12 @@ private: const StoreInfo &storeInfo, int32_t triggerMode, const std::string &prepareTraceId, int32_t user); std::function GetPostEventTask(const std::vector &schemas, CloudInfo &cloud, SyncInfo &info, bool retry, const TraceIds &traceIds); - void DoExceptionalCallback( - const GenAsync &async, GenDetails &details, const StoreInfo &storeInfo, const std::string &prepareTraceId); + void DoExceptionalCallback(const GenAsync &async, GenDetails &details, const StoreInfo &storeInfo, + const std::string &prepareTraceId, int32_t code = GeneralError::E_ERROR); bool InitDefaultUser(int32_t &user); std::function RetryCallback(const StoreInfo &storeInfo, Retryer retryer, int32_t triggerMode, const std::string &prepareTraceId, int32_t user); - static void GetLastResults( - const std::string &storeId, std::map &infos, QueryLastResults &results); + static std::pair GetLastResults(std::map &infos); void BatchUpdateFinishState(const std::vector> &cloudSyncInfos, int32_t code); bool NeedSaveSyncInfo(const QueryKey &queryKey); std::function GetLockChangeHandler(); @@ -161,13 +165,16 @@ private: void AddCompensateSync(const StoreMetaData &meta); static void Report( const std::string &faultType, const std::string &bundleName, int32_t errCode, const std::string &appendix); + void ReportSyncEvent(const DistributedData::SyncEvent &evt, DistributedDataDfx::BizState bizState, int32_t code); + std::pair GetLastSyncInfoFromMeta(const QueryKey &queryKey); + static void SaveLastSyncInfo(const QueryKey &queryKey, CloudLastSyncInfo &&info); static std::atomic genId_; std::shared_ptr executor_; ConcurrentMap actives_; ConcurrentMap activeInfos_; std::shared_ptr syncStrategy_; - ConcurrentMap> lastSyncInfos_; + ConcurrentMap> lastSyncInfos_; std::set kvApps_; ConcurrentMap>> compensateSyncInfos_; }; diff --git a/datamgr_service/services/distributeddataservice/service/cloud/sync_strategies/network_sync_strategy.cpp b/datamgr_service/services/distributeddataservice/service/cloud/sync_strategies/network_sync_strategy.cpp index 9af8d769..ae1cb2a4 100644 --- a/datamgr_service/services/distributeddataservice/service/cloud/sync_strategies/network_sync_strategy.cpp +++ b/datamgr_service/services/distributeddataservice/service/cloud/sync_strategies/network_sync_strategy.cpp @@ -18,7 +18,7 @@ #include "network_sync_strategy.h" #include "device_manager_adapter.h" -#include "network_adapter.h" +#include "network/network_delegate.h" #include "error/general_error.h" #include "log_print.h" #include "metadata/meta_data_manager.h" @@ -53,9 +53,12 @@ NetworkSyncStrategy::~NetworkSyncStrategy() int32_t NetworkSyncStrategy::CheckSyncAction(const StoreInfo &storeInfo) { - if (!NetworkAdapter::GetInstance().IsNetworkAvailable()) { + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { return E_NETWORK_ERROR; } + if (storeInfo.user == 0) { + return E_OK; + } if (storeInfo.user != user_) { strategies_.Clear(); user_ = storeInfo.user; @@ -103,15 +106,15 @@ std::string NetworkSyncStrategy::GetKey(int32_t user, const std::string &bundleN bool NetworkSyncStrategy::Check(uint32_t strategy) { - auto networkType = NetworkAdapter::GetInstance().GetNetworkType(); - if (networkType == NetworkAdapter::NONE) { - networkType = NetworkAdapter::GetInstance().GetNetworkType(true); + auto networkType = NetworkDelegate::GetInstance()->GetNetworkType(); + if (networkType == NetworkDelegate::NONE) { + networkType = NetworkDelegate::GetInstance()->GetNetworkType(true); } switch (networkType) { - case NetworkAdapter::WIFI: - case NetworkAdapter::ETHERNET: + case NetworkDelegate::WIFI: + case NetworkDelegate::ETHERNET: return (strategy & WIFI) == WIFI; - case NetworkAdapter::CELLULAR: + case NetworkDelegate::CELLULAR: return (strategy & CELLULAR) == CELLULAR; default: ZLOGD("verification failed! strategy:%{public}d, networkType:%{public}d", strategy, networkType); diff --git a/datamgr_service/services/distributeddataservice/service/common/BUILD.gn b/datamgr_service/services/distributeddataservice/service/common/BUILD.gn index 3a8e1990..7a493bd1 100644 --- a/datamgr_service/services/distributeddataservice/service/common/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/common/BUILD.gn @@ -31,6 +31,7 @@ ohos_source_set("distributeddata_common") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "include" ] @@ -40,6 +41,7 @@ ohos_source_set("distributeddata_common") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] diff --git a/datamgr_service/services/distributeddataservice/service/config/BUILD.gn b/datamgr_service/services/distributeddataservice/service/config/BUILD.gn index a2ea71cf..9349f811 100644 --- a/datamgr_service/services/distributeddataservice/service/config/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/config/BUILD.gn @@ -39,6 +39,7 @@ ohos_source_set("distributeddata_config") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ @@ -52,6 +53,7 @@ ohos_source_set("distributeddata_config") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] diff --git a/datamgr_service/services/distributeddataservice/service/config/include/config_factory.h b/datamgr_service/services/distributeddataservice/service/config/include/config_factory.h index 727dcec2..bce10b04 100644 --- a/datamgr_service/services/distributeddataservice/service/config/include/config_factory.h +++ b/datamgr_service/services/distributeddataservice/service/config/include/config_factory.h @@ -44,4 +44,4 @@ private: }; } // namespace DistributedData } // namespace OHOS -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_CONFIG_CONFIG_FACTORY_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_CONFIG_CONFIG_FACTORY_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/config/include/model/global_config.h b/datamgr_service/services/distributeddataservice/service/config/include/model/global_config.h index 8bb7b9c6..91f5a237 100644 --- a/datamgr_service/services/distributeddataservice/service/config/include/model/global_config.h +++ b/datamgr_service/services/distributeddataservice/service/config/include/model/global_config.h @@ -46,4 +46,4 @@ public: }; } // namespace DistributedData } // namespace OHOS -#endif // OHOS_DISTRIBUTED_DATA_SERVICES_CONFIG_MODEL_GLOBAL_CONFIG_H +#endif // OHOS_DISTRIBUTED_DATA_SERVICES_CONFIG_MODEL_GLOBAL_CONFIG_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/config/src/model/directory_config.cpp b/datamgr_service/services/distributeddataservice/service/config/src/model/directory_config.cpp index 5ef254e7..9f01b1e7 100644 --- a/datamgr_service/services/distributeddataservice/service/config/src/model/directory_config.cpp +++ b/datamgr_service/services/distributeddataservice/service/config/src/model/directory_config.cpp @@ -22,6 +22,7 @@ bool DirectoryConfig::DirectoryStrategy::Marshal(json &node) const SetValue(node[GET_NAME(pattern)], pattern); SetValue(node[GET_NAME(metaPath)], metaPath); SetValue(node[GET_NAME(autoCreate)], autoCreate); + SetValue(node[GET_NAME(clonePath)], clonePath); return true; } @@ -31,6 +32,7 @@ bool DirectoryConfig::DirectoryStrategy::Unmarshal(const json &node) GetValue(node, GET_NAME(pattern), pattern); GetValue(node, GET_NAME(metaPath), metaPath); GetValue(node, GET_NAME(autoCreate), autoCreate); + GetValue(node, GET_NAME(clonePath), clonePath); return true; } diff --git a/datamgr_service/services/distributeddataservice/service/crypto/BUILD.gn b/datamgr_service/services/distributeddataservice/service/crypto/BUILD.gn index 2b8003be..18b166a9 100644 --- a/datamgr_service/services/distributeddataservice/service/crypto/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/crypto/BUILD.gn @@ -32,6 +32,7 @@ ohos_source_set("distributeddata_crypto") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "include" ] @@ -42,6 +43,7 @@ ohos_source_set("distributeddata_crypto") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] configs = [ ":crypto_public_config" ] public_configs = [ ":crypto_public_config" ] diff --git a/datamgr_service/services/distributeddataservice/service/crypto/include/crypto_manager.h b/datamgr_service/services/distributeddataservice/service/crypto/include/crypto_manager.h index 36c4df71..dca7f96d 100644 --- a/datamgr_service/services/distributeddataservice/service/crypto/include/crypto_manager.h +++ b/datamgr_service/services/distributeddataservice/service/crypto/include/crypto_manager.h @@ -15,26 +15,60 @@ #ifndef OHOS_DISTRIBUTED_DATA_SERVICES_SERVICE_CRYPTO_CRYPTO_MANAGER_H #define OHOS_DISTRIBUTED_DATA_SERVICES_SERVICE_CRYPTO_CRYPTO_MANAGER_H -#include #include +#include #include +#include "metadata/secret_key_meta_data.h" +#include "metadata/store_meta_data.h" #include "visibility.h" namespace OHOS::DistributedData { -enum RootKeys { - ROOT_KEY, - CLONE_KEY, -}; +static constexpr int32_t DEFAULT_ENCRYPTION_LEVEL = 1; +static constexpr const char *DEFAULT_USER = "0"; class API_EXPORT CryptoManager { public: + enum SecretKeyType { + LOCAL_SECRET_KEY, + CLONE_SECRET_KEY, + }; + struct ParamConfig { + std::vector nonce; + uint32_t purpose; + uint32_t storageLevel; + std::string userId; + }; + struct EncryptParams { + std::vector keyAlias; + std::vector nonce; + }; + enum Area : int32_t { + EL0, + EL1, + EL2, + EL3, + EL4, + EL5 + }; static CryptoManager &GetInstance(); int32_t GenerateRootKey(); int32_t CheckRootKey(); std::vector Encrypt(const std::vector &key); - std::vector EncryptCloneKey(const std::vector &key); - bool Decrypt(std::vector &source, std::vector &key); - bool DecryptCloneKey(std::vector &source, std::vector &key); - bool ImportCloneKey(std::vector &key, std::vector &iv); + std::vector Encrypt(const std::vector &key, const EncryptParams &encryptParams); + std::vector Encrypt(const std::vector &key, int32_t area, const std::string &userId); + std::vector Encrypt(const std::vector &key, + int32_t area, const std::string &userId, const EncryptParams &encryptParams + ); + bool Decrypt(std::vector &source, std::vector &key, const EncryptParams &encryptParams); + bool Decrypt(std::vector &source, std::vector &key, int32_t area, const std::string &userId); + bool Decrypt(std::vector &source, std::vector &key, + int32_t area, const std::string &userId, const EncryptParams &encryptParams + ); + bool ImportKey(const std::vector &key, const std::vector &keyAlias); + bool DeleteKey(const std::vector &keyAlias); + bool UpdateSecretKey(const StoreMetaData &meta, const std::vector &password, + SecretKeyType secretKeyType = LOCAL_SECRET_KEY); + bool Decrypt(const StoreMetaData &meta, SecretKeyMetaData &secretKeyMeta, std::vector &key, + SecretKeyType secretKeyType = LOCAL_SECRET_KEY); enum ErrCode : int32_t { SUCCESS, @@ -43,16 +77,24 @@ public: }; private: static constexpr const char *ROOT_KEY_ALIAS = "distributed_db_root_key"; - static constexpr const char *BACKUP_KEY_ALIAS = "distributed_db_backup_key"; static constexpr const char *HKS_BLOB_TYPE_NONCE = "Z5s0Bo571KoqwIi6"; static constexpr const char *HKS_BLOB_TYPE_AAD = "distributeddata"; static constexpr int KEY_SIZE = 32; static constexpr int AES_256_NONCE_SIZE = 32; static constexpr int HOURS_PER_YEAR = (24 * 365); - std::vector EncryptInner(const std::vector &key, const RootKeys type); - bool DecryptInner(std::vector &source, std::vector &key, const RootKeys type); + int32_t GenerateRootKey(uint32_t storageLevel, const std::string &userId); + int32_t CheckRootKey(uint32_t storageLevel, const std::string &userId); + uint32_t GetStorageLevel(int32_t area); + int32_t PrepareRootKey(uint32_t storageLevel, const std::string &userId); + std::vector EncryptInner(const std::vector &key, const SecretKeyType type, int32_t area, + const std::string &userId); + bool DecryptInner(std::vector &source, std::vector &key, int32_t area, + const std::string &userId, std::vector &keyAlias, std::vector &nonce); CryptoManager(); + std::vector vecRootKeyAlias_{}; + std::vector vecNonce_{}; + std::vector vecAad_{}; ~CryptoManager(); std::mutex mutex_; }; diff --git a/datamgr_service/services/distributeddataservice/service/crypto/src/crypto_manager.cpp b/datamgr_service/services/distributeddataservice/service/crypto/src/crypto_manager.cpp index 0f1369b7..d36f34cb 100644 --- a/datamgr_service/services/distributeddataservice/service/crypto/src/crypto_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/crypto/src/crypto_manager.cpp @@ -16,29 +16,24 @@ #include "crypto_manager.h" #include +#include #include "hks_api.h" #include "hks_param.h" #include "log_print.h" +#include "metadata/meta_data_manager.h" #include "securec.h" namespace OHOS::DistributedData { -std::vector backupNonce_{}; -std::vector vecAad_{}; -std::vector vecCloneKeyAlias_{}; -std::vector vecNonce_{}; -std::vector vecRootKeyAlias_{}; +using system_clock = std::chrono::system_clock; CryptoManager::CryptoManager() { vecRootKeyAlias_ = std::vector(ROOT_KEY_ALIAS, ROOT_KEY_ALIAS + strlen(ROOT_KEY_ALIAS)); - vecCloneKeyAlias_ = std::vector(BACKUP_KEY_ALIAS, BACKUP_KEY_ALIAS + strlen(BACKUP_KEY_ALIAS)); vecNonce_ = std::vector(HKS_BLOB_TYPE_NONCE, HKS_BLOB_TYPE_NONCE + strlen(HKS_BLOB_TYPE_NONCE)); vecAad_ = std::vector(HKS_BLOB_TYPE_AAD, HKS_BLOB_TYPE_AAD + strlen(HKS_BLOB_TYPE_AAD)); } -CryptoManager::~CryptoManager() -{ -} +CryptoManager::~CryptoManager() {} CryptoManager &CryptoManager::GetInstance() { @@ -52,28 +47,22 @@ struct HksParam aes256Param[] = { { .tag = HKS_TAG_DIGEST, .uint32Param = HKS_DIGEST_NONE }, { .tag = HKS_TAG_BLOCK_MODE, .uint32Param = HKS_MODE_GCM }, { .tag = HKS_TAG_PADDING, .uint32Param = HKS_PADDING_NONE }, - { .tag = HKS_TAG_AUTH_STORAGE_LEVEL, .uint32Param = HKS_AUTH_STORAGE_LEVEL_DE }, }; -bool AddParams(struct HksParamSet *params, RootKeys type, HksKeyPurpose purpose) +bool AddHksParams(HksParamSet *params, const CryptoManager::ParamConfig ¶mConfig, + const std::vector &vecAad) { - struct HksBlob blobNonce; - struct HksBlob keyName; - if (type == RootKeys::ROOT_KEY) { - blobNonce = { uint32_t(vecNonce_.size()), vecNonce_.data() }; - keyName = { uint32_t(vecRootKeyAlias_.size()), vecRootKeyAlias_.data() }; - } else if (type == RootKeys::CLONE_KEY) { - blobNonce = { uint32_t(backupNonce_.size()), backupNonce_.data() }; - keyName = { uint32_t(vecCloneKeyAlias_.size()), vecCloneKeyAlias_.data() }; - } else { - return false; + struct HksBlob blobAad = { uint32_t(vecAad.size()), const_cast(vecAad.data()) }; + struct HksBlob blobNonce = { uint32_t(paramConfig.nonce.size()), const_cast(paramConfig.nonce.data()) }; + + std::vector hksParam; + hksParam.push_back({ .tag = HKS_TAG_PURPOSE, .uint32Param = paramConfig.purpose }); + hksParam.push_back(HksParam{ .tag = HKS_TAG_NONCE, .blob = blobNonce }); + hksParam.push_back(HksParam{ .tag = HKS_TAG_ASSOCIATED_DATA, .blob = blobAad }); + hksParam.push_back({ .tag = HKS_TAG_AUTH_STORAGE_LEVEL, .uint32Param = paramConfig.storageLevel }); + if (paramConfig.storageLevel > HKS_AUTH_STORAGE_LEVEL_DE) { + hksParam.push_back(HksParam{ .tag = HKS_TAG_SPECIFIC_USER_ID, .int32Param = atoi(paramConfig.userId.c_str()) }); } - struct HksBlob blobAad = { uint32_t(vecAad_.size()), vecAad_.data() }; - struct HksParam hksParam[] = { - { .tag = HKS_TAG_PURPOSE, .uint32Param = purpose }, - { .tag = HKS_TAG_NONCE, .blob = blobNonce }, - { .tag = HKS_TAG_ASSOCIATED_DATA, .blob = blobAad }, - }; auto ret = HksAddParams(params, aes256Param, sizeof(aes256Param) / sizeof(aes256Param[0])); if (ret != HKS_SUCCESS) { @@ -81,7 +70,7 @@ bool AddParams(struct HksParamSet *params, RootKeys type, HksKeyPurpose purpose) HksFreeParamSet(¶ms); return false; } - ret = HksAddParams(params, hksParam, sizeof(hksParam) / sizeof(hksParam[0])); + ret = HksAddParams(params, hksParam.data(), hksParam.size()); if (ret != HKS_SUCCESS) { ZLOGE("HksAddParams failed with error %{public}d", ret); HksFreeParamSet(¶ms); @@ -90,26 +79,28 @@ bool AddParams(struct HksParamSet *params, RootKeys type, HksKeyPurpose purpose) return true; } -int32_t GetRootKeyParams(HksParamSet *¶ms) +int32_t GetRootKeyParams(HksParamSet *¶ms, uint32_t storageLevel, const std::string &userId) { - ZLOGI("GetRootKeyParams."); int32_t ret = HksInitParamSet(¶ms); if (ret != HKS_SUCCESS) { ZLOGE("HksInitParamSet() failed with error %{public}d", ret); return ret; } - struct HksParam hksParam[] = { + std::vector hksParam = { { .tag = HKS_TAG_ALGORITHM, .uint32Param = HKS_ALG_AES }, { .tag = HKS_TAG_KEY_SIZE, .uint32Param = HKS_AES_KEY_SIZE_256 }, { .tag = HKS_TAG_PURPOSE, .uint32Param = HKS_KEY_PURPOSE_ENCRYPT | HKS_KEY_PURPOSE_DECRYPT }, { .tag = HKS_TAG_DIGEST, .uint32Param = 0 }, { .tag = HKS_TAG_PADDING, .uint32Param = HKS_PADDING_NONE }, { .tag = HKS_TAG_BLOCK_MODE, .uint32Param = HKS_MODE_GCM }, - { .tag = HKS_TAG_AUTH_STORAGE_LEVEL, .uint32Param = HKS_AUTH_STORAGE_LEVEL_DE }, + { .tag = HKS_TAG_AUTH_STORAGE_LEVEL, .uint32Param = storageLevel }, }; + if (storageLevel > HKS_AUTH_STORAGE_LEVEL_DE) { + hksParam.emplace_back(HksParam{ .tag = HKS_TAG_SPECIFIC_USER_ID, .int32Param = std::stoi(userId) }); + } - ret = HksAddParams(params, hksParam, sizeof(hksParam) / sizeof(hksParam[0])); + ret = HksAddParams(params, hksParam.data(), hksParam.size()); if (ret != HKS_SUCCESS) { ZLOGE("HksAddParams failed with error %{public}d", ret); HksFreeParamSet(¶ms); @@ -126,32 +117,43 @@ int32_t GetRootKeyParams(HksParamSet *¶ms) int32_t CryptoManager::GenerateRootKey() { - ZLOGI("GenerateRootKey."); + return GenerateRootKey(HKS_AUTH_STORAGE_LEVEL_DE, DEFAULT_USER) == HKS_SUCCESS ? ErrCode::SUCCESS : ErrCode::ERROR; +} + +int32_t CryptoManager::GenerateRootKey(uint32_t storageLevel, const std::string &userId) +{ + ZLOGI("GenerateRootKey, storageLevel=%{public}u, userId=%{public}s", storageLevel, userId.c_str()); struct HksParamSet *params = nullptr; - int32_t ret = GetRootKeyParams(params); - if (ret != HKS_SUCCESS) { - ZLOGE("GetRootKeyParams failed with error %{public}d", ret); + int32_t ret = GetRootKeyParams(params, storageLevel, userId); + if (ret != HKS_SUCCESS || params == nullptr) { + ZLOGE("GetRootKeyParams failed with error %{public}d, storageLevel:%{public}u, userId:%{public}s", ret, + storageLevel, userId.c_str()); return ErrCode::ERROR; } struct HksBlob rootKeyName = { uint32_t(vecRootKeyAlias_.size()), vecRootKeyAlias_.data() }; ret = HksGenerateKey(&rootKeyName, params, nullptr); HksFreeParamSet(¶ms); if (ret == HKS_SUCCESS) { - ZLOGI("GenerateRootKey Succeed."); + ZLOGI("GenerateRootKey Succeed. storageLevel:%{public}u, userId:%{public}s", storageLevel, userId.c_str()); return ErrCode::SUCCESS; } - - ZLOGE("HksGenerateKey failed with error %{public}d", ret); + ZLOGE("HksGenerateKey failed with error %{public}d, storageLevel:%{public}u, userId:%{public}s", ret, storageLevel, + userId.c_str()); return ErrCode::ERROR; } int32_t CryptoManager::CheckRootKey() { ZLOGI("CheckRootKey."); + return CheckRootKey(HKS_AUTH_STORAGE_LEVEL_DE, DEFAULT_USER); +} + +int32_t CryptoManager::CheckRootKey(uint32_t storageLevel, const std::string &userId) +{ struct HksParamSet *params = nullptr; - int32_t ret = GetRootKeyParams(params); + int32_t ret = GetRootKeyParams(params, storageLevel, userId); if (ret != HKS_SUCCESS) { - ZLOGE("GetRootKeyParams failed with error %{public}d", ret); + ZLOGE("GetRootKeyParams failed with error %{public}d, storageLevel: %{public}u", ret, storageLevel); return ErrCode::ERROR; } @@ -161,32 +163,77 @@ int32_t CryptoManager::CheckRootKey() if (ret == HKS_SUCCESS) { return ErrCode::SUCCESS; } - ZLOGE("HksKeyExist failed with error %{public}d", ret); + ZLOGE("HksKeyExist failed with error %{public}d, storageLevel: %{public}u", ret, storageLevel); if (ret == HKS_ERROR_NOT_EXIST) { return ErrCode::NOT_EXIST; } return ErrCode::ERROR; } +uint32_t CryptoManager::GetStorageLevel(int32_t area) +{ + if (area >= EL4 && area <= EL5) { + return HKS_AUTH_STORAGE_LEVEL_ECE; + } + if (area >= EL2 && area <= EL3) { + return HKS_AUTH_STORAGE_LEVEL_CE; + } + return HKS_AUTH_STORAGE_LEVEL_DE; +} + +int32_t CryptoManager::PrepareRootKey(uint32_t storageLevel, const std::string &userId) +{ + if (storageLevel == HKS_AUTH_STORAGE_LEVEL_DE) { + return ErrCode::SUCCESS; + } + auto status = CheckRootKey(storageLevel, userId); + if (status == ErrCode::SUCCESS) { + return ErrCode::SUCCESS; + } + if (status == ErrCode::NOT_EXIST && GenerateRootKey(storageLevel, userId) == ErrCode::SUCCESS) { + ZLOGI("GenerateRootKey success."); + return ErrCode::SUCCESS; + } + ZLOGW("GenerateRootKey failed, storageLevel:%{public}u, userId:%{public}s, status:%{public}d", storageLevel, + userId.c_str(), status); + return status; +} +std::vector CryptoManager::Encrypt(const std::vector &key, int32_t area, const std::string &userId) +{ + EncryptParams encryptParams = { .keyAlias = vecRootKeyAlias_, .nonce = vecNonce_ }; + return Encrypt(key, area, userId, encryptParams); +} + std::vector CryptoManager::Encrypt(const std::vector &key) { - return EncryptInner(key, RootKeys::ROOT_KEY); + EncryptParams encryptParams = { .keyAlias = vecRootKeyAlias_, .nonce = vecNonce_ }; + return Encrypt(key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER, encryptParams); } -std::vector CryptoManager::EncryptCloneKey(const std::vector &key) +std::vector CryptoManager::Encrypt(const std::vector &key, const EncryptParams &encryptParams) { - return EncryptInner(key, RootKeys::CLONE_KEY); + return Encrypt(key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER, encryptParams); } -std::vector CryptoManager::EncryptInner(const std::vector &key, const RootKeys type) +std::vector CryptoManager::Encrypt(const std::vector &key, int32_t area, const std::string &userId, + const EncryptParams &encryptParams) { + uint32_t storageLevel = GetStorageLevel(area); + if (PrepareRootKey(storageLevel, userId) != ErrCode::SUCCESS) { + return {}; + } + struct HksParamSet *params = nullptr; int32_t ret = HksInitParamSet(¶ms); if (ret != HKS_SUCCESS) { ZLOGE("HksInitParamSet() failed with error %{public}d", ret); return {}; } - if (!AddParams(params, type, HKS_KEY_PURPOSE_ENCRYPT)) { + ParamConfig paramConfig = { .nonce = encryptParams.nonce, + .purpose = HKS_KEY_PURPOSE_ENCRYPT, + .storageLevel = storageLevel, + .userId = userId }; + if (!AddHksParams(params, paramConfig, vecAad_)) { return {}; } ret = HksBuildParamSet(¶ms); @@ -198,12 +245,8 @@ std::vector CryptoManager::EncryptInner(const std::vector &key uint8_t cipherBuf[256] = { 0 }; struct HksBlob cipherText = { sizeof(cipherBuf), cipherBuf }; - struct HksBlob keyName; - if (type == RootKeys::ROOT_KEY) { - keyName = { uint32_t(vecRootKeyAlias_.size()), vecRootKeyAlias_.data() }; - } else if (type == RootKeys::CLONE_KEY) { - keyName = { uint32_t(vecCloneKeyAlias_.size()), vecCloneKeyAlias_.data() }; - } + struct HksBlob keyName = { uint32_t(encryptParams.keyAlias.size()), + const_cast(encryptParams.keyAlias.data()) }; struct HksBlob plainKey = { uint32_t(key.size()), const_cast(key.data()) }; ret = HksEncrypt(&keyName, params, &plainKey, &cipherText); (void)HksFreeParamSet(¶ms); @@ -217,18 +260,65 @@ std::vector CryptoManager::EncryptInner(const std::vector &key return encryptedKey; } -bool CryptoManager::Decrypt(std::vector &source, std::vector &key) +bool CryptoManager::UpdateSecretKey(const StoreMetaData &meta, const std::vector &key, + SecretKeyType secretKeyType) { - return DecryptInner(source, key, RootKeys::ROOT_KEY); + if (!meta.isEncrypt) { + return false; + } + SecretKeyMetaData secretKey; + EncryptParams encryptParams = { .keyAlias = vecRootKeyAlias_, .nonce = vecNonce_ }; + secretKey.storeType = meta.storeType; + secretKey.area = meta.area; + secretKey.sKey = Encrypt(key, meta.area, meta.user, encryptParams); + auto time = system_clock::to_time_t(system_clock::now()); + secretKey.time = { reinterpret_cast(&time), reinterpret_cast(&time) + sizeof(time) }; + if (secretKeyType == LOCAL_SECRET_KEY) { + return MetaDataManager::GetInstance().SaveMeta(meta.GetSecretKey(), secretKey, true); + } else { + return MetaDataManager::GetInstance().SaveMeta(meta.GetCloneSecretKey(), secretKey, true); + } } -bool CryptoManager::DecryptCloneKey(std::vector &source, std::vector &key) +bool CryptoManager::Decrypt(const StoreMetaData &meta, SecretKeyMetaData &secretKeyMeta, std::vector &key, + SecretKeyType secretKeyType) { - return DecryptInner(source, key, RootKeys::CLONE_KEY); + EncryptParams encryptParams = { .keyAlias = vecRootKeyAlias_, .nonce = vecNonce_ }; + if (secretKeyMeta.area < 0) { + ZLOGI("Decrypt old secret key"); + if (Decrypt(secretKeyMeta.sKey, key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER, encryptParams)) { + StoreMetaData metaData; + if (MetaDataManager::GetInstance().LoadMeta(meta.GetKey(), metaData, true)) { + ZLOGI("Upgrade secret key"); + UpdateSecretKey(metaData, key, secretKeyType); + } + return true; + } + } else { + return Decrypt(secretKeyMeta.sKey, key, secretKeyMeta.area, meta.user, encryptParams); + } + return false; } -bool CryptoManager::DecryptInner(std::vector &source, std::vector &key, RootKeys type) +bool CryptoManager::Decrypt(std::vector &source, std::vector &key, int32_t area, + const std::string &userId) { + EncryptParams encryptParams = { .keyAlias = vecRootKeyAlias_, .nonce = vecNonce_ }; + return Decrypt(source, key, area, userId, encryptParams); +} + +bool CryptoManager::Decrypt(std::vector &source, std::vector &key, const EncryptParams &encryptParams) +{ + return Decrypt(source, key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER, encryptParams); +} + +bool CryptoManager::Decrypt(std::vector &source, std::vector &key, int32_t area, + const std::string &userId, const EncryptParams &encryptParams) +{ + uint32_t storageLevel = GetStorageLevel(area); + if (PrepareRootKey(storageLevel, userId) != ErrCode::SUCCESS) { + return false; + } struct HksParamSet *params = nullptr; int32_t ret = HksInitParamSet(¶ms); if (ret != HKS_SUCCESS) { @@ -236,8 +326,12 @@ bool CryptoManager::DecryptInner(std::vector &source, std::vector &source, std::vector(encryptParams.keyAlias.data()) }; + ret = HksDecrypt(&keyName, params, &encryptedKeyBlob, &plainKeyBlob); (void)HksFreeParamSet(¶ms); if (ret != HKS_SUCCESS) { @@ -276,9 +367,10 @@ bool BuildImportKeyParams(struct HksParamSet *¶ms) return false; } struct HksParam purposeParam[] = { - {.tag = HKS_TAG_IS_KEY_ALIAS, .boolParam = true}, - {.tag = HKS_TAG_KEY_GENERATE_TYPE, .uint32Param = HKS_KEY_GENERATE_TYPE_DEFAULT}, - {.tag = HKS_TAG_PURPOSE, .uint32Param = HKS_KEY_PURPOSE_ENCRYPT | HKS_KEY_PURPOSE_DECRYPT}, + { .tag = HKS_TAG_IS_KEY_ALIAS, .boolParam = true }, + { .tag = HKS_TAG_KEY_GENERATE_TYPE, .uint32Param = HKS_KEY_GENERATE_TYPE_DEFAULT }, + { .tag = HKS_TAG_PURPOSE, .uint32Param = HKS_KEY_PURPOSE_ENCRYPT | HKS_KEY_PURPOSE_DECRYPT }, + { .tag = HKS_TAG_AUTH_STORAGE_LEVEL, .uint32Param = HKS_AUTH_STORAGE_LEVEL_DE }, }; ret = HksAddParams(params, aes256Param, sizeof(aes256Param) / sizeof(aes256Param[0])); if (ret != HKS_SUCCESS) { @@ -301,20 +393,17 @@ bool BuildImportKeyParams(struct HksParamSet *¶ms) return true; } -bool CryptoManager::ImportCloneKey(std::vector &key, std::vector &iv) +bool CryptoManager::ImportKey(const std::vector &key, const std::vector &keyAlias) { std::lock_guard lock(mutex_); - ZLOGI("ImportCloneKey enter."); - backupNonce_ = std::vector(iv.begin(), iv.end()); - struct HksBlob hksKey = { static_cast(key.size()), key.data()}; + struct HksBlob hksKey = { static_cast(key.size()), const_cast(key.data()) }; struct HksParamSet *params = nullptr; if (!BuildImportKeyParams(params)) { - ZLOGE("Build import key params failed."); return false; } - struct HksBlob backupKeyName = { uint32_t(vecCloneKeyAlias_.size()), vecCloneKeyAlias_.data() }; - int32_t ret = HksImportKey(&backupKeyName, params, &hksKey); + struct HksBlob keyName = { uint32_t(keyAlias.size()), const_cast(keyAlias.data()) }; + int32_t ret = HksImportKey(&keyName, params, &hksKey); if (ret != HKS_SUCCESS) { ZLOGE("Import key failed: %{public}d.", ret); HksFreeParamSet(¶ms); @@ -323,4 +412,19 @@ bool CryptoManager::ImportCloneKey(std::vector &key, std::vector &keyAlias) +{ + struct HksBlob keyName = { uint32_t(keyAlias.size()), const_cast(keyAlias.data()) }; + struct HksParamSet *params = nullptr; + if (!BuildImportKeyParams(params)) { + return false; + } + int32_t ret = HksDeleteKey(&keyName, params); + if (ret != HKS_SUCCESS) { + HksFreeParamSet(¶ms); + return false; + } + return true; +} } // namespace OHOS::DistributedData \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/data_share/BUILD.gn b/datamgr_service/services/distributeddataservice/service/data_share/BUILD.gn index 5e0d4c7d..2964c14d 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/data_share/BUILD.gn @@ -94,11 +94,13 @@ ohos_source_set("data_share_service") { "-Wno-multichar", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] configs = [ ":module_public_config" ] @@ -125,6 +127,7 @@ ohos_source_set("data_share_service") { "hicollie:libhicollie", "hilog:libhilog", "hisysevent:libhisysevent", + "init:libbegetutil", "ipc:ipc_core", "json:nlohmann_json_static", "kv_store:datamgr_common", diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/bundle_mgr_proxy.cpp b/datamgr_service/services/distributeddataservice/service/data_share/common/bundle_mgr_proxy.cpp index 8aeeb003..410408be 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/bundle_mgr_proxy.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/bundle_mgr_proxy.cpp @@ -23,6 +23,8 @@ #include "log_print.h" #include "system_ability_definition.h" #include "uri_utils.h" +#include "ipc_skeleton.h" +#include "hiview_fault_adapter.h" namespace OHOS::DataShare { sptr BundleMgrProxy::GetBundleMgrProxy() @@ -75,19 +77,18 @@ int BundleMgrProxy::GetBundleInfoFromBMS( } auto bmsClient = GetBundleMgrProxy(); if (bmsClient == nullptr) { - RADAR_REPORT(__FUNCTION__, RadarReporter::SILENT_ACCESS, RadarReporter::GET_BMS, - RadarReporter::FAILED, RadarReporter::ERROR_CODE, RadarReporter::GET_BMS_FAILED); ZLOGE("GetBundleMgrProxy is nullptr!"); return E_BMS_NOT_READY; } AppExecFwk::BundleInfo bundleInfo; bool ret; + TimeoutReport timeoutReport({bundleName, "", "", __FUNCTION__, 0}); if (appIndex == 0) { ret = bmsClient->GetBundleInfo( bundleName, AppExecFwk::BundleFlag::GET_BUNDLE_WITH_EXTENSION_INFO, bundleInfo, userId); } else { - ret = bmsClient->GetCloneBundleInfo( - bundleName, static_cast(AppExecFwk::GetBundleInfoFlag::GET_BUNDLE_INFO_WITH_EXTENSION_ABILITY) | + ret = bmsClient->GetCloneBundleInfo(bundleName, + static_cast(AppExecFwk::GetBundleInfoFlag::GET_BUNDLE_INFO_WITH_EXTENSION_ABILITY) | static_cast(AppExecFwk::GetBundleInfoFlag::GET_BUNDLE_INFO_WITH_HAP_MODULE), appIndex, bundleInfo, userId); // when there is no error, the former function returns 1 while the new function returns 0 @@ -98,12 +99,10 @@ int BundleMgrProxy::GetBundleInfoFromBMS( } } } - + timeoutReport.Report(std::to_string(userId), IPCSkeleton::GetCallingPid(), appIndex); if (!ret) { - RADAR_REPORT(__FUNCTION__, RadarReporter::SILENT_ACCESS, RadarReporter::GET_BMS, - RadarReporter::FAILED, RadarReporter::ERROR_CODE, RadarReporter::GET_BUNDLE_INFP_FAILED); ZLOGE("GetBundleInfo failed!bundleName is %{public}s, userId is %{public}d", bundleName.c_str(), userId); - return E_BUNDLE_NAME_NOT_EXIST; + return E_ERROR; } auto [errCode, bundle] = ConvertToDataShareBundle(bundleInfo); if (errCode != E_OK) { diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/db_delegate.h b/datamgr_service/services/distributeddataservice/service/data_share/common/db_delegate.h index 54056e0f..0cb36808 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/db_delegate.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/db_delegate.h @@ -24,6 +24,7 @@ #include "datashare_result_set.h" #include "datashare_values_bucket.h" #include "executor_pool.h" +#include "hiview_fault_adapter.h" #include "metadata/store_meta_data.h" #include "result_set.h" #include "serializable/serializable.h" @@ -127,8 +128,8 @@ public: static std::shared_ptr GetInstance( bool reInit = false, const std::string &dir = "", const std::shared_ptr &executors = nullptr); virtual ~KvDBDelegate() = default; - virtual int32_t Upsert(const std::string &collectionName, const KvData &value) = 0; - virtual int32_t Delete(const std::string &collectionName, const std::string &filter) = 0; + virtual std::pair Upsert(const std::string &collectionName, const KvData &value) = 0; + virtual std::pair Delete(const std::string &collectionName, const std::string &filter) = 0; virtual int32_t Get(const std::string &collectionName, const Id &id, std::string &value) = 0; virtual int32_t Get(const std::string &collectionName, const std::string &filter, const std::string &projection, std::string &result) = 0; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.cpp b/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.cpp index 06f8cc6a..be360b40 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.cpp @@ -40,7 +40,8 @@ const char* g_backupFiles[] = { "dataShare.db.safe", "dataShare.db.undo", }; -const char* BACKUP_SUFFIX = ".backup"; +constexpr const char* BACKUP_SUFFIX = ".backup"; +constexpr std::chrono::milliseconds COPY_TIME_OUT_MS = std::chrono::milliseconds(500); // If isBackUp is true, remove db backup files. Otherwise remove source db files. void KvDelegate::RemoveDbFile(bool isBackUp) const @@ -65,12 +66,15 @@ bool KvDelegate::CopyFile(bool isBackup) std::filesystem::copy_options options = std::filesystem::copy_options::overwrite_existing; std::error_code code; bool ret = true; + int index = 0; for (auto &fileName : g_backupFiles) { std::string src = path_ + "/" + fileName; std::string dst = src; isBackup ? dst.append(BACKUP_SUFFIX) : src.append(BACKUP_SUFFIX); + TimeoutReport timeoutReport({"", "", "", __FUNCTION__, 0}); // If src doesn't exist, error will be returned through `std::error_code` bool copyRet = std::filesystem::copy_file(src, dst, options, code); + timeoutReport.Report(("file index:" + std::to_string(index)), COPY_TIME_OUT_MS); if (!copyRet) { ZLOGE("failed to copy file %{public}s, isBackup %{public}d, err: %{public}s", fileName, isBackup, code.message().c_str()); @@ -78,6 +82,7 @@ bool KvDelegate::CopyFile(bool isBackup) RemoveDbFile(isBackup); break; } + index++; } return ret; } @@ -128,29 +133,30 @@ bool KvDelegate::RestoreIfNeed(int32_t dbStatus) return false; } -int64_t KvDelegate::Upsert(const std::string &collectionName, const std::string &filter, const std::string &value) +std::pair KvDelegate::Upsert(const std::string &collectionName, const std::string &filter, + const std::string &value) { std::lock_guard lock(mutex_); if (!Init()) { ZLOGE("init failed, %{public}s", collectionName.c_str()); - return E_ERROR; + return std::make_pair(E_ERROR, 0); } - int count = GRD_UpsertDoc(db_, collectionName.c_str(), filter.c_str(), value.c_str(), 0); + int32_t count = GRD_UpsertDoc(db_, collectionName.c_str(), filter.c_str(), value.c_str(), 0); if (count <= 0) { ZLOGE("GRD_UpSertDoc failed,status %{public}d", count); RestoreIfNeed(count); - return count; + return std::make_pair(count, 0); } Flush(); - return E_OK; + return std::make_pair(E_OK, count); } -int32_t KvDelegate::Delete(const std::string &collectionName, const std::string &filter) +std::pair KvDelegate::Delete(const std::string &collectionName, const std::string &filter) { std::lock_guard lock(mutex_); if (!Init()) { ZLOGE("init failed, %{public}s", collectionName.c_str()); - return E_ERROR; + return std::make_pair(E_ERROR, 0); } std::vector queryResults; @@ -158,23 +164,25 @@ int32_t KvDelegate::Delete(const std::string &collectionName, const std::string if (status != E_OK) { ZLOGE("db GetBatch failed, %{public}s %{public}d", filter.c_str(), status); // `GetBatch` should decide whether to restore before errors are returned, so skip restoration here. - return status; + return std::make_pair(status, 0); } + int32_t deleteCount = 0; for (auto &result : queryResults) { auto count = GRD_DeleteDoc(db_, collectionName.c_str(), result.c_str(), 0); if (count < 0) { ZLOGE("GRD_DeleteDoc failed,status %{public}d %{public}s", count, result.c_str()); if (RestoreIfNeed(count)) { - return count; + return std::make_pair(count, 0); } continue; } + deleteCount += count; } Flush(); if (queryResults.size() > 0) { ZLOGI("Delete, %{public}s, count %{public}zu", collectionName.c_str(), queryResults.size()); } - return E_OK; + return std::make_pair(E_OK, deleteCount); } bool KvDelegate::Init() @@ -232,7 +240,7 @@ KvDelegate::~KvDelegate() } } -int32_t KvDelegate::Upsert(const std::string &collectionName, const KvData &value) +std::pair KvDelegate::Upsert(const std::string &collectionName, const KvData &value) { std::string id = value.GetId(); if (value.HasVersion() && value.GetVersion() != 0) { @@ -241,7 +249,7 @@ int32_t KvDelegate::Upsert(const std::string &collectionName, const KvData &valu if (value.GetVersion() <= version) { ZLOGE("GetVersion failed,%{public}s id %{private}s %{public}d %{public}d", collectionName.c_str(), id.c_str(), value.GetVersion(), version); - return E_VERSION_NOT_NEWER; + return std::make_pair(E_VERSION_NOT_NEWER, 0); } } } diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.h b/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.h index 3ec92c7c..fce4ecf3 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/kv_delegate.h @@ -28,8 +28,8 @@ class KvDelegate final : public KvDBDelegate { public: KvDelegate(const std::string &path, const std::shared_ptr &executors); ~KvDelegate() override; - int32_t Upsert(const std::string &collectionName, const KvData &value) override; - int32_t Delete(const std::string &collectionName, const std::string &filter) override; + std::pair Upsert(const std::string &collectionName, const KvData &value) override; + std::pair Delete(const std::string &collectionName, const std::string &filter) override; int32_t Get(const std::string &collectionName, const Id &id, std::string &value) override; int32_t Get(const std::string &collectionName, const std::string &filter, const std::string &projection, @@ -41,7 +41,8 @@ public: private: bool Init(); bool GetVersion(const std::string &collectionName, const std::string &filter, int &version); - int64_t Upsert(const std::string &collectionName, const std::string &filter, const std::string &value); + std::pair Upsert(const std::string &collectionName, const std::string &filter, + const std::string &value); void Flush(); bool RestoreIfNeed(int32_t dbStatus); void Backup(); diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/rdb_delegate.cpp b/datamgr_service/services/distributeddataservice/service/data_share/common/rdb_delegate.cpp index 94cd9c2c..de1036ff 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/rdb_delegate.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/rdb_delegate.cpp @@ -20,7 +20,6 @@ #include "datashare_radar_reporter.h" #include "device_manager_adapter.h" #include "extension_connect_adaptor.h" -#include "hiview_fault_adapter.h" #include "int_wrapper.h" #include "metadata/meta_data_manager.h" #include "metadata/store_meta_data.h" @@ -35,6 +34,7 @@ #include "want_params.h" #include "db_delegate.h" #include "log_debug.h" +#include "ipc_skeleton.h" namespace OHOS::DataShare { constexpr static int32_t MAX_RESULTSET_COUNT = 32; @@ -85,7 +85,7 @@ std::pair RdbDelegate::GetConfig(const DistributedData::Sto return std::make_pair(E_DB_NOT_EXIST, config); } std::vector decryptKey; - if (!DistributedData::CryptoManager::GetInstance().Decrypt(secretKeyMeta.sKey, decryptKey)) { + if (!DistributedData::CryptoManager::GetInstance().Decrypt(meta, secretKeyMeta, decryptKey)) { return std::make_pair(E_ERROR, config); }; config.SetEncryptKey(decryptKey); @@ -109,7 +109,10 @@ RdbDelegate::RdbDelegate(const DistributedData::StoreMetaData &meta, int version return; } DefaultOpenCallback callback; + TimeoutReport timeoutReport({meta.bundleName, "", meta.storeId, __FUNCTION__, 0}); store_ = RdbHelper::GetRdbStore(config, version, callback, errCode_); + auto callingPid = IPCSkeleton::GetCallingPid(); + timeoutReport.Report(meta.user, callingPid, -1, meta.instanceId); if (errCode_ != E_OK) { ZLOGW("GetRdbStore failed, errCode is %{public}d, dir is %{public}s", errCode_, DistributedData::Anonymous::Change(meta.dataDir).c_str()); @@ -214,7 +217,7 @@ std::pair> RdbDelegate::Query(const std } int count = resultSetCount.fetch_add(1); ZLOGD_MACRO("start query %{public}d", count); - if (count > MAX_RESULTSET_COUNT && IsLimit(count, callingPid, callingTokenId)) { + if (count >= MAX_RESULTSET_COUNT && IsLimit(count, callingPid, callingTokenId)) { resultSetCount--; return std::make_pair(E_RESULTSET_BUSY, nullptr); } @@ -338,7 +341,7 @@ bool RdbDelegate::IsLimit(int count, int32_t callingPid, uint32_t callingTokenId }); ZLOGE("resultSetCount is full, pid: %{public}d, owner is %{public}s", callingPid, logStr.c_str()); std::string appendix = "callingName:" + HiViewFaultAdapter::GetCallingName(callingTokenId).first; - DataShareFaultInfo faultInfo{RESULTSET_FULL, "callingTokenId:" + std::to_string(callingTokenId), + DataShareFaultInfo faultInfo{HiViewFaultAdapter::resultsetFull, "callingTokenId:" + std::to_string(callingTokenId), "Pid:" + std::to_string(callingPid), "owner:" + logStr, __FUNCTION__, E_RESULTSET_BUSY, appendix}; HiViewFaultAdapter::ReportDataFault(faultInfo); return true; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/common/uri_utils.cpp b/datamgr_service/services/distributeddataservice/service/data_share/common/uri_utils.cpp index 48d4dfce..1953824c 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/common/uri_utils.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/common/uri_utils.cpp @@ -30,7 +30,6 @@ bool URIUtils::GetInfoFromURI(const std::string &uri, UriInfo &uriInfo) std::vector splitUri; SplitStr(uriTemp.GetPath(), "/", splitUri); if (splitUri.size() < PARAM_SIZE) { - ZLOGE("Invalid uri: %{public}s", DistributedData::Anonymous::Change(uri).c_str()); return false; } diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data/published_data.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data/published_data.cpp index 894a8e58..0e73d871 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data/published_data.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data/published_data.cpp @@ -170,7 +170,7 @@ void PublishedData::Delete(const std::string &bundleName, const int32_t userId) ZLOGE("db open failed"); return; } - int32_t status = delegate->Delete(KvDBDelegate::DATA_TABLE, + auto [status, count] = delegate->Delete(KvDBDelegate::DATA_TABLE, "{\"bundleName\":\"" + bundleName + "\", \"userId\": " + std::to_string(userId) + "}"); if (status != E_OK) { ZLOGE("db Delete failed, %{public}s %{public}d", bundleName.c_str(), status); @@ -209,9 +209,9 @@ void PublishedData::ClearAging() } if (data.timestamp < lastValidTime && PublishedDataSubscriberManager::GetInstance() .GetCount(PublishedDataKey(data.key, data.bundleName, data.subscriberId)) == 0) { - status = delegate->Delete(KvDBDelegate::DATA_TABLE, + auto [errorCode, count] = delegate->Delete(KvDBDelegate::DATA_TABLE, Id(PublishedData::GenId(data.key, data.bundleName, data.subscriberId), data.userId)); - if (status != E_OK) { + if (errorCode != E_OK) { ZLOGE("db Delete failed, %{public}s %{public}s", data.key.c_str(), data.bundleName.c_str()); } agingSize++; @@ -249,8 +249,8 @@ void PublishedData::UpdateTimestamp( return; } data.timestamp = now; - status = delegate->Upsert(KvDBDelegate::DATA_TABLE, PublishedData(data)); - if (status == E_OK) { + auto [errorCode, count] = delegate->Upsert(KvDBDelegate::DATA_TABLE, PublishedData(data)); + if (errorCode == E_OK) { ZLOGI("update timestamp %{private}s", data.key.c_str()); } } diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data/template_data.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data/template_data.cpp index baf5dc4d..79219aed 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data/template_data.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data/template_data.cpp @@ -135,13 +135,15 @@ bool TemplateData::Delete(const std::string &bundleName, const int32_t userId) ZLOGE("db open failed"); return false; } - auto status = delegate->Delete(KvDBDelegate::TEMPLATE_TABLE, + auto [status, count] = delegate->Delete(KvDBDelegate::TEMPLATE_TABLE, "{\"bundleName\":\"" + bundleName + "\", \"userId\": " + std::to_string(userId) + "}"); if (status != E_OK) { ZLOGE("db DeleteById failed, %{public}d", status); return false; } - delegate->NotifyBackup(); + if (count > 0) { + delegate->NotifyBackup(); + } return true; } @@ -154,12 +156,14 @@ bool TemplateData::Add(const std::string &uri, const int32_t userId, const std:: return false; } TemplateData data(uri, bundleName, subscriberId, userId, aTemplate); - auto status = delegate->Upsert(KvDBDelegate::TEMPLATE_TABLE, data); + auto [status, count] = delegate->Upsert(KvDBDelegate::TEMPLATE_TABLE, data); if (status != E_OK) { ZLOGE("db Upsert failed, %{public}d", status); return false; } - delegate->NotifyBackup(); + if (count > 0) { + delegate->NotifyBackup(); + } return true; } @@ -171,13 +175,15 @@ bool TemplateData::Delete( ZLOGE("db open failed"); return false; } - auto status = delegate->Delete(KvDBDelegate::TEMPLATE_TABLE, + auto [status, count] = delegate->Delete(KvDBDelegate::TEMPLATE_TABLE, static_cast(Id(TemplateData::GenId(uri, bundleName, subscriberId), userId))); if (status != E_OK) { ZLOGE("db DeleteById failed, %{public}d", status); return false; } - delegate->NotifyBackup(); + if (count > 0) { + delegate->NotifyBackup(); + } return true; } diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.cpp index a45379fd..e713b4ac 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.cpp @@ -149,6 +149,7 @@ int DataProviderConfig::GetFromExtensionProperties(const ProfileInfo &profileInf int DataProviderConfig::GetFromExtension() { + providerInfo_.isFromExtension = true; if (!GetFromUriPath()) { ZLOGE("Uri path failed! uri:%{public}s", URIUtils::Anonymous(providerInfo_.uri).c_str()); return E_URI_NOT_EXIST; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.h b/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.h index 13c2b88c..95d1c1ca 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_provider_config.h @@ -46,6 +46,7 @@ public: std::string type = "rdb"; std::string backup; std::string extensionUri; + bool isFromExtension = false; bool singleton = false; bool hasExtension = false; bool allowEmptyPermission = false; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_db_config.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data_share_db_config.cpp index 79a2f462..544f887f 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_db_config.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_db_config.cpp @@ -27,11 +27,13 @@ #include "metadata/meta_data_manager.h" #include "uri_utils.h" #include "utils/anonymous.h" +#include "ipc_skeleton.h" namespace OHOS::DataShare { std::pair DataShareDbConfig::QueryMetaData( const std::string &bundleName, const std::string &storeName, int32_t userId, int32_t appIndex) { + TimeoutReport timeoutReport({bundleName, "", storeName, __FUNCTION__, 0}); DistributedData::StoreMetaData meta; meta.deviceId = DistributedData::DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid; meta.user = std::to_string(userId); @@ -40,6 +42,8 @@ std::pair DataShareDbConfig::QueryMetaData meta.instanceId = appIndex; DistributedData::StoreMetaData metaData; bool isCreated = DistributedData::MetaDataManager::GetInstance().LoadMeta(meta.GetKey(), metaData, true); + auto callingPid = IPCSkeleton::GetCallingPid(); + timeoutReport.Report(std::to_string(userId), callingPid, appIndex); return std::make_pair(isCreated, metaData); } diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_profile_config.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data_share_profile_config.cpp index ffff2842..9787d9df 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_profile_config.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_profile_config.cpp @@ -35,6 +35,7 @@ namespace DataShare { constexpr const char *PROFILE_FILE_PREFIX = "$profile:"; constexpr const char *SEPARATOR = "/"; static constexpr int PATH_SIZE = 2; +static constexpr int MAX_ALLOWLIST_COUNT = 256; const size_t PROFILE_PREFIX_LEN = strlen(PROFILE_FILE_PREFIX); bool Config::Marshal(json &node) const { @@ -108,6 +109,9 @@ bool ProfileInfo::Unmarshal(const json &node) GetValue(node, GET_NAME(type), type); GetValue(node, GET_NAME(launchInfos), launchInfos); GetValue(node, GET_NAME(allowLists), allowLists); + if (allowLists.size() > MAX_ALLOWLIST_COUNT) { + allowLists.resize(MAX_ALLOWLIST_COUNT); + } GetValue(node, GET_NAME(storeMetaDataFromUri), storeMetaDataFromUri); GetValue(node, GET_NAME(launchForCleanData), launchForCleanData); GetValue(node, GET_NAME(backup), backup); diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.cpp index f6b886f7..02b2ce85 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.cpp @@ -38,7 +38,6 @@ #include "extension_ability_manager.h" #include "hap_token_info.h" #include "hiview_adapter.h" -#include "hiview_fault_adapter.h" #include "if_system_ability_manager.h" #include "ipc_skeleton.h" #include "iservice_registry.h" @@ -58,6 +57,7 @@ #include "utils/anonymous.h" #include "xcollie.h" #include "log_debug.h" +#include "parameters.h" namespace OHOS::DataShare { using FeatureSystem = DistributedData::FeatureSystem; @@ -65,6 +65,8 @@ using DumpManager = OHOS::DistributedData::DumpManager; using ProviderInfo = DataProviderConfig::ProviderInfo; using namespace OHOS::DistributedData; __attribute__((used)) DataShareServiceImpl::Factory DataShareServiceImpl::factory_; +// decimal base +static constexpr int DECIMAL_BASE = 10; class DataShareServiceImpl::SystemAbilityStatusChangeListener : public SystemAbilityStatusChangeStub { public: @@ -103,7 +105,7 @@ std::pair DataShareServiceImpl::InsertEx(const std::string &ur const DataShareValuesBucket &valuesBucket) { std::string func = __FUNCTION__; - XCollie xcollie(func, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(func, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (GetSilentProxyStatus(uri, false) != E_OK) { ZLOGW("silent proxy disable, %{public}s", URIUtils::Anonymous(uri).c_str()); return std::make_pair(ERROR, 0); @@ -112,8 +114,8 @@ std::pair DataShareServiceImpl::InsertEx(const std::string &ur auto callBack = [&uri, &valuesBucket, this, &callingTokenId, &func](ProviderInfo &providerInfo, DistributedData::StoreMetaData &metaData, std::shared_ptr dbDelegate) -> std::pair { - RdbTimeCostInfo rdbTimeCostInfo(providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, - func, callingTokenId); + TimeoutReport timeoutReport({providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, func, + callingTokenId}, true); auto [errCode, ret] = dbDelegate->InsertEx(providerInfo.tableName, valuesBucket); if (errCode == E_OK && ret > 0) { NotifyChange(uri); @@ -121,6 +123,7 @@ std::pair DataShareServiceImpl::InsertEx(const std::string &ur } else { ReportExcuteFault(callingTokenId, providerInfo, errCode, func); } + timeoutReport.Report(); return std::make_pair(errCode, ret); }; return ExecuteEx(uri, extUri, IPCSkeleton::GetCallingTokenID(), false, callBack); @@ -150,7 +153,7 @@ std::pair DataShareServiceImpl::UpdateEx(const std::string &ur const DataSharePredicates &predicate, const DataShareValuesBucket &valuesBucket) { std::string func = __FUNCTION__; - XCollie xcollie(func, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(func, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (GetSilentProxyStatus(uri, false) != E_OK) { ZLOGW("silent proxy disable, %{public}s", URIUtils::Anonymous(uri).c_str()); return std::make_pair(ERROR, 0); @@ -159,8 +162,8 @@ std::pair DataShareServiceImpl::UpdateEx(const std::string &ur auto callBack = [&uri, &predicate, &valuesBucket, this, &callingTokenId, &func](ProviderInfo &providerInfo, DistributedData::StoreMetaData &metaData, std::shared_ptr dbDelegate) -> std::pair { - RdbTimeCostInfo rdbTimeCostInfo(providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, - func, callingTokenId); + TimeoutReport timeoutReport({providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, func, + callingTokenId}, true); auto [errCode, ret] = dbDelegate->UpdateEx(providerInfo.tableName, predicate, valuesBucket); if (errCode == E_OK && ret > 0) { NotifyChange(uri); @@ -168,6 +171,7 @@ std::pair DataShareServiceImpl::UpdateEx(const std::string &ur } else { ReportExcuteFault(callingTokenId, providerInfo, errCode, func); } + timeoutReport.Report(); return std::make_pair(errCode, ret); }; return ExecuteEx(uri, extUri, callingTokenId, false, callBack); @@ -177,7 +181,7 @@ std::pair DataShareServiceImpl::DeleteEx(const std::string &ur const DataSharePredicates &predicate) { std::string func = __FUNCTION__; - XCollie xcollie(func, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(func, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (GetSilentProxyStatus(uri, false) != E_OK) { ZLOGW("silent proxy disable, %{public}s", URIUtils::Anonymous(uri).c_str()); return std::make_pair(ERROR, 0); @@ -186,8 +190,8 @@ std::pair DataShareServiceImpl::DeleteEx(const std::string &ur auto callBack = [&uri, &predicate, this, &callingTokenId, &func](ProviderInfo &providerInfo, DistributedData::StoreMetaData &metaData, std::shared_ptr dbDelegate) -> std::pair { - RdbTimeCostInfo rdbTimeCostInfo(providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, - func, callingTokenId); + TimeoutReport timeoutReport({providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, func, + callingTokenId}, true); auto [errCode, ret] = dbDelegate->DeleteEx(providerInfo.tableName, predicate); if (errCode == E_OK && ret > 0) { NotifyChange(uri); @@ -195,6 +199,7 @@ std::pair DataShareServiceImpl::DeleteEx(const std::string &ur } else { ReportExcuteFault(callingTokenId, providerInfo, errCode, func); } + timeoutReport.Report(); return std::make_pair(errCode, ret); }; return ExecuteEx(uri, extUri, callingTokenId, false, callBack); @@ -205,7 +210,7 @@ std::shared_ptr DataShareServiceImpl::Query(const std::strin { std::string func = __FUNCTION__; XCollie xcollie(std::string(LOG_TAG) + "::" + func, - HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (GetSilentProxyStatus(uri, false) != E_OK) { ZLOGW("silent proxy disable, %{public}s", URIUtils::Anonymous(uri).c_str()); return nullptr; @@ -216,14 +221,15 @@ std::shared_ptr DataShareServiceImpl::Query(const std::strin auto callBack = [&uri, &predicates, &columns, &resultSet, &callingPid, &callingTokenId, &func, this] (ProviderInfo &providerInfo, DistributedData::StoreMetaData &, std::shared_ptr dbDelegate) -> std::pair { - RdbTimeCostInfo rdbTimeCostInfo(providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, - func, callingTokenId); + TimeoutReport timeoutReport({providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, func, + callingTokenId}, true); auto [err, result] = dbDelegate->Query(providerInfo.tableName, predicates, columns, callingPid, callingTokenId); if (err != E_OK) { ReportExcuteFault(callingTokenId, providerInfo, err, func); } resultSet = std::move(result); + timeoutReport.Report(); return std::make_pair(err, E_OK); }; auto [errVal, status] = ExecuteEx(uri, extUri, callingTokenId, true, callBack); @@ -616,7 +622,19 @@ void DataShareServiceImpl::SaveLaunchInfo(const std::string &bundleName, const s const std::string &deviceId) { std::map profileInfos; - if (!DataShareProfileConfig::GetProfileInfo(bundleName, std::stoi(userId), profileInfos)) { + char *endptr = nullptr; + errno = 0; + long userIdLong = strtol(userId.c_str(), &endptr, DECIMAL_BASE); + if (endptr == nullptr || endptr == userId.c_str() || *endptr != '\0') { + ZLOGE("UserId:%{public}s is invalid", userId.c_str()); + return; + } + if (errno == ERANGE || userIdLong >= INT32_MAX || userIdLong <= INT32_MIN) { + ZLOGE("UserId:%{public}s is out of range", userId.c_str()); + return; + } + int32_t currentUserId = static_cast(userIdLong); + if (!DataShareProfileConfig::GetProfileInfo(bundleName, currentUserId, profileInfos)) { ZLOGE("Get profileInfo failed."); return; } @@ -880,7 +898,7 @@ int32_t DataShareServiceImpl::EnableSilentProxy(const std::string &uri, bool ena int32_t DataShareServiceImpl::GetSilentProxyStatus(const std::string &uri, bool isCreateHelper) { XCollie xcollie(std::string(LOG_TAG) + "::" + std::string(__FUNCTION__), - HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); uint32_t callerTokenId = IPCSkeleton::GetCallingTokenID(); if (isCreateHelper) { auto errCode = GetBMSAndMetaDataStatus(uri, callerTokenId); @@ -892,6 +910,9 @@ int32_t DataShareServiceImpl::GetSilentProxyStatus(const std::string &uri, bool } int32_t currentUserId = AccountDelegate::GetInstance()->GetUserByToken(callerTokenId); UriInfo uriInfo; + // GetInfoFromUri will first perform a four-part URI check. Only if the URI contains more than four parts + // is it necessary to continue to check the SilentProxyEnable status. The URI part length is used as an + // indicator to determine whether the SilentProxyEnable status needs to be checked. if (!URIUtils::GetInfoFromURI(uri, uriInfo)) { return E_OK; } @@ -916,79 +937,13 @@ int32_t DataShareServiceImpl::GetSilentProxyStatus(const std::string &uri, bool int32_t DataShareServiceImpl::RegisterObserver(const std::string &uri, const sptr &remoteObj) { - XCollie xcollie(std::string(LOG_TAG) + "::" + std::string(__FUNCTION__), - HiviewDFX::XCOLLIE_FLAG_LOG); - auto callerTokenId = IPCSkeleton::GetCallingTokenID(); - DataProviderConfig providerConfig(uri, callerTokenId); - auto [errCode, providerInfo] = providerConfig.GetProviderInfo(); - if (errCode != E_OK) { - ZLOGE("ProviderInfo failed! token:0x%{public}x,ret:%{public}d,uri:%{public}s", callerTokenId, - errCode, URIUtils::Anonymous(providerInfo.uri).c_str()); - } - if (!CheckAllowList(providerInfo.currentUserId, callerTokenId, providerInfo.allowLists)) { - ZLOGE("CheckAllowList failed, permission denied! token:0x%{public}x, uri:%{public}s", - callerTokenId, URIUtils::Anonymous(providerInfo.uri).c_str()); - return ERROR; - } - if (!providerInfo.allowEmptyPermission && providerInfo.readPermission.empty()) { - ZLOGE("reject permission, tokenId:0x%{public}x, uri:%{public}s", - callerTokenId, URIUtils::Anonymous(uri).c_str()); - } - if (!providerInfo.readPermission.empty() && - !PermitDelegate::VerifyPermission(providerInfo.readPermission, callerTokenId)) { - ZLOGE("Permission denied! token:0x%{public}x, permission:%{public}s, uri:%{public}s", - callerTokenId, providerInfo.readPermission.c_str(), - URIUtils::Anonymous(providerInfo.uri).c_str()); - } - auto obServer = iface_cast(remoteObj); - if (obServer == nullptr) { - ZLOGE("ObServer is nullptr, uri: %{public}s", URIUtils::Anonymous(uri).c_str()); - return ERR_INVALID_VALUE; - } - auto obsMgrClient = AAFwk::DataObsMgrClient::GetInstance(); - if (obsMgrClient == nullptr) { - return ERROR; - } - return obsMgrClient->RegisterObserver(Uri(uri), obServer); + return ERROR; } int32_t DataShareServiceImpl::UnregisterObserver(const std::string &uri, const sptr &remoteObj) { - XCollie xcollie(std::string(LOG_TAG) + "::" + std::string(__FUNCTION__), - HiviewDFX::XCOLLIE_FLAG_LOG); - auto callerTokenId = IPCSkeleton::GetCallingTokenID(); - DataProviderConfig providerConfig(uri, callerTokenId); - auto [errCode, providerInfo] = providerConfig.GetProviderInfo(); - if (errCode != E_OK) { - ZLOGE("ProviderInfo failed! token:0x%{public}x,ret:%{public}d,uri:%{public}s", callerTokenId, - errCode, URIUtils::Anonymous(providerInfo.uri).c_str()); - } - if (!CheckAllowList(providerInfo.currentUserId, callerTokenId, providerInfo.allowLists)) { - ZLOGE("CheckAllowList failed, permission denied! token:0x%{public}x, uri:%{public}s", - callerTokenId, URIUtils::Anonymous(providerInfo.uri).c_str()); - return ERROR; - } - if (!providerInfo.allowEmptyPermission && providerInfo.readPermission.empty()) { - ZLOGE("reject permission, tokenId:0x%{public}x, uri:%{public}s", - callerTokenId, URIUtils::Anonymous(uri).c_str()); - } - if (!providerInfo.readPermission.empty() && - !PermitDelegate::VerifyPermission(providerInfo.readPermission, callerTokenId)) { - ZLOGE("Permission denied! token:0x%{public}x, permission:%{public}s, uri:%{public}s", - callerTokenId, providerInfo.readPermission.c_str(), - URIUtils::Anonymous(providerInfo.uri).c_str()); - } - auto obsMgrClient = AAFwk::DataObsMgrClient::GetInstance(); - if (obsMgrClient == nullptr) { - return ERROR; - } - auto obServer = iface_cast(remoteObj); - if (obServer == nullptr) { - ZLOGE("ObServer is nullptr, uri: %{public}s", URIUtils::Anonymous(uri).c_str()); - return ERR_INVALID_VALUE; - } - return obsMgrClient->UnregisterObserver(Uri(uri), obServer); + return ERROR; } bool DataShareServiceImpl::VerifyAcrossAccountsPermission(int32_t currentUserId, int32_t visitedUserId, @@ -998,7 +953,8 @@ bool DataShareServiceImpl::VerifyAcrossAccountsPermission(int32_t currentUserId, if (currentUserId == 0 || currentUserId == visitedUserId) { return true; } - return PermitDelegate::VerifyPermission(acrossAccountsPermission, callerTokenId); + return system::GetBoolParameter(CONNECT_SUPPORT_CROSS_USER, false) && + PermitDelegate::VerifyPermission(acrossAccountsPermission, callerTokenId); } std::pair DataShareServiceImpl::ExecuteEx(const std::string &uri, const std::string &extUri, @@ -1029,9 +985,9 @@ std::pair DataShareServiceImpl::ExecuteEx(const std::string &u return std::make_pair(ERROR_PERMISSION_DENIED, 0); } std::string permission = isRead ? providerInfo.readPermission : providerInfo.writePermission; - if (!permission.empty() && !PermitDelegate::VerifyPermission(permission, tokenId)) { - ZLOGE("Permission denied! token:0x%{public}x, permission:%{public}s, uri:%{public}s", - tokenId, permission.c_str(), URIUtils::Anonymous(providerInfo.uri).c_str()); + if (!VerifyPermission(providerInfo.bundleName, permission, providerInfo.isFromExtension, tokenId)) { + ZLOGE("Permission denied! token:0x%{public}x, permission:%{public}s,isFromExtension:%{public}d,uri:%{public}s", + tokenId, permission.c_str(), providerInfo.isFromExtension, URIUtils::Anonymous(providerInfo.uri).c_str()); RADAR_REPORT(__FUNCTION__, RadarReporter::SILENT_ACCESS, RadarReporter::PROXY_PERMISSION, RadarReporter::FAILED, RadarReporter::ERROR_CODE, RadarReporter::PERMISSION_DENIED_ERROR); return std::make_pair(ERROR_PERMISSION_DENIED, 0); @@ -1061,7 +1017,7 @@ bool DataShareServiceImpl::CheckAllowList(const uint32_t ¤tUserId, const u return true; } std::string callerBundleName; - int32_t callerAppIndex; + int32_t callerAppIndex = 0; auto [success, type] = GetCallerInfo(callerBundleName, callerAppIndex); if (!success) { ZLOGE("Get caller info failed, token:0x%{public}x", callerTokenId); @@ -1179,8 +1135,35 @@ void DataShareServiceImpl::ReportExcuteFault(uint32_t callingTokenId, DataProvid int32_t errCode, std::string &func) { std::string appendix = "callingName:" + HiViewFaultAdapter::GetCallingName(callingTokenId).first; - DataShareFaultInfo faultInfo = {CURD_FAILED, providerInfo.bundleName, providerInfo.moduleName, + DataShareFaultInfo faultInfo = {HiViewFaultAdapter::curdFailed, providerInfo.bundleName, providerInfo.moduleName, providerInfo.storeName, func, errCode, appendix}; HiViewFaultAdapter::ReportDataFault(faultInfo); } + +bool DataShareServiceImpl::VerifyPermission(const std::string &bundleName, const std::string &permission, + bool isFromExtension, const int32_t tokenId) +{ + // Provider from extension, allows empty permissions, not configured to allow access. + if (isFromExtension) { + if (!permission.empty() && !PermitDelegate::VerifyPermission(permission, tokenId)) { + return false; + } + } else { + Security::AccessToken::HapTokenInfo tokenInfo; + auto result = Security::AccessToken::AccessTokenKit::GetHapTokenInfo(tokenId, tokenInfo); + if (result == Security::AccessToken::RET_SUCCESS && tokenInfo.bundleName == bundleName) { + return true; + } + // Provider from ProxyData, which does not allow empty permissions and cannot be access without configured + if (permission.empty()) { + ZLOGE("Permission empty! token:0x%{public}x, bundleName:%{public}s", tokenId, bundleName.c_str()); + return false; + } + // If the permission is NO_PERMISSION, access is also allowed + if (permission != NO_PERMISSION && !PermitDelegate::VerifyPermission(permission, tokenId)) { + return false; + } + } + return true; +} } // namespace OHOS::DataShare diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.h b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.h index b2d99087..e4d0a7d9 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_impl.h @@ -139,11 +139,15 @@ private: int32_t errCode, std::string &func); bool CheckAllowList(const uint32_t ¤tUserId, const uint32_t &callerTokenId, const std::vector &allowLists); + bool VerifyPermission(const std::string &bundleName, const std::string &permission, + bool isFromExtension, const int32_t tokenId); static Factory factory_; static constexpr int32_t ERROR = -1; static constexpr int32_t ERROR_PERMISSION_DENIED = -2; static constexpr const char *PROXY_URI_SCHEMA = "datashareproxy"; static constexpr const char *EXT_URI_SCHEMA = "datashare://"; + static constexpr const char *NO_PERMISSION = "noPermission"; + static constexpr const char *CONNECT_SUPPORT_CROSS_USER = "const.abilityms.connect_support_cross_user"; PublishStrategy publishStrategy_; GetDataStrategy getDataStrategy_; SubscribeStrategy subscribeStrategy_; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.cpp b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.cpp index bb44ff6c..d0b000ee 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.cpp @@ -18,6 +18,8 @@ #include "data_share_service_stub.h" #include +#include "accesstoken_kit.h" +#include "tokenid_kit.h" #include "data_share_obs_proxy.h" #include "hiview_adapter.h" #include "hiview_fault_adapter.h" @@ -325,6 +327,24 @@ int32_t DataShareServiceStub::OnNotifyConnectDone(MessageParcel &data, MessagePa return 0; } +bool DataShareServiceStub::CheckProxyCallingPermission(uint32_t tokenId) +{ + Security::AccessToken::ATokenTypeEnum tokenType = + Security::AccessToken::AccessTokenKit::GetTokenTypeFlag(tokenId); + return (tokenType == Security::AccessToken::ATokenTypeEnum::TOKEN_NATIVE || + tokenType == Security::AccessToken::ATokenTypeEnum::TOKEN_SHELL); +} + +// GetTokenType use tokenId, and IsSystemApp use fullTokenId, these are different +bool DataShareServiceStub::CheckSystemUidCallingPermission(uint32_t tokenId, uint64_t fullTokenId) +{ + if (CheckProxyCallingPermission(tokenId)) { + return true; + } + // IsSystemAppByFullTokenID here is not IPC + return Security::AccessToken::TokenIdKit::IsSystemAppByFullTokenID(fullTokenId); +} + int DataShareServiceStub::OnRemoteRequest(uint32_t code, MessageParcel &data, MessageParcel &reply) { int tryTimes = TRY_TIMES; @@ -333,6 +353,15 @@ int DataShareServiceStub::OnRemoteRequest(uint32_t code, MessageParcel &data, Me std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_TIME)); } auto callingPid = IPCSkeleton::GetCallingPid(); + if (code >= DATA_SHARE_CMD_SYSTEM_CODE) { + auto fullTokenId = IPCSkeleton::GetCallingFullTokenID(); + if (!CheckSystemUidCallingPermission(IPCSkeleton::GetCallingTokenID(), fullTokenId)) { + ZLOGE("CheckSystemUidCallingPermission fail, token:%{public}" PRIx64 + ", callingPid:%{public}d, code:%{public}u", fullTokenId, callingPid, code); + return E_NOT_SYSTEM_APP; + } + code = code - DATA_SHARE_CMD_SYSTEM_CODE; + } if (code != DATA_SHARE_SERVICE_CMD_QUERY && code != DATA_SHARE_SERVICE_CMD_GET_SILENT_PROXY_STATUS) { ZLOGI("code:%{public}u, callingPid:%{public}d", code, callingPid); } @@ -402,36 +431,12 @@ int32_t DataShareServiceStub::OnGetSilentProxyStatus(MessageParcel &data, Messag int32_t DataShareServiceStub::OnRegisterObserver(MessageParcel &data, MessageParcel &reply) { - std::string uri; - sptr remoteObj; - if (!ITypesUtil::Unmarshal(data, uri, remoteObj)) { - ZLOGE("Unmarshal failed,uri: %{public}s", DistributedData::Anonymous::Change(uri).c_str()); - return IPC_STUB_INVALID_DATA_ERR; - } - int32_t status = RegisterObserver(uri, remoteObj); - if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal failed,status:0x%{public}x,uri:%{public}s", status, - DistributedData::Anonymous::Change(uri).c_str()); - return IPC_STUB_WRITE_PARCEL_ERR; - } - return E_OK; + return DATA_SHARE_ERROR; } int32_t DataShareServiceStub::OnUnregisterObserver(MessageParcel &data, MessageParcel &reply) { - std::string uri; - sptr remoteObj; - if (!ITypesUtil::Unmarshal(data, uri, remoteObj)) { - ZLOGE("Unmarshal failed,uri: %{public}s", DistributedData::Anonymous::Change(uri).c_str()); - return IPC_STUB_INVALID_DATA_ERR; - } - int32_t status = UnregisterObserver(uri, remoteObj); - if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal failed,status:0x%{public}x,uri:%{public}s", status, - DistributedData::Anonymous::Change(uri).c_str()); - return IPC_STUB_WRITE_PARCEL_ERR; - } - return E_OK; + return DATA_SHARE_ERROR; } void DataShareServiceStub::SetServiceReady() diff --git a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.h b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.h index d13b2099..b84c2696 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/data_share_service_stub.h @@ -29,6 +29,8 @@ public: private: static constexpr std::chrono::milliseconds TIME_THRESHOLD = std::chrono::milliseconds(500); static bool CheckInterfaceToken(MessageParcel& data); + bool CheckProxyCallingPermission(uint32_t tokenId); + bool CheckSystemUidCallingPermission(uint32_t tokenId, uint64_t fullTokenId); int32_t OnQuery(MessageParcel& data, MessageParcel& reply); int32_t OnAddTemplate(MessageParcel& data, MessageParcel& reply); int32_t OnDelTemplate(MessageParcel& data, MessageParcel& reply); diff --git a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_adapter.cpp b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_adapter.cpp index 6b1216b3..21e4eba9 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_adapter.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_adapter.cpp @@ -79,6 +79,7 @@ void HiViewAdapter::InvokeData() int64_t maxCostTimes[count]; int64_t totalCostTimes[count]; char* funcCounts[count]; + std::vector funcCountsHolder(count); for (uint32_t i = 0; i < count; i++) { CallerTotalInfo &callerTotalInfo = callerTotalInfos[i]; callerUids[i] = callerTotalInfo.callerUid; @@ -92,7 +93,8 @@ void HiViewAdapter::InvokeData() funcCount += it.first + ":" + std::to_string(it.second) + ","; } funcCount = funcCount.substr(0, funcCount.size() - 1) + "}"; - funcCounts[i] = const_cast(funcCount.c_str()); + funcCountsHolder[i] = std::move(funcCount); + funcCounts[i] = const_cast(funcCountsHolder[i].c_str()); } HiSysEventParam callerUid = { .name = "COLLIE_UID", .t = HISYSEVENT_INT64_ARRAY, diff --git a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.cpp b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.cpp index db04c7eb..5b6bdc45 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.cpp @@ -86,17 +86,57 @@ std::pair HiViewFaultAdapter::GetCallingName(uint32_t callingT return std::make_pair(callingName, result); } -RdbTimeCostInfo::~RdbTimeCostInfo() +void TimeoutReport::Report(const std::string &timeoutAppendix, const std::chrono::milliseconds timeoutms) { auto end = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - if (duration > TIME_OUT_MS) { + std::chrono::milliseconds duration = std::chrono::duration_cast(end - start); + // Used to report DFX timeout faults + if (needFaultReport && duration > HiViewFaultAdapter::dfxTimeOutMs) { + DFXReport(duration); + } + // Used to report log timeout + if (duration > timeoutms) { + int64_t milliseconds = duration.count(); + ZLOGE("over time when doing %{public}s, %{public}s, cost:%{public}" PRIi64 "ms", + dfxInfo.businessType.c_str(), timeoutAppendix.c_str(), milliseconds); + } +} + +void TimeoutReport::Report(const std::string &user, uint32_t callingPid, int32_t appIndex, int32_t instanceId) +{ + auto end = std::chrono::steady_clock::now(); + std::chrono::milliseconds duration = std::chrono::duration_cast(end - start); + // Used to report DFX timeout faults + if (needFaultReport && duration > HiViewFaultAdapter::dfxTimeOutMs) { + DFXReport(duration); + } + // Used to report log timeout + if (duration > HiViewFaultAdapter::timeOutMs) { int64_t milliseconds = duration.count(); - std::string appendix = "callingName:" + HiViewFaultAdapter::GetCallingName(callingTokenId).first; - appendix += ",cost:" + std::to_string(milliseconds) + "ms"; - DataShareFaultInfo faultInfo{TIME_OUT, bundleName, moduleName, storeName, businessType, errorCode, appendix}; - HiViewFaultAdapter::ReportDataFault(faultInfo); + std::string timeoutAppendix = "bundleName: " + dfxInfo.bundleName + ", user: " + user + ", callingPid: " + + std::to_string(callingPid); + if (!dfxInfo.storeName.empty()) { + timeoutAppendix += ", storeName: " + dfxInfo.storeName; + } + if (appIndex != -1) { + timeoutAppendix += ", appIndex: " + std::to_string(appIndex); + } + if (instanceId != -1) { + timeoutAppendix += ", instanceId: " + std::to_string(instanceId); + } + ZLOGE("over time when doing %{public}s, %{public}s, cost:%{public}" PRIi64 "ms", + dfxInfo.businessType.c_str(), timeoutAppendix.c_str(), milliseconds); } } + +void TimeoutReport::DFXReport(const std::chrono::milliseconds &duration) +{ + int64_t milliseconds = duration.count(); + std::string appendix = "callingName:" + HiViewFaultAdapter::GetCallingName(dfxInfo.callingTokenId).first; + appendix += ",cost:" + std::to_string(milliseconds) + "ms"; + DataShareFaultInfo faultInfo{HiViewFaultAdapter::timeOut, dfxInfo.bundleName, dfxInfo.moduleName, + dfxInfo.storeName, dfxInfo.businessType, errorCode, appendix}; + HiViewFaultAdapter::ReportDataFault(faultInfo); +} } } \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.h b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.h index e7a33b27..3b001e62 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/dfx/hiview_fault_adapter.h @@ -21,6 +21,7 @@ #include #include "accesstoken_kit.h" #include "hisysevent_c.h" + namespace OHOS { namespace DataShare { struct DataShareFaultInfo { @@ -33,34 +34,43 @@ struct DataShareFaultInfo { std::string appendix; }; -struct RdbTimeCostInfo { - std::chrono::time_point start = std::chrono::steady_clock::now(); - std::string bundleName; - std::string moduleName; - std::string storeName; - std::string businessType; - int32_t errorCode = 0; - uint64_t callingTokenId; - - explicit RdbTimeCostInfo(const std::string &bundleName, const std::string &moduleName, - const std::string &storeName, const std::string &businessType, - uint64_t tokenId) : bundleName(bundleName), moduleName(moduleName), - storeName(storeName), businessType(businessType), callingTokenId(tokenId) - {} - ~RdbTimeCostInfo(); -}; - - class HiViewFaultAdapter { public: + static constexpr const std::chrono::milliseconds timeOutMs = std::chrono::milliseconds(300); + static constexpr const std::chrono::milliseconds dfxTimeOutMs = std::chrono::milliseconds(2000); + static constexpr const char* timeOut = "TIME_OUT"; + static constexpr const char* resultsetFull = "RESULTSET_FULL"; + static constexpr const char* curdFailed = "CURD_FAILED"; static void ReportDataFault(const DataShareFaultInfo &faultInfo); static std::pair GetCallingName(uint32_t callingTokenid); }; -inline const char* TIME_OUT = "TIME_OUT"; -inline const char* RESULTSET_FULL = "RESULTSET_FULL"; -inline const char* CURD_FAILED = "CURD_FAILED"; -inline const std::chrono::milliseconds TIME_OUT_MS = std::chrono::milliseconds(300); +// TimeoutReport is used for recording the time usage of multiple interfaces; +// It can set up a timeout threshold, and when the time usage exceeds this threshold, will print log; +// If want to record DFX fault report, need to set needFaultReport to true (default is false); +// In the future, it can be separeted to diffent function in order to achieve these functions. +struct TimeoutReport { + struct DfxInfo { + std::string bundleName; + std::string moduleName; + std::string storeName; + std::string businessType; + uint64_t callingTokenId; + }; + + DfxInfo dfxInfo; + int32_t errorCode = 0; + bool needFaultReport; + std::chrono::time_point start = std::chrono::steady_clock::now(); + explicit TimeoutReport(const DfxInfo &dfxInfo, bool needFaultReport = false) + : dfxInfo(dfxInfo), needFaultReport(needFaultReport) + {} + void Report(const std::string &timeoutAppendix = "", + const std::chrono::milliseconds timeoutms = HiViewFaultAdapter::timeOutMs); + void Report(const std::string &user, uint32_t callingPid, int32_t appIndex = -1, int32_t instanceId = -1); + void DFXReport(const std::chrono::milliseconds &duration); + ~TimeoutReport() = default; +}; } } #endif \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/data_share/idata_share_service.h b/datamgr_service/services/distributeddataservice/service/data_share/idata_share_service.h index e57489fe..05e1fd15 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/idata_share_service.h +++ b/datamgr_service/services/distributeddataservice/service/data_share/idata_share_service.h @@ -29,6 +29,7 @@ namespace OHOS::DataShare { class IDataShareService { public: + static constexpr int DATA_SHARE_CMD_SYSTEM_CODE = 100; enum { DATA_SHARE_SERVICE_CMD_QUERY, DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE, @@ -52,7 +53,30 @@ public: DATA_SHARE_SERVICE_CMD_INSERTEX, DATA_SHARE_SERVICE_CMD_DELETEEX, DATA_SHARE_SERVICE_CMD_UPDATEEX, - DATA_SHARE_SERVICE_CMD_MAX + DATA_SHARE_SERVICE_CMD_MAX, + DATA_SHARE_SERVICE_CMD_QUERY_SYSTEM = DATA_SHARE_CMD_SYSTEM_CODE, + DATA_SHARE_SERVICE_CMD_ADD_TEMPLATE_SYSTEM, + DATA_SHARE_SERVICE_CMD_DEL_TEMPLATE_SYSTEM, + DATA_SHARE_SERVICE_CMD_PUBLISH_SYSTEM, + DATA_SHARE_SERVICE_CMD_GET_DATA_SYSTEM, + DATA_SHARE_SERVICE_CMD_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_RDB_SYSTEM, + DATA_SHARE_SERVICE_CMD_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNSUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_ENABLE_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_DISABLE_SUBSCRIBE_PUBLISHED_SYSTEM, + DATA_SHARE_SERVICE_CMD_NOTIFY_SYSTEM, + DATA_SHARE_SERVICE_CMD_NOTIFY_OBSERVERS_SYSTEM, + DATA_SHARE_SERVICE_CMD_SET_SILENT_SWITCH_SYSTEM, + DATA_SHARE_SERVICE_CMD_GET_SILENT_PROXY_STATUS_SYSTEM, + DATA_SHARE_SERVICE_CMD_REGISTER_OBSERVER_SYSTEM, + DATA_SHARE_SERVICE_CMD_UNREGISTER_OBSERVER_SYSTEM, + DATA_SHARE_SERVICE_CMD_INSERTEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_DELETEEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_UPDATEEX_SYSTEM, + DATA_SHARE_SERVICE_CMD_MAX_SYSTEM }; enum { DATA_SHARE_ERROR = -1, DATA_SHARE_OK = 0 }; diff --git a/datamgr_service/services/distributeddataservice/service/data_share/strategies/data_share/load_config_from_data_share_bundle_info_strategy.cpp b/datamgr_service/services/distributeddataservice/service/data_share/strategies/data_share/load_config_from_data_share_bundle_info_strategy.cpp index edec38ac..16fe9151 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/strategies/data_share/load_config_from_data_share_bundle_info_strategy.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/strategies/data_share/load_config_from_data_share_bundle_info_strategy.cpp @@ -107,6 +107,7 @@ bool LoadConfigFromDataShareBundleInfoStrategy::LoadConfigFromUri(std::shared_pt { UriInfo uriInfo; if (!URIUtils::GetInfoFromURI(context->uri, uriInfo)) { + ZLOGE("Invalid uri: %{public}s", DistributedData::Anonymous::Change(context->uri).c_str()); return false; } context->calledBundleName = std::move(uriInfo.bundleName); diff --git a/datamgr_service/services/distributeddataservice/service/data_share/strategies/publish_strategy.cpp b/datamgr_service/services/distributeddataservice/service/data_share/strategies/publish_strategy.cpp index 26b116d4..b0b7ca77 100644 --- a/datamgr_service/services/distributeddataservice/service/data_share/strategies/publish_strategy.cpp +++ b/datamgr_service/services/distributeddataservice/service/data_share/strategies/publish_strategy.cpp @@ -45,7 +45,7 @@ int32_t PublishStrategy::Execute(std::shared_ptr context, const Publish PublishedDataNode node(context->uri, context->calledBundleName, item.subscriberId_, context->currentUserId, PublishedDataNode::MoveTo(value)); PublishedData data(node, context->version); - int32_t status = delegate->Upsert(KvDBDelegate::DATA_TABLE, data); + auto [status, count] = delegate->Upsert(KvDBDelegate::DATA_TABLE, data); if (status != E_OK) { ZLOGE("db Upsert failed, %{public}s %{public}s %{public}d", context->calledBundleName.c_str(), DistributedData::Anonymous::Change(context->uri).c_str(), status); diff --git a/datamgr_service/services/distributeddataservice/service/dumper/BUILD.gn b/datamgr_service/services/distributeddataservice/service/dumper/BUILD.gn index 72b194d7..9ba04db7 100644 --- a/datamgr_service/services/distributeddataservice/service/dumper/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/dumper/BUILD.gn @@ -27,6 +27,7 @@ ohos_source_set("distributeddata_dumper") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "include" ] @@ -37,6 +38,7 @@ ohos_source_set("distributeddata_dumper") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/BUILD.gn b/datamgr_service/services/distributeddataservice/service/kvdb/BUILD.gn index bb3d4525..9f1d96da 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/kvdb/BUILD.gn @@ -27,9 +27,9 @@ ohos_source_set("distributeddata_kvdb") { "${data_service_path}/service/matrix/include", "${data_service_path}/adapter/include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", "${data_service_path}/adapter/include/utils", "${data_service_path}/framework/include", + "${data_service_path}/framework/include/dfx", "${data_service_path}/service/bootstrap/include", "${data_service_path}/service/crypto/include", "${data_service_path}/service/kvdb", @@ -55,6 +55,7 @@ ohos_source_set("distributeddata_kvdb") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] cflags = [ @@ -63,6 +64,7 @@ ohos_source_set("distributeddata_kvdb") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_exporter.cpp b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_exporter.cpp index 8f62255c..b9b4f8c7 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_exporter.cpp +++ b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_exporter.cpp @@ -19,7 +19,7 @@ #include "directory/directory_manager.h" #include "kvdb_general_store.h" #include "log_print.h" -#include "reporter.h" +#include "dfx/reporter.h" namespace OHOS::DistributedKv { using namespace OHOS::DistributedData; using namespace OHOS::DistributedDataDfx; @@ -59,7 +59,7 @@ void KVDBExporter::Exporter(const StoreMetaData &meta, const std::string &backup .append("], backup result [") .append(std::to_string(result)) .append("]"); - Reporter::GetInstance()->BehaviourReporter()->Report( + Reporter::GetInstance()->GetBehaviourReporter()->Report( { meta.account, meta.appId, meta.storeId, BehaviourType::DATABASE_BACKUP, message }); } diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_general_store.cpp b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_general_store.cpp index 0a5daafc..da7a931e 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_general_store.cpp +++ b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_general_store.cpp @@ -20,14 +20,13 @@ #include "app_id_mapping/app_id_mapping_config_manager.h" #include "bootstrap.h" #include "checker/checker_manager.h" -#include "cloud/cloud_sync_finished_event.h" #include "cloud/schema_meta.h" #include "crypto_manager.h" #include "device_manager_adapter.h" #include "device_matrix.h" -#include "dfx_types.h" +#include "dfx/dfx_types.h" +#include "dfx/reporter.h" #include "directory/directory_manager.h" -#include "eventcenter/event_center.h" #include "kvdb_query.h" #include "log_print.h" #include "metadata/meta_data_manager.h" @@ -35,7 +34,6 @@ #include "metadata/store_meta_data_local.h" #include "query_helper.h" #include "rdb_cloud.h" -#include "reporter.h" #include "snapshot/bind_event.h" #include "types.h" #include "user_delegate.h" @@ -110,7 +108,9 @@ KVDBGeneralStore::DBPassword KVDBGeneralStore::GetDBPassword(const StoreMetaData auto storeKey = data.GetSecretKey(); MetaDataManager::GetInstance().LoadMeta(storeKey, secretKey, true); std::vector password; - CryptoManager::GetInstance().Decrypt(secretKey.sKey, password); + StoreMetaData metaData; + MetaDataManager::GetInstance().LoadMeta(data.GetKey(), metaData, true); + CryptoManager::GetInstance().Decrypt(metaData, secretKey, password); dbPassword.SetValue(password.data(), password.size()); password.assign(password.size(), 0); return dbPassword; @@ -171,8 +171,8 @@ KVDBGeneralStore::KVDBGeneralStore(const StoreMetaData &meta) DBStatus status = DBStatus::NOT_FOUND; manager_.SetKvStoreConfig({ meta.dataDir }); std::unique_lock lock(rwMutex_); - manager_.GetKvStore( - meta.storeId, GetDBOption(meta, GetDBPassword(meta)), [&status, this](auto dbStatus, auto *tmpStore) { + manager_.GetKvStore(meta.storeId, GetDBOption(meta, GetDBPassword(meta)), + [&status, this](auto dbStatus, auto *tmpStore) { status = dbStatus; delegate_ = tmpStore; }); @@ -229,8 +229,8 @@ int32_t KVDBGeneralStore::BindSnapshots(std::shared_ptr &bindInfos, const CloudConfig &config) +int32_t KVDBGeneralStore::Bind(const Database &database, const std::map &bindInfos, + const CloudConfig &config) { if (bindInfos.empty()) { ZLOGW("No cloudDB!"); @@ -245,8 +245,8 @@ int32_t KVDBGeneralStore::Bind( if (bindInfo.db_ == nullptr) { return GeneralError::E_INVALID_ARGS; } - dbClouds.insert({ std::to_string(userId), - std::make_shared(bindInfo.db_, nullptr) }); + dbClouds.insert( + { std::to_string(userId), std::make_shared(bindInfo.db_, nullptr) }); bindInfos_.insert(std::make_pair(userId, std::move(bindInfo))); schemas.insert({ std::to_string(userId), dbSchema }); } @@ -321,8 +321,8 @@ int32_t KVDBGeneralStore::Replace(const std::string &table, VBucket &&value) return GeneralError::E_NOT_SUPPORT; } -std::pair> KVDBGeneralStore::Query( - __attribute__((unused)) const std::string &table, const std::string &sql, Values &&args) +std::pair> KVDBGeneralStore::Query(__attribute__((unused)) const std::string &table, + const std::string &sql, Values &&args) { return { GeneralError::E_NOT_SUPPORT, nullptr }; } @@ -479,8 +479,8 @@ void KVDBGeneralStore::SetEqualIdentifier(const std::string &appId, const std::s } } -void KVDBGeneralStore::GetIdentifierParams( - std::vector &devices, const std::vector &uuids, int32_t authType) +void KVDBGeneralStore::GetIdentifierParams(std::vector &devices, const std::vector &uuids, + int32_t authType) { for (const auto &devId : uuids) { if (DMAdapter::GetInstance().IsOHOSType(devId)) { @@ -513,8 +513,8 @@ int32_t KVDBGeneralStore::Clean(const std::vector &devices, int32_t DBStatus status = OK; switch (mode) { case CLOUD_INFO: - status = delegate_->RemoveDeviceData( - "", isPublic_ ? static_cast(CLOUD_DATA) : static_cast(CLOUD_INFO)); + status = delegate_->RemoveDeviceData("", + isPublic_ ? static_cast(CLOUD_DATA) : static_cast(CLOUD_INFO)); break; case CLOUD_DATA: status = delegate_->RemoveDeviceData("", static_cast(CLOUD_DATA)); @@ -580,8 +580,8 @@ int32_t KVDBGeneralStore::AddRef() return ++ref_; } -int32_t KVDBGeneralStore::SetDistributedTables( - const std::vector &tables, int32_t type, const std::vector &references) +int32_t KVDBGeneralStore::SetDistributedTables(const std::vector &tables, int32_t type, + const std::vector &references) { return GeneralError::E_OK; } @@ -632,6 +632,12 @@ void KVDBGeneralStore::ObserverProxy::OnChange(DBOrigin origin, const std::strin auto &info = changeInfo[storeId_][i]; for (auto &priData : data.primaryData[i]) { Watcher::PRIValue value; + if (priData.empty()) { + ZLOGW("priData is empty, store:%{public}s table:%{public}s data change from :%{public}s, i=%{public}d", + Anonymous::Change(storeId_).c_str(), Anonymous::Change(data.tableName).c_str(), + Anonymous::Change(originalId).c_str(), i); + continue; + } Convert(std::move(*(priData.begin())), value); info.push_back(std::move(value)); } diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.cpp b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.cpp index 6e4c7584..f04980cc 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.cpp +++ b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.cpp @@ -49,6 +49,7 @@ #include "utils/constant.h" #include "utils/converter.h" #include "app_id_mapping/app_id_mapping_config_manager.h" +#include "network/network_delegate.h" namespace OHOS::DistributedKv { using namespace OHOS::DistributedData; @@ -60,8 +61,7 @@ using DumpManager = OHOS::DistributedData::DumpManager; using CommContext = OHOS::DistributedData::CommunicatorContext; using SecretKeyMeta = DistributedData::SecretKeyMetaData; static constexpr const char *DEFAULT_USER_ID = "0"; -static constexpr const char *PASTEBOARD_SERVICE = "pasteboard_service"; -static constexpr const char *PASTEBOARD_USER_ID = "100"; +static constexpr const char *KEY_SEPARATOR = "###"; static const size_t SECRET_KEY_COUNT = 2; __attribute__((used)) KVDBServiceImpl::Factory KVDBServiceImpl::factory_; KVDBServiceImpl::Factory::Factory() @@ -122,7 +122,7 @@ void KVDBServiceImpl::Init() meta.storeType > StoreMetaData::StoreType::STORE_KV_END) { return; } - auto watchers = GetWatchers(meta.tokenId, meta.storeId); + auto watchers = GetWatchers(meta.tokenId, meta.storeId, meta.user); auto store = AutoCache::GetInstance().GetStore(meta, watchers); if (store == nullptr) { ZLOGE("store null, storeId:%{public}s", meta.GetStoreAlias().c_str()); @@ -158,10 +158,12 @@ void KVDBServiceImpl::DumpKvServiceInfo(int fd, std::map &storeIds) +Status KVDBServiceImpl::GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) { std::vector metaData; - auto user = AccountDelegate::GetInstance()->GetUserByToken(IPCSkeleton::GetCallingTokenID()); + auto tokenId = IPCSkeleton::GetCallingTokenID(); + auto user = (AccessTokenKit::GetTokenTypeFlag(tokenId) != TOKEN_HAP && subUser != 0) ? subUser : + AccountDelegate::GetInstance()->GetUserByToken(tokenId); auto deviceId = DMAdapter::GetInstance().GetLocalDevice().uuid; auto prefix = StoreMetaData::GetPrefix({ deviceId, std::to_string(user), "default", appId.appId }); auto instanceId = GetInstIndex(IPCSkeleton::GetCallingTokenID(), appId); @@ -176,15 +178,13 @@ Status KVDBServiceImpl::GetStoreIds(const AppId &appId, std::vector &st return SUCCESS; } -Status KVDBServiceImpl::Delete(const AppId &appId, const StoreId &storeId) +Status KVDBServiceImpl::Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); if (metaData.instanceId < 0) { return ILLEGAL_STATE; } - - auto tokenId = IPCSkeleton::GetCallingTokenID(); - syncAgents_.ComputeIfPresent(tokenId, [&appId, &storeId](auto &key, SyncAgent &syncAgent) { + syncAgents_.ComputeIfPresent(metaData.tokenId, [&appId, &storeId](auto &key, SyncAgent &syncAgent) { if (syncAgent.pid_ != IPCSkeleton::GetCallingPid()) { ZLOGW("agent already changed! old pid:%{public}d new pid:%{public}d appId:%{public}s", IPCSkeleton::GetCallingPid(), syncAgent.pid_, appId.appId.c_str()); @@ -203,20 +203,19 @@ Status KVDBServiceImpl::Delete(const AppId &appId, const StoreId &storeId) MetaDataManager::GetInstance().DelMeta(metaData.GetDebugInfoKey(), true); MetaDataManager::GetInstance().DelMeta(metaData.GetCloneSecretKey(), true); PermitDelegate::GetInstance().DelCache(metaData.GetKey()); - AutoCache::GetInstance().CloseStore(tokenId, storeId); + AutoCache::GetInstance().CloseStore(metaData.tokenId, storeId, metaData.user); ZLOGD("appId:%{public}s storeId:%{public}s instanceId:%{public}d", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str(), metaData.instanceId); return SUCCESS; } -Status KVDBServiceImpl::Close(const AppId &appId, const StoreId &storeId) +Status KVDBServiceImpl::Close(const AppId &appId, const StoreId &storeId, int32_t subUser) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); if (metaData.instanceId < 0) { return ILLEGAL_STATE; } - auto tokenId = IPCSkeleton::GetCallingTokenID(); - AutoCache::GetInstance().CloseStore(tokenId, storeId); + AutoCache::GetInstance().CloseStore(metaData.tokenId, storeId, metaData.user); ZLOGD("appId:%{public}s storeId:%{public}s instanceId:%{public}d", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str(), metaData.instanceId); return SUCCESS; @@ -242,11 +241,11 @@ void KVDBServiceImpl::OnAsyncComplete(uint32_t tokenId, uint64_t seqNum, Progres } } -Status KVDBServiceImpl::Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) +Status KVDBServiceImpl::Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData); - auto delay = GetSyncDelayTime(syncInfo.delay, storeId); + auto delay = GetSyncDelayTime(syncInfo.delay, storeId, metaData.user); if (metaData.isAutoSync && syncInfo.seqId == std::numeric_limits::max()) { DeviceMatrix::GetInstance().OnChanged(metaData); StoreMetaDataLocal localMeta; @@ -443,7 +442,8 @@ ProgressDetail KVDBServiceImpl::HandleGenDetails(const GenDetails &details) return progressDetail; } -Status KVDBServiceImpl::SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) +Status KVDBServiceImpl::SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) { if (syncParam.allowedDelayMs > 0 && syncParam.allowedDelayMs < KvStoreSyncManager::SYNC_MIN_DELAY_MS) { return Status::INVALID_ARGUMENT; @@ -451,29 +451,32 @@ Status KVDBServiceImpl::SetSyncParam(const AppId &appId, const StoreId &storeId, if (syncParam.allowedDelayMs > KvStoreSyncManager::SYNC_MAX_DELAY_MS) { return Status::INVALID_ARGUMENT; } - auto tokenId = IPCSkeleton::GetCallingTokenID(); - syncAgents_.Compute(tokenId, [&appId, &storeId, &syncParam](auto &key, SyncAgent &value) { + StoreMetaData meta = GetStoreMetaData(appId, storeId, subUser); + auto key = GenerateKey(meta.user, storeId.storeId); + syncAgents_.Compute(meta.tokenId, [&appId, &key, &syncParam](auto &, SyncAgent &value) { if (value.pid_ != IPCSkeleton::GetCallingPid()) { value.ReInit(IPCSkeleton::GetCallingPid(), appId); } - value.delayTimes_[storeId] = syncParam.allowedDelayMs; + value.delayTimes_[key] = syncParam.allowedDelayMs; return true; }); return SUCCESS; } -Status KVDBServiceImpl::GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) +Status KVDBServiceImpl::GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + KvSyncParam &syncParam) { syncParam.allowedDelayMs = 0; - auto tokenId = IPCSkeleton::GetCallingTokenID(); - syncAgents_.ComputeIfPresent(tokenId, [&appId, &storeId, &syncParam](auto &key, SyncAgent &value) { + StoreMetaData meta = GetStoreMetaData(appId, storeId, subUser); + auto key = GenerateKey(meta.user, storeId.storeId); + syncAgents_.ComputeIfPresent(meta.tokenId, [&appId, &key, &syncParam](auto &, SyncAgent &value) { if (value.pid_ != IPCSkeleton::GetCallingPid()) { ZLOGW("agent already changed! old pid:%{public}d, new pid:%{public}d, appId:%{public}s", IPCSkeleton::GetCallingPid(), value.pid_, appId.appId.c_str()); return true; } - auto it = value.delayTimes_.find(storeId); + auto it = value.delayTimes_.find(key); if (it != value.delayTimes_.end()) { syncParam.allowedDelayMs = it->second; } @@ -482,9 +485,9 @@ Status KVDBServiceImpl::GetSyncParam(const AppId &appId, const StoreId &storeId, return SUCCESS; } -Status KVDBServiceImpl::EnableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceImpl::EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { - StrategyMeta strategyMeta = GetStrategyMeta(appId, storeId); + StrategyMeta strategyMeta = GetStrategyMeta(appId, storeId, subUser); if (strategyMeta.instanceId < 0) { return ILLEGAL_STATE; } @@ -494,9 +497,9 @@ Status KVDBServiceImpl::EnableCapability(const AppId &appId, const StoreId &stor return SUCCESS; } -Status KVDBServiceImpl::DisableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceImpl::DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { - StrategyMeta strategyMeta = GetStrategyMeta(appId, storeId); + StrategyMeta strategyMeta = GetStrategyMeta(appId, storeId, subUser); if (strategyMeta.instanceId < 0) { return ILLEGAL_STATE; } @@ -506,10 +509,10 @@ Status KVDBServiceImpl::DisableCapability(const AppId &appId, const StoreId &sto return SUCCESS; } -Status KVDBServiceImpl::SetCapability(const AppId &appId, const StoreId &storeId, +Status KVDBServiceImpl::SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, const std::vector &local, const std::vector &remote) { - StrategyMeta strategy = GetStrategyMeta(appId, storeId); + StrategyMeta strategy = GetStrategyMeta(appId, storeId, subUser); if (strategy.instanceId < 0) { return ILLEGAL_STATE; } @@ -520,60 +523,67 @@ Status KVDBServiceImpl::SetCapability(const AppId &appId, const StoreId &storeId return SUCCESS; } -Status KVDBServiceImpl::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceImpl::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData); - auto delay = GetSyncDelayTime(syncInfo.delay, storeId); + auto delay = GetSyncDelayTime(syncInfo.delay, storeId, metaData.user); return KvStoreSyncManager::GetInstance()->AddSyncOperation(uintptr_t(metaData.tokenId), delay, std::bind(&KVDBServiceImpl::DoSyncInOrder, this, metaData, syncInfo, std::placeholders::_1, ACTION_SUBSCRIBE), std::bind(&KVDBServiceImpl::DoComplete, this, metaData, syncInfo, RefCount(), std::placeholders::_1)); } -Status KVDBServiceImpl::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceImpl::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData); - auto delay = GetSyncDelayTime(syncInfo.delay, storeId); + auto delay = GetSyncDelayTime(syncInfo.delay, storeId, metaData.user); return KvStoreSyncManager::GetInstance()->AddSyncOperation(uintptr_t(metaData.tokenId), delay, std::bind( &KVDBServiceImpl::DoSyncInOrder, this, metaData, syncInfo, std::placeholders::_1, ACTION_UNSUBSCRIBE), std::bind(&KVDBServiceImpl::DoComplete, this, metaData, syncInfo, RefCount(), std::placeholders::_1)); } -Status KVDBServiceImpl::Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceImpl::Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { if (observer == nullptr) { return INVALID_ARGUMENT; } - auto tokenId = IPCSkeleton::GetCallingTokenID(); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); ZLOGI("appId:%{public}s storeId:%{public}s tokenId:0x%{public}x", appId.appId.c_str(), - Anonymous::Change(storeId.storeId).c_str(), tokenId); + Anonymous::Change(storeId.storeId).c_str(), metaData.tokenId); bool isCreate = false; - syncAgents_.Compute(tokenId, [&appId, &storeId, &observer, &isCreate](auto &key, SyncAgent &agent) { + auto key = GenerateKey(metaData.user, storeId.storeId); + syncAgents_.Compute(metaData.tokenId, [&appId, &key, &observer, &isCreate](auto &, SyncAgent &agent) { if (agent.pid_ != IPCSkeleton::GetCallingPid()) { agent.ReInit(IPCSkeleton::GetCallingPid(), appId); } isCreate = true; auto watcher = std::make_shared(); watcher->SetObserver(observer); - agent.watchers_[storeId.storeId].insert(watcher); + agent.watchers_[key].insert(watcher); return true; }); if (isCreate) { - AutoCache::GetInstance().SetObserver(tokenId, storeId, GetWatchers(tokenId, storeId)); + AutoCache::GetInstance().SetObserver(metaData.tokenId, storeId, + GetWatchers(metaData.tokenId, storeId, metaData.user), metaData.user); } return SUCCESS; } -Status KVDBServiceImpl::Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceImpl::Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { - auto tokenId = IPCSkeleton::GetCallingTokenID(); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); ZLOGI("appId:%{public}s storeId:%{public}s tokenId:0x%{public}x", appId.appId.c_str(), - Anonymous::Change(storeId.storeId).c_str(), tokenId); + Anonymous::Change(storeId.storeId).c_str(), metaData.tokenId); bool destroyed = false; - syncAgents_.ComputeIfPresent(tokenId, [&appId, &storeId, &observer, &destroyed](auto &key, SyncAgent &agent) { - auto iter = agent.watchers_.find(storeId.storeId); + auto key = GenerateKey(metaData.user, storeId.storeId); + syncAgents_.ComputeIfPresent(metaData.tokenId, [&appId, &key, &observer, &destroyed](auto &, SyncAgent &agent) { + auto iter = agent.watchers_.find(key); if (iter == agent.watchers_.end()) { return true; } @@ -585,20 +595,21 @@ Status KVDBServiceImpl::Unsubscribe(const AppId &appId, const StoreId &storeId, } } if (iter->second.size() == 0) { - agent.watchers_.erase(storeId.storeId); + agent.watchers_.erase(key); } return true; }); if (destroyed) { - AutoCache::GetInstance().SetObserver(tokenId, storeId, GetWatchers(tokenId, storeId)); + AutoCache::GetInstance().SetObserver(metaData.tokenId, storeId, + GetWatchers(metaData.tokenId, storeId, metaData.user), metaData.user); } return SUCCESS; } -Status KVDBServiceImpl::GetBackupPassword(const AppId &appId, const StoreId &storeId, +Status KVDBServiceImpl::GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, std::vector> &passwords, int32_t passwordType) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); if (passwordType == KVDBService::PasswordType::BACKUP_SECRET_KEY) { std::vector backupPwd; bool res = BackupManager::GetInstance().GetPassWord(metaData, backupPwd); @@ -613,14 +624,14 @@ Status KVDBServiceImpl::GetBackupPassword(const AppId &appId, const StoreId &sto SecretKeyMetaData secretKey; std::vector password; if (MetaDataManager::GetInstance().LoadMeta(metaData.GetSecretKey(), secretKey, true) && - CryptoManager::GetInstance().Decrypt(secretKey.sKey, password)) { + CryptoManager::GetInstance().Decrypt(metaData, secretKey, password)) { passwords.emplace_back(password); password.assign(password.size(), 0); } std::vector clonePwd; if (MetaDataManager::GetInstance().LoadMeta(metaData.GetCloneSecretKey(), secretKey, true) && - CryptoManager::GetInstance().Decrypt(secretKey.sKey, clonePwd)) { + CryptoManager::GetInstance().Decrypt(metaData, secretKey, clonePwd, CryptoManager::CLONE_SECRET_KEY)) { passwords.emplace_back(clonePwd); clonePwd.assign(clonePwd.size(), 0); } @@ -651,7 +662,7 @@ Status KVDBServiceImpl::SetConfig(const AppId &appId, const StoreId &storeId, co return Status::ERROR; } } - auto stores = AutoCache::GetInstance().GetStoresIfPresent(meta.tokenId, storeId); + auto stores = AutoCache::GetInstance().GetStoresIfPresent(meta.tokenId, storeId, meta.user); for (auto store : stores) { store->SetConfig({ storeConfig.cloudConfig.enableCloud }); } @@ -664,7 +675,7 @@ Status KVDBServiceImpl::BeforeCreate(const AppId &appId, const StoreId &storeId, { ZLOGD("appId:%{public}s storeId:%{public}s to export data", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); - StoreMetaData meta = GetStoreMetaData(appId, storeId); + StoreMetaData meta = GetStoreMetaData(appId, storeId, options.subUser); AddOptions(options, meta); StoreMetaData old; @@ -715,7 +726,7 @@ Status KVDBServiceImpl::AfterCreate( return INVALID_ARGUMENT; } - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, options.subUser); AddOptions(options, metaData); StoreMetaData oldMeta; @@ -743,7 +754,16 @@ Status KVDBServiceImpl::AfterCreate( appIdMeta.appId = metaData.appId; MetaDataManager::GetInstance().SaveMeta(appIdMeta.GetKey(), appIdMeta, true); SaveLocalMetaData(options, metaData); - Upgrade::GetInstance().UpdatePassword(metaData, password); + if (metaData.isEncrypt && CryptoManager::GetInstance().UpdateSecretKey(metaData, password)) { + SecretKeyMetaData secretKey; + if (MetaDataManager::GetInstance().LoadMeta(metaData.GetCloneSecretKey(), secretKey, true) && + secretKey.area < 0) { + std::vector clonePwd; + // Update the encryption method for the key + CryptoManager::GetInstance().Decrypt(metaData, secretKey, clonePwd, CryptoManager::CLONE_SECRET_KEY); + clonePwd.assign(clonePwd.size(), 0); + } + } ZLOGI("appId:%{public}s storeId:%{public}s instanceId:%{public}d type:%{public}d dir:%{public}s " "isCreated:%{public}d dataType:%{public}d", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str(), metaData.instanceId, metaData.storeType, metaData.dataDir.c_str(), isCreated, metaData.dataType); @@ -820,7 +840,7 @@ int32_t KVDBServiceImpl::ResolveAutoLaunch(const std::string &identifier, DBLaun if (identifier != identifierTag && !isTripleIdentifierEqual) { continue; } - auto watchers = GetWatchers(storeMeta.tokenId, storeMeta.storeId); + auto watchers = GetWatchers(storeMeta.tokenId, storeMeta.storeId, storeMeta.user); auto store = AutoCache::GetInstance().GetStore(storeMeta, watchers); if (isTripleIdentifierEqual && store != nullptr) { store->SetEqualIdentifier(storeMeta.appId, storeMeta.storeId, accountId); @@ -866,14 +886,7 @@ void KVDBServiceImpl::AddOptions(const Options &options, StoreMetaData &metaData metaData.appId = CheckerManager::GetInstance().GetAppId(Converter::ConvertToStoreInfo(metaData)); metaData.appType = "harmony"; metaData.hapName = options.hapName; - if (metaData.appId == PASTEBOARD_SERVICE) { - auto userId = metaData.user; - metaData.user = PASTEBOARD_USER_ID; - metaData.dataDir = DirectoryManager::GetInstance().GetStorePath(metaData); - metaData.user = userId; - } else { - metaData.dataDir = DirectoryManager::GetInstance().GetStorePath(metaData); - } + metaData.dataDir = DirectoryManager::GetInstance().GetStorePath(metaData); metaData.schema = options.schema; metaData.account = AccountDelegate::GetInstance()->GetCurrentAccountId(); metaData.isNeedCompress = options.isNeedCompress; @@ -904,7 +917,7 @@ void KVDBServiceImpl::SaveLocalMetaData(const Options &options, const StoreMetaD MetaDataManager::GetInstance().SaveMeta(metaData.GetKeyLocal(), localMetaData, true); } -StoreMetaData KVDBServiceImpl::GetStoreMetaData(const AppId &appId, const StoreId &storeId) +StoreMetaData KVDBServiceImpl::GetStoreMetaData(const AppId &appId, const StoreId &storeId, int32_t subUser) { StoreMetaData metaData; metaData.uid = IPCSkeleton::GetCallingUid(); @@ -913,15 +926,16 @@ StoreMetaData KVDBServiceImpl::GetStoreMetaData(const AppId &appId, const StoreI metaData.bundleName = appId.appId; metaData.deviceId = DMAdapter::GetInstance().GetLocalDevice().uuid; metaData.storeId = storeId.storeId; - auto user = AccountDelegate::GetInstance()->GetUserByToken(metaData.tokenId); - metaData.user = std::to_string(user); + metaData.user = (AccessTokenKit::GetTokenTypeFlag(metaData.tokenId) != TOKEN_HAP && subUser != 0) ? + std::to_string(subUser) : std::to_string(AccountDelegate::GetInstance()->GetUserByToken(metaData.tokenId)); return metaData; } -StrategyMeta KVDBServiceImpl::GetStrategyMeta(const AppId &appId, const StoreId &storeId) +StrategyMeta KVDBServiceImpl::GetStrategyMeta(const AppId &appId, const StoreId &storeId, int32_t subUser) { auto tokenId = IPCSkeleton::GetCallingTokenID(); - auto userId = AccountDelegate::GetInstance()->GetUserByToken(tokenId); + auto userId = (AccessTokenKit::GetTokenTypeFlag(tokenId) != TOKEN_HAP && subUser != 0) ? subUser : + AccountDelegate::GetInstance()->GetUserByToken(tokenId); auto deviceId = DMAdapter::GetInstance().GetLocalDevice().uuid; StrategyMeta strategyMeta(deviceId, std::to_string(userId), appId.appId, storeId.storeId); strategyMeta.instanceId = GetInstIndex(tokenId, appId); @@ -965,6 +979,9 @@ Status KVDBServiceImpl::DoCloudSync(const StoreMetaData &meta, const SyncInfo &s if (instance == nullptr) { return Status::CLOUD_DISABLED; } + if (!NetworkDelegate::GetInstance()->IsNetworkAvailable()) { + return Status::NETWORK_ERROR; + } std::vector users; if (meta.user != StoreMetaData::ROOT_USER) { users.push_back(std::atoi(meta.user.c_str())); @@ -1073,6 +1090,11 @@ bool KVDBServiceImpl::IsNeedMetaSync(const StoreMetaData &meta, const std::vecto isAfterMeta = true; break; } + auto [existLocal, localMask] = DeviceMatrix::GetInstance().GetMask(uuid); + if ((localMask & DeviceMatrix::META_STORE_MASK) == DeviceMatrix::META_STORE_MASK) { + isAfterMeta = true; + break; + } } return isAfterMeta; } @@ -1112,7 +1134,7 @@ Status KVDBServiceImpl::DoSyncBegin(const std::vector &devices, con if (devices.empty()) { return Status::INVALID_ARGUMENT; } - auto watcher = GetWatchers(meta.tokenId, meta.storeId); + auto watcher = GetWatchers(meta.tokenId, meta.storeId, meta.user); RADAR_REPORT(STANDARD_DEVICE_SYNC, OPEN_STORE, RADAR_START, SYNC_STORE_ID, Anonymous::Change(meta.storeId), SYNC_APP_ID, meta.bundleName, CONCURRENT_ID, info.syncId, DATA_TYPE, meta.dataType); auto store = AutoCache::GetInstance().GetStore(meta, watcher); @@ -1222,7 +1244,7 @@ Status KVDBServiceImpl::ConvertDbStatusNative(DBStatus status) } } -uint32_t KVDBServiceImpl::GetSyncDelayTime(uint32_t delay, const StoreId &storeId) +uint32_t KVDBServiceImpl::GetSyncDelayTime(uint32_t delay, const StoreId &storeId, const std::string &subUser) { if (delay != 0) { return std::min(std::max(delay, KvStoreSyncManager::SYNC_MIN_DELAY_MS), KvStoreSyncManager::SYNC_MAX_DELAY_MS); @@ -1233,8 +1255,9 @@ uint32_t KVDBServiceImpl::GetSyncDelayTime(uint32_t delay, const StoreId &storeI return delay; } delay = KvStoreSyncManager::SYNC_DEFAULT_DELAY_MS; - syncAgents_.ComputeIfPresent(IPCSkeleton::GetCallingTokenID(), [&delay, &storeId](auto &, SyncAgent &agent) { - auto it = agent.delayTimes_.find(storeId); + auto key = GenerateKey(subUser, storeId); + syncAgents_.ComputeIfPresent(IPCSkeleton::GetCallingTokenID(), [&delay, &key](auto &, SyncAgent &agent) { + auto it = agent.delayTimes_.find(key); if (it != agent.delayTimes_.end() && it->second != 0) { delay = it->second; } @@ -1401,11 +1424,13 @@ std::vector KVDBServiceImpl::ConvertDevices(const std::vectorsecond) { watchers.insert(watcher); @@ -1459,11 +1484,12 @@ bool KVDBServiceImpl::IsOHOSType(const std::vector &ids) return isOHOSType; } -Status KVDBServiceImpl::RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) +Status KVDBServiceImpl::RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) { - StoreMetaData metaData = GetStoreMetaData(appId, storeId); + StoreMetaData metaData = GetStoreMetaData(appId, storeId, subUser); MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData); - auto watcher = GetWatchers(metaData.tokenId, metaData.storeId); + auto watcher = GetWatchers(metaData.tokenId, metaData.storeId, metaData.user); auto store = AutoCache::GetInstance().GetStore(metaData, watcher); if (store == nullptr) { ZLOGE("GetStore failed! appId:%{public}s storeId:%{public}s dir:%{public}s", metaData.bundleName.c_str(), @@ -1487,4 +1513,13 @@ Status KVDBServiceImpl::RemoveDeviceData(const AppId &appId, const StoreId &stor } return ConvertGeneralErr(GeneralError(ret)); } + +std::string KVDBServiceImpl::GenerateKey(const std::string &userId, const std::string &storeId) const +{ + std::string key = ""; + if (userId.empty() || storeId.empty()) { + return key; + } + return key.append(userId).append(KEY_SEPARATOR).append(storeId); +} } // namespace OHOS::DistributedKv \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.h b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.h index ce87b551..4604f8b5 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.h +++ b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_impl.h @@ -42,28 +42,33 @@ public: using StoreMetaData = OHOS::DistributedData::StoreMetaData; KVDBServiceImpl(); virtual ~KVDBServiceImpl(); - Status GetStoreIds(const AppId &appId, std::vector &storeIds) override; + Status GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) override; Status BeforeCreate(const AppId &appId, const StoreId &storeId, const Options &options) override; Status AfterCreate(const AppId &appId, const StoreId &storeId, const Options &options, const std::vector &password) override; - Status Delete(const AppId &appId, const StoreId &storeId) override; - Status Close(const AppId &appId, const StoreId &storeId) override; + Status Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status Close(const AppId &appId, const StoreId &storeId, int32_t subUser) override; Status CloudSync(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; - Status Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) override; + Status Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) override; Status RegServiceNotifier(const AppId &appId, sptr notifier) override; Status UnregServiceNotifier(const AppId &appId) override; - Status SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) override; - Status GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) override; - Status EnableCapability(const AppId &appId, const StoreId &storeId) override; - Status DisableCapability(const AppId &appId, const StoreId &storeId) override; - Status SetCapability(const AppId &appId, const StoreId &storeId, const std::vector &local, - const std::vector &remote) override; - Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; - Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; - Status Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) override; - Status Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) override; - Status GetBackupPassword(const AppId &appId, const StoreId &storeId, std::vector> &passwords, - int32_t passwordType) override; + Status SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) override; + Status GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, KvSyncParam &syncParam) override; + Status EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::vector &local, const std::vector &remote) override; + Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) override; + Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) override; + Status Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) override; + Status Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) override; + Status GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, + std::vector> &passwords, int32_t passwordType) override; Status NotifyDataChange(const AppId &appId, const StoreId &storeId, uint64_t delay) override; Status PutSwitch(const AppId &appId, const SwitchData &data) override; Status GetSwitch(const AppId &appId, const std::string &networkId, SwitchData &data) override; @@ -75,7 +80,8 @@ public: int32_t OnAppExit(pid_t uid, pid_t pid, uint32_t tokenId, const std::string &appId) override; int32_t ResolveAutoLaunch(const std::string &identifier, DBLaunchParam ¶m) override; int32_t OnUserChange(uint32_t code, const std::string &user, const std::string &account) override; - Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) override; + Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) override; private: using StrategyMeta = OHOS::DistributedData::StrategyMeta; @@ -116,9 +122,9 @@ private: void Init(); void AddOptions(const Options &options, StoreMetaData &metaData); - StoreMetaData GetStoreMetaData(const AppId &appId, const StoreId &storeId); + StoreMetaData GetStoreMetaData(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); StoreMetaData GetDistributedDataMeta(const std::string &deviceId); - StrategyMeta GetStrategyMeta(const AppId &appId, const StoreId &storeId); + StrategyMeta GetStrategyMeta(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); int32_t GetInstIndex(uint32_t tokenId, const AppId &appId); bool IsNeedMetaSync(const StoreMetaData &meta, const std::vector &uuids); Status DoCloudSync(const StoreMetaData &meta, const SyncInfo &syncInfo); @@ -128,7 +134,7 @@ private: Status DoSyncBegin(const std::vector &devices, const StoreMetaData &meta, const SyncInfo &info, const SyncEnd &complete, int32_t type); Status DoComplete(const StoreMetaData &meta, const SyncInfo &info, RefCount refCount, const DBResult &dbResult); - uint32_t GetSyncDelayTime(uint32_t delay, const StoreId &storeId); + uint32_t GetSyncDelayTime(uint32_t delay, const StoreId &storeId, const std::string &subUser = ""); Status ConvertDbStatus(DBStatus status) const; Status ConvertGeneralErr(GeneralError error) const; DBMode ConvertDBMode(SyncMode syncMode) const; @@ -140,7 +146,8 @@ private: DBResult HandleGenBriefDetails(const DistributedData::GenDetails &details); ProgressDetail HandleGenDetails(const DistributedData::GenDetails &details); void OnAsyncComplete(uint32_t tokenId, uint64_t seqNum, ProgressDetail &&detail); - DistributedData::AutoCache::Watchers GetWatchers(uint32_t tokenId, const std::string &storeId); + DistributedData::AutoCache::Watchers GetWatchers(uint32_t tokenId, const std::string &storeId, + const std::string &userId); using SyncResult = std::pair, std::map>; SyncResult ProcessResult(const std::map &results); void SaveLocalMetaData(const Options &options, const StoreMetaData &metaData); @@ -153,6 +160,7 @@ private: Status ConvertDbStatusNative(DBStatus status); bool CompareTripleIdentifier(const std::string &accountId, const std::string &identifier, const StoreMetaData &storeMeta); + std::string GenerateKey(const std::string &userId, const std::string &storeId) const; static Factory factory_; ConcurrentMap syncAgents_; std::shared_ptr executors_; diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_stub.cpp b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_stub.cpp index 78e1e966..37de8906 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_stub.cpp +++ b/datamgr_service/services/distributeddataservice/service/kvdb/kvdb_service_stub.cpp @@ -117,8 +117,14 @@ bool KVDBServiceStub::CheckPermission(uint32_t code, const StoreInfo &storeInfo) int32_t KVDBServiceStub::OnGetStoreIds( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } std::vector storeIds; - int32_t status = GetStoreIds(appId, storeIds); + int32_t status = GetStoreIds(appId, subUser, storeIds); if (!ITypesUtil::Marshal(reply, status, storeIds)) { ZLOGE("Marshal status:0x%{public}d storeIds:%{public}zu", status, storeIds.size()); return IPC_STUB_WRITE_PARCEL_ERR; @@ -166,7 +172,13 @@ int32_t KVDBServiceStub::OnAfterCreate( int32_t KVDBServiceStub::OnDelete(const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { - int32_t status = Delete(appId, storeId); + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } + int32_t status = Delete(appId, storeId, subUser); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -177,7 +189,13 @@ int32_t KVDBServiceStub::OnDelete(const AppId &appId, const StoreId &storeId, Me int32_t KVDBServiceStub::OnClose(const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { - int32_t status = Close(appId, storeId); + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } + int32_t status = Close(appId, storeId, subUser); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -189,12 +207,14 @@ int32_t KVDBServiceStub::OnClose(const AppId &appId, const StoreId &storeId, Mes int32_t KVDBServiceStub::OnSync(const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { SyncInfo syncInfo; - if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.mode, syncInfo.devices, syncInfo.delay, syncInfo.query)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.mode, syncInfo.devices, syncInfo.delay, syncInfo.query, + subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = Sync(appId, storeId, syncInfo); + int32_t status = Sync(appId, storeId, subUser, syncInfo); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -274,12 +294,13 @@ int32_t KVDBServiceStub::OnSetSyncParam( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { KvSyncParam syncParam; - if (!ITypesUtil::Unmarshal(data, syncParam.allowedDelayMs)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, syncParam.allowedDelayMs, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = SetSyncParam(appId, storeId, syncParam); + int32_t status = SetSyncParam(appId, storeId, subUser, syncParam); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -291,8 +312,14 @@ int32_t KVDBServiceStub::OnSetSyncParam( int32_t KVDBServiceStub::OnGetSyncParam( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } KvSyncParam syncParam; - int32_t status = GetSyncParam(appId, storeId, syncParam); + int32_t status = GetSyncParam(appId, storeId, subUser, syncParam); if (!ITypesUtil::Marshal(reply, status, syncParam.allowedDelayMs)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -304,7 +331,13 @@ int32_t KVDBServiceStub::OnGetSyncParam( int32_t KVDBServiceStub::OnEnableCap( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { - int32_t status = EnableCapability(appId, storeId); + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } + int32_t status = EnableCapability(appId, storeId, subUser); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -316,7 +349,13 @@ int32_t KVDBServiceStub::OnEnableCap( int32_t KVDBServiceStub::OnDisableCap( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { - int32_t status = DisableCapability(appId, storeId); + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, subUser)) { + ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), + Anonymous::Change(storeId.storeId).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } + int32_t status = DisableCapability(appId, storeId, subUser); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -330,12 +369,13 @@ int32_t KVDBServiceStub::OnSetCapability( { std::vector local; std::vector remote; - if (!ITypesUtil::Unmarshal(data, local, remote)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, local, remote, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = SetCapability(appId, storeId, local, remote); + int32_t status = SetCapability(appId, storeId, subUser, local, remote); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -348,12 +388,13 @@ int32_t KVDBServiceStub::OnAddSubInfo(const AppId &appId, const StoreId &storeId MessageParcel &reply) { SyncInfo syncInfo; - if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.devices, syncInfo.query)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.devices, syncInfo.query, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = AddSubscribeInfo(appId, storeId, syncInfo); + int32_t status = AddSubscribeInfo(appId, storeId, subUser, syncInfo); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -366,12 +407,13 @@ int32_t KVDBServiceStub::OnRmvSubInfo(const AppId &appId, const StoreId &storeId MessageParcel &reply) { SyncInfo syncInfo; - if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.devices, syncInfo.query)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, syncInfo.seqId, syncInfo.devices, syncInfo.query, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = RmvSubscribeInfo(appId, storeId, syncInfo); + int32_t status = RmvSubscribeInfo(appId, storeId, subUser, syncInfo); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -384,13 +426,14 @@ int32_t KVDBServiceStub::OnSubscribe( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { sptr remoteObj; - if (!ITypesUtil::Unmarshal(data, remoteObj)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, remoteObj, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } auto observer = (remoteObj == nullptr) ? nullptr : iface_cast(remoteObj); - int32_t status = Subscribe(appId, storeId, observer); + int32_t status = Subscribe(appId, storeId, subUser, observer); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -403,13 +446,14 @@ int32_t KVDBServiceStub::OnUnsubscribe( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { sptr remoteObj; - if (!ITypesUtil::Unmarshal(data, remoteObj)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, remoteObj, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } auto observer = (remoteObj == nullptr) ? nullptr : iface_cast(remoteObj); - int32_t status = Unsubscribe(appId, storeId, observer); + int32_t status = Unsubscribe(appId, storeId, subUser, observer); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -422,13 +466,14 @@ int32_t KVDBServiceStub::OnGetBackupPassword( const AppId &appId, const StoreId &storeId, MessageParcel &data, MessageParcel &reply) { int32_t passwordType; - if (!ITypesUtil::Unmarshal(data, passwordType)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, passwordType, subUser)) { ZLOGE("Unmarshal type failed, appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } std::vector> passwords; - int32_t status = GetBackupPassword(appId, storeId, passwords, passwordType); + int32_t status = GetBackupPassword(appId, storeId, subUser, passwords, passwordType); if (!ITypesUtil::Marshal(reply, status, passwords)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); @@ -521,12 +566,13 @@ int32_t KVDBServiceStub::OnRemoveDeviceData(const AppId &appId, const StoreId &s MessageParcel &reply) { std::string device; - if (!ITypesUtil::Unmarshal(data, device)) { + int32_t subUser; + if (!ITypesUtil::Unmarshal(data, device, subUser)) { ZLOGE("Unmarshal appId:%{public}s storeId:%{public}s", appId.appId.c_str(), Anonymous::Change(storeId.storeId).c_str()); return IPC_STUB_INVALID_DATA_ERR; } - int32_t status = RemoveDeviceData(appId, storeId, device); + int32_t status = RemoveDeviceData(appId, storeId, subUser, device); if (!ITypesUtil::Marshal(reply, status)) { ZLOGE("Marshal status:0x%{public}x appId:%{public}s", status, appId.appId.c_str()); return IPC_STUB_WRITE_PARCEL_ERR; diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.cpp b/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.cpp index 11232567..7040dfb8 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.cpp +++ b/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.cpp @@ -19,13 +19,13 @@ #include #include "accesstoken_kit.h" -#include "crypto_manager.h" #include "device_manager_adapter.h" #include "directory/directory_manager.h" #include "kvdb_general_store.h" #include "log_print.h" #include "metadata/meta_data_manager.h" #include "metadata/secret_key_meta_data.h" +#include "utils/anonymous.h" namespace OHOS::DistributedKv { using namespace OHOS::DistributedData; using system_clock = std::chrono::system_clock; @@ -40,7 +40,19 @@ Upgrade &Upgrade::GetInstance() Upgrade::DBStatus Upgrade::UpdateStore(const StoreMeta &old, const StoreMeta &meta, const std::vector &pwd) { - if (old.version < StoreMeta::UUID_CHANGED_TAG && old.storeType == DEVICE_COLLABORATION) { + if (old.isNeedUpdateDeviceId && !old.isEncrypt) { + auto store = GetDBStore(meta, pwd); + if (store == nullptr) { + ZLOGI("get store failed, appId:%{public}s storeId:%{public}s", old.appId.c_str(), + Anonymous::Change(old.storeId).c_str()); + return DBStatus::DB_ERROR; + } + store->OperateDataStatus(static_cast(DistributedDB::DataOperator::UPDATE_TIME) | + static_cast(DistributedDB::DataOperator::RESET_UPLOAD_CLOUD)); + } + + if ((old.version < StoreMeta::UUID_CHANGED_TAG || (old.isNeedUpdateDeviceId && !old.isEncrypt)) && + old.storeType == DEVICE_COLLABORATION) { auto upStatus = Upgrade::GetInstance().UpdateUuid(old, meta, pwd); if (upStatus != DBStatus::OK) { return DBStatus::DB_ERROR; @@ -88,20 +100,6 @@ Upgrade::DBStatus Upgrade::ExportStore(const StoreMeta &old, const StoreMeta &me return DBStatus::OK; } -void Upgrade::UpdatePassword(const StoreMeta &meta, const std::vector &password) -{ - if (!meta.isEncrypt) { - return; - } - - SecretKeyMetaData secretKey; - secretKey.storeType = meta.storeType; - secretKey.sKey = CryptoManager::GetInstance().Encrypt(password); - auto time = system_clock::to_time_t(system_clock::now()); - secretKey.time = { reinterpret_cast(&time), reinterpret_cast(&time) + sizeof(time) }; - MetaDataManager::GetInstance().SaveMeta(meta.GetSecretKey(), secretKey, true); -} - Upgrade::DBStatus Upgrade::UpdateUuid(const StoreMeta &old, const StoreMeta &meta, const std::vector &pwd) { auto kvStore = GetDBStore(meta, pwd); diff --git a/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.h b/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.h index 7d4e9686..62458e23 100644 --- a/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.h +++ b/datamgr_service/services/distributeddataservice/service/kvdb/upgrade.h @@ -41,7 +41,6 @@ public: DBStatus UpdateStore(const StoreMeta &old, const StoreMeta &meta, const std::vector &pwd); DBStatus ExportStore(const StoreMeta &old, const StoreMeta &meta); - void UpdatePassword(const StoreMeta &meta, const std::vector &password); DBStatus UpdateUuid(const StoreMeta &old, const StoreMeta &meta, const std::vector &pwd); API_EXPORT std::string GetEncryptedUuidByMeta(const StoreMeta &meta); diff --git a/datamgr_service/services/distributeddataservice/service/matrix/BUILD.gn b/datamgr_service/services/distributeddataservice/service/matrix/BUILD.gn index 85c4eb2d..3499cbe6 100644 --- a/datamgr_service/services/distributeddataservice/service/matrix/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/matrix/BUILD.gn @@ -35,6 +35,7 @@ ohos_source_set("distributeddata_matrix") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ @@ -49,6 +50,7 @@ ohos_source_set("distributeddata_matrix") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ @@ -58,6 +60,7 @@ ohos_source_set("distributeddata_matrix") { external_deps = [ "device_manager:devicemanagersdk", "hilog:libhilog", + "json:nlohmann_json_static", "kv_store:datamgr_common", ] subsystem_name = "distributeddatamgr" diff --git a/datamgr_service/services/distributeddataservice/service/object/BUILD.gn b/datamgr_service/services/distributeddataservice/service/object/BUILD.gn index 7520d714..29056fe0 100644 --- a/datamgr_service/services/distributeddataservice/service/object/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/object/BUILD.gn @@ -49,6 +49,7 @@ ohos_source_set("distributeddata_object") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ "include" ] @@ -61,6 +62,7 @@ ohos_source_set("distributeddata_object") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ @@ -75,6 +77,7 @@ ohos_source_set("distributeddata_object") { "dfs_service:cloudsync_asset_kit_inner", "dfs_service:distributed_file_daemon_kit_inner", "dmsfwk:distributed_sdk", + "hisysevent:libhisysevent", "kv_store:datamgr_common", "kv_store:distributeddb", ] diff --git a/datamgr_service/services/distributeddataservice/service/object/src/object_manager.cpp b/datamgr_service/services/distributeddataservice/service/object/src/object_manager.cpp index feab5574..3e3c554f 100644 --- a/datamgr_service/services/distributeddataservice/service/object/src/object_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/object/src/object_manager.cpp @@ -250,8 +250,8 @@ int32_t ObjectStoreManager::Retrieve( ObjectRecord results{}; int32_t status = RetrieveFromStore(bundleName, sessionId, results); if (status != OBJECT_SUCCESS) { - ZLOGI("Retrieve from store failed, status: %{public}d", status); - Close(); + ZLOGI("Retrieve from store failed, status: %{public}d, close after one minute.", status); + CloseAfterMinute(); proxy->Completed(ObjectRecord(), false); return status; } diff --git a/datamgr_service/services/distributeddataservice/service/permission/BUILD.gn b/datamgr_service/services/distributeddataservice/service/permission/BUILD.gn index 2795bda9..0f28d277 100644 --- a/datamgr_service/services/distributeddataservice/service/permission/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/permission/BUILD.gn @@ -36,6 +36,7 @@ ohos_source_set("distributeddata_permit") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] include_dirs = [ @@ -51,6 +52,7 @@ ohos_source_set("distributeddata_permit") { "-Wno-c99-designator", "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Oz", + "-fstack-protector-strong", ] deps = [ diff --git a/datamgr_service/services/distributeddataservice/service/rdb/BUILD.gn b/datamgr_service/services/distributeddataservice/service/rdb/BUILD.gn index e8cd0981..ad693d92 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/rdb/BUILD.gn @@ -25,9 +25,9 @@ config("rdb_public_config") { "${data_service_path}/service/permission/include", "${data_service_path}/service/cloud", "${data_service_path}/framework/include", + "${data_service_path}/framework/include/dfx", "${data_service_path}/adapter/include/utils", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] } @@ -46,6 +46,7 @@ ohos_source_set("distributeddata_rdb") { "rdb_cloud.cpp", "rdb_cursor.cpp", "rdb_general_store.cpp", + "rdb_hiview_adapter.cpp", "rdb_notifier_proxy.cpp", "rdb_query.cpp", "rdb_result_set_impl.cpp", @@ -62,6 +63,7 @@ ohos_source_set("distributeddata_rdb") { "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Werror", "-Oz", + "-fstack-protector-strong", ] deps = [ @@ -95,6 +97,7 @@ ohos_source_set("distributeddata_rdb") { cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] subsystem_name = "distributeddatamgr" diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_asset_loader.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_asset_loader.cpp index 93cd875b..801af00c 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_asset_loader.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_asset_loader.cpp @@ -81,10 +81,10 @@ void RdbAssetLoader::UpdateStatus(AssetRecord &assetRecord, const VBucket &asset continue; } for (const auto &asset : *downloadAssets) { - if (assetRecord.status == DBStatus::OK) { - assetRecord.status = ConvertStatus(static_cast(asset.status)); + if (assetRecord.status != DBStatus::OK) { return; } + assetRecord.status = ConvertStatus(static_cast(asset.status)); } } } diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.cpp index 203eec5d..87e15dac 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.cpp @@ -23,14 +23,16 @@ #include "changeevent/remote_change_event.h" #include "cloud/asset_loader.h" #include "cloud/cloud_db.h" +#include "cloud/cloud_mark.h" #include "cloud/cloud_store_types.h" #include "cloud/schema_meta.h" #include "cloud_service.h" #include "commonevent/data_sync_event.h" #include "communicator/device_manager_adapter.h" #include "crypto_manager.h" -#include "dfx_types.h" #include "device_manager_adapter.h" +#include "dfx/dfx_types.h" +#include "dfx/reporter.h" #include "eventcenter/event_center.h" #include "log_print.h" #include "metadata/meta_data_manager.h" @@ -40,7 +42,6 @@ #include "rdb_query.h" #include "rdb_result_set_impl.h" #include "relational_store_manager.h" -#include "reporter.h" #include "snapshot/bind_event.h" #include "utils/anonymous.h" #include "value_proxy.h" @@ -48,7 +49,6 @@ namespace OHOS::DistributedRdb { using namespace DistributedData; using namespace DistributedDB; using namespace NativeRdb; -using namespace CloudData; using namespace std::chrono; using namespace DistributedDataDfx; using DBField = DistributedDB::Field; @@ -57,14 +57,7 @@ using DBSchema = DistributedDB::DataBaseSchema; using ClearMode = DistributedDB::ClearMode; using DBStatus = DistributedDB::DBStatus; using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; -constexpr uint32_t EXECUTE_TIMEOUT = 1000; // ms -#define TIME_STATISTIC(timeout) \ -// DdsTrace(std::string(LOG_TAG "::") + std::string(__FUNCTION__), API_PERFORMANCE_TRACE_ON, \ -// [](const std::string &value, uint64_t delta) { \ -// if (delta > timeout) { \ -// ZLOGW("actual cost:%{public}lu ms, expected cost:%{public}lu ms", delta, timeout); \ -// } \ -// }); + constexpr const char *INSERT = "INSERT INTO "; constexpr const char *REPLACE = "REPLACE INTO "; constexpr const char *VALUES = " VALUES "; @@ -122,6 +115,7 @@ static DistributedSchema GetGaussDistributedSchema(const Database &database) DistributedField dbField; dbField.colName = field.colName; dbField.isP2pSync = IsExistence(field.colName, table.deviceSyncFields); + dbField.isSpecified = field.primary; dbTable.fields.push_back(std::move(dbField)); } } @@ -173,7 +167,7 @@ RdbGeneralStore::RdbGeneralStore(const StoreMetaData &meta) SecretKeyMetaData secretKeyMeta; MetaDataManager::GetInstance().LoadMeta(key, secretKeyMeta, true); std::vector decryptKey; - CryptoManager::GetInstance().Decrypt(secretKeyMeta.sKey, decryptKey); + CryptoManager::GetInstance().Decrypt(meta, secretKeyMeta, decryptKey); option.passwd.SetValue(decryptKey.data(), decryptKey.size()); std::fill(decryptKey.begin(), decryptKey.end(), 0); option.isEncryptedDb = meta.isEncrypt; @@ -202,20 +196,16 @@ RdbGeneralStore::RdbGeneralStore(const StoreMetaData &meta) PragmaData data = static_cast(const_cast(static_cast(&meta.isManualClean))); delegate_->Pragma(PragmaCmd::LOGIC_DELETE_SYNC_DATA, data); } - std::pair tableData = GetDistributedSchema(meta); - if (delegate_ != nullptr && tableData.first) { - delegate_->SetDistributedSchema(GetGaussDistributedSchema(tableData.second)); - } ZLOGI("Get rdb store, tokenId:%{public}u, bundleName:%{public}s, storeName:%{public}s, user:%{public}s," - "isEncrypt:%{public}d, isManualClean:%{public}d, isSearchable:%{public}d", - meta.tokenId, meta.bundleName.c_str(), Anonymous::Change(meta.storeId).c_str(), meta.user.c_str(), - meta.isEncrypt, meta.isManualClean, meta.isSearchable); + "isEncrypt:%{public}d, isManualClean:%{public}d, isSearchable:%{public}d, path:%{public}s", + meta.tokenId, meta.bundleName.c_str(), Anonymous::Change(meta.storeId).c_str(), meta.user.c_str(), + meta.isEncrypt, meta.isManualClean, meta.isSearchable, meta.dataDir.c_str()); } RdbGeneralStore::~RdbGeneralStore() { - ZLOGI("Destruct. BundleName: %{public}s. StoreName: %{public}s. user: %{public}d", - storeInfo_.bundleName.c_str(), Anonymous::Change(storeInfo_.storeName).c_str(), storeInfo_.user); + ZLOGI("Destruct. BundleName: %{public}s. StoreName: %{public}s. user: %{public}d", storeInfo_.bundleName.c_str(), + Anonymous::Change(storeInfo_.storeName).c_str(), storeInfo_.user); manager_.CloseStore(delegate_); delegate_ = nullptr; bindInfo_.loader_ = nullptr; @@ -241,7 +231,6 @@ int32_t RdbGeneralStore::BindSnapshots(std::shared_ptr &bindInfos, const CloudConfig &config) { - TIME_STATISTIC(EXECUTE_TIMEOUT); if (bindInfos.empty()) { return GeneralError::E_OK; } @@ -296,7 +285,6 @@ bool RdbGeneralStore::IsBound(uint32_t user) int32_t RdbGeneralStore::Close(bool isForce) { - TIME_STATISTIC(EXECUTE_TIMEOUT); { std::unique_lock lock(rwMutex_, std::chrono::seconds(isForce ? LOCK_TIMEOUT : 0)); if (!lock) { @@ -332,7 +320,6 @@ int32_t RdbGeneralStore::Close(bool isForce) int32_t RdbGeneralStore::Execute(const std::string &table, const std::string &sql) { - TIME_STATISTIC(EXECUTE_TIMEOUT); std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("Database already closed! database:%{public}s, table:%{public}s, sql:%{public}s", @@ -344,7 +331,7 @@ int32_t RdbGeneralStore::Execute(const std::string &table, const std::string &sq auto status = delegate_->ExecuteSql({ sql, {}, false }, changedData); if (status != DBStatus::OK) { ZLOGE("Execute failed! ret:%{public}d, sql:%{public}s, data size:%{public}zu", status, - Anonymous::Change(sql).c_str(), changedData.size()); + Anonymous::Change(sql).c_str(), changedData.size()); if (status == DBStatus::BUSY) { return GeneralError::E_BUSY; } @@ -373,7 +360,6 @@ size_t RdbGeneralStore::SqlConcatenate(VBucket &value, std::string &strColumnSql int32_t RdbGeneralStore::Insert(const std::string &table, VBuckets &&values) { - TIME_STATISTIC(EXECUTE_TIMEOUT); if (table.empty() || values.size() == 0) { ZLOGE("Insert:table maybe empty:%{public}d,value size is:%{public}zu", table.empty(), values.size()); return GeneralError::E_INVALID_ARGS; @@ -416,8 +402,8 @@ int32_t RdbGeneralStore::Insert(const std::string &table, VBuckets &&values) if (IsPrintLog(status)) { auto time = static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); - ZLOGE("Failed! ret:%{public}d, sql:%{public}s, data size:%{public}zu times %{public}" PRIu64 ".", - status, Anonymous::Change(sql).c_str(), changedData.size(), time); + ZLOGE("Failed! ret:%{public}d, sql:%{public}s, data size:%{public}zu times %{public}" PRIu64 ".", status, + Anonymous::Change(sql).c_str(), changedData.size(), time); } return GeneralError::E_ERROR; } @@ -445,7 +431,6 @@ bool RdbGeneralStore::IsPrintLog(DBStatus status) int32_t RdbGeneralStore::Update(const std::string &table, const std::string &setSql, Values &&values, const std::string &whereSql, Values &&conditions) { - TIME_STATISTIC(EXECUTE_TIMEOUT); if (table.empty()) { ZLOGE("Update: table can't be empty!"); return GeneralError::E_INVALID_ARGS; @@ -478,8 +463,8 @@ int32_t RdbGeneralStore::Update(const std::string &table, const std::string &set } auto status = delegate_->ExecuteSql({ sqlIn, std::move(bindArgs), false }, changedData); if (status != DBStatus::OK) { - ZLOGE("Failed! ret:%{public}d, sql:%{public}s, data size:%{public}zu", status, Anonymous::Change(sqlIn).c_str(), - changedData.size()); + ZLOGE("Failed! ret:%{public}d, sql:%{public}s, data size:%{public}zu", status, + Anonymous::Change(sqlIn).c_str(), changedData.size()); return GeneralError::E_ERROR; } return GeneralError::E_OK; @@ -487,7 +472,6 @@ int32_t RdbGeneralStore::Update(const std::string &table, const std::string &set int32_t RdbGeneralStore::Replace(const std::string &table, VBucket &&value) { - TIME_STATISTIC(EXECUTE_TIMEOUT); if (table.empty() || value.size() == 0) { return GeneralError::E_INVALID_ARGS; } @@ -510,8 +494,8 @@ int32_t RdbGeneralStore::Replace(const std::string &table, VBucket &&value) } auto status = delegate_->ExecuteSql({ sql, std::move(bindArgs) }, changedData); if (status != DBStatus::OK) { - ZLOGE("Replace failed! ret:%{public}d, table:%{public}s, sql:%{public}s, fields:%{public}s", - status, Anonymous::Change(table).c_str(), Anonymous::Change(sql).c_str(), columnSql.c_str()); + ZLOGE("Replace failed! ret:%{public}d, table:%{public}s, sql:%{public}s, fields:%{public}s", status, + Anonymous::Change(table).c_str(), Anonymous::Change(sql).c_str(), columnSql.c_str()); if (status == DBStatus::BUSY) { return GeneralError::E_BUSY; } @@ -525,10 +509,9 @@ int32_t RdbGeneralStore::Delete(const std::string &table, const std::string &sql return 0; } -std::pair> RdbGeneralStore::Query(__attribute__((unused))const std::string &table, +std::pair> RdbGeneralStore::Query(__attribute__((unused)) const std::string &table, const std::string &sql, Values &&args) { - TIME_STATISTIC(EXECUTE_TIMEOUT); std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("Database already closed! database:%{public}s", Anonymous::Change(storeInfo_.storeName).c_str()); @@ -540,7 +523,6 @@ std::pair> RdbGeneralStore::Query(__attribute__ std::pair> RdbGeneralStore::Query(const std::string &table, GenQuery &query) { - TIME_STATISTIC(EXECUTE_TIMEOUT); RdbQuery *rdbQuery = nullptr; auto ret = query.QueryInterface(rdbQuery); if (ret != GeneralError::E_OK || rdbQuery == nullptr) { @@ -559,14 +541,13 @@ std::pair> RdbGeneralStore::Query(const std::st return { GeneralError::E_ERROR, nullptr }; } auto cursor = RemoteQuery(*rdbQuery->GetDevices().begin(), rdbQuery->GetRemoteCondition()); - return { cursor != nullptr ? GeneralError::E_OK : GeneralError::E_ERROR, cursor}; + return { cursor != nullptr ? GeneralError::E_OK : GeneralError::E_ERROR, cursor }; } return { GeneralError::E_ERROR, nullptr }; } int32_t RdbGeneralStore::MergeMigratedData(const std::string &tableName, VBuckets &&values) { - TIME_STATISTIC(EXECUTE_TIMEOUT); std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("Database already closed! database:%{public}s, table:%{public}s", @@ -580,11 +561,10 @@ int32_t RdbGeneralStore::MergeMigratedData(const std::string &tableName, VBucket int32_t RdbGeneralStore::CleanTrackerData(const std::string &tableName, int64_t cursor) { - TIME_STATISTIC(EXECUTE_TIMEOUT); std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("Database already closed! database:%{public}s, table:%{public}s", - Anonymous::Change(storeInfo_.storeName).c_str(), Anonymous::Change(tableName).c_str()); + Anonymous::Change(storeInfo_.storeName).c_str(), Anonymous::Change(tableName).c_str()); return GeneralError::E_ERROR; } @@ -634,7 +614,6 @@ std::pair RdbGeneralStore::DoCloudSync(const Devices &devices, std::pair RdbGeneralStore::Sync(const Devices &devices, GenQuery &query, DetailAsync async, const SyncParam &syncParam) { - TIME_STATISTIC(EXECUTE_TIMEOUT); DistributedDB::Query dbQuery; RdbQuery *rdbQuery = nullptr; bool isPriority = false; @@ -650,8 +629,9 @@ std::pair RdbGeneralStore::Sync(const Devices &devices, GenQue std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("store already closed! devices count:%{public}zu, the 1st:%{public}s, mode:%{public}d, " - "wait:%{public}d", devices.size(), - devices.empty() ? "null" : Anonymous::Change(*devices.begin()).c_str(), syncParam.mode, syncParam.wait); + "wait:%{public}d", + devices.size(), devices.empty() ? "null" : Anonymous::Change(*devices.begin()).c_str(), syncParam.mode, + syncParam.wait); return { GeneralError::E_ALREADY_CLOSED, DBStatus::OK }; } auto dbStatus = DistributedDB::INVALID_ARGS; @@ -665,7 +645,6 @@ std::pair RdbGeneralStore::Sync(const Devices &devices, GenQue std::pair> RdbGeneralStore::PreSharing(GenQuery &query) { - TIME_STATISTIC(EXECUTE_TIMEOUT); RdbQuery *rdbQuery = nullptr; auto ret = query.QueryInterface(rdbQuery); if (ret != GeneralError::E_OK || rdbQuery == nullptr) { @@ -723,8 +702,8 @@ VBuckets RdbGeneralStore::ExtractExtend(VBuckets &values) const return extends; } -std::string RdbGeneralStore::BuildSql( - const std::string &table, const std::string &statement, const std::vector &columns) const +std::string RdbGeneralStore::BuildSql(const std::string &table, const std::string &statement, + const std::vector &columns) const { std::string sql = "select "; sql.append(CLOUD_GID); @@ -742,7 +721,6 @@ std::string RdbGeneralStore::BuildSql( int32_t RdbGeneralStore::Clean(const std::vector &devices, int32_t mode, const std::string &tableName) { - TIME_STATISTIC(EXECUTE_TIMEOUT); if (mode < 0 || mode > CLEAN_MODE_BUTT) { return GeneralError::E_INVALID_ARGS; } @@ -812,8 +790,7 @@ RdbGeneralStore::DBBriefCB RdbGeneralStore::GetDBBriefCB(DetailAsync async) if (!async) { return [](auto &) {}; } - return [async = std::move(async)]( - const std::map> &result) { + return [async = std::move(async)](const std::map> &result) { DistributedData::GenDetails details; for (auto &[key, tables] : result) { auto &value = details[key]; @@ -834,7 +811,7 @@ RdbGeneralStore::DBProcessCB RdbGeneralStore::GetDBProcessCB(DetailAsync async, { std::shared_lock lock(asyncMutex_); return [async, autoAsync = async_, highMode, storeInfo = storeInfo_, flag = syncNotifyFlag_, syncMode, syncId, - rdbCloud = GetRdbCloud()](const std::map &processes) { + rdbCloud = GetRdbCloud()](const std::map &processes) { DistributedData::GenDetails details; for (auto &[id, process] : processes) { bool isDownload = false; @@ -856,9 +833,9 @@ RdbGeneralStore::DBProcessCB RdbGeneralStore::GetDBProcessCB(DetailAsync async, table.download.failed = value.downLoadInfo.failCount; table.download.untreated = table.download.total - table.download.success - table.download.failed; detail.changeCount = (process.process == FINISHED) - ? value.downLoadInfo.insertCount + value.downLoadInfo.updateCount + - value.downLoadInfo.deleteCount - : 0; + ? value.downLoadInfo.insertCount + value.downLoadInfo.updateCount + + value.downLoadInfo.deleteCount + : 0; totalCount += table.download.total; } if (process.process == FINISHED) { @@ -876,8 +853,8 @@ RdbGeneralStore::DBProcessCB RdbGeneralStore::GetDBProcessCB(DetailAsync async, async(details); } - if (highMode == AUTO_SYNC_MODE && autoAsync - && (details.empty() || details.begin()->second.code != E_SYNC_TASK_MERGED)) { + if (highMode == AUTO_SYNC_MODE && autoAsync && + (details.empty() || details.begin()->second.code != E_SYNC_TASK_MERGED)) { autoAsync(details); } }; @@ -945,13 +922,49 @@ int32_t RdbGeneralStore::SetDistributedTables(const std::vector &ta properties.push_back({ reference.sourceTable, reference.targetTable, reference.refFields }); } auto status = delegate_->SetReference(properties); - if (status != DistributedDB::DBStatus::OK) { + if (status != DistributedDB::DBStatus::OK && status != DistributedDB::DBStatus::PROPERTY_CHANGED) { ZLOGE("distributed table set reference failed, err:%{public}d", status); return GeneralError::E_ERROR; } + auto [exist, database] = GetDistributedSchema(observer_.meta_); + if (exist) { + delegate_->SetDistributedSchema(GetGaussDistributedSchema(database)); + } + CloudMark metaData(storeInfo_); + if (MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData, true) && metaData.isClearWaterMark) { + DistributedDB::ClearMetaDataOption option{ .mode = DistributedDB::ClearMetaDataMode::CLOUD_WATERMARK }; + auto ret = delegate_->ClearMetaData(option); + if (ret != DBStatus::OK) { + ZLOGE("clear watermark failed, err:%{public}d", ret); + return GeneralError::E_ERROR; + } + MetaDataManager::GetInstance().DelMeta(metaData.GetKey(), true); + auto event = std::make_unique(CloudEvent::UPGRADE_SCHEMA, storeInfo_); + EventCenter::GetInstance().PostEvent(std::move(event)); + ZLOGI("clear watermark success, bundleName:%{public}s, storeName:%{public}s", storeInfo_.bundleName.c_str(), + Anonymous::Change(storeInfo_.storeName).c_str()); + } return GeneralError::E_OK; } +void RdbGeneralStore::SetConfig(const StoreConfig &storeConfig) +{ + std::shared_lock lock(rwMutex_); + if (delegate_ == nullptr) { + ZLOGE("database already closed!, tableMode is :%{public}d", + storeConfig.tableMode.has_value() ? static_cast(storeConfig.tableMode.value()) : -1); + } + if (storeConfig.tableMode.has_value()) { + RelationalStoreDelegate::StoreConfig config; + if (storeConfig.tableMode == DistributedTableMode::COLLABORATION) { + config.tableMode = DistributedDB::DistributedTableMode::COLLABORATION; + } else if (storeConfig.tableMode == DistributedTableMode::SPLIT_BY_DEVICE) { + config.tableMode = DistributedDB::DistributedTableMode::SPLIT_BY_DEVICE; + } + delegate_->SetStoreConfig(config); + } +} + int32_t RdbGeneralStore::SetTrackerTable(const std::string &tableName, const std::set &trackerColNames, const std::set &extendColNames, bool isForceUpgrade) { @@ -968,8 +981,8 @@ int32_t RdbGeneralStore::SetTrackerTable(const std::string &tableName, const std return GeneralError::E_WITH_INVENTORY_DATA; } if (status != DBStatus::OK) { - ZLOGE("Set tracker table failed! ret:%{public}d, database:%{public}s, tables name:%{public}s", - status, Anonymous::Change(storeInfo_.storeName).c_str(), Anonymous::Change(tableName).c_str()); + ZLOGE("Set tracker table failed! ret:%{public}d, database:%{public}s, tables name:%{public}s", status, + Anonymous::Change(storeInfo_.storeName).c_str(), Anonymous::Change(tableName).c_str()); return GeneralError::E_ERROR; } return GeneralError::E_OK; @@ -1070,8 +1083,8 @@ void RdbGeneralStore::OnSyncFinish(const StoreInfo &storeInfo, uint32_t flag, ui return; } StoreInfo info = storeInfo; - auto evt = std::make_unique(std::move(info), syncMode, DataSyncEvent::DataSyncStatus::FINISH, - traceId); + auto evt = + std::make_unique(std::move(info), syncMode, DataSyncEvent::DataSyncStatus::FINISH, traceId); EventCenter::GetInstance().PostEvent(std::move(evt)); } @@ -1103,16 +1116,15 @@ std::vector RdbGeneralStore::GetIntersection(std::vector res; for (auto &it : syncTables) { - if (localTables.count(it) && - localTables.count(RelationalStoreManager::GetDistributedLogTableName(it))) { + if (localTables.count(it) && localTables.count(RelationalStoreManager::GetDistributedLogTableName(it))) { res.push_back(std::move(it)); } } return res; } -void RdbGeneralStore::ObserverProxy::PostDataChange(const StoreMetaData &meta, - const std::vector &tables, ChangeType type) +void RdbGeneralStore::ObserverProxy::PostDataChange(const StoreMetaData &meta, const std::vector &tables, + ChangeType type) { RemoteChangeEvent::DataInfo info; info.userId = meta.user; @@ -1154,9 +1166,9 @@ void RdbGeneralStore::ObserverProxy::OnChange(DBOrigin origin, const std::string ZLOGD("store:%{public}s table:%{public}s data change from :%{public}s", Anonymous::Change(storeId_).c_str(), Anonymous::Change(data.tableName).c_str(), Anonymous::Change(originalId).c_str()); GenOrigin genOrigin; - genOrigin.origin = (origin == DBOrigin::ORIGIN_LOCAL) - ? GenOrigin::ORIGIN_LOCAL - : (origin == DBOrigin::ORIGIN_CLOUD) ? GenOrigin::ORIGIN_CLOUD : GenOrigin::ORIGIN_NEARBY; + genOrigin.origin = (origin == DBOrigin::ORIGIN_LOCAL) ? GenOrigin::ORIGIN_LOCAL + : (origin == DBOrigin::ORIGIN_CLOUD) ? GenOrigin::ORIGIN_CLOUD + : GenOrigin::ORIGIN_NEARBY; genOrigin.dataType = data.type == DistributedDB::ASSET ? GenOrigin::ASSET_DATA : GenOrigin::BASIC_DATA; genOrigin.id.push_back(originalId); genOrigin.store = storeId_; @@ -1167,6 +1179,12 @@ void RdbGeneralStore::ObserverProxy::OnChange(DBOrigin origin, const std::string auto &info = changeInfo[data.tableName][i]; for (auto &priData : data.primaryData[i]) { Watcher::PRIValue value; + if (priData.empty()) { + ZLOGW("priData is empty, store:%{public}s table:%{public}s data change from :%{public}s, i=%{public}d", + Anonymous::Change(storeId_).c_str(), Anonymous::Change(data.tableName).c_str(), + Anonymous::Change(originalId).c_str(), i); + continue; + } Convert(std::move(*(priData.begin())), value); if (notifyFlag || origin != DBOrigin::ORIGIN_CLOUD || i != DistributedDB::OP_DELETE) { info.push_back(std::move(value)); @@ -1192,7 +1210,6 @@ void RdbGeneralStore::ObserverProxy::OnChange(DBOrigin origin, const std::string std::pair RdbGeneralStore::LockCloudDB() { - TIME_STATISTIC(EXECUTE_TIMEOUT); auto rdbCloud = GetRdbCloud(); if (rdbCloud == nullptr) { return { GeneralError::E_ERROR, 0 }; @@ -1202,7 +1219,6 @@ std::pair RdbGeneralStore::LockCloudDB() int32_t RdbGeneralStore::UnLockCloudDB() { - TIME_STATISTIC(EXECUTE_TIMEOUT); auto rdbCloud = GetRdbCloud(); if (rdbCloud == nullptr) { return GeneralError::E_ERROR; @@ -1218,7 +1234,6 @@ std::shared_ptr RdbGeneralStore::GetRdbCloud() const bool RdbGeneralStore::IsFinished(SyncId syncId) const { - TIME_STATISTIC(EXECUTE_TIMEOUT); std::shared_lock lock(rwMutex_); if (delegate_ == nullptr) { ZLOGE("database already closed! database:%{public}s", Anonymous::Change(storeInfo_.storeName).c_str()); @@ -1249,7 +1264,7 @@ Executor::Task RdbGeneralStore::GetFinishTask(SyncId syncId) }); if (cb != nullptr) { ZLOGW("database:%{public}s syncId:%{public}" PRIu64 " miss finished. ", - Anonymous::Change(storeInfo_.storeName).c_str(), syncId); + Anonymous::Change(storeInfo_.storeName).c_str(), syncId); std::map result; result.insert({ "", { DistributedDB::FINISHED, DBStatus::DB_ERROR } }); cb(result); @@ -1274,7 +1289,7 @@ void RdbGeneralStore::RemoveTasks() tasks_->EraseIf([&cbs, &taskIds, store = storeInfo_.storeName](SyncId syncId, const FinishTask &task) { if (task.cb != nullptr) { ZLOGW("database:%{public}s syncId:%{public}" PRIu64 " miss finished. ", Anonymous::Change(store).c_str(), - syncId); + syncId); } cbs.push_back(std::move(task.cb)); taskIds.push_back(task.taskId); @@ -1290,7 +1305,7 @@ void RdbGeneralStore::RemoveTasks() } }; if (executor_ != nullptr) { - for (auto taskId: taskIds) { + for (auto taskId : taskIds) { executor_->Remove(taskId, true); } executor_->Execute([cbs, func]() { @@ -1322,4 +1337,14 @@ RdbGeneralStore::DBProcessCB RdbGeneralStore::GetCB(SyncId syncId) return; }; } + +int32_t RdbGeneralStore::UpdateDBStatus() +{ + std::shared_lock lock(rwMutex_); + if (delegate_ == nullptr) { + ZLOGE("Database already closed! database:%{public}s", Anonymous::Change(storeInfo_.storeName).c_str()); + return GeneralError::E_ALREADY_CLOSED; + } + return delegate_->OperateDataStatus(static_cast(DataOperator::UPDATE_TIME)); +} } // namespace OHOS::DistributedRdb \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.h index 8d2a4079..98b5d2ed 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_general_store.h @@ -80,6 +80,7 @@ public: int32_t RegisterDetailProgressObserver(DetailAsync async) override; int32_t UnregisterDetailProgressObserver() override; int32_t Close(bool isForce = false) override; + void SetConfig(const StoreConfig &storeConfig) override; int32_t AddRef() override; int32_t Release() override; int32_t BindSnapshots(std::shared_ptr>> bindAssets) override; @@ -87,6 +88,7 @@ public: int32_t CleanTrackerData(const std::string &tableName, int64_t cursor) override; std::pair LockCloudDB() override; int32_t UnLockCloudDB() override; + int32_t UpdateDBStatus() override; private: RdbGeneralStore(const RdbGeneralStore& rdbGeneralStore); diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.cpp new file mode 100644 index 00000000..ea00117e --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "RdbHiviewAdapter" + +#include "rdb_hiview_adapter.h" +#include "log_print.h" +#include +#include + +namespace OHOS::DistributedRdb { + +static constexpr const char *DISTRIBUTED_DATAMGR = "DISTDATAMGR"; +static constexpr const char *STATISTIC_EVENT = "RDB_STATISTIC"; +constexpr int32_t WAIT_TIME = 60 * 60 * 24; // 24h + +RdbHiViewAdapter& RdbHiViewAdapter::GetInstance() +{ + static RdbHiViewAdapter instance; + return instance; +} + +void RdbHiViewAdapter::ReportStatistic(const RdbStatEvent &stat) +{ + if (executors_ == nullptr) { + return; + } + statEvents_.Compute(stat, [&stat](const RdbStatEvent &key, uint32_t &value) { + value++; + return true; + }); +} + +void RdbHiViewAdapter::InvokeSync() +{ + auto statEvents = std::move(statEvents_); + uint32_t count = statEvents.Size(); + if (count == 0) { + return; + } + + uint32_t statTypes[count]; + const char* bundleNames[count]; + const char* storeNames[count]; + uint32_t subTypes[count]; + uint32_t costTimes[count]; + uint32_t occurTimes[count]; + uint32_t index = 0; + + statEvents.ForEach([&statTypes, &bundleNames, &storeNames, &subTypes, &costTimes, &occurTimes, &index]( + const RdbStatEvent &key, uint32_t &value) { + statTypes[index] = key.statType; + bundleNames[index] = key.bundleName.c_str(); + storeNames[index] = key.storeName.c_str(); + subTypes[index] = key.subType; + costTimes[index] = key.costTime; + occurTimes[index] = value; + index++; + return false; + }); + + HiSysEventParam params[] = { + { .name = "TYPE", .t = HISYSEVENT_UINT32_ARRAY, .v = { .array = statTypes }, .arraySize = count }, + { .name = "BUNDLE_NAME", .t = HISYSEVENT_STRING_ARRAY, .v = { .array = bundleNames }, .arraySize = count }, + { .name = "STORE_NAME", .t = HISYSEVENT_STRING_ARRAY, .v = { .array = storeNames }, .arraySize = count }, + { .name = "PARAM_TYPE1", .t = HISYSEVENT_UINT32_ARRAY, .v = { .array = subTypes }, .arraySize = count }, + { .name = "PARAM_TYPE2", .t = HISYSEVENT_UINT32_ARRAY, .v = { .array = costTimes }, .arraySize = count }, + { .name = "TIMES", .t = HISYSEVENT_UINT32_ARRAY, .v = { .array = occurTimes }, .arraySize = count }, + }; + auto size = sizeof(params) / sizeof(params[0]); + OH_HiSysEvent_Write(DISTRIBUTED_DATAMGR, STATISTIC_EVENT, HISYSEVENT_STATISTIC, params, size); +} + +void RdbHiViewAdapter::StartTimerThread() +{ + if (executors_ == nullptr) { + return; + } + if (running_.exchange(true)) { + return; + } + auto interval = std::chrono::seconds(WAIT_TIME); + auto fun = [this]() { + InvokeSync(); + }; + executors_->Schedule(fun, interval); +} + +void RdbHiViewAdapter::SetThreadPool(std::shared_ptr executors) +{ + executors_ = executors; + StartTimerThread(); +} + +} // namespace OHOS::DistributedRdb \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.h new file mode 100644 index 00000000..8f4213f2 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_hiview_adapter.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATAMGR_SERVICE_RDB_HIVIEW_ADAPTER_H +#define DATAMGR_SERVICE_RDB_HIVIEW_ADAPTER_H + +#include "concurrent_map.h" +#include "executor_pool.h" +#include "hisysevent.h" +#include "rdb_types.h" + +namespace OHOS::DistributedRdb { + +class RdbHiViewAdapter { +public: + static RdbHiViewAdapter &GetInstance(); + void ReportStatistic(const RdbStatEvent &stat); + void SetThreadPool(std::shared_ptr executors); + +private: + std::shared_ptr executors_ = nullptr; + ConcurrentMap statEvents_; + void InvokeSync(); + void StartTimerThread(); + std::atomic running_ = false; +}; +} // namespace OHOS::DistributedRdb +#endif // DATAMGR_SERVICE_RDB_HIVIEW_ADAPTER_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.cpp index 298e0998..994a98a4 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.cpp @@ -381,4 +381,4 @@ void RdbQuery::AssetsOnly(const RdbPredicateOperation &operation) query_.AssetsOnly(assetsMap); predicates_->EqualTo(operation.field_, object); } -} // namespace OHOS::DistributedRdb +} // namespace OHOS::DistributedRdb \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.h index 5f985a28..94d9460f 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_query.h @@ -119,4 +119,4 @@ private: DistributedData::QueryNodes queryNodes_; }; } // namespace OHOS::DistributedRdb -#endif // OHOS_DISTRIBUTED_DATA_DATAMGR_SERVICE_RDB_QUERY_H +#endif // OHOS_DISTRIBUTED_DATA_DATAMGR_SERVICE_RDB_QUERY_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_impl.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_impl.h index 388bdc30..c8c07017 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_impl.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_impl.h @@ -25,7 +25,15 @@ #include "value_proxy.h" #include "store/general_value.h" +#define RDB_UTILS_PUSH_WARNING _Pragma("GCC diagnostic push") +#define RDB_UTILS_POP_WARNING _Pragma("GCC diagnostic pop") +#define RDB_UTILS_DISABLE_WARNING_INTERNAL2(warningName) #warningName +#define RDB_UTILS_DISABLE_WARNING(warningName) \ + _Pragma( \ + RDB_UTILS_DISABLE_WARNING_INTERNAL2(GCC diagnostic ignored warningName)) + namespace OHOS::DistributedRdb { + class RdbResultSetImpl final : public NativeRdb::ResultSet { public: using ValueProxy = DistributedData::ValueProxy; @@ -85,6 +93,8 @@ private: }; mutable std::shared_mutex mutex_ {}; +RDB_UTILS_PUSH_WARNING +RDB_UTILS_DISABLE_WARNING("-Wc99-designator") static constexpr ColumnType COLUMNTYPES[DistributedData::TYPE_MAX] = { [DistributedData::TYPE_INDEX] = ColumnType::TYPE_NULL, [DistributedData::TYPE_INDEX] = ColumnType::TYPE_INTEGER, @@ -95,6 +105,7 @@ private: [DistributedData::TYPE_INDEX] = ColumnType::TYPE_BLOB, [DistributedData::TYPE_INDEX] = ColumnType::TYPE_BLOB, }; +RDB_UTILS_POP_WARNING std::shared_ptr resultSet_; std::atomic current_ = -1; int32_t count_ = 0; diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_stub.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_stub.h index 9931418f..bd8ab6cc 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_stub.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_result_set_stub.h @@ -20,6 +20,7 @@ #include "irdb_result_set.h" #include "result_set.h" +#include "rdb_result_set_impl.h" namespace OHOS::DistributedRdb { class RdbResultSetStub : public IRemoteStub { @@ -50,6 +51,8 @@ private: static bool CheckInterfaceToken(MessageParcel &data); using RequestHandle = int (RdbResultSetStub::*)(MessageParcel &, MessageParcel &); +RDB_UTILS_PUSH_WARNING +RDB_UTILS_DISABLE_WARNING("-Wc99-designator") static constexpr RequestHandle HANDLERS[Code::CMD_MAX] = { [Code::CMD_GET_ALL_COLUMN_NAMES] = &RdbResultSetStub::OnGetAllColumnNames, [Code::CMD_GET_COLUMN_COUNT] = &RdbResultSetStub::OnGetColumnCount, @@ -70,6 +73,7 @@ private: [Code::CMD_GET_SIZE] = &RdbResultSetStub::OnGetSize, [Code::CMD_CLOSE] = &RdbResultSetStub::OnClose }; +RDB_UTILS_POP_WARNING std::shared_ptr resultSet_; }; } // namespace OHOS::DistributedRdb diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_schema_config.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_schema_config.cpp index 1f5557f9..8bfff9e0 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_schema_config.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_schema_config.cpp @@ -31,7 +31,7 @@ namespace OHOS::DistributedRdb { using namespace OHOS::Global::Resource; using namespace OHOS::DistributedData; using Serializable = DistributedData::Serializable; -constexpr const char *SCHEMA_PATH = "arkdata/schema/schema.json"; +constexpr const char *SCHEMA_PATH = "arkdata/schema/sync_schema.json"; bool RdbSchemaConfig::GetDistributedSchema(const StoreMetaData &meta, Database &database) { @@ -92,10 +92,10 @@ bool RdbSchemaConfig::GetSchemaFromHap( resMgr->AddResource(hapInfo.hapPath.c_str()); size_t length = 0; std::unique_ptr fileContent; -// int err = resMgr->GetRawFileFromHap(SCHEMA_PATH, length, fileContent); -// if (err != 0) { -// continue; -// } + int err = resMgr->GetRawFileFromHap(SCHEMA_PATH, length, fileContent); + if (err != 0) { + continue; + } std::string jsonData(fileContent.get(), fileContent.get() + length); DbSchema databases; databases.Unmarshall(jsonData); diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.cpp index 7c0181c9..886de4a0 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.cpp @@ -55,6 +55,7 @@ #include "xcollie.h" #include "permit_delegate.h" #include "bootstrap.h" +#include "rdb_hiview_adapter.h" using OHOS::DistributedData::Anonymous; using OHOS::DistributedData::CheckerManager; using OHOS::DistributedData::MetaDataManager; @@ -201,7 +202,7 @@ std::string RdbServiceImpl::ObtainDistributedTableName(const std::string &device int32_t RdbServiceImpl::InitNotifier(const RdbSyncerParam ¶m, const sptr notifier) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (!CheckAccess(param.bundleName_, "")) { ZLOGE("bundleName:%{public}s. Permission error", param.bundleName_.c_str()); return RDB_ERROR; @@ -238,7 +239,7 @@ std::shared_ptr RdbServiceImpl::GetStore(const Rd return store; } -void RdbServiceImpl::UpdateSyncMeta(const StoreMetaData &meta, const StoreMetaData &localMeta) +void RdbServiceImpl::UpdateMeta(const StoreMetaData &meta, const StoreMetaData &localMeta, AutoCache::Store store) { StoreMetaData syncMeta; bool isCreatedSync = MetaDataManager::GetInstance().LoadMeta(meta.GetKey(), syncMeta); @@ -249,6 +250,12 @@ void RdbServiceImpl::UpdateSyncMeta(const StoreMetaData &meta, const StoreMetaDa syncMeta.isEncrypt, meta.isEncrypt, syncMeta.area, meta.area); MetaDataManager::GetInstance().SaveMeta(meta.GetKey(), localMeta); } + Database dataBase; + if (RdbSchemaConfig::GetDistributedSchema(localMeta, dataBase) && !dataBase.name.empty() && + !dataBase.bundleName.empty()) { + MetaDataManager::GetInstance().SaveMeta(dataBase.GetKey(), dataBase, true); + store->SetConfig({false, GeneralStore::DistributedTableMode::COLLABORATION}); + } } int32_t RdbServiceImpl::SetDistributedTables(const RdbSyncerParam ¶m, const std::vector &tables, @@ -272,27 +279,25 @@ int32_t RdbServiceImpl::SetDistributedTables(const RdbSyncerParam ¶m, const Anonymous::Change(param.storeName_).c_str()); return RDB_ERROR; } - if (type == DistributedTableType::DISTRIBUTED_DEVICE) { - UpdateSyncMeta(meta, localMeta); - Database dataBase; - if (RdbSchemaConfig::GetDistributedSchema(localMeta, dataBase) && !dataBase.name.empty() && - !dataBase.bundleName.empty()) { - MetaDataManager::GetInstance().SaveMeta(dataBase.GetKey(), dataBase, true); - } - } else if (type == DistributedTableType::DISTRIBUTED_CLOUD) { - if (localMeta.asyncDownloadAsset != param.asyncDownloadAsset_) { - localMeta.asyncDownloadAsset = param.asyncDownloadAsset_; - ZLOGI("update meta, bundleName:%{public}s, storeName:%{public}s, asyncDownloadAsset?[%{public}d]", - param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str(), localMeta.asyncDownloadAsset); - MetaDataManager::GetInstance().SaveMeta(localMeta.GetKey(), localMeta, true); - } - } auto store = GetStore(param); if (store == nullptr) { ZLOGE("bundleName:%{public}s, storeName:%{public}s. GetStore failed", param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); return RDB_ERROR; } + if (type == DistributedTableType::DISTRIBUTED_DEVICE) { + UpdateMeta(meta, localMeta, store); + } else if (type == DistributedTableType::DISTRIBUTED_CLOUD) { + if (localMeta.asyncDownloadAsset != param.asyncDownloadAsset_ || localMeta.enableCloud != param.enableCloud_) { + ZLOGI("update meta, bundleName:%{public}s, storeName:%{public}s, asyncDownloadAsset? [%{public}d -> " + "%{public}d],enableCloud? [%{public}d -> %{public}d]", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str(), localMeta.asyncDownloadAsset, param.asyncDownloadAsset_, + localMeta.enableCloud, param.enableCloud_); + localMeta.asyncDownloadAsset = param.asyncDownloadAsset_; + localMeta.enableCloud = param.enableCloud_; + MetaDataManager::GetInstance().SaveMeta(localMeta.GetKey(), localMeta, true); + } + } std::vector relationships; for (const auto &reference : references) { DistributedData::Reference relationship = { reference.sourceTable, reference.targetTable, reference.refFields }; @@ -379,9 +384,16 @@ std::pair> RdbServiceImpl::R } auto store = GetStore(param); if (store == nullptr) { - ZLOGE("store is null"); + ZLOGE("bundleName:%{public}s, storeName:%{public}s. GetStore failed", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str()); return { RDB_ERROR, nullptr }; } + StoreMetaData meta = GetStoreMetaData(param); + std::vector devices = {DmAdapter::GetInstance().ToUUID(device)}; + if (IsNeedMetaSync(meta, devices) && !MetaDataManager::GetInstance().Sync(devices, [](auto &results) {}, true)) { + ZLOGW("bundleName:%{public}s, storeName:%{public}s. meta sync failed", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str()); + } RdbQuery rdbQuery; rdbQuery.MakeRemoteQuery(DmAdapter::GetInstance().ToUUID(device), sql, ValueProxy::Convert(selectionArgs)); auto [errCode, cursor] = store->Query("", rdbQuery); @@ -463,6 +475,11 @@ bool RdbServiceImpl::IsNeedMetaSync(const StoreMetaData &meta, const std::vector isAfterMeta = true; break; } + auto [existLocal, localMask] = DeviceMatrix::GetInstance().GetMask(uuid); + if ((localMask & DeviceMatrix::META_STORE_MASK) == DeviceMatrix::META_STORE_MASK) { + isAfterMeta = true; + break; + } } return isAfterMeta; } @@ -571,8 +588,9 @@ int32_t RdbServiceImpl::Subscribe(const RdbSyncerParam ¶m, const SubscribeOp return true; }); if (isCreate) { + auto subUser = GetSubUser(param.subUser_); AutoCache::GetInstance().SetObserver(tokenId, RemoveSuffix(param.storeName_), - GetWatchers(tokenId, param.storeName_)); + GetWatchers(tokenId, param.storeName_), subUser); } return RDB_OK; } @@ -600,8 +618,9 @@ int32_t RdbServiceImpl::UnSubscribe(const RdbSyncerParam ¶m, const Subscribe return true; }); if (destroyed) { + auto subUser = GetSubUser(param.subUser_); AutoCache::GetInstance().SetObserver(tokenId, RemoveSuffix(param.storeName_), - GetWatchers(tokenId, param.storeName_)); + GetWatchers(tokenId, param.storeName_), subUser); } return RDB_OK; } @@ -652,9 +671,10 @@ int32_t RdbServiceImpl::UnregisterAutoSyncCallback(const RdbSyncerParam& param, int32_t RdbServiceImpl::Delete(const RdbSyncerParam ¶m) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto tokenId = IPCSkeleton::GetCallingTokenID(); - AutoCache::GetInstance().CloseStore(tokenId, RemoveSuffix(param.storeName_)); + auto subUser = GetSubUser(param.subUser_); + AutoCache::GetInstance().CloseStore(tokenId, RemoveSuffix(param.storeName_), subUser); RdbSyncerParam tmpParam = param; HapTokenInfo hapTokenInfo; AccessTokenKit::GetHapTokenInfo(tokenId, hapTokenInfo); @@ -668,6 +688,7 @@ int32_t RdbServiceImpl::Delete(const RdbSyncerParam ¶m) MetaDataManager::GetInstance().DelMeta(storeMeta.GetBackupSecretKey(), true); MetaDataManager::GetInstance().DelMeta(storeMeta.GetAutoLaunchKey(), true); MetaDataManager::GetInstance().DelMeta(storeMeta.GetDebugInfoKey(), true); + MetaDataManager::GetInstance().DelMeta(storeMeta.GetDfxInfoKey(), true); MetaDataManager::GetInstance().DelMeta(storeMeta.GetCloneSecretKey(), true); return RDB_OK; } @@ -718,7 +739,7 @@ std::pair> RdbServiceImpl::AllocResource(StoreI int32_t RdbServiceImpl::BeforeOpen(RdbSyncerParam ¶m) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (!CheckAccess(param.bundleName_, param.storeName_)) { ZLOGE("bundleName:%{public}s, storeName:%{public}s. Permission error", param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); @@ -751,7 +772,7 @@ void RdbServiceImpl::SetReturnParam(StoreMetaData &metadata, RdbSyncerParam &par int32_t RdbServiceImpl::AfterOpen(const RdbSyncerParam ¶m) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (!CheckAccess(param.bundleName_, param.storeName_)) { ZLOGE("bundleName:%{public}s, storeName:%{public}s. Permission error", param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); @@ -763,9 +784,9 @@ int32_t RdbServiceImpl::AfterOpen(const RdbSyncerParam ¶m) if (!isCreated || meta != old) { Upgrade(param, old); ZLOGI("meta bundle:%{public}s store:%{public}s type:%{public}d->%{public}d encrypt:%{public}d->%{public}d " - "area:%{public}d->%{public}d", - meta.bundleName.c_str(), meta.GetStoreAlias().c_str(), old.storeType, meta.storeType, - old.isEncrypt, meta.isEncrypt, old.area, meta.area); + "area:%{public}d->%{public}d", meta.bundleName.c_str(), meta.GetStoreAlias().c_str(), old.storeType, + meta.storeType, old.isEncrypt, meta.isEncrypt, old.area, meta.area); + meta.isNeedUpdateDeviceId = isCreated && !TryUpdateDeviceId(param, old, meta); MetaDataManager::GetInstance().SaveMeta(meta.GetKey(), meta, true); AutoLaunchMetaData launchData; if (!MetaDataManager::GetInstance().LoadMeta(meta.GetAutoLaunchKey(), launchData, true)) { @@ -780,30 +801,41 @@ int32_t RdbServiceImpl::AfterOpen(const RdbSyncerParam ¶m) SaveDebugInfo(meta, param); SavePromiseInfo(meta, param); + SaveDfxInfo(meta, param); AppIDMetaData appIdMeta; appIdMeta.bundleName = meta.bundleName; appIdMeta.appId = meta.appId; if (!MetaDataManager::GetInstance().SaveMeta(appIdMeta.GetKey(), appIdMeta, true)) { ZLOGE("meta bundle:%{public}s store:%{public}s type:%{public}d->%{public}d encrypt:%{public}d->%{public}d " - "area:%{public}d->%{public}d", - meta.bundleName.c_str(), meta.GetStoreAlias().c_str(), old.storeType, meta.storeType, - old.isEncrypt, meta.isEncrypt, old.area, meta.area); + "area:%{public}d->%{public}d", meta.bundleName.c_str(), meta.GetStoreAlias().c_str(), old.storeType, + meta.storeType, old.isEncrypt, meta.isEncrypt, old.area, meta.area); return RDB_ERROR; } if (param.isEncrypt_ && !param.password_.empty()) { - auto ret = SetSecretKey(param, meta); - if (ret != RDB_OK) { + if (SetSecretKey(param, meta) != RDB_OK) { ZLOGE("Set secret key failed, bundle:%{public}s store:%{public}s", meta.bundleName.c_str(), meta.GetStoreAlias().c_str()); return RDB_ERROR; } + UpgradeCloneSecretKey(meta); + } + GetSchema(param); + return RDB_OK; +} + +int32_t RdbServiceImpl::ReportStatistic(const RdbSyncerParam& param, const RdbStatEvent &statEvent) +{ + if (!CheckAccess(param.bundleName_, param.storeName_)) { + ZLOGE("bundleName:%{public}s, storeName:%{public}s. Permission error", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str()); + return RDB_ERROR; } - GetCloudSchema(param); + RdbHiViewAdapter::GetInstance().ReportStatistic(statEvent); return RDB_OK; } -void RdbServiceImpl::GetCloudSchema(const RdbSyncerParam ¶m) +void RdbServiceImpl::GetSchema(const RdbSyncerParam ¶m) { if (executors_ != nullptr) { StoreInfo storeInfo; @@ -832,7 +864,11 @@ StoreMetaData RdbServiceImpl::GetStoreMetaData(const RdbSyncerParam ¶m) metaData.bundleName = param.bundleName_; metaData.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; metaData.storeId = RemoveSuffix(param.storeName_); - metaData.user = std::to_string(user); + if (AccessTokenKit::GetTokenTypeFlag(metaData.tokenId) != TOKEN_HAP && param.subUser_ != 0) { + metaData.user = std::to_string(param.subUser_); + } else { + metaData.user = std::to_string(user); + } metaData.storeType = param.type_; metaData.securityLevel = param.level_; metaData.area = param.area_; @@ -854,7 +890,8 @@ int32_t RdbServiceImpl::SetSecretKey(const RdbSyncerParam ¶m, const StoreMet { SecretKeyMetaData newSecretKey; newSecretKey.storeType = meta.storeType; - newSecretKey.sKey = CryptoManager::GetInstance().Encrypt(param.password_); + newSecretKey.area = meta.area; + newSecretKey.sKey = CryptoManager::GetInstance().Encrypt(param.password_, meta.area, meta.user); if (newSecretKey.sKey.empty()) { ZLOGE("encrypt work key error."); return RDB_ERROR; @@ -864,6 +901,18 @@ int32_t RdbServiceImpl::SetSecretKey(const RdbSyncerParam ¶m, const StoreMet return MetaDataManager::GetInstance().SaveMeta(meta.GetSecretKey(), newSecretKey, true) ? RDB_OK : RDB_ERROR; } +bool RdbServiceImpl::UpgradeCloneSecretKey(const StoreMetaData &meta) +{ + SecretKeyMetaData secretKey; + if (MetaDataManager::GetInstance().LoadMeta(meta.GetCloneSecretKey(), secretKey, true) && secretKey.area < 0) { + std::vector clonePwd; + // Update the encryption method for the key + CryptoManager::GetInstance().Decrypt(meta, secretKey, clonePwd, CryptoManager::CLONE_SECRET_KEY); + clonePwd.assign(clonePwd.size(), 0); + } + return true; +} + int32_t RdbServiceImpl::Upgrade(const RdbSyncerParam ¶m, const StoreMetaData &old) { if (old.storeType == RDB_DEVICE_COLLABORATION && old.version < StoreMetaData::UUID_CHANGED_TAG) { @@ -903,7 +952,7 @@ bool RdbServiceImpl::GetDBPassword(const StoreMetaData &metaData, DistributedDB: DistributedData::SecretKeyMetaData secretKeyMeta; MetaDataManager::GetInstance().LoadMeta(key, secretKeyMeta, true); std::vector decryptKey; - CryptoManager::GetInstance().Decrypt(secretKeyMeta.sKey, decryptKey); + CryptoManager::GetInstance().Decrypt(metaData, secretKeyMeta, decryptKey); if (password.SetValue(decryptKey.data(), decryptKey.size()) != DistributedDB::CipherPassword::OK) { std::fill(decryptKey.begin(), decryptKey.end(), 0); ZLOGE("Set secret key value failed. len is (%{public}d)", int32_t(decryptKey.size())); @@ -943,6 +992,7 @@ std::pair RdbServiceImpl::GetInstIndexAndUser(uint32_t tokenId int32_t RdbServiceImpl::OnBind(const BindInfo &bindInfo) { executors_ = bindInfo.executors; + RdbHiViewAdapter::GetInstance().SetThreadPool(executors_); return 0; } @@ -1204,7 +1254,7 @@ int RdbServiceImpl::DoDataChangeSync(const StoreInfo &storeInfo, const RdbChange int32_t RdbServiceImpl::NotifyDataChange( const RdbSyncerParam ¶m, const RdbChangedData &rdbChangedData, const RdbNotifyConfig &rdbNotifyConfig) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (!CheckAccess(param.bundleName_, param.storeName_)) { ZLOGE("bundleName:%{public}s, storeName:%{public}s. Permission error", param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); @@ -1275,7 +1325,7 @@ bool RdbServiceImpl::IsPostImmediately(const int32_t callingPid, const RdbNotify int32_t RdbServiceImpl::SetSearchable(const RdbSyncerParam& param, bool isSearchable) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); if (!CheckAccess(param.bundleName_, param.storeName_)) { ZLOGE("bundleName:%{public}s, storeName:%{public}s. Permission error", param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); @@ -1307,7 +1357,8 @@ int32_t RdbServiceImpl::Disable(const RdbSyncerParam ¶m) { auto tokenId = IPCSkeleton::GetCallingTokenID(); auto storeId = RemoveSuffix(param.storeName_); - AutoCache::GetInstance().Disable(tokenId, storeId); + auto userId = GetSubUser(param.subUser_); + AutoCache::GetInstance().Disable(tokenId, storeId, userId); return RDB_OK; } @@ -1315,7 +1366,8 @@ int32_t RdbServiceImpl::Enable(const RdbSyncerParam ¶m) { auto tokenId = IPCSkeleton::GetCallingTokenID(); auto storeId = RemoveSuffix(param.storeName_); - AutoCache::GetInstance().Enable(tokenId, storeId); + auto userId = GetSubUser(param.subUser_); + AutoCache::GetInstance().Enable(tokenId, storeId, userId); return RDB_OK; } @@ -1337,8 +1389,9 @@ int32_t RdbServiceImpl::GetPassword(const RdbSyncerParam ¶m, std::vectorQueryForegroundUserId(dfxInfo.curUserId_); + } + return RDB_OK; +} + +int32_t RdbServiceImpl::SaveDfxInfo(const StoreMetaData &metaData, const RdbSyncerParam ¶m) +{ + DistributedData::StoreDfxInfo dfxMeta; + dfxMeta.lastOpenTime = param.dfxInfo_.lastOpenTime_; + MetaDataManager::GetInstance().SaveMeta(metaData.GetDfxInfoKey(), dfxMeta, true); + return RDB_OK; +} + int32_t RdbServiceImpl::SavePromiseInfo(const StoreMetaData &metaData, const RdbSyncerParam ¶m) { if (param.tokenIds_.empty() && param.uids_.empty()) { @@ -1472,7 +1562,7 @@ int32_t RdbServiceImpl::SavePromiseInfo(const StoreMetaData &metaData, const Rdb int32_t RdbServiceImpl::VerifyPromiseInfo(const RdbSyncerParam ¶m) { - XCollie xcollie(__FUNCTION__, HiviewDFX::XCOLLIE_FLAG_LOG | HiviewDFX::XCOLLIE_FLAG_RECOVERY); + XCollie xcollie(__FUNCTION__, XCollie::XCOLLIE_LOG | XCollie::XCOLLIE_RECOVERY); auto meta = GetStoreMetaData(param); auto tokenId = IPCSkeleton::GetCallingTokenID(); auto uid = IPCSkeleton::GetCallingUid(); @@ -1510,6 +1600,37 @@ int32_t RdbServiceImpl::VerifyPromiseInfo(const RdbSyncerParam ¶m) return RDB_OK; } +std::string RdbServiceImpl::GetSubUser(const int32_t subUser) +{ + std::string userId = ""; + if (AccessTokenKit::GetTokenTypeFlag(IPCSkeleton::GetCallingTokenID()) != TOKEN_HAP && subUser != 0) { + userId = std::to_string(subUser); + } + return userId; +} + +bool RdbServiceImpl::TryUpdateDeviceId(const RdbSyncerParam ¶m, const StoreMetaData &oldMeta, + StoreMetaData &meta) +{ + StoreMetaData syncMeta; + if (oldMeta.isNeedUpdateDeviceId && oldMeta.storeType >= StoreMetaData::StoreType::STORE_RELATIONAL_BEGIN && + oldMeta.storeType <= StoreMetaData::StoreType::STORE_RELATIONAL_END && + MetaDataManager::GetInstance().LoadMeta(meta.GetKey(), syncMeta)) { + auto store = GetStore(param); + if (store == nullptr) { + ZLOGE("store is null, bundleName:%{public}s storeName:%{public}s", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str()); + return false; + } + auto errCode = store->UpdateDBStatus(); + if (errCode != RDB_OK) { + ZLOGE("Update failed errCode %{public}d", errCode); + return false; + } + } + return true; +} + void RdbServiceImpl::RegisterEvent() { auto process = [this](const Event &event) { diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.h index e5ca813a..0c06e926 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_impl.h @@ -102,6 +102,8 @@ public: int32_t AfterOpen(const RdbSyncerParam ¶m) override; + int32_t ReportStatistic(const RdbSyncerParam ¶m, const RdbStatEvent &statEvent) override; + int32_t GetPassword(const RdbSyncerParam ¶m, std::vector> &password) override; std::pair LockCloudContainer(const RdbSyncerParam ¶m) override; @@ -110,6 +112,8 @@ public: int32_t GetDebugInfo(const RdbSyncerParam ¶m, std::map &debugInfo) override; + int32_t GetDfxInfo(const RdbSyncerParam ¶m, DistributedRdb::RdbDfxInfo &dfxInfo) override; + int32_t VerifyPromiseInfo(const RdbSyncerParam ¶m) override; private: @@ -117,6 +121,7 @@ private: using StaticActs = DistributedData::StaticActs; using DBStatus = DistributedDB::DBStatus; using SyncResult = std::pair, std::map>; + using AutoCache = DistributedData::AutoCache; struct SyncAgent { SyncAgent() = default; explicit SyncAgent(const std::string &bundleName); @@ -169,10 +174,10 @@ private: int DoSync(const RdbSyncerParam ¶m, const Option &option, const PredicatesMemo &predicates, const AsyncDetail &async); - + int DoAutoSync( const std::vector &devices, const Database &dataBase, std::vector tableNames); - + std::vector GetReuseDevice(const std::vector &devices); int DoOnlineSync(const std::vector &devices, const Database &dataBase); @@ -185,7 +190,7 @@ private: bool CheckAccess(const std::string& bundleName, const std::string& storeName); std::shared_ptr GetStore(const RdbSyncerParam& param); - + std::shared_ptr GetStore(const StoreMetaData &storeMetaData); void OnAsyncComplete(uint32_t tokenId, pid_t pid, uint32_t seqNum, Details &&result); @@ -209,9 +214,11 @@ private: static std::pair GetInstIndexAndUser(uint32_t tokenId, const std::string &bundleName); + static std::string GetSubUser(const int32_t subUser); + static bool GetDBPassword(const StoreMetaData &metaData, DistributedDB::CipherPassword &password); - void GetCloudSchema(const RdbSyncerParam ¶m); + void GetSchema(const RdbSyncerParam ¶m); void SetReturnParam(StoreMetaData &metadata, RdbSyncerParam ¶m); @@ -223,15 +230,20 @@ private: int32_t SaveDebugInfo(const StoreMetaData &metaData, const RdbSyncerParam ¶m); + int32_t SaveDfxInfo(const StoreMetaData &metaData, const RdbSyncerParam ¶m); + int32_t SavePromiseInfo(const StoreMetaData &metaData, const RdbSyncerParam ¶m); int32_t PostSearchEvent(int32_t evtId, const RdbSyncerParam& param, DistributedData::SetSearchableEvent::EventInfo &eventInfo); - + bool IsPostImmediately(const int32_t callingPid, const RdbNotifyConfig &rdbNotifyConfig, StoreInfo &storeInfo, DistributedData::DataChangeEvent::EventInfo &eventInfo, const std::string &storeName); + void UpdateMeta(const StoreMetaData &meta, const StoreMetaData &localMeta, AutoCache::Store store); + + bool UpgradeCloneSecretKey(const StoreMetaData &meta); - void UpdateSyncMeta(const StoreMetaData &meta, const StoreMetaData &localMeta); + bool TryUpdateDeviceId(const RdbSyncerParam ¶m, const StoreMetaData &oldMeta, StoreMetaData &meta); static Factory factory_; ConcurrentMap syncAgents_; diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.cpp b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.cpp index c6daa96a..737b216b 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.cpp +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.cpp @@ -75,6 +75,23 @@ int32_t RdbServiceStub::OnAfterOpen(MessageParcel &data, MessageParcel &reply) return RDB_OK; } +int32_t RdbServiceStub::OnReportStatistic(MessageParcel& data, MessageParcel& reply) +{ + RdbSyncerParam param; + RdbStatEvent statEvent; + if (!ITypesUtil::Unmarshal(data, param, statEvent)) { + ZLOGE("Unmarshal bundleName_:%{public}s storeName_:%{public}s", param.bundleName_.c_str(), + Anonymous::Change(param.storeName_).c_str()); + return IPC_STUB_INVALID_DATA_ERR; + } + auto status = ReportStatistic(param, statEvent); + if (!ITypesUtil::Marshal(reply, status)) { + ZLOGE("Marshal status:0x%{public}x", status); + return IPC_STUB_WRITE_PARCEL_ERR; + } + return RDB_OK; +} + int32_t RdbServiceStub::OnDelete(MessageParcel &data, MessageParcel &reply) { RdbSyncerParam param; @@ -218,7 +235,12 @@ int32_t RdbServiceStub::OnRemoteDoRemoteQuery(MessageParcel& data, MessageParcel } auto [status, resultSet] = RemoteQuery(param, device, sql, selectionArgs); - sptr object = new RdbResultSetStub(resultSet); + sptr object = new(std::nothrow) RdbResultSetStub(resultSet); + if (object == nullptr) { + ZLOGE("Failed to create RdbResultSetStub object.bundleName:%{public}s storeName:%{public}s", + param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); + return ERR_NULL_OBJECT; + } if (!ITypesUtil::Marshal(reply, status, object->AsObject())) { ZLOGE("Marshal status:0x%{public}x", status); return IPC_STUB_WRITE_PARCEL_ERR; @@ -332,7 +354,12 @@ int32_t RdbServiceStub::OnRemoteQuerySharingResource(MessageParcel& data, Messag } auto [status, resultSet] = QuerySharingResource(param, predicates, columns); - sptr object = new RdbResultSetStub(resultSet); + sptr object = new(std::nothrow) RdbResultSetStub(resultSet); + if (object == nullptr) { + ZLOGE("Failed to create RdbResultSetStub object.bundleName:%{public}s storeName:%{public}s", + param.bundleName_.c_str(), Anonymous::Change(param.storeName_).c_str()); + return ERR_NULL_OBJECT; + } if (!ITypesUtil::Marshal(reply, status, object->AsObject())) { ZLOGE("Marshal status:0x%{public}x", status); return IPC_STUB_WRITE_PARCEL_ERR; @@ -443,6 +470,22 @@ int32_t RdbServiceStub::OnGetDebugInfo(MessageParcel &data, MessageParcel &reply return RDB_OK; } +int32_t RdbServiceStub::OnGetDfxInfo(MessageParcel &data, MessageParcel &reply) +{ + RdbSyncerParam param; + if (!ITypesUtil::Unmarshal(data, param)) { + ZLOGE("Unmarshal failed"); + return IPC_STUB_INVALID_DATA_ERR; + } + RdbDfxInfo dfxInfo; + auto status = GetDfxInfo(param, dfxInfo); + if (!ITypesUtil::Marshal(reply, status, dfxInfo)) { + ZLOGE("Marshal status:0x%{public}x", status); + return IPC_STUB_WRITE_PARCEL_ERR; + } + return RDB_OK; +} + int32_t RdbServiceStub::OnVerifyPromiseInfo(MessageParcel &data, MessageParcel &reply) { RdbSyncerParam param; diff --git a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.h b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.h index 9164eb0a..1f9a75d1 100644 --- a/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.h +++ b/datamgr_service/services/distributeddataservice/service/rdb/rdb_service_stub.h @@ -20,6 +20,7 @@ #include "rdb_service.h" #include "rdb_notifier.h" #include "feature/feature_system.h" +#include "rdb_result_set_impl.h" namespace OHOS::DistributedRdb { using RdbServiceCode = OHOS::DistributedRdb::RelationalStore::RdbServiceInterfaceCode; @@ -64,6 +65,8 @@ private: int32_t OnAfterOpen(MessageParcel& data, MessageParcel& reply); + int32_t OnReportStatistic(MessageParcel& data, MessageParcel& reply); + int32_t OnDisable(MessageParcel& data, MessageParcel& reply); int32_t OnEnable(MessageParcel& data, MessageParcel& reply); @@ -76,9 +79,13 @@ private: int32_t OnGetDebugInfo(MessageParcel& data, MessageParcel& reply); + int32_t OnGetDfxInfo(MessageParcel& data, MessageParcel& reply); + int32_t OnVerifyPromiseInfo(MessageParcel& data, MessageParcel& reply); using RequestHandle = int (RdbServiceStub::*)(MessageParcel &, MessageParcel &); +RDB_UTILS_PUSH_WARNING +RDB_UTILS_DISABLE_WARNING("-Wc99-designator") static constexpr RequestHandle HANDLERS[static_cast(RdbServiceCode::RDB_SERVICE_CMD_MAX)] = { [static_cast(RdbServiceCode::RDB_SERVICE_CMD_OBTAIN_TABLE)] = &RdbServiceStub::OnRemoteObtainDistributedTableName, @@ -112,8 +119,12 @@ private: &RdbServiceStub::OnUnlockCloudContainer, [static_cast(RdbServiceCode::RDB_SERVICE_CMD_GET_DEBUG_INFO)] = &RdbServiceStub::OnGetDebugInfo, [static_cast(RdbServiceCode::RDB_SERVICE_CMD_VERIFY_PROMISE_INFO)] = - &RdbServiceStub::OnVerifyPromiseInfo + &RdbServiceStub::OnVerifyPromiseInfo, + [static_cast(RdbServiceCode::RDB_SERVICE_CMD_REPORT_STAT)] = + &RdbServiceStub::OnReportStatistic, + [static_cast(RdbServiceCode::RDB_SERVICE_CMD_GET_DFX_INFO)] = &RdbServiceStub::OnGetDfxInfo, }; +RDB_UTILS_POP_WARNING }; } // namespace OHOS::DistributedRdb #endif diff --git a/datamgr_service/services/distributeddataservice/service/test/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/BUILD.gn index 77e91514..97804c0c 100644 --- a/datamgr_service/services/distributeddataservice/service/test/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/BUILD.gn @@ -14,7 +14,7 @@ import("//build/ohos_var.gni") import("//build/test.gni") import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -module_output_path = "datamgr_service/distributeddatafwk" +module_output_path = "datamgr_service/datamgr_service/distributeddatafwk" ############################################################################### config("module_private_config") { @@ -39,7 +39,6 @@ config("module_private_config") { "${data_service_path}/service/data_share/subscriber_managers", "${data_service_path}/service/data_share", "${data_service_path}/service/matrix/include/", - "${data_service_path}/service/network", "${data_service_path}/service/kvdb", "${data_service_path}/service/object/include", "${data_service_path}/service/permission/include", @@ -52,7 +51,6 @@ config("module_private_config") { "${relational_store_path}/interfaces/inner_api/cloud_data/include", "${relational_store_path}/interfaces/inner_api/common_type/include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] cflags = [ "-Werror" ] defines = [ @@ -103,6 +101,7 @@ ohos_unittest("CloudDataTest") { "${data_service_path}/service/rdb/rdb_cloud.cpp", "${data_service_path}/service/rdb/rdb_cursor.cpp", "${data_service_path}/service/rdb/rdb_general_store.cpp", + "${data_service_path}/service/rdb/rdb_hiview_adapter.cpp", "${data_service_path}/service/rdb/rdb_notifier_proxy.cpp", "${data_service_path}/service/rdb/rdb_query.cpp", "${data_service_path}/service/rdb/rdb_result_set_impl.cpp", @@ -113,7 +112,6 @@ ohos_unittest("CloudDataTest") { "${data_service_path}/service/rdb/rdb_watcher.cpp", "${data_service_path}/service/test/mock/checker_mock.cpp", "cloud_data_test.cpp", - "network_adapter_test.cpp", ] configs = [ ":module_private_config" ] @@ -127,12 +125,15 @@ ohos_unittest("CloudDataTest") { "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", "c_utils:utils", + "device_manager:devicemanagersdk", "hicollie:libhicollie", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", "kv_store:distributeddb", + "netmanager_base:net_conn_manager_if", "relational_store:native_rdb", "resource_management:global_resmgr", "samgr:samgr_proxy", @@ -141,7 +142,7 @@ ohos_unittest("CloudDataTest") { deps = [ "${data_service_path}/adapter/account:distributeddata_account", "${data_service_path}/adapter/communicator:distributeddata_communicator", - "${data_service_path}/service/network:distributeddata_network", + "${data_service_path}/adapter/schema_helper:distributeddata_schema_helper", "../../framework:distributeddatasvcfwk", "mock:distributeddata_mock_static", "//third_party/googletest:gtest_main", @@ -196,6 +197,7 @@ ohos_unittest("CloudServiceImplTest") { "${data_service_path}/service/rdb/rdb_cloud.cpp", "${data_service_path}/service/rdb/rdb_cursor.cpp", "${data_service_path}/service/rdb/rdb_general_store.cpp", + "${data_service_path}/service/rdb/rdb_hiview_adapter.cpp", "${data_service_path}/service/rdb/rdb_notifier_proxy.cpp", "${data_service_path}/service/rdb/rdb_query.cpp", "${data_service_path}/service/rdb/rdb_result_set_impl.cpp", @@ -219,12 +221,15 @@ ohos_unittest("CloudServiceImplTest") { "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", "c_utils:utils", + "device_manager:devicemanagersdk", "hicollie:libhicollie", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", "kv_store:distributeddb", + "netmanager_base:net_conn_manager_if", "relational_store:native_rdb", "resource_management:global_resmgr", "samgr:samgr_proxy", @@ -233,7 +238,7 @@ ohos_unittest("CloudServiceImplTest") { deps = [ "${data_service_path}/adapter/account:distributeddata_account", "${data_service_path}/adapter/communicator:distributeddata_communicator", - "${data_service_path}/service/network:distributeddata_network", + "${data_service_path}/adapter/schema_helper:distributeddata_schema_helper", "../../framework:distributeddatasvcfwk", "mock:distributeddata_mock_static", "//third_party/googletest:gtest_main", @@ -391,6 +396,11 @@ ohos_unittest("DeviceMatrixTest") { configs = [ ":module_private_config" ] + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + external_deps = [ "ability_base:base", "ability_base:want", @@ -500,7 +510,6 @@ ohos_unittest("KVDBGeneralStoreAbnormalTest") { "${data_service_path}/service/permission/include", "${data_service_path}/service/test/mock", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] configs = [] @@ -521,6 +530,7 @@ ohos_unittest("KVDBGeneralStoreAbnormalTest") { "googletest:gtest_main", "hilog:libhilog", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", "kv_store:distributeddb", "relational_store:native_rdb", @@ -663,6 +673,7 @@ ohos_unittest("ObjectAssetLoaderTest") { "hilog:libhilog", "hisysevent:libhisysevent", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", ] @@ -701,6 +712,7 @@ ohos_unittest("ObjectAssetMachineTest") { "hilog:libhilog", "hisysevent:libhisysevent", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", ] @@ -750,15 +762,22 @@ ohos_unittest("ObjectDmsHandlerTest") { ohos_unittest("ObjectManagerTest") { module_out_path = module_output_path sources = [ + "${data_service_path}/service/common/common_types_utils.cpp", "${data_service_path}/service/common/value_proxy.cpp", "../object/src/object_asset_loader.cpp", "../object/src/object_asset_machine.cpp", "../object/src/object_callback_proxy.cpp", "../object/src/object_data_listener.cpp", + "../object/src/object_dms_handler.cpp", "../object/src/object_manager.cpp", + "../object/src/object_service_impl.cpp", + "../object/src/object_service_stub.cpp", "../object/src/object_snapshot.cpp", + "../object/src/object_types_utils.cpp", "mock/kv_store_nb_delegate_mock.cpp", "object_manager_test.cpp", + "object_service_impl_test.cpp", + "object_service_stub_test.cpp", ] include_dirs = [ @@ -773,6 +792,7 @@ ohos_unittest("ObjectManagerTest") { external_deps = [ "access_token:libaccesstoken_sdk", + "access_token:libtoken_setproc", "access_token:libtokenid_sdk", "c_utils:utils", "data_object:distributeddataobject_impl", @@ -782,6 +802,7 @@ ohos_unittest("ObjectManagerTest") { "hilog:libhilog", "hisysevent:libhisysevent", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", "kv_store:distributeddb", "relational_store:native_rdb", @@ -823,6 +844,7 @@ ohos_unittest("ObjectSnapshotTest") { "hilog:libhilog", "hisysevent:libhisysevent", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", ] @@ -854,7 +876,6 @@ ohos_unittest("MetaDataTest") { "${kv_store_path}/frameworks/innerkitsimpl/distributeddatafwk/include", "${kv_store_path}/frameworks/innerkitsimpl/distributeddatasvc/include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] configs = [ ":module_private_config" ] @@ -895,10 +916,11 @@ ohos_unittest("UdmfRunTimeStoreTest") { "${data_service_path}/app/src", "${data_service_path}/service/kvdb", "${data_service_path}/service/udmf", + "${data_service_path}/service/udmf/store", + "${data_service_path}/service/udmf/preprocess", "${kv_store_path}/frameworks/innerkitsimpl/distributeddatafwk/include", "${kv_store_path}/frameworks/innerkitsimpl/distributeddatasvc/include", "${kv_store_path}/frameworks/innerkitsimpl/kvdb/include", - "${data_service_path}/adapter/include/dfx", ] configs = [ ":module_private_config" ] @@ -909,6 +931,8 @@ ohos_unittest("UdmfRunTimeStoreTest") { "access_token:libaccesstoken_sdk", "access_token:libnativetoken", "access_token:libtoken_setproc", + "bundle_framework:appexecfwk_base", + "bundle_framework:appexecfwk_core", "c_utils:utils", "dataclassification:data_transit_mgr", "dsoftbus:softbus_client", @@ -917,6 +941,7 @@ ohos_unittest("UdmfRunTimeStoreTest") { "image_framework:image", "ipc:ipc_core", "kv_store:distributeddata_inner", + "samgr:samgr_proxy", "udmf:udmf_client", ] @@ -929,6 +954,11 @@ ohos_unittest("UdmfRunTimeStoreTest") { "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_shared", ] + + defines = [ + "private=public", + "protected=public", + ] } ohos_unittest("DataShareServiceImplTest") { @@ -994,11 +1024,14 @@ ohos_unittest("DataShareServiceImplTest") { "${data_service_path}/service/kvdb/user_delegate.cpp", "${data_service_path}/service/permission/src/permission_validator.cpp", "${data_service_path}/service/permission/src/permit_delegate.cpp", + "data_share_common_test.cpp", "data_share_obs_proxy_test.cpp", "data_share_profile_config_test.cpp", "data_share_service_impl_test.cpp", "data_share_service_stub_test.cpp", "data_share_subscriber_managers_test.cpp", + "data_share_types_util_test.cpp", + "kv_dalegate_test.cpp", ] configs = [ ":module_private_config" ] @@ -1034,7 +1067,9 @@ ohos_unittest("DataShareServiceImplTest") { "hilog:libhilog", "hisysevent:libhisysevent", "huks:libhukssdk", + "init:libbegetutil", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", "kv_store:distributeddb", "relational_store:native_rdb", @@ -1058,18 +1093,25 @@ ohos_unittest("KvdbServiceImplTest") { sources = [ "${data_service_path}/app/src/kvstore_meta_manager.cpp", "kvdb_service_impl_test.cpp", + "kvdb_service_stub_unittest.cpp", "kvdb_service_test.cpp", + "mock/access_token_mock.cpp", + "mock/checker_mock.cpp", + "mock/meta_data_manager_mock.cpp", ] include_dirs = [ "${data_service_path}/app/src", "${data_service_path}/service/kvdb", "${data_service_path}/adapter/include/account", + "${data_service_path}/framework/include", + "${data_service_path}/framework/include/cloud", + "${data_service_path}/framework/include/eventcenter", + "${data_service_path}/service/test/mock", "${kv_store_path}/frameworks/innerkitsimpl/kvdb/include", "${kv_store_path}/frameworks/innerkitsimpl/distributeddatafwk/include", "${kv_store_path}/frameworks/innerkitsimpl/distributeddatasvc/include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] configs = [ ":module_private_config" ] @@ -1086,6 +1128,7 @@ ohos_unittest("KvdbServiceImplTest") { "c_utils:utils", "dataclassification:data_transit_mgr", "device_auth:deviceauth_sdk", + "googletest:gmock_main", "hilog:libhilog", "hisysevent:libhisysevent", "ipc:ipc_core", @@ -1099,6 +1142,7 @@ ohos_unittest("KvdbServiceImplTest") { "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service/kvdb:distributeddata_kvdb", "${kv_store_path}/frameworks/libs/distributeddb:distributeddb", + "mock:distributeddata_mock_static", "//third_party/googletest:gtest_main", ] } @@ -1161,6 +1205,46 @@ ohos_unittest("UdmfServiceImplTest") { "ability_runtime:uri_permission_mgr", "access_token:libaccesstoken_sdk", "app_file_service:remote_file_share_native", + "bundle_framework:appexecfwk_base", + "bundle_framework:appexecfwk_core", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "json:nlohmann_json_static", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + "relational_store:native_rdb", + "samgr:samgr_proxy", + "udmf:udmf_client", + ] +} + +ohos_unittest("UdmfServiceStubTest") { + module_out_path = module_output_path + sources = [ "udmf_service_stub_test.cpp" ] + + include_dirs = [ + "${data_service_path}/service/udmf/store", + "${data_service_path}/service/udmf", + "${data_service_path}/framework/include", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + deps = [ + "${data_service_path}/service:distributeddatasvc", + "${data_service_path}/service/udmf:udmf_server", + ] + + external_deps = [ + "ability_runtime:uri_permission_mgr", + "access_token:libaccesstoken_sdk", + "app_file_service:remote_file_share_native", + "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", "c_utils:utils", "device_manager:devicemanagersdk", @@ -1169,6 +1253,89 @@ ohos_unittest("UdmfServiceImplTest") { "kv_store:distributeddata_inner", "kv_store:distributeddb", "relational_store:native_rdb", + "samgr:samgr_proxy", + "udmf:udmf_client", + ] +} + +ohos_unittest("UdmfServiceStubMockTest") { + module_out_path = module_output_path + sources = [ + "${data_service_path}/framework/account/account_delegate.cpp", + "${data_service_path}/framework/checker/checker_manager.cpp", + "${data_service_path}/framework/dfx/reporter.cpp", + "${data_service_path}/framework/directory/directory_manager.cpp", + "${data_service_path}/framework/feature/feature_system.cpp", + "${data_service_path}/framework/metadata/appid_meta_data.cpp", + "${data_service_path}/framework/metadata/auto_launch_meta_data.cpp", + "${data_service_path}/framework/metadata/capability_range.cpp", + "${data_service_path}/framework/metadata/meta_data_manager.cpp", + "${data_service_path}/framework/metadata/secret_key_meta_data.cpp", + "${data_service_path}/framework/metadata/store_debug_info.cpp", + "${data_service_path}/framework/metadata/store_meta_data.cpp", + "${data_service_path}/framework/metadata/store_meta_data_local.cpp", + "${data_service_path}/framework/metadata/strategy_meta_data.cpp", + "${data_service_path}/framework/serializable/serializable.cpp", + "${data_service_path}/framework/utils/anonymous.cpp", + "${data_service_path}/framework/utils/constant.cpp", + "${data_service_path}/framework/utils/corrupt_reporter.cpp", + "${data_service_path}/framework/utils/crypto.cpp", + "${data_service_path}/service/udmf/lifecycle/lifecycle_manager.cpp", + "${data_service_path}/service/udmf/lifecycle/lifecycle_policy.cpp", + "${data_service_path}/service/udmf/permission/checker_manager.cpp", + "${data_service_path}/service/udmf/permission/uri_permission_manager.cpp", + "${data_service_path}/service/udmf/preprocess/data_handler.cpp", + "${data_service_path}/service/udmf/preprocess/preprocess_utils.cpp", + "${data_service_path}/service/udmf/store/runtime_store.cpp", + "${data_service_path}/service/udmf/store/store_account_observer.cpp", + "${data_service_path}/service/udmf/store/store_cache.cpp", + "${data_service_path}/service/udmf/udmf_service_impl.cpp", + "${data_service_path}/service/udmf/udmf_service_stub.cpp", + "udmf_service_stub_mock_test.cpp", + ] + + include_dirs = [ + "${data_service_path}/service/udmf/store", + "${data_service_path}/service/udmf", + "${data_service_path}/framework/include", + "${data_service_path}/service/test/mock", + "${data_service_path}/service/udmf/permission", + "${data_service_path}/framework/include/dfx", + "${data_service_path}/service/udmf/preprocess", + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include/account", + "${data_service_path}/service/udmf/store", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + deps = [ "${data_service_path}/service:distributeddatasvc" ] + + external_deps = [ + "ability_base:zuri", + "ability_runtime:uri_permission_mgr", + "access_token:libaccesstoken_sdk", + "access_token:libnativetoken", + "access_token:libtoken_setproc", + "access_token:libtokenid_sdk", + "app_file_service:remote_file_share_native", + "bundle_framework:appexecfwk_base", + "bundle_framework:appexecfwk_core", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + "hitrace:libhitracechain", + "ipc:ipc_core", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + "relational_store:native_rdb", + "samgr:samgr_proxy", "udmf:udmf_client", ] } @@ -1255,6 +1422,382 @@ ohos_unittest("PermissionValidatorTest") { defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] } +ohos_unittest("PermissionValidatorMockTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/service/permission/include", + "${data_service_path}/service/test/mock", + ] + + sources = [ + "${data_service_path}/service/permission/src/permission_validator.cpp", + "mock/access_token_mock.cpp", + "permission_validator_mock_test.cpp", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + deps = [ + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "c_utils:utils", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + ] + defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] +} + +ohos_unittest("PermitDelegateMockTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/framework/include", + "${data_service_path}/service/kvdb", + "${data_service_path}/service/permission/include", + "${data_service_path}/service/test/mock", + ] + + sources = [ + "${data_service_path}/service/kvdb/user_delegate.cpp", + "${data_service_path}/service/permission/src/permission_validator.cpp", + "${data_service_path}/service/permission/src/permit_delegate.cpp", + "mock/access_token_mock.cpp", + "mock/meta_data_manager_mock.cpp", + "permit_delegate_mock_test.cpp", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + cflags_cc = cflags + defines = [ + "TEST_ON_DEVICE", + "OPENSSL_SUPPRESS_DEPRECATED", + ] + + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "json:nlohmann_json_static", + "kv_store:distributeddata_inner", + "kv_store:distributeddb", + ] +} + +ohos_unittest("BootStrapMockTest") { + sanitize = { + cfi = true + cfi_cross_dso = true + debug = false + } + + include_dirs = [ + "${data_service_path}/service/backup/include", + "${data_service_path}/service/bootstrap/include/", + "${data_service_path}/service/config/include/", + "${data_service_path}/service/config/include/model", + ] + + module_out_path = module_output_path + sources = [ + "${data_service_path}/service/bootstrap/src/bootstrap.cpp", + "${data_service_path}/service/config/src/model/app_id_mapping_config.cpp", + "${data_service_path}/service/config/src/model/backup_config.cpp", + "${data_service_path}/service/config/src/model/checker_config.cpp", + "${data_service_path}/service/config/src/model/cloud_config.cpp", + "${data_service_path}/service/config/src/model/component_config.cpp", + "${data_service_path}/service/config/src/model/directory_config.cpp", + "${data_service_path}/service/config/src/model/global_config.cpp", + "${data_service_path}/service/config/src/model/network_config.cpp", + "${data_service_path}/service/config/src/model/protocol_config.cpp", + "${data_service_path}/service/config/src/model/thread_config.cpp", + "bootstrap_mock_test.cpp", + "mock/config_factory_mock.cpp", + ] + + external_deps = [ + "bundle_framework:appexecfwk_core", + "googletest:gtest_main", + "hilog:libhilog", + "kv_store:distributeddata_inner", + ] + + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/service/backup:distributeddata_backup", + "../../framework:distributeddatasvcfwk", + ] + + cflags = [ + "-Werror", + "-Dprivate=public", + "-Dprotected=public", + ] + + cflags_cc = cflags + + defines = [ + "TEST_ON_DEVICE", + "OPENSSL_SUPPRESS_DEPRECATED", + ] +} + +ohos_unittest("QueryHelperUnitTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/adapter/include/utils", + "${data_service_path}/service/kvdb", + ] + + sources = [ + "${data_service_path}/service/kvdb/query_helper.cpp", + "query_helper_test.cpp", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "access_token:libtokenid_sdk", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "hisysevent:libhisysevent", + "ipc:ipc_core", + "json:nlohmann_json_static", + "kv_store:datamgr_common", + "kv_store:distributeddb", + "kv_store:kvdb_inner_lite", + ] +} + +ohos_unittest("AuthDelegateMockTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/adapter/include/communicator/", + "${data_service_path}/framework/include/", + "${data_service_path}/service/kvdb/", + "${data_service_path}/service/test/mock", + ] + + sources = [ + "${data_service_path}/service/kvdb/auth_delegate.cpp", + "auth_delegate_mock_test.cpp", + "mock/device_manager_adapter_mock.cpp", + "mock/user_delegate_mock.cpp", + ] + + external_deps = [ + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "json:nlohmann_json_static", + "kv_store:distributeddata_inner", + ] + + deps = [ + "${data_service_path}/framework:distributeddatasvcfwk", + "${data_service_path}/service:distributeddatasvc", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + cflags_cc = cflags + + defines = [ + "TEST_ON_DEVICE", + "OPENSSL_SUPPRESS_DEPRECATED", + ] +} + +ohos_unittest("UpgradeMockTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/adapter/include/communicator/", + "${data_service_path}/framework/include/", + "${data_service_path}/service/bootstrap/include/", + "${data_service_path}/service/crypto/include/", + "${data_service_path}/service/kvdb/", + "${data_service_path}/service/matrix/include/", + "${data_service_path}/service/rdb/", + "${data_service_path}/service/test/mock", + ] + + sources = [ + "${data_service_path}/service/kvdb/kvdb_general_store.cpp", + "${data_service_path}/service/kvdb/upgrade.cpp", + "mock/access_token_mock.cpp", + "upgrade_mock_test.cpp", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "huks:libhukssdk", + "json:nlohmann_json_static", + "kv_store:datamgr_common", + "kv_store:distributeddb", + ] + + deps = [ + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/adapter/utils:distributeddata_utils", + "${data_service_path}/framework:distributeddatasvcfwk", + "${data_service_path}/service/bootstrap:distributeddata_bootstrap", + "${data_service_path}/service/crypto:distributeddata_crypto", + "${data_service_path}/service/matrix:distributeddata_matrix", + "${data_service_path}/service/rdb:distributeddata_rdb", + "${data_service_path}/service/test/mock:distributeddata_mock_static", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + cflags_cc = cflags +} + +ohos_unittest("UserDelegateMockTest") { + module_out_path = module_output_path + + include_dirs = [ + "${data_service_path}/adapter/include/communicator/", + "${data_service_path}/framework/include/", + "${data_service_path}/service/kvdb/", + "${data_service_path}/service/test/mock", + ] + + sources = [ + "${data_service_path}/service/kvdb/user_delegate.cpp", + "mock/account_delegate_mock.cpp", + "mock/device_manager_adapter_mock.cpp", + "user_delegate_mock_test.cpp", + ] + + external_deps = [ + "access_token:libaccesstoken_sdk", + "c_utils:utils", + "device_manager:devicemanagersdk", + "googletest:gmock_main", + "googletest:gtest_main", + "hilog:libhilog", + "json:nlohmann_json_static", + "kv_store:datamgr_common", + "kv_store:distributeddb", + ] + + deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + cflags_cc = cflags +} + +ohos_unittest("BackupManagerServiceTest") { + sanitize = { + cfi = true + cfi_cross_dso = true + debug = false + } + module_out_path = module_output_path + sources = [ + "${data_service_path}/app/src/kvstore_meta_manager.cpp", + "backup_manager_service_test.cpp", + ] + + include_dirs = [ + "${data_service_path}/adapter/include/communicator", + "${data_service_path}/app/src", + "${data_service_path}/service/bootstrap/include/", + "${data_service_path}/service/common", + "${data_service_path}/service/crypto/include", + "${data_service_path}/service/matrix/include/", + "${data_service_path}/service/rdb", + "${data_service_path}/service/kvdb", + "${data_service_path}/service/backup/include/", + ] + + cflags = [ + "-Dprivate=public", + "-Dprotected=public", + ] + + external_deps = [ + "ability_base:base", + "ability_base:want", + "access_token:libaccesstoken_sdk", + "access_token:libnativetoken", + "c_utils:utils", + "dataclassification:data_transit_mgr", + "device_manager:devicemanagersdk", + "googletest:gtest_main", + "hilog:libhilog", + "hisysevent:libhisysevent", + "ipc:ipc_core", + "kv_store:distributeddata_inner", + "kv_store:distributeddata_mgr", + "kv_store:distributeddb", + ] + + deps = [ + "${data_service_path}/adapter/account:distributeddata_account", + "${data_service_path}/adapter/communicator:distributeddata_communicator", + "${data_service_path}/framework:distributeddatasvcfwk", + "${data_service_path}/service/backup:distributeddata_backup", + "${data_service_path}/service/kvdb:distributeddata_kvdb", + ] +} + ############################################################################### group("unittest") { testonly = true @@ -1270,6 +1813,8 @@ group("unittest") { ":CloudDataTest", ":CloudServiceImplTest", ":CloudTest", + "ohos_test:copy_ohos_test", + "testCloud:testCloud", ] } @@ -1278,6 +1823,8 @@ group("unittest") { ":UdmfCheckerManagerTest", ":UdmfRunTimeStoreTest", ":UdmfServiceImplTest", + ":UdmfServiceStubMockTest", + ":UdmfServiceStubTest", ] } @@ -1300,9 +1847,14 @@ group("unittest") { if (datamgr_service_kvdb) { deps += [ + ":AuthDelegateMockTest", + ":BackupManagerServiceTest", ":KVDBGeneralStoreAbnormalTest", ":KVDBGeneralStoreTest", ":KvdbServiceImplTest", + ":QueryHelperUnitTest", + ":UpgradeMockTest", + ":UserDelegateMockTest", ] } @@ -1311,12 +1863,15 @@ group("unittest") { } deps += [ + ":BootStrapMockTest", ":CryptoManagerTest", ":DeviceMatrixTest", ":DirectoryManagerTest", ":DumpHelperTest", ":MetaDataTest", + ":PermissionValidatorMockTest", ":PermissionValidatorTest", + ":PermitDelegateMockTest", ":ValueProxyServiceTest", ] } diff --git a/datamgr_service/services/distributeddataservice/service/test/auth_delegate_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/auth_delegate_mock_test.cpp new file mode 100644 index 00000000..1d6c08d7 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/auth_delegate_mock_test.cpp @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "user_delegate_mock.h" +#include "device_manager_adapter_mock.h" +#include "auth_delegate.h" +#include "types.h" + +namespace OHOS::DistributedData { +using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; +using DistributedData::UserStatus; +using AclParams = OHOS::AppDistributedKv::AclParams; +using namespace testing; +using namespace std; +class AuthDelegateMockTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +public: + static inline shared_ptr devMgrAdapterMock = nullptr; + static inline shared_ptr userDelegateMock = nullptr; + static inline AuthHandler *authHandler = nullptr; +}; + +void AuthDelegateMockTest::SetUpTestCase(void) +{ + authHandler = AuthDelegate::GetInstance(); + devMgrAdapterMock = make_shared(); + BDeviceManagerAdapter::deviceManagerAdapter = devMgrAdapterMock; + userDelegateMock = make_shared(); + BUserDelegate::userDelegate = userDelegateMock; +} + +void AuthDelegateMockTest::TearDownTestCase(void) +{ + BDeviceManagerAdapter::deviceManagerAdapter = nullptr; + devMgrAdapterMock = nullptr; + userDelegateMock = nullptr; + BUserDelegate::userDelegate = nullptr; + authHandler = nullptr; +} + +void AuthDelegateMockTest::SetUp(void) +{} + +void AuthDelegateMockTest::TearDown(void) +{} + +/** +* @tc.name: CheckAccess001 +* @tc.desc: Check access. +* @tc.type: FUNC +* @tc.require: +* @tc.author: caozhijun +*/ +HWTEST_F(AuthDelegateMockTest, CheckAccess001, testing::ext::TestSize.Level0) +{ + ASSERT_NE(authHandler, nullptr); + int localUserId = 1; + int peerUserId = 1; + std::string peerDevId = "CheckAccess001"; + AclParams aclParams; + UserStatus local; + local.id = localUserId; + local.isActive = true; + UserStatus remote; + remote.id = peerUserId; + remote.isActive = true; + std::vector localUsers; + std::vector peerUsers; + localUsers.push_back(local); + peerUsers.push_back(remote); + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(false)); + auto result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_TRUE(result.first); +} + +/** +* @tc.name: CheckAccess002 +* @tc.desc: Check access. +* @tc.type: FUNC +* @tc.require: +* @tc.author: caozhijun +*/ +HWTEST_F(AuthDelegateMockTest, CheckAccess002, testing::ext::TestSize.Level0) +{ + ASSERT_NE(authHandler, nullptr); + int localUserId = 1; + int peerUserId = 1; + std::string peerDevId = "CheckAccess002"; + AclParams aclParams; + aclParams.authType = static_cast(DistributedKv::AuthType::DEFAULT); + UserStatus local; + local.id = localUserId; + local.isActive = true; + std::vector localUsers = { local }; + std::vector peerUsers(localUsers); + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(true)); + EXPECT_CALL(*devMgrAdapterMock, IsSameAccount(_, _)).WillOnce(Return(true)); + auto result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_TRUE(result.first); + + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(true)); + EXPECT_CALL(*devMgrAdapterMock, IsSameAccount(_, _)).WillOnce(Return(false)); + EXPECT_CALL(*devMgrAdapterMock, CheckAccessControl(_, _)).WillOnce(Return(true)); + result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_TRUE(result.first); + + aclParams.accCaller.bundleName = "com.AuthDelegateMockTest.app"; + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(true)); + EXPECT_CALL(*devMgrAdapterMock, IsSameAccount(_, _)).WillOnce(Return(false)); + EXPECT_CALL(*devMgrAdapterMock, CheckAccessControl(_, _)).WillOnce(Return(false)); + result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_FALSE(result.first); +} + +/** +* @tc.name: CheckAccess003 +* @tc.desc: Check access. +* @tc.type: FUNC +* @tc.require: +* @tc.author: caozhijun +*/ +HWTEST_F(AuthDelegateMockTest, CheckAccess003, testing::ext::TestSize.Level0) +{ + ASSERT_NE(authHandler, nullptr); + int localUserId = 1; + int peerUserId = 1; + std::string peerDevId = "CheckAccess003"; + AclParams aclParams; + aclParams.authType = static_cast(DistributedKv::AuthType::IDENTICAL_ACCOUNT); + UserStatus local; + local.id = localUserId; + local.isActive = true; + std::vector localUsers = { local }; + std::vector peerUsers(localUsers); + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(true)); + EXPECT_CALL(*devMgrAdapterMock, IsSameAccount(_, _)).WillOnce(Return(true)); + auto result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_TRUE(result.first); + + aclParams.authType += 1; + aclParams.accCaller.bundleName = "com.AuthDelegateTest.app"; + EXPECT_CALL(*userDelegateMock, GetLocalUserStatus()).WillOnce(Return(localUsers)); + EXPECT_CALL(*userDelegateMock, GetRemoteUserStatus(_)).WillOnce(Return(peerUsers)); + EXPECT_CALL(*devMgrAdapterMock, IsOHOSType(_)).WillOnce(Return(true)); + result = authHandler->CheckAccess(localUserId, peerUserId, peerDevId, aclParams); + EXPECT_FALSE(result.second); +} +} diff --git a/datamgr_service/services/distributeddataservice/service/test/backup_manager_service_test.cpp b/datamgr_service/services/distributeddataservice/service/test/backup_manager_service_test.cpp new file mode 100644 index 00000000..fac7bdbc --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/backup_manager_service_test.cpp @@ -0,0 +1,328 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#define LOG_TAG "BackupManagerServiceTest" + +#include +#include +#include +#include "backup_manager.h" +#include "backuprule/backup_rule_manager.h" +#include "bootstrap.h" +#include "crypto_manager.h" +#include "device_manager_adapter.h" +#include "directory/directory_manager.h" +#include "log_print.h" +#include "metadata/meta_data_manager.h" +#include "kvstore_meta_manager.h" +#include "types.h" + +using namespace testing::ext; +using namespace std; +using namespace OHOS::DistributedData; +namespace OHOS::Test { +namespace DistributedDataTest { +static constexpr uint32_t SKEY_SIZE = 32; +static constexpr int MICROSEC_TO_SEC_TEST = 1000; +static constexpr const char *TEST_BACKUP_BUNDLE = "test_backup_bundleName"; +static constexpr const char *TEST_BACKUP_STOREID = "test_backup_storeId"; +class BackupManagerServiceTest : public testing::Test { +public: + class TestRule : public BackupRuleManager::BackupRule { + public: + TestRule() + { + BackupRuleManager::GetInstance().RegisterPlugin("TestRule", [this]() -> auto { + return this; + }); + } + bool CanBackup() override + { + return false; + } + }; + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + static std::vector Random(uint32_t len); + void InitMetaData(); + StoreMetaData metaData_; +}; + +void BackupManagerServiceTest::SetUpTestCase(void) +{ + auto executors = std::make_shared(1, 0); + Bootstrap::GetInstance().LoadComponents(); + Bootstrap::GetInstance().LoadDirectory(); + Bootstrap::GetInstance().LoadCheckers(); + DeviceManagerAdapter::GetInstance().Init(executors); + DistributedKv::KvStoreMetaManager::GetInstance().BindExecutor(executors); + DistributedKv::KvStoreMetaManager::GetInstance().InitMetaParameter(); + DistributedKv::KvStoreMetaManager::GetInstance().InitMetaListener(); +} + +void BackupManagerServiceTest::TearDownTestCase() +{ +} + +void BackupManagerServiceTest::SetUp() +{ +} + +void BackupManagerServiceTest::TearDown() +{ +} + +void BackupManagerServiceTest::InitMetaData() +{ + metaData_.deviceId = DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid; + metaData_.bundleName = TEST_BACKUP_BUNDLE; + metaData_.appId = TEST_BACKUP_BUNDLE; + metaData_.user = "0"; + metaData_.area = OHOS::DistributedKv::EL1; + metaData_.instanceId = 0; + metaData_.isAutoSync = true; + metaData_.storeType = DistributedKv::KvStoreType::SINGLE_VERSION; + metaData_.storeId = TEST_BACKUP_STOREID; + metaData_.dataDir = "/data/service/el1/public/database/" + std::string(TEST_BACKUP_BUNDLE) + "/kvdb"; + metaData_.securityLevel = OHOS::DistributedKv::SecurityLevel::S2; +} + +std::vector BackupManagerServiceTest::Random(uint32_t len) +{ + std::random_device randomDevice; + std::uniform_int_distribution distribution(0, std::numeric_limits::max()); + std::vector key(len); + for (uint32_t i = 0; i < len; i++) { + key[i] = static_cast(distribution(randomDevice)); + } + return key; +} + +/** +* @tc.name: Init +* @tc.desc: Init testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, Init, TestSize.Level1) +{ + BackupManagerServiceTest::InitMetaData(); + StoreMetaData meta1; + meta1 = metaData_; + meta1.isBackup = false; + meta1.isDirty = false; + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(StoreMetaData::GetPrefix( + { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid }), meta1, true)); + BackupManager::GetInstance().Init(); + + StoreMetaData meta2; + meta2 = metaData_; + meta2.isBackup = true; + meta2.isDirty = false; + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(StoreMetaData::GetPrefix( + { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid }), meta2, true)); + BackupManager::GetInstance().Init(); + + StoreMetaData meta3; + meta3 = metaData_; + meta3.isBackup = false; + meta3.isDirty = true; + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(StoreMetaData::GetPrefix( + { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid }), meta3, true)); + BackupManager::GetInstance().Init(); + + StoreMetaData meta4; + meta4 = metaData_; + meta4.isBackup = true; + meta4.isDirty = true; + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(StoreMetaData::GetPrefix( + { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid }), meta4, true)); + BackupManager::GetInstance().Init(); +} + +/** +* @tc.name: RegisterExporter +* @tc.desc: RegisterExporter testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, RegisterExporter, TestSize.Level1) +{ + int32_t type = DistributedKv::KvStoreType::DEVICE_COLLABORATION; + BackupManager::Exporter exporter = + [](const StoreMetaData &meta, const std::string &backupPath, bool &result) + { result = true; }; + BackupManager instance; + instance.RegisterExporter(type, exporter); + EXPECT_FALSE(instance.exporters_[type] == nullptr); + instance.RegisterExporter(type, exporter); +} + +/** +* @tc.name: BackSchedule +* @tc.desc: BackSchedule testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, BackSchedule, TestSize.Level1) +{ + std::shared_ptr executors = std::make_shared(0, 1); + BackupManager instance; + instance.executors_ = nullptr; + instance.BackSchedule(executors); + EXPECT_FALSE(instance.executors_ == nullptr); +} + +/** +* @tc.name: CanBackup001 +* @tc.desc: CanBackup testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, CanBackup001, TestSize.Level1) +{ + BackupManager instance; + instance.backupSuccessTime_ = 0; + instance.backupInternal_ = 0; + bool status = instance.CanBackup(); // false false + EXPECT_TRUE(status); + + instance.backupInternal_ = MICROSEC_TO_SEC_TEST; + status = instance.CanBackup(); // true false + EXPECT_TRUE(status); + + instance.backupSuccessTime_ = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + instance.backupInternal_ = 0; + status = instance.CanBackup(); // false true + EXPECT_TRUE(status); + + instance.backupSuccessTime_ = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + instance.backupInternal_ = MICROSEC_TO_SEC_TEST; + status = instance.CanBackup(); // true true + EXPECT_FALSE(status); +} + +/** +* @tc.name: CanBackup +* @tc.desc: CanBackup testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, CanBackup, TestSize.Level1) +{ + BackupManagerServiceTest::TestRule(); + std::vector rule = { "TestRule" }; + BackupRuleManager::GetInstance().LoadBackupRules(rule); + EXPECT_FALSE(BackupRuleManager::GetInstance().CanBackup()); + bool status = BackupManager::GetInstance().CanBackup(); + EXPECT_FALSE(status); +} + +/** +* @tc.name: SaveData +* @tc.desc: SaveData testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, SaveData, TestSize.Level1) +{ + std::string path = "test"; + std::string key = "test"; + std::vector randomKey = BackupManagerServiceTest::Random(SKEY_SIZE); + SecretKeyMetaData secretKey; + secretKey.storeType = DistributedKv::KvStoreType::SINGLE_VERSION; + secretKey.sKey = randomKey; + EXPECT_EQ(secretKey.sKey.size(), SKEY_SIZE); + BackupManager::GetInstance().SaveData(path, key, secretKey); + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(key, secretKey, true)); + randomKey.assign(randomKey.size(), 0); +} + +/** +* @tc.name: GetClearType001 +* @tc.desc: GetClearType testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, GetClearType001, TestSize.Level1) +{ + StoreMetaData meta; + MetaDataManager::GetInstance().SaveMeta(StoreMetaData::GetPrefix( + { DeviceManagerAdapter::GetInstance().GetLocalDevice().uuid }), metaData_, true); + BackupManager::ClearType status = BackupManager::GetInstance().GetClearType(meta); + EXPECT_EQ(status, BackupManager::ClearType::DO_NOTHING); +} + +/** +* @tc.name: GetClearType002 +* @tc.desc: GetClearType testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, GetClearType002, TestSize.Level1) +{ + BackupManagerServiceTest::InitMetaData(); + StoreMetaData meta; + meta = metaData_; + EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(meta.GetSecretKey(), meta, true)); + + SecretKeyMetaData dbPassword; + EXPECT_TRUE(MetaDataManager::GetInstance().LoadMeta(meta.GetSecretKey(), dbPassword, true)); + SecretKeyMetaData backupPassword; + EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(meta.GetBackupSecretKey(), backupPassword, true)); + EXPECT_FALSE(dbPassword.sKey != backupPassword.sKey); + BackupManager::ClearType status = BackupManager::GetInstance().GetClearType(meta); + EXPECT_EQ(status, BackupManager::ClearType::DO_NOTHING); +} + +/** +* @tc.name: GetPassWord +* @tc.desc: GetPassWord testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, GetPassWord, TestSize.Level1) +{ + StoreMetaData meta; + std::vector password; + bool status = BackupManager::GetInstance().GetPassWord(meta, password); + EXPECT_FALSE(status); +} + +/** +* @tc.name: IsFileExist +* @tc.desc: IsFileExist testing exception branching scenarios. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong +*/ +HWTEST_F(BackupManagerServiceTest, IsFileExist, TestSize.Level1) +{ + std::string path; + bool status = BackupManager::GetInstance().IsFileExist(path); + EXPECT_FALSE(status); +} +} // namespace DistributedDataTest +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/permission/test/unittest/permission_validator_test.cpp b/datamgr_service/services/distributeddataservice/service/test/bootstrap_mock_test.cpp similarity index 32% rename from datamgr_service/services/distributeddataservice/adapter/permission/test/unittest/permission_validator_test.cpp rename to datamgr_service/services/distributeddataservice/service/test/bootstrap_mock_test.cpp index 46934841..c9887503 100644 --- a/datamgr_service/services/distributeddataservice/adapter/permission/test/unittest/permission_validator_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/bootstrap_mock_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,57 +13,61 @@ * limitations under the License. */ +#include "bootstrap.h" #include +#include -#include "permission_validator.h" -#include "utils/crypto.h" - -using namespace testing::ext; -using namespace OHOS::DistributedKv; -using namespace OHOS::DistributedData; - -class PermissionValidatorTest : public testing::Test { +namespace OHOS::DistributedData { +using namespace testing; +class BootStrapMockTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); void SetUp(); void TearDown(); +private: + static constexpr const char *defaultLabel = "distributeddata"; + static constexpr const char *defaultMeta = "service_meta"; }; -void PermissionValidatorTest::SetUpTestCase(void) +void BootStrapMockTest::SetUpTestCase(void) {} -void PermissionValidatorTest::TearDownTestCase(void) +void BootStrapMockTest::TearDownTestCase(void) {} -void PermissionValidatorTest::SetUp(void) +void BootStrapMockTest::SetUp(void) {} -void PermissionValidatorTest::TearDown(void) +void BootStrapMockTest::TearDown(void) {} - -/** - * @tc.name: TestPermissionValidate001 - * @tc.desc: test if CheckPermission can return correct permission. - * @tc.type: FUNC - * @tc.require: AR000CQDUT - * @tc.author: liqiao - */ -HWTEST_F(PermissionValidatorTest, TestPermissionValidate001, TestSize.Level0) + + /** + * @tc.name: GetProcessLabel + * @tc.desc: Get process label. + * @tc.type: FUNC + */ +HWTEST_F(BootStrapMockTest, GetProcessLabel, testing::ext::TestSize.Level0) { - uint32_t tokenId = 0; - EXPECT_FALSE(PermissionValidator::GetInstance().CheckSyncPermission(tokenId)); + std::string proLabel = Bootstrap::GetInstance().GetProcessLabel(); + EXPECT_TRUE(proLabel == BootStrapMockTest::defaultLabel); } - -/** - * @tc.name: TestPermissionValidate003 - * @tc.desc: test if account id sha256. - * @tc.type: FUNC - * @tc.require: AR000DPSH0 AR000DPSEC - * @tc.author: liqiao - */ -HWTEST_F(PermissionValidatorTest, TestPermissionValidate002, TestSize.Level0) + + /** + * @tc.name: GetMetaDBNameAndLoadMultiItem + * @tc.desc: Get meta db name and load other items. + * @tc.type: FUNC + */ +HWTEST_F(BootStrapMockTest, GetMetaDBName, testing::ext::TestSize.Level0) { - std::string userId = "ohos"; - EXPECT_NE(Crypto::Sha256("ohos"), userId); + std::string metaDbName = Bootstrap::GetInstance().GetMetaDBName(); + Bootstrap::GetInstance().LoadComponents(); + Bootstrap::GetInstance().LoadCheckers(); + Bootstrap::GetInstance().LoadDirectory(); + Bootstrap::GetInstance().LoadCloud(); + Bootstrap::GetInstance().LoadAppIdMappings(); + std::shared_ptr executors = std::make_shared(0, 1); + Bootstrap::GetInstance().LoadBackup(executors); + EXPECT_TRUE(metaDbName == BootStrapMockTest::defaultMeta); } +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/cloud_data_test.cpp b/datamgr_service/services/distributeddataservice/service/test/cloud_data_test.cpp index 7b9d2acf..a41c6b73 100644 --- a/datamgr_service/services/distributeddataservice/service/test/cloud_data_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/cloud_data_test.cpp @@ -20,8 +20,10 @@ #include "account/account_delegate.h" #include "bootstrap.h" #include "checker_mock.h" +#include "cloud/cloud_mark.h" #include "cloud/change_event.h" #include "cloud/cloud_event.h" +#include "cloud/cloud_last_sync_info.h" #include "cloud/cloud_report.h" #include "cloud/cloud_server.h" #include "cloud/cloud_share_event.h" @@ -43,7 +45,8 @@ #include "metadata/store_meta_data_local.h" #include "mock/db_store_mock.h" #include "mock/general_store_mock.h" -#include "network_adapter.h" +#include "network/network_delegate.h" +#include "network_delegate_mock.h" #include "rdb_query.h" #include "rdb_service.h" #include "rdb_service_impl.h" @@ -59,6 +62,7 @@ using namespace OHOS::DistributedData; using namespace OHOS::Security::AccessToken; using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; using Querykey = OHOS::CloudData::QueryKey; +using QueryLastResults = OHOS::CloudData::QueryLastResults; using CloudSyncInfo = OHOS::CloudData::CloudSyncInfo; using SharingCfm = OHOS::CloudData::SharingUtil::SharingCfm; using Confirmation = OHOS::CloudData::Confirmation; @@ -83,11 +87,16 @@ void AllocHapToken(const HapPolicyParams &policy) namespace OHOS::Test { namespace DistributedDataTest { +static constexpr const int32_t SCHEMA_VERSION = 101; +static constexpr const int32_t EVT_USER = 102; +static constexpr const char *TEST_TRACE_ID = "123456789"; static constexpr const char *TEST_CLOUD_BUNDLE = "test_cloud_bundleName"; static constexpr const char *TEST_CLOUD_APPID = "test_cloud_appid"; static constexpr const char *TEST_CLOUD_STORE = "test_cloud_store"; +static constexpr const char *TEST_CLOUD_STORE_1 = "test_cloud_store1"; static constexpr const char *TEST_CLOUD_ID = "test_cloud_id"; static constexpr const char *TEST_CLOUD_TABLE = "teat_cloud_table"; +static constexpr const char *COM_EXAMPLE_TEST_CLOUD = "com.example.testCloud"; static constexpr const char *TEST_CLOUD_DATABASE_ALIAS_1 = "test_cloud_database_alias_1"; static constexpr const char *TEST_CLOUD_DATABASE_ALIAS_2 = "test_cloud_database_alias_2"; static constexpr const char *PERMISSION_CLOUDDATA_CONFIG = "ohos.permission.CLOUDDATA_CONFIG"; @@ -95,6 +104,7 @@ static constexpr const char *PERMISSION_GET_NETWORK_INFO = "ohos.permission.GET_ static constexpr const char *PERMISSION_DISTRIBUTED_DATASYNC = "ohos.permission.DISTRIBUTED_DATASYNC"; static constexpr const char *PERMISSION_ACCESS_SERVICE_DM = "ohos.permission.ACCESS_SERVICE_DM"; static constexpr const char *PERMISSION_MANAGE_LOCAL_ACCOUNTS = "ohos.permission.MANAGE_LOCAL_ACCOUNTS"; +static constexpr const char *PERMISSION_GET_BUNDLE_INFO = "ohos.permission.GET_BUNDLE_INFO_PRIVILEGED"; PermissionDef GetPermissionDef(const std::string &permission) { PermissionDef def = { .permissionName = permission, @@ -135,6 +145,7 @@ protected: static StoreMetaData metaData_; static CloudInfo cloudInfo_; static DistributedData::CheckerMock checker_; + static NetworkDelegateMock delegate_; }; class CloudServerMock : public CloudServer { @@ -186,6 +197,7 @@ CloudInfo CloudDataTest::cloudInfo_; std::shared_ptr CloudDataTest::cloudServiceImpl_ = std::make_shared(); DistributedData::CheckerMock CloudDataTest::checker_; +NetworkDelegateMock CloudDataTest::delegate_; void CloudDataTest::InitMetaData() { @@ -258,12 +270,13 @@ void CloudDataTest::SetUpTestCase(void) .domain = "test.domain", .permList = { GetPermissionDef(PERMISSION_CLOUDDATA_CONFIG), GetPermissionDef(PERMISSION_GET_NETWORK_INFO), GetPermissionDef(PERMISSION_DISTRIBUTED_DATASYNC), GetPermissionDef(PERMISSION_ACCESS_SERVICE_DM), - GetPermissionDef(PERMISSION_MANAGE_LOCAL_ACCOUNTS) }, + GetPermissionDef(PERMISSION_MANAGE_LOCAL_ACCOUNTS), GetPermissionDef(PERMISSION_GET_BUNDLE_INFO) }, .permStateList = { GetPermissionStateFull(PERMISSION_CLOUDDATA_CONFIG), GetPermissionStateFull(PERMISSION_GET_NETWORK_INFO), GetPermissionStateFull(PERMISSION_DISTRIBUTED_DATASYNC), GetPermissionStateFull(PERMISSION_ACCESS_SERVICE_DM), - GetPermissionStateFull(PERMISSION_MANAGE_LOCAL_ACCOUNTS)} }; + GetPermissionStateFull(PERMISSION_MANAGE_LOCAL_ACCOUNTS), + GetPermissionStateFull(PERMISSION_GET_BUNDLE_INFO)} }; g_selfTokenID = GetSelfTokenID(); AllocHapToken(policy); size_t max = 12; @@ -275,10 +288,10 @@ void CloudDataTest::SetUpTestCase(void) Bootstrap::GetInstance().LoadCheckers(); auto dmExecutor = std::make_shared(max, min); DeviceManagerAdapter::GetInstance().Init(dmExecutor); + NetworkDelegate::RegisterNetworkInstance(&delegate_); InitCloudInfo(); InitMetaData(); InitSchemaMeta(); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::WIFI); } void CloudDataTest::TearDownTestCase() @@ -379,6 +392,7 @@ HWTEST_F(CloudDataTest, QueryStatistics003, TestSize.Level0) if (store != nullptr) { std::map entry = { { "inserted", 1 }, { "updated", 2 }, { "normal", 3 } }; store->MakeCursor(entry); + store->SetEqualIdentifier("", ""); } return store; }; @@ -476,7 +490,7 @@ HWTEST_F(CloudDataTest, QueryLastSyncInfo003, TestSize.Level0) /** * @tc.name: QueryLastSyncInfo004 -* @tc.desc: The query last sync info interface failed when switch is close. +* @tc.desc: The query last sync info interface * @tc.type: FUNC * @tc.require: */ @@ -492,12 +506,15 @@ HWTEST_F(CloudDataTest, QueryLastSyncInfo004, TestSize.Level0) cloudServiceImpl_->QueryLastSyncInfo(TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); EXPECT_EQ(status, CloudData::CloudService::SUCCESS); EXPECT_TRUE(!result.empty()); - EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code = E_CLOUD_DISABLED); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code == E_BLOCKED_BY_NETWORK_STRATEGY); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].startTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].finishTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].syncStatus == 1); } /** * @tc.name: QueryLastSyncInfo005 -* @tc.desc: The query last sync info interface failed when app cloud switch is close. +* @tc.desc: The query last sync info interface * @tc.type: FUNC * @tc.require: */ @@ -516,12 +533,15 @@ HWTEST_F(CloudDataTest, QueryLastSyncInfo005, TestSize.Level0) cloudServiceImpl_->QueryLastSyncInfo(TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); EXPECT_EQ(status, CloudData::CloudService::SUCCESS); EXPECT_TRUE(!result.empty()); - EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code = E_CLOUD_DISABLED); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code == E_BLOCKED_BY_NETWORK_STRATEGY); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].startTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].finishTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].syncStatus == 1); } /** * @tc.name: QueryLastSyncInfo006 -* @tc.desc: The query last sync info interface failed when schema is invalid. +* @tc.desc: The query last sync info interface * @tc.type: FUNC * @tc.require: */ @@ -541,6 +561,307 @@ HWTEST_F(CloudDataTest, QueryLastSyncInfo006, TestSize.Level0) EXPECT_TRUE(result.empty()); } +/** +* @tc.name: QueryLastSyncInfo007 +* @tc.desc: The query last sync info interface +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo007, TestSize.Level0) +{ + int32_t user = 100; + int64_t startTime = 123456789; + int64_t finishTime = 123456799; + CloudLastSyncInfo lastSyncInfo; + lastSyncInfo.id = TEST_CLOUD_ID; + lastSyncInfo.storeId = TEST_CLOUD_DATABASE_ALIAS_1; + lastSyncInfo.startTime = startTime; + lastSyncInfo.finishTime = finishTime; + lastSyncInfo.syncStatus = 1; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfo, true); + + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); + + CloudData::SyncManager sync; + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1 } }); + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(!result.empty()); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code == -1); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].startTime == startTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].finishTime == finishTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].syncStatus == 1); +} + +/** +* @tc.name: QueryLastSyncInfo008 +* @tc.desc: The query last sync info interface +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo008, TestSize.Level0) +{ + int32_t user = 100; + int64_t startTime = 123456789; + int64_t finishTime = 123456799; + CloudLastSyncInfo lastSyncInfo; + lastSyncInfo.id = TEST_CLOUD_ID; + lastSyncInfo.storeId = TEST_CLOUD_DATABASE_ALIAS_1; + lastSyncInfo.startTime = startTime; + lastSyncInfo.finishTime = finishTime; + lastSyncInfo.syncStatus = 1; + lastSyncInfo.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfo, true); + CloudLastSyncInfo lastSyncInfo1; + lastSyncInfo1.id = TEST_CLOUD_ID; + lastSyncInfo1.storeId = TEST_CLOUD_DATABASE_ALIAS_2; + lastSyncInfo1.startTime = startTime; + lastSyncInfo1.finishTime = finishTime; + lastSyncInfo1.syncStatus = 1; + lastSyncInfo1.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_2), lastSyncInfo1, true); + + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + ""), lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); + + CloudData::SyncManager sync; + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1 }, { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_2} }); + + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(result.size() == 2); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].code == 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].startTime == startTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].finishTime == finishTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_1].syncStatus == 1); + + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].code == 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].startTime == startTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].finishTime == finishTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].syncStatus == 1); +} + +/** +* @tc.name: QueryLastSyncInfo009 +* @tc.desc: The query last sync info interface failed when schema is invalid. +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo009, TestSize.Level0) +{ + int32_t user = 100; + int64_t startTime = 123456789; + int64_t finishTime = 123456799; + CloudLastSyncInfo lastSyncInfo; + lastSyncInfo.id = TEST_CLOUD_ID; + lastSyncInfo.storeId = TEST_CLOUD_DATABASE_ALIAS_1; + lastSyncInfo.startTime = startTime; + lastSyncInfo.finishTime = finishTime; + lastSyncInfo.syncStatus = 1; + lastSyncInfo.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfo, true); + CloudLastSyncInfo lastSyncInfo1; + lastSyncInfo1.id = TEST_CLOUD_ID; + lastSyncInfo1.storeId = TEST_CLOUD_DATABASE_ALIAS_2; + lastSyncInfo1.startTime = startTime; + lastSyncInfo1.finishTime = finishTime; + lastSyncInfo1.syncStatus = 1; + lastSyncInfo1.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_2), lastSyncInfo1, true); + + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, ""), + lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); + + CloudData::SyncManager sync; + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_2} }); + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(result.size() == 1); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].code == 0); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].startTime == startTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].finishTime == finishTime); + EXPECT_TRUE(result[TEST_CLOUD_DATABASE_ALIAS_2].syncStatus == 1); +} + +/** +* @tc.name: QueryLastSyncInfo010 +* @tc.desc: The query last sync info interface failed +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo010, TestSize.Level0) +{ + int32_t user = 100; + int64_t startTime = 123456789; + int64_t finishTime = 123456799; + CloudLastSyncInfo lastSyncInfo; + lastSyncInfo.id = TEST_CLOUD_ID; + lastSyncInfo.storeId = TEST_CLOUD_DATABASE_ALIAS_1; + lastSyncInfo.startTime = startTime; + lastSyncInfo.finishTime = finishTime; + lastSyncInfo.syncStatus = 1; + lastSyncInfo.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfo, true); + CloudLastSyncInfo lastSyncInfo1; + lastSyncInfo1.id = TEST_CLOUD_ID; + lastSyncInfo1.storeId = TEST_CLOUD_DATABASE_ALIAS_2; + lastSyncInfo1.startTime = startTime; + lastSyncInfo1.finishTime = finishTime; + lastSyncInfo1.syncStatus = 1; + lastSyncInfo1.code = 0; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_2), lastSyncInfo1, true); + + CloudData::SyncManager sync; + CloudData::SyncManager::SyncInfo info(user, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, "1234"} }); + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(result.empty()); +} + +/** +* @tc.name: QueryLastSyncInfo011 +* @tc.desc: The query last sync info interface +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo011, TestSize.Level0) +{ + schemaMeta_.databases[1].name = TEST_CLOUD_STORE_1; + MetaDataManager::GetInstance().SaveMeta(cloudInfo_.GetSchemaKey(TEST_CLOUD_BUNDLE), schemaMeta_, true); + int32_t user = 100; + CloudData::SyncManager sync; + CloudData::SyncManager::SyncInfo info(user, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); + info.syncId_ = 0; + CloudInfo cloud; + cloud.user = info.user_; + auto cloudSyncInfos = sync.GetCloudSyncInfo(info, cloud); + sync.UpdateStartSyncInfo(cloudSyncInfos); + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, TEST_CLOUD_STORE }, + { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, TEST_CLOUD_STORE_1} }); + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(result.size() == 2); + EXPECT_TRUE(result[TEST_CLOUD_STORE].code == 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE].startTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE].finishTime == 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE].syncStatus == 0); + + EXPECT_TRUE(result[TEST_CLOUD_STORE_1].code == 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE_1].startTime != 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE_1].finishTime == 0); + EXPECT_TRUE(result[TEST_CLOUD_STORE_1].syncStatus == 0); +} + +/** +* @tc.name: QueryLastSyncInfo012 +* @tc.desc: The query last sync info interface failed. +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, QueryLastSyncInfo012, TestSize.Level0) +{ + int32_t user = 100; + int64_t startTime = 123456789; + int64_t finishTime = 123456799; + CloudLastSyncInfo lastSyncInfo; + lastSyncInfo.id = TEST_CLOUD_ID; + lastSyncInfo.storeId = TEST_CLOUD_DATABASE_ALIAS_1; + lastSyncInfo.startTime = startTime; + lastSyncInfo.finishTime = finishTime; + lastSyncInfo.syncStatus = 1; + MetaDataManager::GetInstance().SaveMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfo, true); + + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + TEST_CLOUD_DATABASE_ALIAS_1), lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); + + CloudData::SyncManager sync; + CloudData::SyncManager::SyncInfo info(user, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); + auto [status, result] = sync.QueryLastSyncInfo({ { user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, "1234"} }); + EXPECT_EQ(status, CloudData::CloudService::SUCCESS); + EXPECT_TRUE(result.empty()); +} + +/** +* @tc.name: GetStores +* @tc.desc: Test GetStores function +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, GetStores, TestSize.Level0) +{ + auto cloudServerMock = std::make_shared(); + auto user = AccountDelegate::GetInstance()->GetUserByToken(OHOS::IPCSkeleton::GetCallingTokenID()); + auto [status, cloudInfo] = cloudServerMock->GetServerInfo(user, true); + SchemaMeta schemaMeta; + MetaDataManager::GetInstance().LoadMeta(cloudInfo.GetSchemaKey(TEST_CLOUD_BUNDLE), schemaMeta, true); + EXPECT_TRUE(!schemaMeta.GetStores().empty()); +} + +/** +* @tc.name: UpdateStartSyncInfo +* @tc.desc: Test UpdateStartSyncInfo +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, UpdateStartSyncInfo, TestSize.Level0) +{ + int32_t user = 100; + CloudData::SyncManager sync; + CloudData::SyncManager::SyncInfo info(user, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); + CloudInfo cloud; + cloud.user = info.user_; + auto cloudSyncInfos = sync.GetCloudSyncInfo(info, cloud); + sync.UpdateStartSyncInfo(cloudSyncInfos); + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + ""), lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); + printf("code: %d", lastSyncInfos[0].code); + EXPECT_TRUE(lastSyncInfos[0].code == -1); + EXPECT_TRUE(lastSyncInfos[0].startTime != 0); + EXPECT_TRUE(lastSyncInfos[0].finishTime != 0); + EXPECT_TRUE(lastSyncInfos[0].syncStatus == 1); +} + +/** +* @tc.name: UpdateStartSyncInfo +* @tc.desc: Test UpdateStartSyncInfo +* @tc.type: FUNC +* @tc.require: + */ +HWTEST_F(CloudDataTest, UpdateFinishSyncInfo, TestSize.Level0) +{ + int32_t user = 100; + CloudData::SyncManager sync; + CloudData::SyncManager::SyncInfo info(user, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1); + CloudInfo cloud; + cloud.user = info.user_; + auto cloudSyncInfos = sync.GetCloudSyncInfo(info, cloud); + sync.UpdateStartSyncInfo(cloudSyncInfos); + sync.UpdateFinishSyncInfo({ user, TEST_CLOUD_ID, TEST_CLOUD_BUNDLE, TEST_CLOUD_DATABASE_ALIAS_1 }, 0, 0); + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, TEST_CLOUD_BUNDLE, + ""), lastSyncInfos, true); + EXPECT_TRUE(!lastSyncInfos.empty()); +} + /** * @tc.name: Share * @tc.desc: @@ -1510,7 +1831,9 @@ HWTEST_F(CloudDataTest, GetPostEventTask, TestSize.Level0) std::map traceIds; auto task = sync.GetPostEventTask(schemas, cloudInfo_, info, true, traceIds); task(); - EXPECT_TRUE(sync.lastSyncInfos_.Empty()); + std::vector lastSyncInfos; + MetaDataManager::GetInstance().LoadMeta(CloudLastSyncInfo::GetKey(user, "test", "test"), lastSyncInfos, true); + EXPECT_TRUE(lastSyncInfos.size() == 0); } /** @@ -1606,7 +1929,7 @@ HWTEST_F(CloudDataTest, GetCloudSyncInfo, TestSize.Level0) cloud.enableCloud = false; CloudData::SyncManager::SyncInfo info(cloudInfo_.user); MetaDataManager::GetInstance().DelMeta(cloudInfo_.GetKey(), true); - info.bundleName_ = "test"; + info.bundleName_ = TEST_CLOUD_BUNDLE; auto ret = sync.GetCloudSyncInfo(info, cloud); EXPECT_TRUE(!ret.empty()); } @@ -1832,14 +2155,10 @@ HWTEST_F(CloudDataTest, GetAppSchemaFromServer, TestSize.Level0) { int32_t userId = CloudServerMock::INVALID_USER_ID; std::string bundleName; - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NONE); - NetworkAdapter::GetInstance().expireTime_ = - std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) - .count() + - 1000; + delegate_.isNetworkAvailable_ = false; auto [status, meta] = cloudServiceImpl_->GetAppSchemaFromServer(userId, bundleName); EXPECT_EQ(status, CloudData::CloudService::NETWORK_ERROR); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::WIFI); + delegate_.isNetworkAvailable_ = true; std::tie(status, meta) = cloudServiceImpl_->GetAppSchemaFromServer(userId, bundleName); EXPECT_EQ(status, CloudData::CloudService::SCHEMA_INVALID); userId = 100; @@ -1888,15 +2207,11 @@ HWTEST_F(CloudDataTest, GetCloudInfo001, TestSize.Level0) int32_t userId = 1000; auto [status, cloudInfo] = cloudServiceImpl_->GetCloudInfo(userId); EXPECT_EQ(status, CloudData::CloudService::ERROR); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::NONE); - NetworkAdapter::GetInstance().expireTime_ = - std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) - .count() + - 1000; MetaDataManager::GetInstance().DelMeta(cloudInfo_.GetKey(), true); + delegate_.isNetworkAvailable_ = false; std::tie(status, cloudInfo) = cloudServiceImpl_->GetCloudInfo(cloudInfo_.user); EXPECT_EQ(status, CloudData::CloudService::NETWORK_ERROR); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::WIFI); + delegate_.isNetworkAvailable_ = true; } /** @@ -2182,5 +2497,315 @@ HWTEST_F(CloudDataTest, GetPriorityLevel004, TestSize.Level1) DistributedRdb::PredicatesMemo memo; rdbServiceImpl.DoCloudSync(param, option, memo, nullptr); } + +/** +* @tc.name: UpdateSchemaFromHap001 +* @tc.desc: Test the UpdateSchemaFromHap with invalid user +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateSchemaFromHap001, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo info = { .user = -1, .instIndex = 0, .bundleName = TEST_CLOUD_BUNDLE,}; + auto ret = cloudServiceImpl_->UpdateSchemaFromHap(info); + EXPECT_EQ(ret, Status::ERROR); +} + +/** +* @tc.name: UpdateSchemaFromHap002 +* @tc.desc: Test the UpdateSchemaFromHap with invalid bundleName +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateSchemaFromHap002, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo info = { .user = cloudInfo_.user, .instIndex = 0, .bundleName = "", }; + auto ret = cloudServiceImpl_->UpdateSchemaFromHap(info); + EXPECT_EQ(ret, Status::ERROR); +} + +/** +* @tc.name: UpdateSchemaFromHap003 +* @tc.desc: Test the UpdateSchemaFromHap with the schema application is not configured +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateSchemaFromHap003, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo info = { + .user = cloudInfo_.user, .instIndex = 0, .bundleName = TEST_CLOUD_BUNDLE, + }; + auto ret = cloudServiceImpl_->UpdateSchemaFromHap(info); + EXPECT_EQ(ret, Status::SUCCESS); + SchemaMeta schemaMeta; + std::string schemaKey = CloudInfo::GetSchemaKey(info.user, info.bundleName, info.instIndex); + ASSERT_TRUE(MetaDataManager::GetInstance().LoadMeta(schemaKey, schemaMeta, true)); + EXPECT_EQ(schemaMeta.version, schemaMeta_.version); +} + +/** +* @tc.name: UpdateSchemaFromHap004 +* @tc.desc: Test the UpdateSchemaFromHap with valid parameter +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateSchemaFromHap004, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudInfo::AppInfo exampleAppInfo; + exampleAppInfo.bundleName = COM_EXAMPLE_TEST_CLOUD; + exampleAppInfo.appId = COM_EXAMPLE_TEST_CLOUD; + exampleAppInfo.version = 1; + exampleAppInfo.cloudSwitch = true; + CloudInfo cloudInfo; + MetaDataManager::GetInstance().LoadMeta(cloudInfo_.GetKey(), cloudInfo, true); + cloudInfo.apps[COM_EXAMPLE_TEST_CLOUD] = std::move(exampleAppInfo); + MetaDataManager::GetInstance().SaveMeta(cloudInfo_.GetKey(), cloudInfo, true); + CloudData::CloudServiceImpl::HapInfo info = { + .user = cloudInfo_.user, .instIndex = 0, .bundleName = COM_EXAMPLE_TEST_CLOUD, + }; + auto ret = cloudServiceImpl_->UpdateSchemaFromHap(info); + EXPECT_EQ(ret, Status::SUCCESS); + SchemaMeta schemaMeta; + std::string schemaKey = CloudInfo::GetSchemaKey(info.user, info.bundleName, info.instIndex); + ASSERT_TRUE(MetaDataManager::GetInstance().LoadMeta(schemaKey, schemaMeta, true)); + EXPECT_EQ(schemaMeta.version, SCHEMA_VERSION); +} + +/** +* @tc.name: UpdateClearWaterMark001 +* @tc.desc: Test UpdateClearWaterMark001 the database.version not found. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateClearWaterMark001, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo hapInfo = { + .user = cloudInfo_.user, .instIndex = 0, .bundleName = TEST_CLOUD_BUNDLE + }; + SchemaMeta::Database database; + database.name = TEST_CLOUD_STORE; + database.version = 1; + SchemaMeta schemaMeta; + schemaMeta.version = 1; + schemaMeta.databases.push_back(database); + + SchemaMeta::Database database1; + database1.name = TEST_CLOUD_STORE_1; + database1.version = 2; + SchemaMeta newSchemaMeta; + newSchemaMeta.version = 0; + newSchemaMeta.databases.push_back(database1); + cloudServiceImpl_->UpdateClearWaterMark(hapInfo, newSchemaMeta, schemaMeta); + + CloudMark metaData; + metaData.bundleName = hapInfo.bundleName; + metaData.userId = hapInfo.user; + metaData.index = hapInfo.instIndex; + metaData.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; + metaData.storeId = database1.name; + ASSERT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData, true)); +} + +/** +* @tc.name: UpdateClearWaterMark002 +* @tc.desc: Test UpdateClearWaterMark002 the same database.version +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateClearWaterMark002, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo hapInfo = { + .user = cloudInfo_.user, .instIndex = 0, .bundleName = TEST_CLOUD_BUNDLE + }; + SchemaMeta::Database database; + database.name = TEST_CLOUD_STORE; + database.version = 1; + SchemaMeta schemaMeta; + schemaMeta.version = 1; + schemaMeta.databases.push_back(database); + + SchemaMeta::Database database1; + database1.name = TEST_CLOUD_STORE; + database1.version = 1; + SchemaMeta newSchemaMeta; + newSchemaMeta.version = 0; + newSchemaMeta.databases.push_back(database1); + cloudServiceImpl_->UpdateClearWaterMark(hapInfo, newSchemaMeta, schemaMeta); + + CloudMark metaData; + metaData.bundleName = hapInfo.bundleName; + metaData.userId = hapInfo.user; + metaData.index = hapInfo.instIndex; + metaData.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; + metaData.storeId = database1.name; + ASSERT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData, true)); +} + +/** +* @tc.name: UpdateClearWaterMark003 +* @tc.desc: Test UpdateClearWaterMark003 the different database.version +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, UpdateClearWaterMark003, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + CloudData::CloudServiceImpl::HapInfo hapInfo = { + .user = cloudInfo_.user, .instIndex = 0, .bundleName = TEST_CLOUD_BUNDLE + }; + SchemaMeta::Database database; + database.name = TEST_CLOUD_STORE; + database.version = 1; + SchemaMeta schemaMeta; + schemaMeta.version = 1; + schemaMeta.databases.push_back(database); + + SchemaMeta::Database database1; + database1.name = TEST_CLOUD_STORE; + database1.version = 2; + SchemaMeta newSchemaMeta; + newSchemaMeta.version = 0; + newSchemaMeta.databases.push_back(database1); + cloudServiceImpl_->UpdateClearWaterMark(hapInfo, newSchemaMeta, schemaMeta); + + CloudMark metaData; + metaData.bundleName = hapInfo.bundleName; + metaData.userId = hapInfo.user; + metaData.index = hapInfo.instIndex; + metaData.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; + metaData.storeId = database1.name; + ASSERT_TRUE(MetaDataManager::GetInstance().LoadMeta(metaData.GetKey(), metaData, true)); + EXPECT_TRUE(metaData.isClearWaterMark); + MetaDataManager::GetInstance().DelMeta(metaData.GetKey(), true); +} + +/** +* @tc.name: GetPrepareTraceId +* @tc.desc: Test GetPrepareTraceId && GetUser +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(CloudDataTest, GetPrepareTraceId, TestSize.Level0) +{ + SyncParam syncParam; + syncParam.prepareTraceId = TEST_TRACE_ID; + syncParam.user = EVT_USER; + auto async = [](const GenDetails &details) {}; + SyncEvent::EventInfo eventInfo(syncParam, true, nullptr, async); + StoreInfo storeInfo; + SyncEvent evt(storeInfo, std::move(eventInfo)); + EXPECT_EQ(evt.GetUser(), EVT_USER); + EXPECT_EQ(evt.GetPrepareTraceId(), TEST_TRACE_ID); +} + +/** +* @tc.name: TryUpdateDeviceId001 +* @tc.desc: TryUpdateDeviceId test +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(CloudDataTest, TryUpdateDeviceId001, TestSize.Level1) +{ + DistributedRdb::RdbServiceImpl rdbServiceImpl; + DistributedRdb::RdbSyncerParam param{ .bundleName_ = TEST_CLOUD_BUNDLE, .storeName_ = TEST_CLOUD_STORE }; + StoreMetaData oldMeta; + oldMeta.deviceId = "oldUuidtest"; + oldMeta.user = "100"; + oldMeta.bundleName = "test_appid_001"; + oldMeta.storeId = "test_storeid_001"; + oldMeta.isNeedUpdateDeviceId = true; + oldMeta.storeType = StoreMetaData::StoreType::STORE_RELATIONAL_BEGIN; + bool isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta); + EXPECT_EQ(isSuccess, true); + StoreMetaData meta1 = oldMeta; + auto ret = rdbServiceImpl.TryUpdateDeviceId(param, oldMeta, meta1); + EXPECT_EQ(ret, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey()); +} + +/** +* @tc.name: TryUpdateDeviceId002 +* @tc.desc: TryUpdateDeviceId test +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(CloudDataTest, TryUpdateDeviceId002, TestSize.Level1) +{ + DistributedRdb::RdbServiceImpl rdbServiceImpl; + DistributedRdb::RdbSyncerParam param{ .bundleName_ = TEST_CLOUD_BUNDLE, .storeName_ = TEST_CLOUD_STORE }; + StoreMetaData oldMeta; + oldMeta.deviceId = "oldUuidtest"; + oldMeta.user = "100"; + oldMeta.bundleName = "test_appid_001"; + oldMeta.storeId = "test_storeid_001"; + oldMeta.isNeedUpdateDeviceId = false; + oldMeta.storeType = StoreMetaData::StoreType::STORE_RELATIONAL_BEGIN; + bool isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta); + EXPECT_EQ(isSuccess, true); + StoreMetaData meta1 = oldMeta; + auto ret = rdbServiceImpl.TryUpdateDeviceId(param, oldMeta, meta1); + EXPECT_EQ(ret, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey()); +} + +/** +* @tc.name: TryUpdateDeviceId003 +* @tc.desc: TryUpdateDeviceId test +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(CloudDataTest, TryUpdateDeviceId003, TestSize.Level1) +{ + DistributedRdb::RdbServiceImpl rdbServiceImpl; + DistributedRdb::RdbSyncerParam param{ .bundleName_ = TEST_CLOUD_BUNDLE, .storeName_ = TEST_CLOUD_STORE }; + StoreMetaData oldMeta; + oldMeta.deviceId = "oldUuidtest"; + oldMeta.user = "100"; + oldMeta.bundleName = "test_appid_001"; + oldMeta.storeId = "test_storeid_001"; + oldMeta.isNeedUpdateDeviceId = true; + oldMeta.storeType = StoreMetaData::StoreType::STORE_RELATIONAL_END; + bool isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta); + EXPECT_EQ(isSuccess, true); + StoreMetaData meta1 = oldMeta; + auto ret = rdbServiceImpl.TryUpdateDeviceId(param, oldMeta, meta1); + EXPECT_EQ(ret, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey()); +} + +/** +* @tc.name: TryUpdateDeviceId004 +* @tc.desc: TryUpdateDeviceId test +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(CloudDataTest, TryUpdateDeviceId004, TestSize.Level1) +{ + DistributedRdb::RdbServiceImpl rdbServiceImpl; + DistributedRdb::RdbSyncerParam param{ .bundleName_ = TEST_CLOUD_BUNDLE, .storeName_ = TEST_CLOUD_STORE }; + StoreMetaData oldMeta; + oldMeta.deviceId = "oldUuidtest"; + oldMeta.user = "100"; + oldMeta.bundleName = "test_appid_001"; + oldMeta.storeId = "test_storeid_001"; + oldMeta.isNeedUpdateDeviceId = false; + oldMeta.storeType = StoreMetaData::StoreType::STORE_RELATIONAL_END; + bool isSuccess = MetaDataManager::GetInstance().SaveMeta(oldMeta.GetKey(), oldMeta); + EXPECT_EQ(isSuccess, true); + StoreMetaData meta1 = oldMeta; + auto ret = rdbServiceImpl.TryUpdateDeviceId(param, oldMeta, meta1); + EXPECT_EQ(ret, true); + MetaDataManager::GetInstance().DelMeta(oldMeta.GetKey()); +} } // namespace DistributedDataTest } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/cloud_service_impl_test.cpp b/datamgr_service/services/distributeddataservice/service/test/cloud_service_impl_test.cpp index b8fd653a..bad6767d 100644 --- a/datamgr_service/services/distributeddataservice/service/test/cloud_service_impl_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/cloud_service_impl_test.cpp @@ -1,17 +1,17 @@ /* -* Copyright (c) 2024 Huawei Device Co., Ltd. -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #define LOG_TAG "CloudServiceImplTest" #include "cloud_service_impl.h" @@ -19,34 +19,22 @@ #include #include "accesstoken_kit.h" -#include "account/account_delegate.h" #include "bootstrap.h" -#include "checker_mock.h" -#include "cloud/change_event.h" -#include "cloud/cloud_event.h" #include "cloud/cloud_server.h" -#include "cloud/cloud_share_event.h" #include "cloud/schema_meta.h" #include "cloud_types.h" -#include "cloud_types_util.h" #include "cloud_value_util.h" #include "communicator/device_manager_adapter.h" -#include "device_matrix.h" -#include "eventcenter/event_center.h" -#include "feature/feature_system.h" -#include "ipc_skeleton.h" #include "log_print.h" -#include "metadata/meta_data_manager.h" #include "metadata/store_meta_data.h" #include "metadata/store_meta_data_local.h" #include "mock/db_store_mock.h" #include "mock/general_store_mock.h" -#include "network_adapter.h" +#include "model/component_config.h" +#include "network/network_delegate.h" +#include "network_delegate_mock.h" #include "rdb_query.h" -#include "rdb_service.h" -#include "rdb_service_impl.h" #include "rdb_types.h" -#include "store/auto_cache.h" #include "store/store_info.h" #include "token_setproc.h" @@ -71,9 +59,11 @@ public: void TearDown(); static std::shared_ptr cloudServiceImpl_; + static NetworkDelegateMock delegate_; }; std::shared_ptr CloudServiceImplTest::cloudServiceImpl_ = std::make_shared(); +NetworkDelegateMock CloudServiceImplTest::delegate_; void CloudServiceImplTest::SetUpTestCase(void) { @@ -81,6 +71,7 @@ void CloudServiceImplTest::SetUpTestCase(void) size_t min = 5; auto executor = std::make_shared(max, min); DeviceManagerAdapter::GetInstance().Init(executor); + NetworkDelegate::RegisterNetworkInstance(&delegate_); } void CloudServiceImplTest::TearDownTestCase() {} @@ -90,10 +81,10 @@ void CloudServiceImplTest::SetUp() {} void CloudServiceImplTest::TearDown() {} /** -* @tc.name: EnableCloud001 -* @tc.desc: Test EnableCloud functions with user is invalid. -* @tc.type: FUNC -* @tc.require: + * @tc.name: EnableCloud001 + * @tc.desc: Test EnableCloud functions with user is invalid. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, EnableCloud001, TestSize.Level0) { @@ -105,10 +96,10 @@ HWTEST_F(CloudServiceImplTest, EnableCloud001, TestSize.Level0) } /** -* @tc.name: DisableCloud001 -* @tc.desc: Test DisableCloud functions with user is invalid. -* @tc.type: FUNC -* @tc.require: + * @tc.name: DisableCloud001 + * @tc.desc: Test DisableCloud functions with user is invalid. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, DisableCloud001, TestSize.Level0) { @@ -118,10 +109,10 @@ HWTEST_F(CloudServiceImplTest, DisableCloud001, TestSize.Level0) } /** -* @tc.name: ChangeAppSwitch001 -* @tc.desc: Test ChangeAppSwitch functions with user is invalid. -* @tc.type: FUNC -* @tc.require: + * @tc.name: ChangeAppSwitch001 + * @tc.desc: Test ChangeAppSwitch functions with user is invalid. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ChangeAppSwitch001, TestSize.Level0) { @@ -132,10 +123,10 @@ HWTEST_F(CloudServiceImplTest, ChangeAppSwitch001, TestSize.Level0) } /** -* @tc.name: Clean001 -* @tc.desc: Test Clean functions with user is invalid. -* @tc.type: FUNC -* @tc.require: + * @tc.name: Clean001 + * @tc.desc: Test Clean functions with user is invalid. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, Clean001, TestSize.Level0) { @@ -147,10 +138,10 @@ HWTEST_F(CloudServiceImplTest, Clean001, TestSize.Level0) } /** -* @tc.name: NotifyDataChange001 -* @tc.desc: Test the EnableCloud function in case it doesn't get cloudInfo. -* @tc.type: FUNC -* @tc.require: + * @tc.name: NotifyDataChange001 + * @tc.desc: Test the EnableCloud function in case it doesn't get cloudInfo. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, NotifyDataChange001, TestSize.Level0) { @@ -160,10 +151,10 @@ HWTEST_F(CloudServiceImplTest, NotifyDataChange001, TestSize.Level0) } /** -* @tc.name: ExecuteStatistics001 -* @tc.desc: Test the ExecuteStatistics function if the package name does not support CloudSync. -* @tc.type: FUNC -* @tc.require: + * @tc.name: ExecuteStatistics001 + * @tc.desc: Test the ExecuteStatistics function if the package name does not support CloudSync. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ExecuteStatistics001, TestSize.Level0) { @@ -183,10 +174,10 @@ HWTEST_F(CloudServiceImplTest, ExecuteStatistics001, TestSize.Level0) } /** -* @tc.name: QueryStatistics001 -* @tc.desc: When metadata is not supported store test the QueryStatistics function. -* @tc.type: FUNC -* @tc.require: + * @tc.name: QueryStatistics001 + * @tc.desc: When metadata is not supported store test the QueryStatistics function. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, QueryStatistics001, TestSize.Level0) { @@ -199,10 +190,10 @@ HWTEST_F(CloudServiceImplTest, QueryStatistics001, TestSize.Level0) } /** -* @tc.name: QueryLastSyncInfo001 -* @tc.desc: Test QueryLastSyncInfo functions with invalid parameter. -* @tc.type: FUNC -* @tc.require: + * @tc.name: QueryLastSyncInfo001 + * @tc.desc: Test QueryLastSyncInfo functions with invalid parameter. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, QueryLastSyncInfo001, TestSize.Level0) { @@ -213,10 +204,10 @@ HWTEST_F(CloudServiceImplTest, QueryLastSyncInfo001, TestSize.Level0) } /** -* @tc.name: OnBind001 -* @tc.desc: Test OnBind functions with invalid parameter. -* @tc.type: FUNC -* @tc.require: + * @tc.name: OnBind001 + * @tc.desc: Test OnBind functions with invalid parameter. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, OnBind001, TestSize.Level0) { @@ -226,10 +217,10 @@ HWTEST_F(CloudServiceImplTest, OnBind001, TestSize.Level0) } /** -* @tc.name: UpdateSchema001 -* @tc.desc: Test UpdateSchema001 functions with invalid parameter. -* @tc.type: FUNC -* @tc.require: + * @tc.name: UpdateSchema001 + * @tc.desc: Test UpdateSchema001 functions with invalid parameter. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, UpdateSchema001, TestSize.Level0) { @@ -240,40 +231,38 @@ HWTEST_F(CloudServiceImplTest, UpdateSchema001, TestSize.Level0) } /** -* @tc.name: GetAppSchemaFromServer001 -* @tc.desc: Test GetAppSchemaFromServer functions not support CloudService. -* @tc.type: FUNC -* @tc.require: + * @tc.name: GetAppSchemaFromServer001 + * @tc.desc: Test GetAppSchemaFromServer functions not support CloudService. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, GetAppSchemaFromServer001, TestSize.Level0) { ZLOGI("CloudServiceImplTest GetAppSchemaFromServer001 start"); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::WIFI); int user = -1; auto [status, result] = cloudServiceImpl_->GetAppSchemaFromServer(user, TEST_CLOUD_BUNDLE); EXPECT_EQ(status, CloudData::CloudService::SERVER_UNAVAILABLE); } /** -* @tc.name: GetCloudInfoFromServer001 -* @tc.desc: Test GetCloudInfoFromServer functions not support CloudService. -* @tc.type: FUNC -* @tc.require: + * @tc.name: GetCloudInfoFromServer001 + * @tc.desc: Test GetCloudInfoFromServer functions not support CloudService. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, GetCloudInfoFromServer001, TestSize.Level0) { ZLOGI("CloudServiceImplTest GetCloudInfoFromServer001 start"); - NetworkAdapter::GetInstance().SetNet(NetworkAdapter::WIFI); int user = -1; auto [status, result] = cloudServiceImpl_->GetCloudInfoFromServer(user); EXPECT_EQ(status, CloudData::CloudService::SERVER_UNAVAILABLE); } /** -* @tc.name: ReleaseUserInfo001 -* @tc.desc: Test ReleaseUserInfo functions with invalid parameter. -* @tc.type: FUNC -* @tc.require: + * @tc.name: ReleaseUserInfo001 + * @tc.desc: Test ReleaseUserInfo functions with invalid parameter. + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ReleaseUserInfo001, TestSize.Level0) { @@ -284,10 +273,10 @@ HWTEST_F(CloudServiceImplTest, ReleaseUserInfo001, TestSize.Level0) } /** -* @tc.name: DoSubscribe -* @tc.desc: Test the DoSubscribe with not support CloudService -* @tc.type: FUNC -* @tc.require: + * @tc.name: DoSubscribe + * @tc.desc: Test the DoSubscribe with not support CloudService + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, DoSubscribe, TestSize.Level0) { @@ -297,10 +286,10 @@ HWTEST_F(CloudServiceImplTest, DoSubscribe, TestSize.Level0) } /** -* @tc.name: Share001 -* @tc.desc: Test the Share with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: Share001 + * @tc.desc: Test the Share with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, Share001, TestSize.Level0) { @@ -312,10 +301,10 @@ HWTEST_F(CloudServiceImplTest, Share001, TestSize.Level0) } /** -* @tc.name: Unshare001 -* @tc.desc: Test the Unshare with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: Unshare001 + * @tc.desc: Test the Unshare with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, Unshare001, TestSize.Level0) { @@ -327,10 +316,10 @@ HWTEST_F(CloudServiceImplTest, Unshare001, TestSize.Level0) } /** -* @tc.name: Exit001 -* @tc.desc: Test the Exit with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: Exit001 + * @tc.desc: Test the Exit with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, Exit001, TestSize.Level0) { @@ -341,10 +330,10 @@ HWTEST_F(CloudServiceImplTest, Exit001, TestSize.Level0) } /** -* @tc.name: ChangePrivilege001 -* @tc.desc: Test the ChangePrivilege with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: ChangePrivilege001 + * @tc.desc: Test the ChangePrivilege with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ChangePrivilege001, TestSize.Level0) { @@ -356,10 +345,10 @@ HWTEST_F(CloudServiceImplTest, ChangePrivilege001, TestSize.Level0) } /** -* @tc.name: Query001 -* @tc.desc: Test the Query with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: Query001 + * @tc.desc: Test the Query with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, Query001, TestSize.Level0) { @@ -370,10 +359,10 @@ HWTEST_F(CloudServiceImplTest, Query001, TestSize.Level0) } /** -* @tc.name: QueryByInvitation001 -* @tc.desc: Test the QueryByInvitation with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: QueryByInvitation001 + * @tc.desc: Test the QueryByInvitation with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, QueryByInvitation001, TestSize.Level0) { @@ -384,10 +373,10 @@ HWTEST_F(CloudServiceImplTest, QueryByInvitation001, TestSize.Level0) } /** -* @tc.name: ConfirmInvitation001 -* @tc.desc: Test the ConfirmInvitation with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: ConfirmInvitation001 + * @tc.desc: Test the ConfirmInvitation with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ConfirmInvitation001, TestSize.Level0) { @@ -399,10 +388,10 @@ HWTEST_F(CloudServiceImplTest, ConfirmInvitation001, TestSize.Level0) } /** -* @tc.name: ChangeConfirmation001 -* @tc.desc: Test the ChangeConfirmation with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: ChangeConfirmation001 + * @tc.desc: Test the ChangeConfirmation with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, ChangeConfirmation001, TestSize.Level0) { @@ -414,10 +403,10 @@ HWTEST_F(CloudServiceImplTest, ChangeConfirmation001, TestSize.Level0) } /** -* @tc.name: GetSharingHandle001 -* @tc.desc: Test the GetSharingHandle with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: GetSharingHandle001 + * @tc.desc: Test the GetSharingHandle with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, GetSharingHandle001, TestSize.Level0) { @@ -427,10 +416,10 @@ HWTEST_F(CloudServiceImplTest, GetSharingHandle001, TestSize.Level0) } /** -* @tc.name: SetCloudStrategy001 -* @tc.desc: Test the SetCloudStrategy with get hapInfo failed -* @tc.type: FUNC -* @tc.require: + * @tc.name: SetCloudStrategy001 + * @tc.desc: Test the SetCloudStrategy with get hapInfo failed + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, SetCloudStrategy001, TestSize.Level0) { @@ -443,10 +432,10 @@ HWTEST_F(CloudServiceImplTest, SetCloudStrategy001, TestSize.Level0) } /** -* @tc.name: SetGlobalCloudStrategy001 -* @tc.desc: Test the SetGlobalCloudStrategy with get hapInfo failed -* @tc.type: FUNC -* @tc.require: + * @tc.name: SetGlobalCloudStrategy001 + * @tc.desc: Test the SetGlobalCloudStrategy with get hapInfo failed + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, SetGlobalCloudStrategy001, TestSize.Level0) { @@ -459,10 +448,10 @@ HWTEST_F(CloudServiceImplTest, SetGlobalCloudStrategy001, TestSize.Level0) } /** -* @tc.name: CheckNotifyConditions001 -* @tc.desc: Test the CheckNotifyConditions with invalid parameters -* @tc.type: FUNC -* @tc.require: + * @tc.name: CheckNotifyConditions001 + * @tc.desc: Test the CheckNotifyConditions with invalid parameters + * @tc.type: FUNC + * @tc.require: */ HWTEST_F(CloudServiceImplTest, CheckNotifyConditions, TestSize.Level0) { @@ -480,5 +469,83 @@ HWTEST_F(CloudServiceImplTest, CheckNotifyConditions, TestSize.Level0) status = cloudServiceImpl_->CheckNotifyConditions(TEST_CLOUD_APPID, TEST_CLOUD_BUNDLE, cloudInfo); EXPECT_EQ(status, CloudData::CloudService::CLOUD_DISABLE_SWITCH); } + +/** + * @tc.name: GetDfxFaultType + * @tc.desc: test GetDfxFaultType function + * @tc.type: FUNC + * @tc.require: + * @tc.author: + */ +HWTEST_F(CloudServiceImplTest, GetDfxFaultType, TestSize.Level0) +{ + ASSERT_NE(cloudServiceImpl_, nullptr); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::ENABLE_CLOUD), "ENABLE_CLOUD"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::DISABLE_CLOUD), "DISABLE_CLOUD"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::SWITCH_ON), "SWITCH_ON"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::SWITCH_OFF), "SWITCH_OFF"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::QUERY_SYNC_INFO), "QUERY_SYNC_INFO"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::USER_CHANGE), "USER_CHANGE"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::USER_UNLOCK), "USER_UNLOCK"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::NETWORK_RECOVERY), "NETWORK_RECOVERY"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::SERVICE_INIT), "SERVICE_INIT"); + EXPECT_EQ(cloudServiceImpl_->GetDfxFaultType(CloudSyncScene::ACCOUNT_STOP), "SYNC_TASK"); +} + +class ComponentConfigTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void ComponentConfigTest::SetUpTestCase(void) {} + +void ComponentConfigTest::TearDownTestCase() {} + +void ComponentConfigTest::SetUp() {} + +void ComponentConfigTest::TearDown() {} + +/** + * @tc.name: CapabilityRange + * @tc.desc: test CapabilityRange Marshal function + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(ComponentConfigTest, ComponentConfig, TestSize.Level0) +{ + DistributedData::ComponentConfig config; + config.description = "description"; + config.lib = "lib"; + config.constructor = "constructor"; + config.destructor = "destructor"; + config.params = ""; + Serializable::json node; + EXPECT_EQ(config.params.empty(), true); + + EXPECT_EQ(config.Marshal(node), true); + EXPECT_EQ(node["description"], config.description); + EXPECT_EQ(node["lib"], config.lib); + EXPECT_EQ(node["constructor"], config.constructor); + EXPECT_EQ(node["destructor"], config.destructor); + + DistributedData::ComponentConfig componentConfig; + componentConfig.description = "description"; + componentConfig.lib = "lib"; + componentConfig.constructor = "constructor"; + componentConfig.destructor = "destructor"; + componentConfig.params = "params"; + Serializable::json node1; + EXPECT_EQ(componentConfig.params.empty(), false); + + EXPECT_EQ(componentConfig.Marshal(node1), true); + EXPECT_EQ(node["description"], componentConfig.description); + EXPECT_EQ(node["lib"], componentConfig.lib); + EXPECT_EQ(node["constructor"], componentConfig.constructor); + EXPECT_EQ(node["destructor"], componentConfig.destructor); +} } // namespace DistributedDataTest } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/crypto_manager_test.cpp b/datamgr_service/services/distributeddataservice/service/test/crypto_manager_test.cpp index 3e935ff9..4b42a2cb 100644 --- a/datamgr_service/services/distributeddataservice/service/test/crypto_manager_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/crypto_manager_test.cpp @@ -17,24 +17,32 @@ #include +#include "crypto_manager.h" #include "gtest/gtest.h" +#include "metadata/secret_key_meta_data.h" +#include "metadata/store_meta_data.h" #include "types.h" using namespace testing::ext; using namespace OHOS::DistributedData; +namespace OHOS::Test { +namespace DistributedDataTest { class CryptoManagerTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); void SetUp(){}; void TearDown(){}; + static std::vector Random(uint32_t len); protected: static std::vector randomKey; - static std::vector Random(uint32_t len); }; static const uint32_t KEY_LENGTH = 32; static const uint32_t ENCRYPT_KEY_LENGTH = 48; +static constexpr int32_t EL2 = 2; +static constexpr int32_t EL4 = 4; +static constexpr const char *TEST_USERID = "100"; std::vector CryptoManagerTest::randomKey; void CryptoManagerTest::SetUpTestCase(void) { @@ -92,7 +100,7 @@ HWTEST_F(CryptoManagerTest, CheckRootKey, TestSize.Level0) */ HWTEST_F(CryptoManagerTest, Encrypt001, TestSize.Level0) { - auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey); + auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); EXPECT_EQ(encryptKey.size(), ENCRYPT_KEY_LENGTH); encryptKey.assign(encryptKey.size(), 0); } @@ -106,8 +114,51 @@ HWTEST_F(CryptoManagerTest, Encrypt001, TestSize.Level0) */ HWTEST_F(CryptoManagerTest, Encrypt002, TestSize.Level0) { - auto encryptKey = CryptoManager::GetInstance().Encrypt({ }); + auto encryptKey = CryptoManager::GetInstance().Encrypt({ }, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); + EXPECT_TRUE(encryptKey.empty()); +} + +/** +* @tc.name: Encrypt003 +* @tc.desc: Check root key fail; +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, Encrypt003, TestSize.Level0) +{ + auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, EL2, DEFAULT_USER); + EXPECT_TRUE(encryptKey.empty()); + encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, EL4, DEFAULT_USER); + EXPECT_TRUE(encryptKey.empty()); + + encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, EL2, TEST_USERID); + // check interact across local accounts permission failed + EXPECT_TRUE(encryptKey.empty()); + encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, EL4, TEST_USERID); + // check interact across local accounts permission failed + EXPECT_TRUE(encryptKey.empty()); +} + +/** +* @tc.name: Encrypt004 +* @tc.desc: Encrypt clone key, but root key not imported; +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, Encrypt004, TestSize.Level0) +{ + const std::string str = "distributed_db_backup_key"; + auto cloneKey = std::vector(str.begin(), str.end()); + std::vector nonce{}; + CryptoManager::EncryptParams params = { .keyAlias = cloneKey, .nonce = nonce }; + auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, params); EXPECT_TRUE(encryptKey.empty()); + std::vector key; + + auto result = CryptoManager::GetInstance().Decrypt(encryptKey, key, params); + ASSERT_FALSE(result); } /** @@ -119,9 +170,9 @@ HWTEST_F(CryptoManagerTest, Encrypt002, TestSize.Level0) */ HWTEST_F(CryptoManagerTest, DecryptKey001, TestSize.Level0) { - auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey); + auto encryptKey = CryptoManager::GetInstance().Encrypt(randomKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); std::vector key; - auto result = CryptoManager::GetInstance().Decrypt(encryptKey, key); + auto result = CryptoManager::GetInstance().Decrypt(encryptKey, key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); EXPECT_TRUE(result); EXPECT_EQ(key.size(), KEY_LENGTH); encryptKey.assign(encryptKey.size(), 0); @@ -137,7 +188,7 @@ HWTEST_F(CryptoManagerTest, DecryptKey001, TestSize.Level0) HWTEST_F(CryptoManagerTest, DecryptKey002, TestSize.Level0) { std::vector key; - auto result = CryptoManager::GetInstance().Decrypt(randomKey, key); + auto result = CryptoManager::GetInstance().Decrypt(randomKey, key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); EXPECT_FALSE(result); EXPECT_TRUE(key.empty()); } @@ -153,7 +204,96 @@ HWTEST_F(CryptoManagerTest, DecryptKey003, TestSize.Level0) { std::vector srcKey {}; std::vector key; - auto result = CryptoManager::GetInstance().Decrypt(srcKey, key); + auto result = CryptoManager::GetInstance().Decrypt(srcKey, key, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); + EXPECT_FALSE(result); + EXPECT_TRUE(key.empty()); +} + +/** +* @tc.name: DecryptKey004 +* @tc.desc: Check root key fail; +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, DecryptKey004, TestSize.Level0) +{ + std::vector key; + auto result = CryptoManager::GetInstance().Decrypt(randomKey, key, EL2, DEFAULT_USER); + EXPECT_FALSE(result); + EXPECT_TRUE(key.empty()); + result = CryptoManager::GetInstance().Decrypt(randomKey, key, EL4, DEFAULT_USER); EXPECT_FALSE(result); EXPECT_TRUE(key.empty()); -} \ No newline at end of file +} + +/** +* @tc.name: Decrypt001 +* @tc.desc: SecretKeyMetaData is old. +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, Decrypt001, TestSize.Level0) +{ + SecretKeyMetaData secretKeyMeta; + StoreMetaData metaData; + std::vector key; + secretKeyMeta.sKey = CryptoManager::GetInstance().Encrypt(randomKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); + auto result = CryptoManager::GetInstance().Decrypt(metaData, secretKeyMeta, key); + EXPECT_TRUE(result); +} + +/** +* @tc.name: Decrypt002 +* @tc.desc: SecretKeyMetaData is new. +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, Decrypt002, TestSize.Level0) +{ + SecretKeyMetaData secretKeyMeta; + secretKeyMeta.area = 1; + StoreMetaData metaData; + std::vector key; + secretKeyMeta.sKey = CryptoManager::GetInstance().Encrypt(randomKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); + auto result = CryptoManager::GetInstance().Decrypt(metaData, secretKeyMeta, key); + EXPECT_TRUE(result); + for (int8_t i = 0; i < randomKey.size(); i++) { + EXPECT_EQ(randomKey[i], key[i]); + } +} + +/** +* @tc.name: UpdatePassword001 +* @tc.desc: The data is unencrypted. +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, UpdatePassword001, TestSize.Level0) +{ + StoreMetaData metaData; + std::vector key; + EXPECT_FALSE(CryptoManager::GetInstance().UpdateSecretKey(metaData, key)); +} + +/** +* @tc.name: UpdatePassword002 +* @tc.desc: The data is encrypted. +* @tc.type: FUNC +* @tc.require: +* @tc.author: yanhui +*/ +HWTEST_F(CryptoManagerTest, UpdatePassword002, TestSize.Level0) +{ + StoreMetaData metaData; + metaData.isEncrypt = true; + metaData.area = DEFAULT_ENCRYPTION_LEVEL; + // MetaDataManager not initialized + EXPECT_FALSE(CryptoManager::GetInstance().UpdateSecretKey(metaData, randomKey)); + EXPECT_FALSE(CryptoManager::GetInstance().UpdateSecretKey(metaData, randomKey, CryptoManager::CLONE_SECRET_KEY)); +} +} // namespace DistributedDataTest +} // namespace OHOS::Test diff --git a/datamgr_service/services/distributeddataservice/service/test/data_share_common_test.cpp b/datamgr_service/services/distributeddataservice/service/test/data_share_common_test.cpp new file mode 100644 index 00000000..ad14f7d3 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/data_share_common_test.cpp @@ -0,0 +1,92 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#define LOG_TAG "DataShareCommonTest" + +#include +#include +#include "data_share_profile_config.h" +#include "div_strategy.h" +#include "log_print.h" +#include "strategy.h" + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::DataShare; +class DataShareCommonTest : public testing::Test { +public: + static constexpr int64_t USER_TEST = 100; + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +/** +* @tc.name: DivStrategy001 +* @tc.desc: test DivStrategy function when three parameters are all nullptr +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareCommonTest, DivStrategy001, TestSize.Level1) +{ + ZLOGI("DataShareCommonTest DivStrategy001 start"); + std::shared_ptr check = nullptr; + std::shared_ptr trueAction = nullptr; + std::shared_ptr falseAction = nullptr; + DivStrategy divStrategy(check, trueAction, falseAction); + auto context = std::make_shared(); + bool result = divStrategy.operator()(context); + EXPECT_EQ(result, false); + ZLOGI("DataShareCommonTest DivStrategy001 end"); +} + +/** +* @tc.name: DivStrategy002 +* @tc.desc: test DivStrategy function when trueAction and falseAction are nullptr +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareCommonTest, DivStrategy002, TestSize.Level1) +{ + ZLOGI("DataShareCommonTest DivStrategy002 start"); + std::shared_ptr check = std::make_shared(); + std::shared_ptr trueAction = nullptr; + std::shared_ptr falseAction = nullptr; + DivStrategy divStrategy(check, trueAction, falseAction); + auto context = std::make_shared(); + bool result = divStrategy.operator()(context); + EXPECT_EQ(result, false); + ZLOGI("DataShareCommonTest DivStrategy002 end"); +} + +/** +* @tc.name: DivStrategy003 +* @tc.desc: test DivStrategy function when only falseAction is nullptr +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareCommonTest, DivStrategy003, TestSize.Level1) +{ + ZLOGI("DataShareCommonTest DivStrategy003 start"); + std::shared_ptr check = std::make_shared(); + std::shared_ptr trueAction = std::make_shared(); + std::shared_ptr falseAction = nullptr; + DivStrategy divStrategy(check, trueAction, falseAction); + auto context = std::make_shared(); + bool result = divStrategy.operator()(context); + EXPECT_EQ(result, false); + ZLOGI("DataShareCommonTest DivStrategy003 end"); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/data_share_service_impl_test.cpp b/datamgr_service/services/distributeddataservice/service/test/data_share_service_impl_test.cpp index f9aaed5e..a14d505b 100644 --- a/datamgr_service/services/distributeddataservice/service/test/data_share_service_impl_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/data_share_service_impl_test.cpp @@ -1,5 +1,5 @@ /* -* Copyright (c) 2024 Huawei Device Co., Ltd. +* Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -38,6 +38,7 @@ std::string TBL_NAME0 = "name0"; std::string TBL_NAME1 = "name1"; std::string BUNDLE_NAME = "ohos.datasharetest.demo"; namespace OHOS::Test { +using OHOS::DataShare::LogLabel; class DataShareServiceImplTest : public testing::Test { public: static constexpr int64_t USER_TEST = 100; @@ -458,40 +459,36 @@ HWTEST_F(DataShareServiceImplTest, OnInitialize, TestSize.Level1) } /** -* @tc.name: NotifyObserver -* @tc.desc: test NotifyObserver no GetCallerBundleName scene +* @tc.name: GetCallerInfo001 +* @tc.desc: test GetCallerInfo function when succeeded in getting tokenID * @tc.type: FUNC * @tc.require:SQL */ -HWTEST_F(DataShareServiceImplTest, NotifyObserver, TestSize.Level1) +HWTEST_F(DataShareServiceImplTest, GetCallerInfo001, TestSize.Level1) { + ZLOGI("DataShareServiceImplTest GetCallerInfo001 start"); DataShareServiceImpl dataShareServiceImpl; - std::string uri = SLIENT_ACCESS_URI; - sptr remoteObj; - auto tokenId = AccessTokenKit::GetHapTokenID(USER_TEST, "ohos.datasharetest.demo", 0); - AccessTokenKit::DeleteToken(tokenId); - - auto result = dataShareServiceImpl.RegisterObserver(uri, remoteObj); - EXPECT_EQ(result, ERR_INVALID_VALUE); - dataShareServiceImpl.NotifyObserver(uri); - result = dataShareServiceImpl.UnregisterObserver(uri, remoteObj); - EXPECT_EQ(result, ERR_INVALID_VALUE); + int32_t appIndex = 1; + auto result = dataShareServiceImpl.GetCallerInfo(BUNDLE_NAME, appIndex); + EXPECT_EQ(result.first, true); + ZLOGI("DataShareServiceImplTest GetCallerInfo001 end"); } /** -* @tc.name: RegisterObserver -* @tc.desc: test RegisterObserver abnormal scene +* @tc.name: GetCallerInfo002 +* @tc.desc: test GetCallerInfo function when failed to get tokenID * @tc.type: FUNC * @tc.require:SQL */ -HWTEST_F(DataShareServiceImplTest, RegisterObserver, TestSize.Level1) +HWTEST_F(DataShareServiceImplTest, GetCallerInfo002, TestSize.Level1) { + ZLOGI("DataShareServiceImplTest GetCallerInfo002 start"); DataShareServiceImpl dataShareServiceImpl; - sptr remoteObj; - auto result = dataShareServiceImpl.RegisterObserver("", remoteObj); - EXPECT_EQ(result, ERR_INVALID_VALUE); - dataShareServiceImpl.NotifyObserver(""); - result = dataShareServiceImpl.UnregisterObserver("", remoteObj); - EXPECT_EQ(result, ERR_INVALID_VALUE); + int32_t appIndex = 1; + auto tokenId = AccessTokenKit::GetHapTokenID(USER_TEST, "ohos.datasharetest.demo", 0); + AccessTokenKit::DeleteToken(tokenId); + auto result = dataShareServiceImpl.GetCallerInfo(BUNDLE_NAME, appIndex); + EXPECT_EQ(result.first, false); + ZLOGI("DataShareServiceImplTest GetCallerInfo002 end"); } } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/data_share_service_stub_test.cpp b/datamgr_service/services/distributeddataservice/service/test/data_share_service_stub_test.cpp index 9e3a5514..a31921b5 100644 --- a/datamgr_service/services/distributeddataservice/service/test/data_share_service_stub_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/data_share_service_stub_test.cpp @@ -207,36 +207,4 @@ HWTEST_F(DataShareServiceStubTest, OnEnableRdbSubs001, TestSize.Level1) result = dataShareServiceStub->OnDisableRdbSubs(request, reply); EXPECT_EQ(result, IDataShareService::DATA_SHARE_ERROR); } - -/** -* @tc.name: OnRegisterObserver001 -* @tc.desc: test OnRegisterObserver function abnormal scene -* @tc.type: FUNC -* @tc.require:SQL -*/ -HWTEST_F(DataShareServiceStubTest, OnRegisterObserver001, TestSize.Level1) -{ - uint8_t value = TEST_DATA; - uint8_t *data = &value; - size_t size = TEST_SIZE; - MessageParcel request; - request.WriteInterfaceToken(INTERFACE_TOKEN); - request.WriteBuffer(data, size); - request.RewindRead(0); - MessageParcel reply; - auto result = dataShareServiceStub->OnRegisterObserver(request, reply); - EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); - - result = dataShareServiceStub->OnNotifyObserver(request, reply); - EXPECT_EQ(result, IDataShareService::DATA_SHARE_ERROR); - - result = dataShareServiceStub->OnSetSilentSwitch(request, reply); - EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); - - result = dataShareServiceStub->OnGetSilentProxyStatus(request, reply); - EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); - - result = dataShareServiceStub->OnUnregisterObserver(request, reply); - EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); -} } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/data_share_types_util_test.cpp b/datamgr_service/services/distributeddataservice/service/test/data_share_types_util_test.cpp new file mode 100644 index 00000000..2294f3bc --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/data_share_types_util_test.cpp @@ -0,0 +1,174 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#define LOG_TAG "DataShareTypesUtilTest" + +#include +#include + +#include "data_share_types_util.h" +#include "data_share_obs_proxy.h" +#include "log_print.h" + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::DataShare; +std::string BUNDLE = "ohos.datasharetest.demo"; +constexpr int64_t SUB_ID = 100; + +class DataShareTypesUtilTest : public testing::Test { +public: + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +RdbChangeNode CreateChangeNode() +{ + TemplateId tplId; + tplId.subscriberId_ = SUB_ID; + tplId.bundleName_ = BUNDLE; + + RdbChangeNode node; + node.uri_ = std::string(""); + node.templateId_ = tplId; + node.data_ = std::vector(); + node.isSharedMemory_ = false; + node.memory_ = nullptr; + node.size_ = 0; + + return node; +} + +/** +* @tc.name: Unmarshalling_001 +* @tc.desc: test Unmarshalling function and abnormal scene +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Unmarshalling_001, TestSize.Level1) +{ + ZLOGI("Unmarshalling_001 starts"); + Data data; + data.version_ = 0; + MessageParcel msgParcel; + // msgParcel is empty, nothing to unmarshalling + bool retVal = ITypesUtil::Unmarshalling(data, msgParcel); + EXPECT_EQ(retVal, false); + ZLOGI("Unmarshalling_001 ends"); +} + +/** +* @tc.name: Unmarshalling_002 +* @tc.desc: test Unmarshalling function and abnormal scene +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Unmarshalling_002, TestSize.Level1) +{ + ZLOGI("Unmarshalling_002 starts"); + AshmemNode node; + MessageParcel msgParcel; + bool retVal = ITypesUtil::Unmarshalling(node, msgParcel); + EXPECT_EQ(retVal, true); + ZLOGI("Unmarshalling_002 ends"); +} + +/** +* @tc.name: Unmarshalling_003 +* @tc.desc: test Unmarshalling function abnormal scene +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Unmarshalling_003, TestSize.Level1) +{ + ZLOGI("Unmarshalling_003 starts"); + ITypesUtil::Predicates predicates; + MessageParcel msgParcel; + bool retVal = ITypesUtil::Unmarshalling(predicates, msgParcel); + EXPECT_EQ(retVal, false); + ZLOGI("Unmarshalling_003 ends"); +} + +/** +* @tc.name: Marshalling_001 +* @tc.desc: test Marshalling function abnormal scene +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Marshalling_001, TestSize.Level1) +{ + ZLOGI("Marshalling_001 starts"); + RdbChangeNode node = CreateChangeNode(); + node.isSharedMemory_ = true; + node.memory_ = nullptr; + MessageParcel msgParcel; + bool retVal = ITypesUtil::Marshalling(node, msgParcel); + EXPECT_EQ(retVal, false); + ZLOGI("Marshalling_001 ends"); +} + +/** +* @tc.name: Marshalling_002 +* @tc.desc: test Marshalling function +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Marshalling_002, TestSize.Level1) +{ + ZLOGI("Marshalling_002 starts"); + RdbChangeNode node = CreateChangeNode(); + node.isSharedMemory_ = false; + node.memory_ = nullptr; + MessageParcel msgParcel; + bool retVal = ITypesUtil::Marshalling(node, msgParcel); + EXPECT_EQ(retVal, true); + ZLOGI("Marshalling_002 ends"); +} + +/** +* @tc.name: Marshalling_003 +* @tc.desc: test Marshalling function +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(DataShareTypesUtilTest, Marshalling_003, TestSize.Level1) +{ + ZLOGI("Marshalling_003 starts"); + RdbChangeNode node = CreateChangeNode(); + OHOS::sptr fake = nullptr; + RdbObserverProxy proxy(fake); + int retCreate = proxy.CreateAshmem(node); + EXPECT_EQ(retCreate, DataShare::E_OK); + + int len = 10; + int intLen = 4; + int offset = 0; + int ret = proxy.WriteAshmem(node, (void *)&len, intLen, offset); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(offset, intLen); + + // read from the start + const void *read = node.memory_->ReadFromAshmem(intLen, 0); + EXPECT_NE(read, nullptr); + + node.isSharedMemory_ = true; + + MessageParcel msgParcel; + bool retVal = ITypesUtil::Marshalling(node, msgParcel); + EXPECT_EQ(retVal, true); + ZLOGI("Marshalling_003 ends"); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/device_matrix_test.cpp b/datamgr_service/services/distributeddataservice/service/test/device_matrix_test.cpp index 9eb491b1..66e25b24 100644 --- a/datamgr_service/services/distributeddataservice/service/test/device_matrix_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/device_matrix_test.cpp @@ -20,7 +20,6 @@ #include "device_manager_adapter.h" #include "eventcenter/event_center.h" #include "feature/feature_system.h" -#include "gtest/gtest.h" #include "ipc_skeleton.h" #include "matrix_event.h" #include "metadata/meta_data_manager.h" @@ -28,6 +27,7 @@ #include "mock/checker_mock.h" #include "mock/db_store_mock.h" #include "types.h" +#include "gtest/gtest.h" using namespace testing::ext; using namespace OHOS::DistributedData; using namespace OHOS::DistributedKv; @@ -45,7 +45,7 @@ protected: struct Result { uint16_t mask_ = DeviceMatrix::INVALID_LEVEL; std::string deviceId_; - Result(){}; + Result() {}; }; static constexpr const char *TEST_DEVICE = "14a0a92a428005db27c40bad46bf145fede38ec37effe0347cd990fcb031f320"; static constexpr const char *TEST_BUNDLE = "matrix_test"; @@ -54,10 +54,14 @@ protected: void InitRemoteMatrixMeta(); void InitMetaData(); - static inline std::vector> staticStores_ = { { "bundle0", "store0" }, - { "bundle1", "store0" } }; - static inline std::vector> dynamicStores_ = { { "bundle0", "store1" }, - { "bundle3", "store0" } }; + static inline std::vector> staticStores_ = { + { "bundle0", "store0" }, + { "bundle1", "store0" } + }; + static inline std::vector> dynamicStores_ = { + { "bundle0", "store1" }, + { "bundle3", "store0" } + }; static BlockData isFinished_; static std::shared_ptr dbStoreMock_; static uint32_t selfToken_; @@ -147,12 +151,12 @@ void DeviceMatrixTest::InitMetaData() } /** -* @tc.name: GetMetaStoreCode -* @tc.desc: get the meta data store mask code; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: GetMetaStoreCode + * @tc.desc: get the meta data store mask code; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, GetMetaStoreCode, TestSize.Level0) { StoreMetaData meta; @@ -165,12 +169,12 @@ HWTEST_F(DeviceMatrixTest, GetMetaStoreCode, TestSize.Level0) } /** -* @tc.name: GetAllCode -* @tc.desc: get all dynamic store mask code; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: GetAllCode + * @tc.desc: get all dynamic store mask code; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, GetAllCode, TestSize.Level0) { StoreMetaData meta = metaData_; @@ -183,12 +187,12 @@ HWTEST_F(DeviceMatrixTest, GetAllCode, TestSize.Level0) } /** -* @tc.name: GetOtherStoreCode -* @tc.desc: get the other store mask code; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: GetOtherStoreCode + * @tc.desc: get the other store mask code; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, GetOtherStoreCode, TestSize.Level0) { StoreMetaData meta = metaData_; @@ -197,12 +201,12 @@ HWTEST_F(DeviceMatrixTest, GetOtherStoreCode, TestSize.Level0) } /** -* @tc.name: BroadcastMeta -* @tc.desc: broadcast the meta store change; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: BroadcastMeta + * @tc.desc: broadcast the meta store change; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, BroadcastMeta, TestSize.Level0) { DeviceMatrix::DataLevel level = { @@ -213,12 +217,12 @@ HWTEST_F(DeviceMatrixTest, BroadcastMeta, TestSize.Level0) } /** -* @tc.name: BroadcastFirst -* @tc.desc: broadcast all stores change; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: BroadcastFirst + * @tc.desc: broadcast all stores change; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, BroadcastFirst, TestSize.Level0) { StoreMetaData meta = metaData_; @@ -234,12 +238,12 @@ HWTEST_F(DeviceMatrixTest, BroadcastFirst, TestSize.Level0) } /** -* @tc.name: BroadcastOthers -* @tc.desc: broadcast the device profile store change; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: BroadcastOthers + * @tc.desc: broadcast the device profile store change; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, BroadcastOthers, TestSize.Level0) { StoreMetaData meta = metaData_; @@ -252,12 +256,12 @@ HWTEST_F(DeviceMatrixTest, BroadcastOthers, TestSize.Level0) } /** -* @tc.name: BroadcastAll -* @tc.desc: broadcast the all store change; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: BroadcastAll + * @tc.desc: broadcast the all store change; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, BroadcastAll, TestSize.Level0) { DeviceMatrix::DataLevel level = { @@ -287,12 +291,12 @@ HWTEST_F(DeviceMatrixTest, BroadcastAll, TestSize.Level0) } /** -* @tc.name: UpdateMatrixMeta -* @tc.desc: update the remote matrix meta the all store change; -* @tc.type: FUNC -* @tc.require: -* @tc.author: blue sky -*/ + * @tc.name: UpdateMatrixMeta + * @tc.desc: update the remote matrix meta the all store change; + * @tc.type: FUNC + * @tc.require: + * @tc.author: blue sky + */ HWTEST_F(DeviceMatrixTest, UpdateMatrixMeta, TestSize.Level0) { MatrixMetaData metaData; @@ -301,8 +305,8 @@ HWTEST_F(DeviceMatrixTest, UpdateMatrixMeta, TestSize.Level0) metaData.deviceId = TEST_DEVICE; metaData.origin = MatrixMetaData::Origin::REMOTE_RECEIVED; metaData.dynamicInfo = { TEST_BUNDLE, dynamicStores_[0].first }; - MetaDataManager::GetInstance().Subscribe(MatrixMetaData::GetPrefix({ TEST_DEVICE }), - [](const std::string &, const std::string &value, int32_t flag) { + MetaDataManager::GetInstance().Subscribe( + MatrixMetaData::GetPrefix({ TEST_DEVICE }), [](const std::string &, const std::string &value, int32_t flag) { if (flag != MetaDataManager::INSERT && flag != MetaDataManager::UPDATE) { return true; } @@ -337,4 +341,566 @@ HWTEST_F(DeviceMatrixTest, UpdateMatrixMeta, TestSize.Level0) mask = DeviceMatrix::GetInstance().OnBroadcast(TEST_DEVICE, level); ASSERT_EQ(mask.first, 0x0); MetaDataManager::GetInstance().Unsubscribe(MatrixMetaData::GetPrefix({ TEST_DEVICE })); +} + +/** + * @tc.name: Online + * @tc.desc: The test equipment is operated both online and offline. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, Online, TestSize.Level1) +{ + RefCount refCount; + std::string device = "Online"; + DeviceMatrix::GetInstance().Online(device, refCount); + + DeviceMatrix::GetInstance().Offline(device); + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().Online(device, refCount)); +} + +/** + * @tc.name: Offline + * @tc.desc: Switch between offline and online status of test equipment. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, Offline, TestSize.Level1) +{ + RefCount refCount; + std::string device = "Offline"; + DeviceMatrix::GetInstance().Offline(device); + + DeviceMatrix::GetInstance().Online(device, refCount); + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().Offline(device)); +} + +/** + * @tc.name: OnBroadcast + * @tc.desc: OnBroadcast testing exceptions. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnBroadcast, TestSize.Level1) +{ + std::string device; + DeviceMatrix::DataLevel dataLevel; + EXPECT_TRUE(!dataLevel.IsValid()); + auto mask = DeviceMatrix::GetInstance().OnBroadcast(device, dataLevel); + EXPECT_EQ(mask.first, DeviceMatrix::INVALID_LEVEL); // true true + + dataLevel = { + .dynamic = DeviceMatrix::META_STORE_MASK, + }; + EXPECT_FALSE(!dataLevel.IsValid()); + mask = DeviceMatrix::GetInstance().OnBroadcast(device, dataLevel); + EXPECT_EQ(mask.first, DeviceMatrix::INVALID_LEVEL); // true false + + EXPECT_FALSE(!dataLevel.IsValid()); + mask = DeviceMatrix::GetInstance().OnBroadcast(TEST_DEVICE, dataLevel); + EXPECT_EQ(mask.first, DeviceMatrix::META_STORE_MASK); // false false + + DeviceMatrix::DataLevel dataLevels; + EXPECT_TRUE(!dataLevels.IsValid()); + mask = DeviceMatrix::GetInstance().OnBroadcast(device, dataLevels); + EXPECT_EQ(mask.first, DeviceMatrix::INVALID_LEVEL); // false true +} + +/** + * @tc.name: ConvertStatics + * @tc.desc: ConvertStatics testing exceptions. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, ConvertStatics, TestSize.Level1) +{ + DeviceMatrix deviceMatrix; + DistributedData::MatrixMetaData meta; + uint16_t mask = 0; + uint16_t result = deviceMatrix.ConvertStatics(meta, mask); + EXPECT_EQ(result, 0); + + mask = 0xFFFF; + result = deviceMatrix.ConvertStatics(meta, mask); + EXPECT_EQ(result, 0); + + meta.version = 4; + meta.dynamic = 0x1F; + meta.deviceId = TEST_DEVICE; + meta.origin = MatrixMetaData::Origin::REMOTE_RECEIVED; + meta.dynamicInfo = { TEST_BUNDLE, dynamicStores_[0].first }; + result = deviceMatrix.ConvertStatics(meta, DeviceMatrix::INVALID_LEVEL); + EXPECT_EQ(result, 0); +} + +/** + * @tc.name: SaveSwitches + * @tc.desc: SaveSwitches testing exceptions. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, SaveSwitches, TestSize.Level1) +{ + DeviceMatrix deviceMatrix; + std::string device; + DeviceMatrix::DataLevel dataLevel; + dataLevel.switches = DeviceMatrix::INVALID_VALUE; + dataLevel.switchesLen = DeviceMatrix::INVALID_LENGTH; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.SaveSwitches(device, dataLevel)); + + device = "SaveSwitches"; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.SaveSwitches(device, dataLevel)); + + dataLevel.switches = 0; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.SaveSwitches(device, dataLevel)); + + dataLevel.switchesLen = 0; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.SaveSwitches(device, dataLevel)); +} + +/** + * @tc.name: Broadcast + * @tc.desc: Broadcast testing exceptions. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, Broadcast, TestSize.Level1) +{ + DeviceMatrix deviceMatrix; + DeviceMatrix::DataLevel dataLevel; + EXPECT_FALSE(deviceMatrix.lasts_.IsValid()); + EXPECT_NO_FATAL_FAILURE(deviceMatrix.Broadcast(dataLevel)); + + dataLevel.statics = 0; + dataLevel.dynamic = 0; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.Broadcast(dataLevel)); + + deviceMatrix.lasts_.statics = 0; + deviceMatrix.lasts_.dynamic = 0; + EXPECT_TRUE(deviceMatrix.lasts_.IsValid()); + EXPECT_NO_FATAL_FAILURE(deviceMatrix.Broadcast(dataLevel)); + + DeviceMatrix::DataLevel dataLevels; + dataLevel = dataLevels; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.Broadcast(dataLevel)); +} + +/** + * @tc.name: UpdateConsistentMeta + * @tc.desc: UpdateConsistentMeta testing exceptions. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, UpdateConsistentMeta, TestSize.Level1) +{ + DeviceMatrix deviceMatrix; + std::string device = "device"; + DeviceMatrix::Mask remote; + remote.statics = 0; + remote.dynamic = 0; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.UpdateConsistentMeta(device, remote)); + + remote.statics = 0x1; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.UpdateConsistentMeta(device, remote)); + + remote.dynamic = 0x1; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.UpdateConsistentMeta(device, remote)); + + remote.statics = 0; + EXPECT_NO_FATAL_FAILURE(deviceMatrix.UpdateConsistentMeta(device, remote)); +} + +/** + * @tc.name: OnChanged001 + * @tc.desc: Test the DeviceMatrix::OnChanged method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnChanged001, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.dataType = static_cast(DeviceMatrix::LevelType::STATICS - 1); + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(metaData)); + + metaData.dataType = DeviceMatrix::LevelType::BUTT; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(metaData)); + + metaData.bundleName = "distributeddata"; + metaData.tokenId = selfToken_; + metaData.storeId = "service_meta"; + metaData.dataType = 1; + metaData.dataType = DeviceMatrix::LevelType::STATICS; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(metaData)); + + StoreMetaData meta = metaData_; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(meta)); +} + +/** + * @tc.name: OnChanged002 + * @tc.desc: Test the DeviceMatrix::OnChanged method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnChanged002, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.bundleName = "distributeddata"; + metaData.tokenId = selfToken_; + metaData.storeId = "service_meta"; + metaData.dataType = 1; + auto code = DeviceMatrix::GetInstance().GetCode(metaData); + DeviceMatrix::LevelType type = static_cast(DeviceMatrix::LevelType::STATICS - 1); + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(code, type)); + + type = DeviceMatrix::LevelType::BUTT; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(code, type)); + + StoreMetaData meta = metaData_; + code = DeviceMatrix::GetInstance().GetCode(meta); + EXPECT_EQ(code, 0); + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().OnChanged(code, type)); +} + +/** + * @tc.name: OnExchanged001 + * @tc.desc: Test the DeviceMatrix::OnExchanged method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnExchanged001, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.bundleName = "distributeddata"; + metaData.tokenId = selfToken_; + metaData.storeId = "service_meta"; + metaData.dataType = 1; + auto code = DeviceMatrix::GetInstance().GetCode(metaData); + DeviceMatrix::LevelType type = static_cast(DeviceMatrix::LevelType::STATICS - 1); + std::string device; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, code, type, DeviceMatrix::ChangeType::CHANGE_REMOTE)); + + device = "OnExchanged"; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, code, type, DeviceMatrix::ChangeType::CHANGE_REMOTE)); + + type = DeviceMatrix::LevelType::BUTT; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, code, type, DeviceMatrix::ChangeType::CHANGE_REMOTE)); +} + +/** + * @tc.name: OnExchanged002 + * @tc.desc: Test the DeviceMatrix::OnExchanged method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnExchanged002, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.dataType = static_cast(DeviceMatrix::LevelType::STATICS - 1); + std::string device = "OnExchanged"; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, metaData, DeviceMatrix::ChangeType::CHANGE_REMOTE)); + + metaData.dataType = DeviceMatrix::LevelType::BUTT; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, metaData, DeviceMatrix::ChangeType::CHANGE_REMOTE)); +} + +/** + * @tc.name: OnExchanged003 + * @tc.desc: Test the DeviceMatrix::OnExchanged method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, OnExchanged003, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.bundleName = "distributeddata"; + metaData.tokenId = selfToken_; + metaData.storeId = "service_meta"; + metaData.dataType = 1; + std::string device = "OnExchanged"; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, metaData, DeviceMatrix::ChangeType::CHANGE_REMOTE)); + + StoreMetaData meta = metaData_; + EXPECT_NO_FATAL_FAILURE( + DeviceMatrix::GetInstance().OnExchanged(device, meta, DeviceMatrix::ChangeType::CHANGE_REMOTE)); +} + +/** + * @tc.name: GetCode + * @tc.desc: Test the DeviceMatrix::GetCode method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetCode, TestSize.Level1) +{ + StoreMetaData metaData; + metaData.bundleName = "distributeddata"; + metaData.tokenId = selfToken_; + metaData.storeId = "storeId"; + metaData.dataType = DeviceMatrix::LevelType::BUTT; + auto code = DeviceMatrix::GetInstance().GetCode(metaData); + EXPECT_EQ(code, 0); +} + +/** + * @tc.name: GetMask001 + * @tc.desc: Test the DeviceMatrix::GetMask method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetMask001, TestSize.Level1) +{ + std::string device = "GetMask"; + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + std::pair mask = DeviceMatrix::GetInstance().GetMask(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: GetMask002 + * @tc.desc: Test the DeviceMatrix::GetMask method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetMask002, TestSize.Level1) +{ + std::string device = "GetMask"; + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + RefCount refCount; + DeviceMatrix::GetInstance().Online(device, refCount); + std::pair mask = DeviceMatrix::GetInstance().GetMask(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::DYNAMIC; + mask = DeviceMatrix::GetInstance().GetMask(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::BUTT; + mask = DeviceMatrix::GetInstance().GetMask(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: GetRemoteMask001 + * @tc.desc: Test the DeviceMatrix::GetRemoteMask method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetRemoteMask001, TestSize.Level1) +{ + std::string device = "GetRemoteMask"; + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + std::pair mask = DeviceMatrix::GetInstance().GetRemoteMask(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: GetRemoteMask002 + * @tc.desc: Test the DeviceMatrix::GetRemoteMask method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetRemoteMask002, TestSize.Level1) +{ + std::string device = "GetRemoteMask"; + StoreMetaData meta = metaData_; + auto code = DeviceMatrix::GetInstance().GetCode(meta); + DeviceMatrix::DataLevel level = { + .dynamic = code, + }; + DeviceMatrix::GetInstance().OnBroadcast(device, level); + + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + std::pair mask = DeviceMatrix::GetInstance().GetRemoteMask(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::DYNAMIC; + mask = DeviceMatrix::GetInstance().GetRemoteMask(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::BUTT; + mask = DeviceMatrix::GetInstance().GetRemoteMask(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: GetRecvLevel + * @tc.desc: Test the DeviceMatrix::GetRecvLevel method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetRecvLevel, TestSize.Level1) +{ + std::string device = "GetRemoteMask"; + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + std::pair mask = DeviceMatrix::GetInstance().GetRecvLevel(device, type); + EXPECT_EQ(mask.first, true); + + device = TEST_DEVICE; + mask = DeviceMatrix::GetInstance().GetRecvLevel(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::DYNAMIC; + mask = DeviceMatrix::GetInstance().GetRecvLevel(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::BUTT; + mask = DeviceMatrix::GetInstance().GetRecvLevel(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: GetConsLevel + * @tc.desc: Test the DeviceMatrix::GetConsLevel method. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, GetConsLevel, TestSize.Level1) +{ + std::string device = "GetRemoteMask"; + DeviceMatrix::LevelType type = DeviceMatrix::LevelType::STATICS; + std::pair mask = DeviceMatrix::GetInstance().GetConsLevel(device, type); + EXPECT_EQ(mask.first, false); + + device = TEST_DEVICE; + mask = DeviceMatrix::GetInstance().GetConsLevel(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::DYNAMIC; + mask = DeviceMatrix::GetInstance().GetConsLevel(device, type); + EXPECT_EQ(mask.first, true); + + type = DeviceMatrix::LevelType::BUTT; + mask = DeviceMatrix::GetInstance().GetConsLevel(device, type); + EXPECT_EQ(mask.first, false); +} + +/** + * @tc.name: UpdateLevel + * @tc.desc: Test the DeviceMatrix::UpdateLevel method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, UpdateLevel, TestSize.Level1) +{ + uint16_t level = 0; + DeviceMatrix::LevelType type = static_cast(DeviceMatrix::LevelType::STATICS - 1); + std::string device; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().UpdateLevel(device, level, type)); + + device = "OnExchanged"; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().UpdateLevel(device, level, type)); + + type = DeviceMatrix::LevelType::BUTT; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().UpdateLevel(device, level, type)); + + type = DeviceMatrix::LevelType::STATICS; + device = TEST_DEVICE; + EXPECT_NO_FATAL_FAILURE(DeviceMatrix::GetInstance().UpdateLevel(device, level, type)); +} + +/** + * @tc.name: IsDynamic + * @tc.desc: Test the DeviceMatrix::IsDynamic method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, IsDynamic, TestSize.Level1) +{ + StoreMetaData meta; + meta.bundleName = "distributeddata"; + meta.tokenId = selfToken_; + meta.storeId = ""; + meta.dataType = DeviceMatrix::LevelType::STATICS; + bool isDynamic = DeviceMatrix::GetInstance().IsDynamic(meta); + EXPECT_EQ(isDynamic, false); + + meta.dataType = DeviceMatrix::LevelType::DYNAMIC; + isDynamic = DeviceMatrix::GetInstance().IsDynamic(meta); + EXPECT_EQ(isDynamic, false); + + meta.storeId = "service_meta"; + isDynamic = DeviceMatrix::GetInstance().IsDynamic(meta); + EXPECT_EQ(isDynamic, true); + + meta.tokenId = 1; + isDynamic = DeviceMatrix::GetInstance().IsDynamic(meta); + EXPECT_EQ(isDynamic, false); + + meta.storeId = ""; + isDynamic = DeviceMatrix::GetInstance().IsDynamic(meta); + EXPECT_EQ(isDynamic, false); +} + +/** + * @tc.name: IsValid + * @tc.desc: Test the DeviceMatrix::IsValid method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(DeviceMatrixTest, IsValid, TestSize.Level1) +{ + DistributedData::DeviceMatrix::DataLevel dataLevel; + EXPECT_EQ(dataLevel.IsValid(), false); + + dataLevel.dynamic = 0; + dataLevel.statics = 0; + dataLevel.switches = 0; + dataLevel.switchesLen = 0; + EXPECT_EQ(dataLevel.IsValid(), true); +} + +class MatrixEventTest : public testing::Test { +public: + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +/** + * @tc.name: IsValid + * @tc.desc: Test the MatrixEvent::IsValid method exception scenario. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suoqilong + */ +HWTEST_F(MatrixEventTest, IsValid, TestSize.Level1) +{ + DistributedData::MatrixEvent::MatrixData matrixData; + EXPECT_EQ(matrixData.IsValid(), false); + + matrixData.dynamic = 0; + matrixData.statics = 0; + matrixData.switches = 0; + matrixData.switchesLen = 0; + EXPECT_EQ(matrixData.IsValid(), true); } \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/cloudservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/cloudservicestub_fuzzer/BUILD.gn index 2826516f..3cb8f10b 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/cloudservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/cloudservicestub_fuzzer/BUILD.gn @@ -49,7 +49,6 @@ ohos_fuzztest("CloudServiceStubFuzzTest") { "${relational_store_path}/interfaces/inner_api/common_type/include", "//third_party/json/single_include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] fuzz_config_file = @@ -97,6 +96,7 @@ ohos_fuzztest("CloudServiceStubFuzzTest") { "${data_service_path}/service/rdb/rdb_cloud.cpp", "${data_service_path}/service/rdb/rdb_cursor.cpp", "${data_service_path}/service/rdb/rdb_general_store.cpp", + "${data_service_path}/service/rdb/rdb_hiview_adapter.cpp", "${data_service_path}/service/rdb/rdb_notifier_proxy.cpp", "${data_service_path}/service/rdb/rdb_query.cpp", "${data_service_path}/service/rdb/rdb_result_set_impl.cpp", @@ -110,10 +110,11 @@ ohos_fuzztest("CloudServiceStubFuzzTest") { deps = [ "${data_service_path}/adapter/account:distributeddata_account", + "${data_service_path}/adapter/network:distributeddata_network", + "${data_service_path}/adapter/schema_helper:distributeddata_schema_helper", "${data_service_path}/adapter/utils:distributeddata_utils", "${data_service_path}/framework:distributeddatasvcfwk", "${data_service_path}/service:distributeddatasvc", - "${data_service_path}/service/network:distributeddata_network", "${kv_store_distributeddb_path}:distributeddb", ] @@ -132,6 +133,7 @@ ohos_fuzztest("CloudServiceStubFuzzTest") { "device_manager:devicemanagersdk", "hicollie:libhicollie", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/datashareservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/datashareservicestub_fuzzer/BUILD.gn index 110dc065..67dd0941 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/datashareservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/datashareservicestub_fuzzer/BUILD.gn @@ -125,6 +125,7 @@ ohos_fuzztest("DataShareServiceStubFuzzTest") { "hilog:libhilog", "hisysevent:libhisysevent", "huks:libhukssdk", + "init:libbegetutil", "ipc:ipc_core", "kv_store:distributeddata_inner", "kv_store:distributeddb", diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/dumphelper_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/dumphelper_fuzzer/BUILD.gn index 3e39786a..2fd16555 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/dumphelper_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/dumphelper_fuzzer/BUILD.gn @@ -70,6 +70,7 @@ ohos_fuzztest("DumpHelperFuzzTest") { "device_auth:deviceauth_sdk", "device_manager:devicemanagersdk", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/kvdbservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/kvdbservicestub_fuzzer/BUILD.gn index aeedc92b..458b5897 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/kvdbservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/kvdbservicestub_fuzzer/BUILD.gn @@ -44,7 +44,6 @@ ohos_fuzztest("KvdbServiceStubFuzzTest") { "${relational_store_path}/interfaces/inner_api/common_type/include", "//third_party/json/single_include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", "${data_service_path}/adapter/include/utils", ] diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/objectservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/objectservicestub_fuzzer/BUILD.gn index fb2775fb..336b0dbf 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/objectservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/objectservicestub_fuzzer/BUILD.gn @@ -113,6 +113,7 @@ ohos_fuzztest("ObjectServiceStubFuzzTest") { "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", + "json:nlohmann_json_static", "kv_store:distributeddata_inner", "kv_store:distributeddata_mgr", ] diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbresultsetstub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbresultsetstub_fuzzer/BUILD.gn index b7dffe3f..984eddf3 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbresultsetstub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbresultsetstub_fuzzer/BUILD.gn @@ -75,6 +75,7 @@ ohos_fuzztest("RdbResultSetStubFuzzTest") { "device_auth:deviceauth_sdk", "device_manager:devicemanagersdk", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbservicestub_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbservicestub_fuzzer/BUILD.gn index 22538632..e51d7851 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbservicestub_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/rdbservicestub_fuzzer/BUILD.gn @@ -49,7 +49,6 @@ ohos_fuzztest("RdbServiceStubFuzzTest") { "${relational_store_path}/interfaces/inner_api/common_type/include", "//third_party/json/single_include", "${data_service_path}/adapter/include/communicator", - "${data_service_path}/adapter/include/dfx", ] fuzz_config_file = @@ -86,6 +85,7 @@ ohos_fuzztest("RdbServiceStubFuzzTest") { "${data_service_path}/service/rdb/rdb_cloud.cpp", "${data_service_path}/service/rdb/rdb_cursor.cpp", "${data_service_path}/service/rdb/rdb_general_store.cpp", + "${data_service_path}/service/rdb/rdb_hiview_adapter.cpp", "${data_service_path}/service/rdb/rdb_notifier_proxy.cpp", "${data_service_path}/service/rdb/rdb_query.cpp", "${data_service_path}/service/rdb/rdb_result_set_impl.cpp", @@ -120,6 +120,7 @@ ohos_fuzztest("RdbServiceStubFuzzTest") { "device_manager:devicemanagersdk", "hicollie:libhicollie", "hilog:libhilog", + "hisysevent:libhisysevent", "huks:libhukssdk", "ipc:ipc_core", "kv_store:distributeddata_inner", diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/BUILD.gn index 067d82c0..27778e7e 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/BUILD.gn @@ -48,6 +48,7 @@ ohos_fuzztest("UdmfServiceFuzzTest") { "${data_service_path}/adapter/account:distributeddata_account", "${data_service_path}/adapter/communicator:distributeddata_communicator", "${data_service_path}/framework:distributeddatasvcfwk", + "${data_service_path}/service/bootstrap:distributeddata_bootstrap", "${data_service_path}/service/udmf:udmf_server", ] @@ -55,13 +56,17 @@ ohos_fuzztest("UdmfServiceFuzzTest") { "ability_base:zuri", "ability_runtime:uri_permission_mgr", "access_token:libaccesstoken_sdk", + "access_token:libtoken_setproc", + "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", "c_utils:utils", "hilog:libhilog", + "hisysevent:libhisysevent", "ipc:ipc_core", "kv_store:distributeddata_inner", "kv_store:distributeddata_mgr", "kv_store:distributeddb", + "samgr:samgr_proxy", "udmf:udmf_client", ] } diff --git a/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/udmfservice_fuzzer.cpp b/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/udmfservice_fuzzer.cpp index 4a820073..1d5b03a1 100644 --- a/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/udmfservice_fuzzer.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/fuzztest/udmfservice_fuzzer/udmfservice_fuzzer.cpp @@ -15,13 +15,15 @@ #include "udmfservice_fuzzer.h" -#include -#include - +#include "accesstoken_kit.h" +#include "distributeddata_udmf_ipc_interface_code.h" +#include "itypes_util.h" #include "ipc_skeleton.h" -#include "udmf_service_impl.h" #include "message_parcel.h" #include "securec.h" +#include "token_setproc.h" +#include "udmf_service_impl.h" +#include "udmf_types_util.h" using namespace OHOS::UDMF; @@ -31,6 +33,24 @@ constexpr uint32_t CODE_MIN = 0; constexpr uint32_t CODE_MAX = static_cast(UdmfServiceInterfaceCode::CODE_BUTT) + 1; constexpr size_t NUM_MIN = 5; constexpr size_t NUM_MAX = 12; +static constexpr int ID_LEN = 32; +static constexpr int MINIMUM = 48; + +QueryOption GenerateFuzzQueryOption(const uint8_t* data, size_t size) +{ + std::vector groupId(ID_LEN, '0'); + size_t length = groupId.size() > size ? size : groupId.size(); + for (size_t i = 0; i < length; ++i) { + groupId[i] = data[i] % MINIMUM + MINIMUM; + } + std::string groupIdStr(groupId.begin(), groupId.end()); + UnifiedKey udKey = UnifiedKey("drag", "com.test.demo", groupIdStr); + QueryOption query = { + .key = udKey.GetUnifiedKey(), + .intention = Intention::UD_INTENTION_DRAG + }; + return query; +} bool OnRemoteRequestFuzz(const uint8_t* data, size_t size) { @@ -45,10 +65,214 @@ bool OnRemoteRequestFuzz(const uint8_t* data, size_t size) request.WriteBuffer(data, size); request.RewindRead(0); MessageParcel reply; - std::shared_ptr udmfServiceStub = std::make_shared(); - udmfServiceStub->OnRemoteRequest(code, request, reply); + udmfServiceImpl->OnRemoteRequest(code, request, reply); return true; } + +void SetDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + CustomOption option1 = {.intention = Intention::UD_INTENTION_DRAG}; + std::string svalue(data, data + size); + UnifiedData data1; + std::shared_ptr obj = std::make_shared(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = svalue; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared(FILE_URI, obj); + data1.AddRecord(record); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, option1, data1); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::SET_DATA), request, reply); +} + +void GetDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::GET_DATA), request, reply); +} + +void GetBatchDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::GET_BATCH_DATA), request, reply); +} + +void UpdateDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + std::string svalue(data, data + size); + UnifiedData data1; + std::shared_ptr obj = std::make_shared(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = svalue; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared(FILE_URI, obj); + data1.AddRecord(record); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query, data1); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::UPDATE_DATA), request, reply); +} + +void DeleteDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::DELETE_DATA), request, reply); +} + +void GetSummaryFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query); + MessageParcel reply; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::GET_SUMMARY), request, reply); +} + +void AddPrivilegeDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + + Privilege privilege = { + .tokenId = 1, + .readPermission = "read", + .writePermission = "write" + }; + + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(request, query, privilege); + MessageParcel replyUpdate; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::ADD_PRIVILEGE), + request, replyUpdate); +} + +void SyncDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + std::vector devices = { "11", "22" }; + MessageParcel requestUpdate; + requestUpdate.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(requestUpdate, query, devices); + MessageParcel replyUpdate; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::SYNC), + requestUpdate, replyUpdate); +} + +void IsRemoteDataFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + QueryOption query = GenerateFuzzQueryOption(data, size); + MessageParcel requestUpdate; + requestUpdate.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(requestUpdate, query); + MessageParcel replyUpdate; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::IS_REMOTE_DATA), + requestUpdate, replyUpdate); +} + +void ObtainAsynProcessFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + std::vector groupId(ID_LEN, '0'); + size_t length = groupId.size() > size ? size : groupId.size(); + for (size_t i = 0; i < length; ++i) { + groupId[i] = data[i] % MINIMUM + MINIMUM; + } + std::string businessUdKey(groupId.begin(), groupId.end()); + AsyncProcessInfo processInfo = { + .businessUdKey = businessUdKey, + }; + MessageParcel requestUpdate; + requestUpdate.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(requestUpdate, processInfo); + MessageParcel replyUpdate; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::OBTAIN_ASYN_PROCESS), + requestUpdate, replyUpdate); +} + +void ClearAsynProcessFuzz(const uint8_t *data, size_t size) +{ + std::shared_ptr udmfServiceImpl = std::make_shared(); + std::shared_ptr executor = std::make_shared(NUM_MAX, NUM_MIN); + udmfServiceImpl->OnBind( + { "UdmfServiceStubFuzz", static_cast(IPCSkeleton::GetSelfTokenID()), std::move(executor) }); + std::vector groupId(ID_LEN, '0'); + size_t length = groupId.size() > size ? size : groupId.size(); + for (size_t i = 0; i < length; ++i) { + groupId[i] = data[i] % MINIMUM + MINIMUM; + } + std::string businessUdKey(groupId.begin(), groupId.end()); + MessageParcel requestUpdate; + requestUpdate.WriteInterfaceToken(INTERFACE_TOKEN); + ITypesUtil::Marshal(requestUpdate, businessUdKey); + MessageParcel replyUpdate; + udmfServiceImpl->OnRemoteRequest(static_cast(UdmfServiceInterfaceCode::CLEAR_ASYN_PROCESS_BY_KEY), + requestUpdate, replyUpdate); +} +} + +/* Fuzzer entry point */ +extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) +{ + OHOS::Security::AccessToken::AccessTokenID tokenId = + OHOS::Security::AccessToken::AccessTokenKit::GetHapTokenID(100, "com.ohos.dlpmanager", 0); + SetSelfTokenID(tokenId); + return 0; } /* Fuzzer entry point */ @@ -59,6 +283,20 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) } OHOS::OnRemoteRequestFuzz(data, size); + OHOS::SetDataFuzz(data, size); + OHOS::GetDataFuzz(data, size); + + OHOS::GetBatchDataFuzz(data, size); + OHOS::UpdateDataFuzz(data, size); + OHOS::DeleteDataFuzz(data, size); + + OHOS::GetSummaryFuzz(data, size); + OHOS::AddPrivilegeDataFuzz(data, size); + OHOS::SyncDataFuzz(data, size); + + OHOS::IsRemoteDataFuzz(data, size); + OHOS::ObtainAsynProcessFuzz(data, size); + OHOS::ClearAsynProcessFuzz(data, size); return 0; } \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/kv_dalegate_test.cpp b/datamgr_service/services/distributeddataservice/service/test/kv_dalegate_test.cpp new file mode 100644 index 00000000..26bba5c0 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/kv_dalegate_test.cpp @@ -0,0 +1,115 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#define LOG_TAG "KvDelegateTest" + +#include +#include +#include "data_share_profile_config.h" +#include "executor_pool.h" +#include "grd_error.h" +#include "kv_delegate.h" +#include "log_print.h" + +namespace OHOS::Test { +using namespace testing::ext; +using namespace OHOS::DataShare; +class KvDelegateTest : public testing::Test { +public: + static constexpr int64_t USER_TEST = 100; + static void SetUpTestCase(void){}; + static void TearDownTestCase(void){}; + void SetUp(){}; + void TearDown(){}; +}; + +const char* g_backupFiles[] = { + "dataShare.db", + "dataShare.db.redo", + "dataShare.db.safe", + "dataShare.db.undo", +}; +const char* BACKUP_SUFFIX = ".backup"; +std::shared_ptr executors = std::make_shared(5, 3); + +/** +* @tc.name: RestoreIfNeedt001 +* @tc.desc: test RestoreIfNeed function when dbstatus is GRD_INVALID_FILE_FORMAT +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(KvDelegateTest, RestoreIfNeed001, TestSize.Level1) +{ + ZLOGI("KvDelegateTest RestoreIfNeed001 start"); + int32_t dbstatus = GRD_INVALID_FILE_FORMAT; + std::string path = "path/to/your/db"; + KvDelegate kvDelegate(path, executors); + bool result = kvDelegate.RestoreIfNeed(dbstatus); + EXPECT_EQ(result, true); + ZLOGI("KvDelegateTest RestoreIfNeed001 end"); +} + +/** +* @tc.name: RestoreIfNeedt002 +* @tc.desc: test RestoreIfNeed function when dbstatus is GRD_REBUILD_DATABASE +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(KvDelegateTest, RestoreIfNeed002, TestSize.Level1) +{ + ZLOGI("KvDelegateTest RestoreIfNeed002 start"); + int32_t dbstatus = GRD_REBUILD_DATABASE; + std::string path = "path/to/your/db"; + KvDelegate kvDelegate(path, executors); + bool result = kvDelegate.RestoreIfNeed(dbstatus); + EXPECT_EQ(result, true); + ZLOGI("KvDelegateTest RestoreIfNeed002 end"); +} + +/** +* @tc.name: RestoreIfNeedt003 +* @tc.desc: test RestoreIfNeed function when dbstatus is GRD_TIME_OUT +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(KvDelegateTest, RestoreIfNeed003, TestSize.Level1) +{ + ZLOGI("KvDelegateTest RestoreIfNeed003 start"); + int32_t dbstatus = GRD_TIME_OUT; + std::string path = "path/to/your/db"; + KvDelegate kvDelegate(path, executors); + bool result = kvDelegate.RestoreIfNeed(dbstatus); + EXPECT_EQ(result, false); + ZLOGI("KvDelegateTest RestoreIfNeed003 end"); +} + +/** +* @tc.name: GetVersion001 +* @tc.desc: test GetVersion function when get version failed +* @tc.type: FUNC +* @tc.require:SQL +*/ +HWTEST_F(KvDelegateTest, GetVersion001, TestSize.Level1) +{ + ZLOGI("KvDelegateTest GetVersion001 start"); + std::string path = "path/to/your/db"; + KvDelegate kvDelegate(path, executors); + std::string collectionname = "testname"; + std::string filter = "testfilter"; + int version = 0; + bool result = kvDelegate.GetVersion(collectionname, filter, version); + EXPECT_EQ(result, false); + ZLOGI("KvDelegateTest GetVersion001 end"); +} +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/kvdb_general_store_test.cpp b/datamgr_service/services/distributeddataservice/service/test/kvdb_general_store_test.cpp index 8466ecce..6da7e1d4 100644 --- a/datamgr_service/services/distributeddataservice/service/test/kvdb_general_store_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/kvdb_general_store_test.cpp @@ -44,6 +44,9 @@ using StoreMetaData = OHOS::DistributedData::StoreMetaData; using SecurityLevel = OHOS::DistributedKv::SecurityLevel; using KVDBGeneralStore = OHOS::DistributedKv::KVDBGeneralStore; using DMAdapter = OHOS::DistributedData::DeviceManagerAdapter; +using DBInterceptedData = DistributedDB::InterceptedData; +using StoreId = OHOS::DistributedKv::StoreId; +using AppId = OHOS::DistributedKv::AppId; namespace OHOS::Test { namespace DistributedDataTest { class KVDBGeneralStoreTest : public testing::Test { @@ -52,7 +55,6 @@ public: static void TearDownTestCase(void); void SetUp(); void TearDown(); - protected: static constexpr const char *bundleName = "test_distributeddata"; static constexpr const char *storeName = "test_service_meta"; @@ -183,7 +185,7 @@ HWTEST_F(KVDBGeneralStoreTest, GetDBPasswordTest_002, TestSize.Level0) std::vector randomKey = Random(KEY_LENGTH); SecretKeyMetaData secretKey; secretKey.storeType = metaData_.storeType; - secretKey.sKey = CryptoManager::GetInstance().Encrypt(randomKey); + secretKey.sKey = CryptoManager::GetInstance().Encrypt(randomKey, DEFAULT_ENCRYPTION_LEVEL, DEFAULT_USER); EXPECT_EQ(secretKey.sKey.size(), ENCRYPT_KEY_LENGTH); EXPECT_TRUE(MetaDataManager::GetInstance().SaveMeta(metaData_.GetSecretKey(), secretKey, true)); @@ -755,5 +757,294 @@ HWTEST_F(KVDBGeneralStoreTest, Query002, TestSize.Level1) EXPECT_EQ(errCode, GeneralError::E_NOT_SUPPORT); EXPECT_EQ(result, nullptr); } + +/** +* @tc.name: GetDBOption +* @tc.desc: GetDBOption from meta and dbPassword +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(KVDBGeneralStoreTest, GetDBOption001, TestSize.Level0) +{ + metaData_.isEncrypt = false; + metaData_.appId = "distributeddata"; + auto dbPassword = KVDBGeneralStore::GetDBPassword(metaData_); + auto dbOption = KVDBGeneralStore::GetDBOption(metaData_, dbPassword); + EXPECT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKeyLocal(), metaData_, true)); + EXPECT_EQ(metaData_.appId, Bootstrap::GetInstance().GetProcessLabel()); + EXPECT_EQ(dbOption.createIfNecessary, false); + EXPECT_EQ(dbOption.isMemoryDb, false); + EXPECT_EQ(dbOption.isEncryptedDb, metaData_.isEncrypt); + EXPECT_EQ(dbOption.isNeedCompressOnSync, metaData_.isNeedCompress); + EXPECT_EQ(dbOption.schema, metaData_.schema); + EXPECT_EQ(dbOption.conflictResolvePolicy, DistributedDB::LAST_WIN); + EXPECT_EQ(dbOption.createDirByStoreIdOnly, true); + EXPECT_EQ(dbOption.secOption, KVDBGeneralStore::GetDBSecurity(metaData_.securityLevel)); +} + +/** +* @tc.name: Sync +* @tc.desc: Sync test the functionality of different branches. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, Sync001, TestSize.Level0) +{ + mkdir(("/data/service/el1/public/database/" + std::string(bundleName)).c_str(), + (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)); + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + uint32_t syncMode = GeneralStore::SyncMode::CLOUD_BEGIN; + uint32_t highMode = GeneralStore::HighMode::MANUAL_SYNC_MODE; + auto mixMode = GeneralStore::MixMode(syncMode, highMode); + std::string kvQuery = "querytest"; + DistributedKv::KVDBQuery query(kvQuery); + SyncParam syncParam{}; + syncParam.mode = mixMode; + KvStoreNbDelegateMock mockDelegate; + store->delegate_ = &mockDelegate; + EXPECT_NE(store->delegate_, nullptr); + auto ret = store->Sync({}, query, [](const GenDetails &result) {}, syncParam); + EXPECT_EQ(ret.first, GeneralError::E_NOT_SUPPORT); + GeneralStore::StoreConfig storeConfig; + storeConfig.enableCloud_ = false; + store->SetConfig(storeConfig); + ret = store->Sync({}, query, [](const GenDetails &result) {}, syncParam); + EXPECT_EQ(ret.first, GeneralError::E_NOT_SUPPORT); + + std::vector devices = { "device1", "device2" }; + syncMode = GeneralStore::SyncMode::CLOUD_END; + mixMode = GeneralStore::MixMode(syncMode, highMode); + syncParam.mode = mixMode; + ret = store->Sync({}, query, [](const GenDetails &result) {}, syncParam); + EXPECT_EQ(ret.first, GeneralError::E_INVALID_ARGS); + + syncMode = GeneralStore::SyncMode::NEARBY_PULL; + mixMode = GeneralStore::MixMode(syncMode, highMode); + syncParam.mode = mixMode; + ret = store->Sync(devices, query, [](const GenDetails &result) {}, syncParam); + EXPECT_EQ(ret.first, GeneralError::E_INVALID_ARGS); + store->delegate_ = nullptr; + delete store; +} + + +/** +* @tc.name: SetEqualIdentifier +* @tc.desc: SetEqualIdentifier test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(KVDBGeneralStoreTest, SetEqualIdentifier, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + std::vector sameAccountDevs{"account01", "account02", "account03"}; + std::vector uuids{"uuidtest01", "uuidtest02", "uuidtest03"}; + AppId appId01 = { "ohos.kvdbservice.test01" }; + StoreId storeId01 = { "meta_test_storeid" }; + std::string account = "account"; + store->SetEqualIdentifier(appId01, storeId01, account); + EXPECT_EQ(uuids.empty(), false); + EXPECT_EQ(sameAccountDevs.empty(), false); + delete store; +} + +/** +* @tc.name: GetIdentifierParams +* @tc.desc: GetIdentifierParams test. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, GetIdentifierParams001, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + std::vector sameAccountDevs{"account01", "account02", "account03"}; + std::vector uuids{"uuidtest01", "uuidtest02", "uuidtest03"}; + store->GetIdentifierParams(sameAccountDevs, uuids, 1); // + for (const auto &devId : uuids) { + EXPECT_EQ(DMAdapter::GetInstance().IsOHOSType(devId), false); + EXPECT_EQ(DMAdapter::GetInstance().GetAuthType(devId), 0); // + } + EXPECT_EQ(sameAccountDevs.empty(), false); + delete store; +} + +/** +* @tc.name: ConvertStatus +* @tc.desc: ConvertStatus test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(KVDBGeneralStoreTest, ConvertStatus001, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + auto ret = store->ConvertStatus(DBStatus::NOT_SUPPORT); + EXPECT_EQ(ret, GeneralError::E_NOT_SUPPORT); + DBStatus unknownStatus = static_cast(0xDEAD); + ret = store->ConvertStatus(unknownStatus); + EXPECT_EQ(ret, GeneralError::E_ERROR); + delete store; +} + +/** +* @tc.name: GetDBProcessCB +* @tc.desc: GetDBProcessCB test. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, GetDBProcessCB, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + bool asyncCalled = false; + auto async = [&asyncCalled](const DistributedData::GenDetails &details) { + asyncCalled = true; + EXPECT_TRUE(details.empty()); + }; + + std::map processes; + auto callback = store->GetDBProcessCB(async); + callback(processes); + EXPECT_TRUE(asyncCalled); + delete store; +} + +/** +* @tc.name: GetDBProcessCB +* @tc.desc: GetDBProcessCB test. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, GetDBProcessCB001, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + GeneralStore::DetailAsync async; + EXPECT_EQ(async, nullptr); + auto process = store->GetDBProcessCB(async); + EXPECT_NE(process, nullptr); + auto asyncs = [](const GenDetails &result) {}; + EXPECT_NE(asyncs, nullptr); + process = store->GetDBProcessCB(asyncs); + EXPECT_NE(process, nullptr); + store->delegate_ = nullptr; + delete store; +} + +/** +* @tc.name: GetDBProcessCB +* @tc.desc: GetDBProcessCB test. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, GetDBProcessCB002, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + std::map processes = {{"test_id", {}}}; + GeneralStore::DetailAsync async; + EXPECT_EQ(async, nullptr); + auto callback = store->GetDBProcessCB(async); + callback(processes); + delete store; +} + +/** +* @tc.name: Delete +* @tc.desc: Delete test the function. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, SetDBPushDataInterceptor, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + KvStoreNbDelegateMock mockDelegate; + store->delegate_ = &mockDelegate; + EXPECT_NE(store->delegate_, nullptr); + store->SetDBPushDataInterceptor(DistributedKv::KvStoreType::MULTI_VERSION); + store->SetDBReceiveDataInterceptor(DistributedKv::KvStoreType::MULTI_VERSION); + EXPECT_NE(metaData_.storeType, DistributedKv::KvStoreType::MULTI_VERSION); + + store->SetDBPushDataInterceptor(DistributedKv::KvStoreType::SINGLE_VERSION); + store->SetDBReceiveDataInterceptor(DistributedKv::KvStoreType::SINGLE_VERSION); + EXPECT_EQ(metaData_.storeType, DistributedKv::KvStoreType::SINGLE_VERSION); + store->delegate_ = nullptr; + delete store; +} + +/** +* @tc.name: CloseTest +* @tc.desc: Close kvdb general store +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, CloseTest001, TestSize.Level0) +{ + auto store = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store, nullptr); + EXPECT_EQ(store->delegate_, nullptr); + auto ret = store->Close(); + EXPECT_EQ(ret, GeneralError::E_OK); + + bool isForce = false; + ret = store->Close(isForce); + EXPECT_EQ(ret, DBStatus::OK); + delete store; +} + +/** +* @tc.name: ConstructorTest +* @tc.desc: Test KVDBGeneralStore constructor with different scenarios +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(KVDBGeneralStoreTest, ConstructorTest, TestSize.Level0) +{ + // Test manager initialization with same appId + metaData_.appId = Bootstrap::GetInstance().GetProcessLabel(); + auto store1 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store1, nullptr); + delete store1; + + // Test manager initialization with different appId + metaData_.appId = "different_app_id"; + auto store2 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store2, nullptr); + delete store2; + + // Test GetKvStore failure path + metaData_.storeId = "invalid_store"; + auto store3 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store3, nullptr); + EXPECT_EQ(store3->delegate_, nullptr); + delete store3; + + // Test observer registration + metaData_.storeId = storeName; + metaData_.storeType = DistributedKv::KvStoreType::SINGLE_VERSION; + auto store4 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store4, nullptr); + delete store4; + + // Test remote push notification setup + metaData_.storeType = DistributedKv::KvStoreType::DEVICE_COLLABORATION; + auto store5 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store5, nullptr); + delete store5; + + // Test auto-sync configuration + metaData_.isAutoSync = true; + auto store6 = new (std::nothrow) KVDBGeneralStore(metaData_); + ASSERT_NE(store6, nullptr); + EXPECT_EQ(store6->delegate_, nullptr); + delete store6; +} } // namespace DistributedDataTest } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/kvdb_service_impl_test.cpp b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_impl_test.cpp index cdb512aa..c2f49cc0 100644 --- a/datamgr_service/services/distributeddataservice/service/test/kvdb_service_impl_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_impl_test.cpp @@ -16,18 +16,28 @@ #include "kvdb_service_impl.h" #include +#include #include -#include "accesstoken_kit.h" #include "bootstrap.h" #include "checker/checker_manager.h" +#include "cloud/cloud_event.h" +#include "cloud/cloud_server.h" #include "device_manager_adapter.h" #include "distributed_kv_data_manager.h" +#include "eventcenter/event_center.h" #include "ipc_skeleton.h" #include "kvdb_service_stub.h" +#include "kvdb_query.h" #include "kvstore_death_recipient.h" #include "kvstore_meta_manager.h" +#include "kvstore_sync_manager.h" #include "log_print.h" +#include +#include "mock/access_token_mock.h" +#include "mock/meta_data_manager_mock.h" +#include "network/network_delegate.h" +#include "network_delegate_mock.h" #include "nativetoken_kit.h" #include "token_setproc.h" #include "types.h" @@ -52,13 +62,19 @@ using SyncMode = OHOS::DistributedKv::SyncMode; using SyncAction = OHOS::DistributedKv::KVDBServiceImpl::SyncAction; using SwitchState = OHOS::DistributedKv::SwitchState; using UserId = OHOS::DistributedKv::UserId; +using StoreMetaData = OHOS::DistributedData::StoreMetaData; +using SyncEnd = OHOS::DistributedKv::KvStoreSyncManager::SyncEnd; +using DBResult = std::map; +using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; static OHOS::DistributedKv::StoreId storeId = { "kvdb_test_storeid" }; static OHOS::DistributedKv::AppId appId = { "ohos.test.kvdb" }; - +static constexpr const char *TEST_USER = "0"; namespace OHOS::Test { namespace DistributedDataTest { class KvdbServiceImplTest : public testing::Test { public: + static inline std::shared_ptr accTokenMock = nullptr; + static inline std::shared_ptr metaDataManagerMock = nullptr; static constexpr size_t NUM_MIN = 5; static constexpr size_t NUM_MAX = 12; static DistributedKvDataManager manager; @@ -66,7 +82,7 @@ public: static UserId userId; std::shared_ptr kvStore; - + std::shared_ptr executors_; static AppId appId; static StoreId storeId64; static StoreId storeId65; @@ -81,8 +97,22 @@ public: protected: std::shared_ptr kvdbServiceImpl_; + static NetworkDelegateMock delegate_; + StoreMetaData metaData_; + Options options_; +}; + +class CloudServerMock : public CloudServer { +public: + virtual ~CloudServerMock() = default; + bool IsSupportCloud(int32_t userId); }; +bool CloudServerMock::IsSupportCloud(int32_t userId) +{ + return true; +} + OHOS::DistributedKv::DistributedKvDataManager KvdbServiceImplTest::manager; Options KvdbServiceImplTest::create; UserId KvdbServiceImplTest::userId; @@ -90,6 +120,7 @@ UserId KvdbServiceImplTest::userId; AppId KvdbServiceImplTest::appId; StoreId KvdbServiceImplTest::storeId64; StoreId KvdbServiceImplTest::storeId65; +NetworkDelegateMock KvdbServiceImplTest::delegate_; void KvdbServiceImplTest::RemoveAllStore(DistributedKvDataManager &manager) { @@ -116,6 +147,12 @@ void KvdbServiceImplTest::SetUpTestCase(void) storeId65.storeId = "a000000000b000000000c000000000d000000000e000000000f000000000g000" "a000000000b000000000c000000000d000000000e000000000f000000000g0000"; RemoveAllStore(manager); + + accTokenMock = std::make_shared(); + BAccessTokenKit::accessTokenkit = accTokenMock; + metaDataManagerMock = std::make_shared(); + BMetaDataManager::metaDataManager = metaDataManagerMock; + NetworkDelegate::RegisterNetworkInstance(&delegate_); } void KvdbServiceImplTest::TearDownTestCase() @@ -123,11 +160,25 @@ void KvdbServiceImplTest::TearDownTestCase() RemoveAllStore(manager); (void)remove((create.baseDir + "/kvdb").c_str()); (void)remove(create.baseDir.c_str()); + + accTokenMock = nullptr; + BAccessTokenKit::accessTokenkit = nullptr; + metaDataManagerMock = nullptr; + BMetaDataManager::metaDataManager = nullptr; } void KvdbServiceImplTest::SetUp(void) { kvdbServiceImpl_ = std::make_shared(); + + options_.isNeedCompress = true; + metaData_.deviceId = DmAdapter::GetInstance().GetLocalDevice().uuid; + metaData_.bundleName = appId.appId; + metaData_.storeId = storeId.storeId; + metaData_.user = TEST_USER; + metaData_.tokenId = OHOS::IPCSkeleton::GetCallingTokenID(); + metaData_.version = 1; + MetaDataManager::GetInstance().DelMeta(metaData_.GetKey()); } void KvdbServiceImplTest::TearDown(void) @@ -135,6 +186,13 @@ void KvdbServiceImplTest::TearDown(void) RemoveAllStore(manager); } +void SyncEndCallback(const std::map &statusMap) +{ + for (const auto &pair : statusMap) { + ZLOGI("Key: %{public}s, Status: %{public}d", pair.first.c_str(), pair.second); + } +} + KvdbServiceImplTest::KvdbServiceImplTest(void) {} /** @@ -172,10 +230,39 @@ HWTEST_F(KvdbServiceImplTest, KvdbServiceImpl001, TestSize.Level0) status = kvdbServiceImpl_->UnsubscribeSwitchData(appId); EXPECT_EQ(status, Status::SUCCESS); - status = kvdbServiceImpl_->Close(appId, id1); + + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return + (ATokenTypeEnum::TOKEN_NATIVE)).WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + status = kvdbServiceImpl_->Close(appId, id1, 0); EXPECT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: OnInitialize001 +* @tc.desc: OnInitialize function test. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, OnInitialize001, TestSize.Level0) +{ + std::string device = "OH_device_test"; + int32_t result = kvdbServiceImpl_->OnInitialize(); + EXPECT_EQ(result, Status::SUCCESS); + DistributedData::StoreInfo storeInfo; + storeInfo.bundleName = appId.appId; + storeInfo.storeName = storeId.storeId; + storeInfo.instanceId = 10; + storeInfo.user = 100; + auto event = std::make_unique(CloudEvent::CLOUD_SYNC, storeInfo); + EXPECT_NE(event, nullptr); + result = EventCenter::GetInstance().PostEvent(move(event)); + EXPECT_EQ(result, 1); // CODE_SYNC + auto event1 = std::make_unique(CloudEvent::CLEAN_DATA, storeInfo); + EXPECT_NE(event1, nullptr); + result = EventCenter::GetInstance().PostEvent(move(event1)); + EXPECT_EQ(result, 1); // CODE_SYNC +} + /** * @tc.name: GetStoreIdsTest001 * @tc.desc: GetStoreIds @@ -201,7 +288,7 @@ HWTEST_F(KvdbServiceImplTest, GetStoreIdsTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status, Status::SUCCESS); std::vector storeIds; - status = kvdbServiceImpl_->GetStoreIds(appId, storeIds); + status = kvdbServiceImpl_->GetStoreIds(appId, 0, storeIds); ASSERT_EQ(status, Status::SUCCESS); } @@ -216,14 +303,14 @@ HWTEST_F(KvdbServiceImplTest, GetStoreIdsTest002, TestSize.Level0) ZLOGI("GetStoreIdsTest002 start"); std::vector storeIds; AppId appId01; - auto status = kvdbServiceImpl_->GetStoreIds(appId01, storeIds); + auto status = kvdbServiceImpl_->GetStoreIds(appId01, 0, storeIds); ZLOGI("GetStoreIdsTest002 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } /** * @tc.name: DeleteTest001 -* @tc.desc: GetStoreIds +* @tc.desc: Delete Test * @tc.type: FUNC * @tc.author: wangbin */ @@ -233,14 +320,16 @@ HWTEST_F(KvdbServiceImplTest, DeleteTest001, TestSize.Level0) Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); - auto status = kvdbServiceImpl_->Delete(appId, storeId); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return + (ATokenTypeEnum::TOKEN_NATIVE)).WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->Delete(appId, storeId, 0); ZLOGI("DeleteTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } /** * @tc.name: DeleteTest002 -* @tc.desc: GetStoreIds +* @tc.desc: Delete Test * @tc.type: FUNC * @tc.author: wangbin */ @@ -249,11 +338,83 @@ HWTEST_F(KvdbServiceImplTest, DeleteTest002, TestSize.Level0) ZLOGI("DeleteTest002 start"); AppId appId01 = { "ohos.kvdbserviceimpl.test01" }; StoreId storeId01 = { "meta_test_storeid" }; - auto status = kvdbServiceImpl_->Delete(appId01, storeId01); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return + (ATokenTypeEnum::TOKEN_NATIVE)).WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->Delete(appId01, storeId01, 0); ZLOGI("DeleteTest002 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: DeleteTest003 +* @tc.desc: Delete function test. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, DeleteTest003, TestSize.Level0) +{ + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + EXPECT_CALL(*accTokenMock, GetHapTokenInfo(testing::_, testing::_)).WillOnce(testing::Return(-1)) + .WillRepeatedly(testing::Return(-1)); + int32_t status = kvdbServiceImpl_->Delete(appId, storeId, 0); + EXPECT_EQ(status, DistributedKv::ILLEGAL_STATE); +} + +/** +* @tc.name: CloseTest001 +* @tc.desc: Close function test. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, CloseTest001, TestSize.Level0) +{ + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + EXPECT_CALL(*accTokenMock, GetHapTokenInfo(testing::_, testing::_)).WillOnce(testing::Return(-1)) + .WillRepeatedly(testing::Return(-1)); + int32_t status = kvdbServiceImpl_->Close(appId, storeId, 0); + EXPECT_EQ(status, DistributedKv::ILLEGAL_STATE); +} + +/** +* @tc.name: OnAsyncCompleteTest001 +* @tc.desc: OnAsyncComplete function test. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, OnAsyncCompleteTest001, TestSize.Level0) +{ + DistributedKv::Statistic upload; + upload.failed = 1; // test + upload.success = 1; // test + upload.total = 1; // test + upload.untreated = 1; // test + DistributedKv::Statistic download; + download.failed = 1; // test + download.success = 1; // test + download.total = 1; // test + download.untreated = 1; // test + DistributedKv::TableDetail details; + details.download = download; + details.upload = upload; + DistributedKv::ProgressDetail detail; + detail.code = 1; // test + detail.progress = 1; // test + detail.details = details; + sptr notifier; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.pid_ = 1; // test + syncAgent.switchesObserverCount_ = 1; // test + syncAgent.appId_ = { "ohos.OnAsyncCompleteTest001.kvdb" }; + syncAgent.notifier_ = notifier; + kvdbServiceImpl_->syncAgents_.Insert(100, syncAgent); // test + kvdbServiceImpl_->OnAsyncComplete(1, 1, std::move(detail)); // test + EXPECT_EQ(kvdbServiceImpl_->syncAgents_.Find(1).first, false); + kvdbServiceImpl_->OnAsyncComplete(100, 1, std::move(detail)); // test + EXPECT_EQ(kvdbServiceImpl_->syncAgents_.Find(100).first, true); +} + /** * @tc.name: syncTest001 * @tc.desc: GetStoreIds @@ -267,7 +428,7 @@ HWTEST_F(KvdbServiceImplTest, syncTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); SyncInfo syncInfo; - auto status = kvdbServiceImpl_->Sync(appId, storeId, syncInfo); + auto status = kvdbServiceImpl_->Sync(appId, storeId, 0, syncInfo); ZLOGI("syncTest001 status = :%{public}d", status); ASSERT_NE(status, Status::SUCCESS); } @@ -291,38 +452,71 @@ HWTEST_F(KvdbServiceImplTest, RegisterSyncCallbackTest001, TestSize.Level0) } /** -* @tc.name: UnregisterSyncCallbackTest001 -* @tc.desc: GetStoreIds +* @tc.name: UnregServiceNotifierTest001 +* @tc.desc: UnregServiceNotifier test * @tc.type: FUNC * @tc.author: wangbin */ -HWTEST_F(KvdbServiceImplTest, UnregisterSyncCallbackTest001, TestSize.Level0) +HWTEST_F(KvdbServiceImplTest, UnregServiceNotifierTest001, TestSize.Level0) { - ZLOGI("UnregisterSyncCallbackTest001 start"); Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); + sptr notifier; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.pid_ = 1; // test + syncAgent.switchesObserverCount_ = 1; // test + syncAgent.appId_ = { "ohos.OnAsyncCompleteTest001.kvdb" }; + syncAgent.notifier_ = notifier; + kvdbServiceImpl_->syncAgents_.Insert(IPCSkeleton::GetCallingTokenID(), syncAgent); auto status = kvdbServiceImpl_->UnregServiceNotifier(appId); - ZLOGI("UnregisterSyncCallbackTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: HandleGenDetailsTest001 +* @tc.desc: HandleGenDetails test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, HandleGenDetailsTest001, TestSize.Level0) +{ + DistributedData::GenDetails details; + ASSERT_EQ(details.begin(), details.end()); + DistributedKv::ProgressDetail progressDetails = kvdbServiceImpl_->HandleGenDetails(details); + GenProgressDetail detail; + detail.progress = GenProgress::SYNC_IN_PROGRESS; + detail.code = GenProgress::SYNC_IN_PROGRESS; + details.insert_or_assign("test", detail); + progressDetails = kvdbServiceImpl_->HandleGenDetails(details); + ASSERT_EQ(detail.details.begin(), detail.details.end()); + ASSERT_EQ(progressDetails.progress, GenProgress::SYNC_IN_PROGRESS); + std::map gentabledetail; + GenTableDetail tabledetail; + gentabledetail.insert_or_assign("test", tabledetail); + detail.details = gentabledetail; + progressDetails = kvdbServiceImpl_->HandleGenDetails(details); + ASSERT_NE(detail.details.begin(), detail.details.end()); + ASSERT_EQ(progressDetails.code, GenProgress::SYNC_IN_PROGRESS); +} + /** * @tc.name: SetSyncParamTest001 -* @tc.desc: GetStoreIds +* @tc.desc: SetSyncParam test * @tc.type: FUNC * @tc.author: wangbin */ HWTEST_F(KvdbServiceImplTest, SetSyncParamTest001, TestSize.Level0) { - ZLOGI("SetSyncParamTest001 start"); Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); - OHOS::DistributedKv::KvSyncParam const syncparam; - auto status = kvdbServiceImpl_->SetSyncParam(appId, storeId, syncparam); - ZLOGI("SetSyncParamTest001 status = :%{public}d", status); + OHOS::DistributedKv::KvSyncParam syncparam; + auto status = kvdbServiceImpl_->SetSyncParam(appId, storeId, 0, syncparam); ASSERT_EQ(status, Status::SUCCESS); + syncparam.allowedDelayMs = DistributedKv::KvStoreSyncManager::SYNC_MAX_DELAY_MS + 1; + status = kvdbServiceImpl_->SetSyncParam(appId, storeId, 0, syncparam); + ASSERT_EQ(status, Status::INVALID_ARGUMENT); } /** @@ -338,31 +532,230 @@ HWTEST_F(KvdbServiceImplTest, GetSyncParamTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); OHOS::DistributedKv::KvSyncParam syncparam; - auto status = kvdbServiceImpl_->GetSyncParam(appId, storeId, syncparam); + auto status = kvdbServiceImpl_->GetSyncParam(appId, storeId, 0, syncparam); ZLOGI("GetSyncParamTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } /** * @tc.name: EnableCapabilityTest001 -* @tc.desc: GetStoreIds +* @tc.desc: EnableCapability test * @tc.type: FUNC * @tc.author: wangbin */ HWTEST_F(KvdbServiceImplTest, EnableCapabilityTest001, TestSize.Level0) { - ZLOGI("EnableCapabilityTest001 start"); Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); - auto status = kvdbServiceImpl_->EnableCapability(appId, storeId); - ZLOGI("EnableCapabilityTest001 status = :%{public}d", status); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->EnableCapability(appId, storeId, 0); + ASSERT_EQ(status, Status::SUCCESS); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + status = kvdbServiceImpl_->EnableCapability(appId, storeId, 0); + ASSERT_EQ(status, Status::ILLEGAL_STATE); +} + +/** +* @tc.name: GetInstIndexTest001 +* @tc.desc: GetInstIndex test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, GetInstIndexTest001, TestSize.Level0) +{ + Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status1, Status::SUCCESS); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + auto status = kvdbServiceImpl_->GetInstIndex(100, appId); + ASSERT_EQ(status, -1); +} + +/** +* @tc.name: IsNeedMetaSyncTest001 +* @tc.desc: IsNeedMetaSync test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, IsNeedMetaSyncTest001, TestSize.Level0) +{ + Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status1, Status::SUCCESS); + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + std::vector uuids{"uuidtest01"}; + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(true)).WillRepeatedly(testing::Return(true)); + auto status = kvdbServiceImpl_->IsNeedMetaSync(meta, uuids); + ASSERT_EQ(status, false); + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(false)).WillRepeatedly(testing::Return(false)); + status = kvdbServiceImpl_->IsNeedMetaSync(meta, uuids); + ASSERT_EQ(status, true); +} + +/** +* @tc.name: GetDistributedDataMetaTest001 +* @tc.desc: GetDistributedDataMeta test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, GetDistributedDataMetaTest001, TestSize.Level0) +{ + Status status = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status, Status::SUCCESS); + std::string deviceId = "KvdbServiceImplTest_deviceId"; + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(false)).WillRepeatedly(testing::Return(false)); + auto meta = kvdbServiceImpl_->GetDistributedDataMeta(deviceId); + ASSERT_EQ(meta.user, "0"); + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(true)).WillRepeatedly(testing::Return(true)); + meta = kvdbServiceImpl_->GetDistributedDataMeta(deviceId); + ASSERT_EQ(meta.deviceId, deviceId); +} + +/** +* @tc.name: ProcessResultTest001 +* @tc.desc: ProcessResult test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, ProcessResultTest001, TestSize.Level0) +{ + Status status = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status, Status::SUCCESS); + std::map results; + results.insert_or_assign("uuidtest01", DBStatus::DB_ERROR); + results.insert_or_assign("uuidtest02", DBStatus::OK); + auto result = kvdbServiceImpl_->ProcessResult(results); + std::vector devices = result.first; + auto it = std::find(devices.begin(), devices.end(), "uuidtest02"); + ASSERT_NE(it, devices.end()); +} + +/** +* @tc.name: DoSyncBeginTest001 +* @tc.desc: DoSyncBegin test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, DoSyncBeginTest001, TestSize.Level0) +{ + Status status = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status, Status::SUCCESS); + std::vector device1{"uuidtest01"}; + std::vector device2; + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + SyncInfo syncInfo; + syncInfo.devices = { "device1", "device2" }; + syncInfo.query = "query"; + SyncEnd syncEnd = SyncEndCallback; + std::map statusMap; + statusMap.insert_or_assign("DoSyncBeginTest001", DBStatus::OK); + syncEnd(statusMap); + status = kvdbServiceImpl_->DoSyncBegin(device2, meta, syncInfo, syncEnd, DBStatus::OK); + ASSERT_EQ(status, Status::INVALID_ARGUMENT); + status = kvdbServiceImpl_->DoSyncBegin(device1, meta, syncInfo, syncEnd, DBStatus::OK); + ASSERT_EQ(status, Status::ERROR); +} + +/** +* @tc.name: DoCompleteTest001 +* @tc.desc: DoComplete test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, DoCompleteTest001, TestSize.Level0) +{ + Status status = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status, Status::SUCCESS); + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + SyncInfo syncInfo; + syncInfo.devices = { "device1", "device2" }; + syncInfo.query = "query"; + syncInfo.seqId = 1; // test + RefCount refCount; + DBResult dbResult; + dbResult.insert_or_assign("DoCompleteTest_1", DBStatus::OK); + dbResult.insert_or_assign("DoCompleteTest_1", DBStatus::DB_ERROR); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + status = kvdbServiceImpl_->DoComplete(meta, syncInfo, refCount, dbResult); + ASSERT_EQ(status, Status::SUCCESS); + syncInfo.seqId = std::numeric_limits::max(); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + status = kvdbServiceImpl_->DoComplete(meta, syncInfo, refCount, dbResult); ASSERT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: ConvertDbStatusNativeTest001 +* @tc.desc: ConvertDbStatusNative test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, ConvertDbStatusNativeTest001, TestSize.Level0) +{ + auto status = kvdbServiceImpl_->ConvertDbStatusNative(DBStatus::DB_ERROR); + ASSERT_EQ(status, Status::DB_ERROR); + status = kvdbServiceImpl_->ConvertDbStatusNative(DBStatus::COMM_FAILURE); + ASSERT_EQ(status, Status::DEVICE_NOT_ONLINE); + DBStatus dbstatus = static_cast(DBStatus::OK - 1); + status = kvdbServiceImpl_->ConvertDbStatusNative(dbstatus); + ASSERT_EQ(status, - 1); +} + +/** +* @tc.name: GetSyncDelayTimeTest001 +* @tc.desc: GetSyncDelayTime test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, GetSyncDelayTimeTest001, TestSize.Level0) +{ + auto status = kvdbServiceImpl_->GetSyncDelayTime(1, storeId); + ASSERT_EQ(status, DistributedKv::KvStoreSyncManager::SYNC_MIN_DELAY_MS); + status = kvdbServiceImpl_->GetSyncDelayTime(0, storeId); + ASSERT_EQ(status, 0); +} + +/** +* @tc.name: ConvertGeneralErrTest +* @tc.desc: ConvertGeneralErr test the return result of input with different values. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, ConvertGeneralErrTest, TestSize.Level0) +{ + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_DB_ERROR), Status::DB_ERROR); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_OK), Status::SUCCESS); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_ARGS), Status::INVALID_ARGUMENT); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_RECORD_NOT_FOUND), Status::KEY_NOT_FOUND); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_VALUE_FIELDS), Status::INVALID_VALUE_FIELDS); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_FIELD_TYPE), Status::INVALID_FIELD_TYPE); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_CONSTRAIN_VIOLATION), Status::CONSTRAIN_VIOLATION); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_FORMAT), Status::INVALID_FORMAT); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_QUERY_FORMAT), Status::INVALID_QUERY_FORMAT); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_INVALID_QUERY_FIELD), Status::INVALID_QUERY_FIELD); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_NOT_SUPPORT), Status::NOT_SUPPORT); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_TIME_OUT), Status::TIME_OUT); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_OVER_MAX_LIMITS), Status::OVER_MAX_LIMITS); + EXPECT_EQ(kvdbServiceImpl_->ConvertGeneralErr(GeneralError::E_SECURITY_LEVEL_ERROR), Status::SECURITY_LEVEL_ERROR); +} + /** * @tc.name: DisableCapabilityTest001 -* @tc.desc: GetStoreIds +* @tc.desc: DisableCapability test * @tc.type: FUNC * @tc.author: wangbin */ @@ -372,14 +765,20 @@ HWTEST_F(KvdbServiceImplTest, DisableCapabilityTest001, TestSize.Level0) Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); - auto status = kvdbServiceImpl_->DisableCapability(appId, storeId); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->DisableCapability(appId, storeId, 0); ZLOGI("DisableCapabilityTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + status = kvdbServiceImpl_->EnableCapability(appId, storeId, 0); + ASSERT_EQ(status, Status::ILLEGAL_STATE); } /** * @tc.name: SetCapabilityTest001 -* @tc.desc: GetStoreIds +* @tc.desc: SetCapability test * @tc.type: FUNC * @tc.author: wangbin */ @@ -391,14 +790,20 @@ HWTEST_F(KvdbServiceImplTest, SetCapabilityTest001, TestSize.Level0) ASSERT_EQ(status1, Status::SUCCESS); std::vector local; std::vector remote; - auto status = kvdbServiceImpl_->SetCapability(appId, storeId, local, remote); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->SetCapability(appId, storeId, 0, local, remote); ZLOGI("SetCapabilityTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)).WillOnce(testing::Return(ATokenTypeEnum::TOKEN_HAP)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_HAP)); + status = kvdbServiceImpl_->EnableCapability(appId, storeId, 0); + ASSERT_EQ(status, Status::ILLEGAL_STATE); } /** * @tc.name: AddSubscribeInfoTest001 -* @tc.desc: GetStoreIds +* @tc.desc: AddSubscribeInfo test * @tc.type: FUNC * @tc.author: wangbin */ @@ -409,14 +814,17 @@ HWTEST_F(KvdbServiceImplTest, AddSubscribeInfoTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); SyncInfo syncInfo; - auto status = kvdbServiceImpl_->AddSubscribeInfo(appId, storeId, syncInfo); + EXPECT_CALL(*accTokenMock, GetTokenTypeFlag(testing::_)) + .WillOnce(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)) + .WillRepeatedly(testing::Return(ATokenTypeEnum::TOKEN_NATIVE)); + auto status = kvdbServiceImpl_->AddSubscribeInfo(appId, storeId, 0, syncInfo); ZLOGI("AddSubscribeInfoTest001 status = :%{public}d", status); ASSERT_NE(status, Status::SUCCESS); } /** * @tc.name: RmvSubscribeInfoTest001 -* @tc.desc: GetStoreIds +* @tc.desc: RmvSubscribeInfo test * @tc.type: FUNC * @tc.author: wangbin */ @@ -427,14 +835,14 @@ HWTEST_F(KvdbServiceImplTest, RmvSubscribeInfoTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); SyncInfo syncInfo; - auto status = kvdbServiceImpl_->RmvSubscribeInfo(appId, storeId, syncInfo); + auto status = kvdbServiceImpl_->RmvSubscribeInfo(appId, storeId, 0, syncInfo); ZLOGI("RmvSubscribeInfoTest001 status = :%{public}d", status); ASSERT_NE(status, Status::SUCCESS); } /** * @tc.name: SubscribeTest001 -* @tc.desc: GetStoreIds +* @tc.desc: Subscribe test * @tc.type: FUNC * @tc.author: wangbin */ @@ -445,14 +853,14 @@ HWTEST_F(KvdbServiceImplTest, SubscribeTest001, TestSize.Level0) ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); sptr observer; - auto status = kvdbServiceImpl_->Subscribe(appId, storeId, observer); + auto status = kvdbServiceImpl_->Subscribe(appId, storeId, 0, observer); ZLOGI("SubscribeTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::INVALID_ARGUMENT); } /** * @tc.name: UnsubscribeTest001 -* @tc.desc: GetStoreIds +* @tc.desc: Unsubscribe test * @tc.type: FUNC * @tc.author: wangbin */ @@ -462,46 +870,75 @@ HWTEST_F(KvdbServiceImplTest, UnsubscribeTest001, TestSize.Level0) Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); + sptr notifier; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.pid_ = 1; // test + syncAgent.switchesObserverCount_ = 1; // test + syncAgent.appId_ = { "ohos.OnAsyncCompleteTest001.kvdb" }; + syncAgent.notifier_ = notifier; + auto tokenId = IPCSkeleton::GetCallingTokenID(); + kvdbServiceImpl_->syncAgents_.Insert(tokenId, syncAgent); sptr observer; - auto status = kvdbServiceImpl_->Unsubscribe(appId, storeId, observer); + auto status = kvdbServiceImpl_->Unsubscribe(appId, storeId, 0, observer); ZLOGI("UnsubscribeTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); } /** * @tc.name: GetBackupPasswordTest001 -* @tc.desc: GetStoreIds +* @tc.desc: GetBackupPassword test * @tc.type: FUNC * @tc.author: wangbin */ HWTEST_F(KvdbServiceImplTest, GetBackupPasswordTest001, TestSize.Level0) { - ZLOGI("GetBackupPasswordTest001 start"); auto status = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status, Status::SUCCESS); + std::vector> password; + status = kvdbServiceImpl_->GetBackupPassword + (appId, storeId, 0, password, DistributedKv::KVDBService::PasswordType::BACKUP_SECRET_KEY); + ASSERT_EQ(status, Status::ERROR); + status = kvdbServiceImpl_->GetBackupPassword + (appId, storeId, 0, password, DistributedKv::KVDBService::PasswordType::SECRET_KEY); + ASSERT_EQ(status, Status::ERROR); + status = kvdbServiceImpl_->GetBackupPassword + (appId, storeId, 0, password, DistributedKv::KVDBService::PasswordType::BUTTON); + ASSERT_EQ(status, Status::ERROR); } /** * @tc.name: BeforeCreateTest001 -* @tc.desc: GetStoreIds +* @tc.desc: BeforeCreate test * @tc.type: FUNC * @tc.author: wangbin */ HWTEST_F(KvdbServiceImplTest, BeforeCreateTest001, TestSize.Level0) { - ZLOGI("BeforeCreateTest001 start"); Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status1, Status::SUCCESS); - auto status = kvdbServiceImpl_->BeforeCreate(appId, storeId, create); - ZLOGI("BeforeCreateTest001 status = :%{public}d", status); + Options creates; + creates.createIfMissing = true; + creates.encrypt = false; + creates.securityLevel = OHOS::DistributedKv::S1; + creates.autoSync = true; + creates.kvStoreType = OHOS::DistributedKv::SINGLE_VERSION; + creates.area = OHOS::DistributedKv::EL1; + creates.baseDir = std::string("/data/service/el1/public/database/") + appId.appId; + creates.cloudConfig.enableCloud = true; + kvdbServiceImpl_->executors_ = std::make_shared(1, 1); + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(false)).WillRepeatedly(testing::Return(false)); + auto status = kvdbServiceImpl_->BeforeCreate(appId, storeId, creates); + ASSERT_NE(status, Status::STORE_META_CHANGED); + kvdbServiceImpl_->executors_ = nullptr; ASSERT_EQ(status, Status::SUCCESS); } /** * @tc.name: AfterCreateTest001 -* @tc.desc: GetStoreIds +* @tc.desc: AfterCreate test * @tc.type: FUNC * @tc.author: wangbin */ @@ -515,11 +952,21 @@ HWTEST_F(KvdbServiceImplTest, AfterCreateTest001, TestSize.Level0) auto status = kvdbServiceImpl_->AfterCreate(appId, storeId, create, password); ZLOGI("AfterCreateTest001 status = :%{public}d", status); ASSERT_EQ(status, Status::SUCCESS); + AppId appIds; + appIds.appId = ""; + status = kvdbServiceImpl_->AfterCreate(appIds, storeId, create, password); + ASSERT_EQ(status, Status::INVALID_ARGUMENT); + StoreId storeIds; + storeIds.storeId = ""; + status = kvdbServiceImpl_->AfterCreate(appId, storeIds, create, password); + ASSERT_EQ(status, Status::INVALID_ARGUMENT); + status = kvdbServiceImpl_->AfterCreate(appIds, storeIds, create, password); + ASSERT_EQ(status, Status::INVALID_ARGUMENT); } /** * @tc.name: OnAppExitTest001 -* @tc.desc: GetStoreIds +* @tc.desc: OnAppExit test * @tc.type: FUNC * @tc.author: wangbin */ @@ -550,9 +997,24 @@ HWTEST_F(KvdbServiceImplTest, OnAppExitTest001, TestSize.Level0) ASSERT_EQ(status, Status::SUCCESS); } +/** +* @tc.name: CompareTripleIdentifierTest001 +* @tc.desc: CompareTripleIdentifier test +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, CompareTripleIdentifierTest001, TestSize.Level0) +{ + std::string accountId = "accountIdTest"; + std::string identifier = "identifierTest"; + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + auto status = kvdbServiceImpl_->CompareTripleIdentifier(accountId, identifier, meta); + ASSERT_EQ(status, false); +} + /** * @tc.name: OnUserChangeTest001 -* @tc.desc: GetStoreIds +* @tc.desc: OnUserChange test * @tc.type: FUNC * @tc.author: wangbin */ @@ -569,7 +1031,7 @@ HWTEST_F(KvdbServiceImplTest, OnUserChangeTest001, TestSize.Level0) /** * @tc.name: OnReadyTest001 -* @tc.desc: GetStoreIds +* @tc.desc: OnReady test * @tc.type: FUNC * @tc.author: wangbin */ @@ -592,9 +1054,9 @@ HWTEST_F(KvdbServiceImplTest, OnReadyTest001, TestSize.Level0) */ HWTEST_F(KvdbServiceImplTest, ResolveAutoLaunch, TestSize.Level0) { - StoreId id1; - id1.storeId = "id1"; - Status status = manager.GetSingleKvStore(create, appId, id1, kvStore); + StoreId id; + id.storeId = "id"; + Status status = manager.GetSingleKvStore(create, appId, id, kvStore); EXPECT_NE(kvStore, nullptr); EXPECT_EQ(status, Status::SUCCESS); std::string identifier = "identifier"; @@ -611,6 +1073,20 @@ HWTEST_F(KvdbServiceImplTest, ResolveAutoLaunch, TestSize.Level0) EXPECT_EQ(result, Status::SUCCESS); } +/** +* @tc.name: IsRemoteChange +* @tc.desc: IsRemoteChange function test. +* @tc.type: FUNC +* @tc.author: wangbin +*/ +HWTEST_F(KvdbServiceImplTest, IsRemoteChangeTest, TestSize.Level0) +{ + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + std::string devices= "IsRemoteChangeTest"; + auto changes = kvdbServiceImpl_->IsRemoteChange(meta, devices); + EXPECT_EQ(changes, true); +} + /** * @tc.name: PutSwitch * @tc.desc: PutSwitch function test. @@ -619,9 +1095,9 @@ HWTEST_F(KvdbServiceImplTest, ResolveAutoLaunch, TestSize.Level0) */ HWTEST_F(KvdbServiceImplTest, PutSwitch, TestSize.Level0) { - StoreId id1; - id1.storeId = "id1"; - Status status = manager.GetSingleKvStore(create, appId, id1, kvStore); + StoreId id; + id.storeId = "id1"; + Status status = manager.GetSingleKvStore(create, appId, id, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status, Status::SUCCESS); DistributedKv::SwitchData switchData; @@ -656,9 +1132,9 @@ HWTEST_F(KvdbServiceImplTest, PutSwitch, TestSize.Level0) */ HWTEST_F(KvdbServiceImplTest, DoCloudSync, TestSize.Level0) { - StoreId id1; - id1.storeId = "id1"; - Status status = manager.GetSingleKvStore(create, appId, id1, kvStore); + StoreId id; + id.storeId = "id1"; + Status status = manager.GetSingleKvStore(create, appId, id, kvStore); ASSERT_NE(kvStore, nullptr); ASSERT_EQ(status, Status::SUCCESS); StoreMetaData metaData; @@ -793,5 +1269,225 @@ HWTEST_F(KvdbServiceImplTest, GetSyncMode, TestSize.Level0) status = kvdbServiceImpl_->GetSyncMode(false, false); EXPECT_EQ(status, SyncMode::PUSH_PULL); } + +/** +* @tc.name: DoCloudSync02 +* @tc.desc: DoCloudSync02 function test. +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, DoCloudSync02, TestSize.Level0) +{ + delegate_.isNetworkAvailable_ = false; + auto cloudServerMock = new CloudServerMock(); + CloudServer::RegisterCloudInstance(cloudServerMock); + StoreMetaData metaData; + metaData.enableCloud = true; + SyncInfo syncInfo; + auto status = kvdbServiceImpl_->DoCloudSync(metaData, syncInfo); + EXPECT_EQ(status, OHOS::DistributedKv::Status::NETWORK_ERROR); +} + +/** +* @tc.name: DoCloudSync +* @tc.desc: DoCloudSync error function test. +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, DoCloudSync01, TestSize.Level0) +{ + StoreId id; + id.storeId = "id1"; + Status status = manager.GetSingleKvStore(create, appId, id, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status, Status::SUCCESS); + StoreMetaData metaData; + metaData.enableCloud = false; + SyncInfo syncInfo; + status = kvdbServiceImpl_->DoCloudSync(metaData, syncInfo); + EXPECT_EQ(status, Status::NOT_SUPPORT); +} + +/** +* @tc.name: OnAsyncCompleteTest002 +* @tc.desc: OnAsyncComplete function test. +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, OnAsyncCompleteTest002, TestSize.Level0) +{ + DistributedKv::Statistic upload; + upload.failed = 1; // test + upload.success = 1; // test + upload.total = 1; // test + upload.untreated = 1; // test + DistributedKv::Statistic download; + download.failed = 1; // test + download.success = 1; // test + download.total = 1; // test + download.untreated = 1; // test + DistributedKv::TableDetail details; + details.download = download; + details.upload = upload; + DistributedKv::ProgressDetail detail; + detail.code = 1; // test + detail.progress = 1; // test + detail.details = details; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + sptr notifier = nullptr; + syncAgent.pid_ = 1; // test + syncAgent.switchesObserverCount_ = 1; // test + syncAgent.appId_ = { "ohos.OnAsyncCompleteTest.kvdb" }; + syncAgent.notifier_ = notifier; + EXPECT_EQ(notifier, nullptr); + uint32_t tokenId = 2; + uint64_t seqNum = 1; + kvdbServiceImpl_->syncAgents_.Insert(tokenId, syncAgent); + kvdbServiceImpl_->OnAsyncComplete(tokenId, seqNum, std::move(detail)); + EXPECT_TRUE(kvdbServiceImpl_->syncAgents_.Find(tokenId).first); + kvdbServiceImpl_->OnAsyncComplete(0, 1, std::move(detail)); + EXPECT_TRUE(kvdbServiceImpl_->syncAgents_.Find(tokenId).first); +} + +/** +* @tc.name: OnAsyncCompleteTest003 +* @tc.desc: OnAsyncComplete function test. +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, OnAsyncCompleteTest003, TestSize.Level0) +{ + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + sptr notifier; + syncAgent.pid_ = 1; // test + syncAgent.switchesObserverCount_ = 1; // test + syncAgent.appId_ = { "ohos.OnAsyncCompleteTest001.kvdb" }; + syncAgent.notifier_ = notifier; + DistributedKv::ProgressDetail detail; + auto tokenId = IPCSkeleton::GetCallingTokenID(); + kvdbServiceImpl_->syncAgents_.Insert(tokenId, syncAgent); + kvdbServiceImpl_->OnAsyncComplete(tokenId, 1, std::move(detail)); + EXPECT_TRUE(kvdbServiceImpl_->syncAgents_.Find(tokenId).first); +} + +/** +* @tc.name: DeleteTest004 +* @tc.desc: Delete Test +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, DeleteTest004, TestSize.Level0) +{ + ZLOGI("DeleteTest004 start"); + AppId appId = { "ohos.kvdbserviceimpl.test" }; + StoreId storeId = { "meta_test_storeid" }; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.pid_ = 1; + auto tokenId = IPCSkeleton::GetCallingTokenID(); + auto status = kvdbServiceImpl_->Delete(appId, storeId, 0); + ZLOGI("DeleteTest002 status = :%{public}d", status); + EXPECT_NE(tokenId, syncAgent.pid_); + ASSERT_EQ(status, Status::SUCCESS); +} + +/** +* @tc.name: syncTest002 +* @tc.desc: Sync +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, syncTest002, TestSize.Level0) +{ + ZLOGI("syncTest002 start"); + StoreMetaData metaData; + auto mm = std::numeric_limits::max(); + metaData.isAutoSync = true; + SyncInfo syncInfo; + syncInfo.devices = { "device1", "device2" }; + syncInfo.query = "query"; + syncInfo.seqId = mm; // test + auto status = kvdbServiceImpl_->Sync(appId, storeId, 0, syncInfo); + ASSERT_EQ(syncInfo.seqId, std::numeric_limits::max()); + ZLOGI("syncTest002 status = :%{public}d", status); + ASSERT_NE(status, Status::SUCCESS); +} + +/** +* @tc.name: syncTest003 +* @tc.desc: Sync +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, syncTest003, TestSize.Level0) +{ + ZLOGI("syncTest003 start"); + StoreMetaData meta = kvdbServiceImpl_->GetStoreMetaData(appId, storeId); + StoreMetaData metaData; + metaData.isAutoSync = true; + StoreMetaDataLocal localMeta; + PolicyValue value; + value.type = OHOS::DistributedKv::PolicyType::IMMEDIATE_SYNC_ON_ONLINE; + localMeta.policies = { std::move(value) }; + SyncInfo syncInfo; + syncInfo.seqId = std::numeric_limits::max(); + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(true)).WillRepeatedly(testing::Return(true)); + auto status = kvdbServiceImpl_->Sync(appId, storeId, 0, syncInfo); + EXPECT_EQ(localMeta.HasPolicy(DistributedKv::IMMEDIATE_SYNC_ON_CHANGE), false); + EXPECT_NE(status, Status::SUCCESS); + EXPECT_CALL(*metaDataManagerMock, LoadMeta(testing::_, testing::_, testing::_)) + .WillOnce(testing::Return(false)).WillRepeatedly(testing::Return(false)); + status = kvdbServiceImpl_->Sync(appId, storeId, 0, syncInfo); + EXPECT_EQ(localMeta.HasPolicy(DistributedKv::IMMEDIATE_SYNC_ON_ONLINE), true); +} + +/** +* @tc.name: GetSyncParamTest001 +* @tc.desc: GetStoreIds +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, GetSyncParamTest002, TestSize.Level0) +{ + ZLOGI("GetSyncParamTest001 start"); + Status status1 = manager.GetSingleKvStore(create, appId, storeId, kvStore); + ASSERT_NE(kvStore, nullptr); + ASSERT_EQ(status1, Status::SUCCESS); + OHOS::DistributedKv::KvSyncParam syncparam; + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.pid_ = 1; + auto tokenId = IPCSkeleton::GetCallingTokenID(); + EXPECT_NE(tokenId, syncAgent.pid_); + auto status = kvdbServiceImpl_->GetSyncParam(appId, storeId, 0, syncparam); + ZLOGI("GetSyncParamTest002 status = :%{public}d", status); + ASSERT_EQ(status, Status::SUCCESS); +} + +/** +* @tc.name: SubscribeSwitchData +* @tc.desc: SubscribeSwitchData function test. +* @tc.type: FUNC +* @tc.author: +*/ +HWTEST_F(KvdbServiceImplTest, SubscribeSwitchData, TestSize.Level0) +{ + options_.isNeedCompress = false; + std::vector password {}; + StoreMetaData metaData; + auto status = kvdbServiceImpl_->AfterCreate(appId, storeId, options_, password); + ASSERT_EQ(status, Status::SUCCESS); + auto tokenId = IPCSkeleton::GetCallingTokenID(); + DistributedKv::KVDBServiceImpl::SyncAgent syncAgent; + syncAgent.switchesObserverCount_ = 1; + syncAgent.pid_ = tokenId; + syncAgent.notifier_ = nullptr; + status = kvdbServiceImpl_->SubscribeSwitchData(appId); + EXPECT_EQ(status, Status::SUCCESS); + ASSERT_FALSE(MetaDataManager::GetInstance().LoadMeta(metaData_.GetKey(), metaData)); + kvdbServiceImpl_->syncAgents_.Insert(IPCSkeleton::GetCallingTokenID(), syncAgent); + EXPECT_EQ(tokenId, syncAgent.pid_); + status = kvdbServiceImpl_->UnregServiceNotifier(appId); + ASSERT_EQ(status, Status::SUCCESS); +} } // namespace DistributedDataTest -} // namespace OHOS::Test \ No newline at end of file +} // namespace OHOS::Test diff --git a/datamgr_service/services/distributeddataservice/service/test/kvdb_service_stub_unittest.cpp b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_stub_unittest.cpp new file mode 100644 index 00000000..bf41f55f --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_stub_unittest.cpp @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "KVDBServiceStubTest " + +#include "kvdb_service_stub.h" +#include "kvdb_service_impl.h" +#include "gtest/gtest.h" +#include "ipc_skeleton.h" +#include "kv_types_util.h" +#include "log_print.h" +#include "types.h" +#include "device_matrix.h" +#include "bootstrap.h" +#include "itypes_util.h" + +using namespace OHOS; +using namespace testing; +using namespace testing::ext; +using namespace OHOS::DistributedData; +using KVDBServiceStub = OHOS::DistributedKv::KVDBServiceStub; +using StoreId = OHOS::DistributedKv::StoreId; +using AppId = OHOS::DistributedKv::AppId; +using Options = OHOS::DistributedKv::Options; +const std::u16string INTERFACE_TOKEN = u"OHOS.DistributedKv.IKvStoreDataService"; +static OHOS::DistributedKv::StoreId storeId = { "kvdb_test_storeid" }; +static OHOS::DistributedKv::AppId appId = { "ohos.test.kvdb" }; +namespace OHOS::Test { +namespace DistributedDataTest { +class KVDBServiceStubTest : public testing::Test { +public: + static void SetUpTestCase(void) {}; + static void TearDownTestCase(void) {}; + void SetUp() {}; + void TearDown() {}; +}; +std::shared_ptr kvdbServiceImpl = std::make_shared(); +std::shared_ptr kvdbServiceStub = kvdbServiceImpl; + +/** + * @tc.name: OnRemoteRequest_Test_001 + * @tc.desc: Test OnRemoteRequest + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnRemoteRequest, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + uint32_t code = 0; + auto result = kvdbServiceStub->OnRemoteRequest(code, data, reply); + EXPECT_EQ(result, -1); +} + +/** + * @tc.name: OnRemoteRequest_Test_001 + * @tc.desc: Test OnRemoteRequest + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnRemoteRequest001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + uint32_t code = static_cast(DistributedKv::KVDBServiceInterfaceCode::TRANS_HEAD) - 1; + auto result = kvdbServiceStub->OnRemoteRequest(code, data, reply); + EXPECT_EQ(result, -1); + + // Test code greater than TRANS_BUTT + code = static_cast(DistributedKv::KVDBServiceInterfaceCode::TRANS_BUTT); + result = kvdbServiceStub->OnRemoteRequest(code, data, reply); + EXPECT_EQ(result, -1); +} + +/** + * @tc.name: GetStoreInfo + * @tc.desc: Test GetStoreInfo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, GetStoreInfo, TestSize.Level1) +{ + MessageParcel data; + auto [status, info] = kvdbServiceStub->GetStoreInfo(0, data); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: GetStoreInfo + * @tc.desc: Test GetStoreInfo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, GetStoreInfo001, TestSize.Level1) +{ + MessageParcel data; + AppId appId = {"test_app"}; + StoreId storeId = {"test_store"}; + auto [status, info] = kvdbServiceStub->GetStoreInfo(0, data); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); + EXPECT_EQ(info.bundleName, ""); + EXPECT_EQ(info.storeId, ""); +} + +/** + * @tc.name: CheckPermission + * @tc.desc: Test CheckPermission + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, CheckPermission, TestSize.Level1) +{ + KVDBServiceStub::StoreInfo info; + info.bundleName = "test_bundleName"; + info.storeId = "test_storeId"; + uint32_t code = static_cast(DistributedKv::KVDBServiceInterfaceCode::TRANS_PUT_SWITCH); + auto status = kvdbServiceStub->CheckPermission(code, info); + EXPECT_FALSE(status); + + code = static_cast(DistributedKv::KVDBServiceInterfaceCode::TRANS_UNSUBSCRIBE_SWITCH_DATA); + status = kvdbServiceStub->CheckPermission(code, info); + EXPECT_FALSE(status); +} + +/** + * @tc.name: CheckPermission + * @tc.desc: Test CheckPermission + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, CheckPermission001, TestSize.Level1) +{ + KVDBServiceStub::StoreInfo info; + info.bundleName = "validApp"; + info.storeId = "validStore"; + EXPECT_FALSE(kvdbServiceStub->CheckPermission(0, info)); +} + + +/** + * @tc.name: OnBeforeCreate + * @tc.desc: Test OnBeforeCreate + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnBeforeCreate, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStoreId"}; + auto status = kvdbServiceStub->OnBeforeCreate(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnAfterCreate + * @tc.desc: Test OnAfterCreate + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnAfterCreate, TestSize.Level1) +{ + MessageParcel data; + data.WriteInterfaceToken(INTERFACE_TOKEN); + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnAfterCreate(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnSync + * @tc.desc: Test OnSync + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnSync, TestSize.Level1) +{ + MessageParcel data; + data.WriteInterfaceToken(INTERFACE_TOKEN); + MessageParcel reply; + AppId appId = {"testAppId01"}; + StoreId storeId = {"testStoreId01"}; + auto status = kvdbServiceStub->OnSync(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnRegServiceNotifier + * @tc.desc: Test OnRegServiceNotifier + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnRegServiceNotifier, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnRegServiceNotifier(appId, storeId, data, reply); + EXPECT_EQ(status, ERR_NONE); +} + +/** + * @tc.name: OnSetSyncParam + * @tc.desc: Test OnSetSyncParam + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnSetSyncParam, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnSetSyncParam(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnAddSubInfo + * @tc.desc: Test OnAddSubInfo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnAddSubInfo, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnAddSubInfo(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); + + status = kvdbServiceStub->OnRmvSubInfo(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnRmvSubInfo + * @tc.desc: Test OnRmvSubInfo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnRmvSubInfo, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnRmvSubInfo(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnPutSwitch + * @tc.desc: Test OnPutSwitch + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnPutSwitch, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnPutSwitch(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); + + status = kvdbServiceStub->OnGetSwitch(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnGetBackupPassword + * @tc.desc: Test OnGetBackupPassword + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnGetBackupPassword, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"testApp"}; + StoreId storeId = {"testStore"}; + auto status = kvdbServiceStub->OnGetBackupPassword(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnSubscribeSwitchData + * @tc.desc: Test OnSubscribeSwitchData + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnSubscribeSwitchData, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"appId"}; + StoreId storeId = {"storeId"}; + auto status = kvdbServiceStub->OnSubscribeSwitchData(appId, storeId, data, reply); + EXPECT_EQ(status, ERR_NONE); + + status = kvdbServiceStub->OnUnsubscribeSwitchData(appId, storeId, data, reply); + EXPECT_EQ(status, ERR_NONE); +} + +/** + * @tc.name: OnSetConfig + * @tc.desc: Test OnSetConfig + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnSetConfig, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"appId"}; + StoreId storeId = {"storeId"}; + auto status = kvdbServiceStub->OnSetConfig(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} + +/** + * @tc.name: OnRemoveDeviceData + * @tc.desc: Test OnRemoveDeviceData + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(KVDBServiceStubTest, OnRemoveDeviceData, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + AppId appId = {"appId"}; + StoreId storeId = {"storeId"}; + auto status = kvdbServiceStub->OnRemoveDeviceData(appId, storeId, data, reply); + EXPECT_EQ(status, IPC_STUB_INVALID_DATA_ERR); +} +} // namespace DistributedDataTest +} // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/kvdb_service_test.cpp b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_test.cpp index 2317e82b..c1a589f4 100644 --- a/datamgr_service/services/distributeddataservice/service/test/kvdb_service_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/kvdb_service_test.cpp @@ -15,29 +15,36 @@ #include -#include "accesstoken_kit.h" #include "auth_delegate.h" #include "bootstrap.h" +#include "change_notification.h" #include "crypto_manager.h" #include "device_manager_adapter.h" #include "directory/directory_manager.h" +#include "ikvstore_observer.h" +#include "iremote_broker.h" +#include "iremote_proxy.h" #include "kvdb_general_store.h" #include "kvdb_notifier_proxy.h" #include "kvdb_watcher.h" #include "kvstore_meta_manager.h" #include "kvstore_sync_manager.h" #include "log_print.h" -#include "metadata/meta_data_manager.h" #include "metadata/secret_key_meta_data.h" #include "metadata/store_meta_data.h" #include "metadata/store_meta_data_local.h" +#include "mock/access_token_mock.h" +#include "mock/meta_data_manager_mock.h" #include "query_helper.h" +#include "types.h" #include "upgrade.h" #include "user_delegate.h" using namespace testing::ext; using namespace DistributedDB; +using namespace OHOS; using namespace OHOS::DistributedData; +using namespace OHOS::Security::AccessToken; using StoreMetaData = OHOS::DistributedData::StoreMetaData; using DBPassword = DistributedDB::CipherPassword; using DBStatus = DistributedDB::DBStatus; @@ -49,8 +56,10 @@ namespace OHOS::Test { namespace DistributedDataTest { class UpgradeTest : public testing::Test { public: - static void SetUpTestCase(void){}; - static void TearDownTestCase(void){}; + static inline std::shared_ptr accessTokenKitMock = nullptr; + static inline std::shared_ptr metaDataManagerMock = nullptr; + static void SetUpTestCase(void); + static void TearDownTestCase(void); void SetUp(); void TearDown(){}; protected: @@ -60,6 +69,22 @@ protected: StoreMetaData metaData_; }; +void UpgradeTest::SetUpTestCase(void) +{ + accessTokenKitMock = std::make_shared(); + BAccessTokenKit::accessTokenkit = accessTokenKitMock; + metaDataManagerMock = std::make_shared(); + BMetaDataManager::metaDataManager = metaDataManagerMock; +} + +void UpgradeTest::TearDownTestCase() +{ + accessTokenKitMock = nullptr; + BAccessTokenKit::accessTokenkit = nullptr; + metaDataManagerMock = nullptr; + BMetaDataManager::metaDataManager = nullptr; +} + void UpgradeTest::InitMetaData() { metaData_.bundleName = bundleName; @@ -118,13 +143,26 @@ protected: class AuthHandlerTest : public testing::Test { public: - static void SetUpTestCase(void){}; - static void TearDownTestCase(void){}; + static inline std::shared_ptr metaDataManagerMock = nullptr; + static void SetUpTestCase(void); + static void TearDownTestCase(void); void SetUp(){}; void TearDown(){}; protected: }; +void AuthHandlerTest::SetUpTestCase(void) +{ + metaDataManagerMock = std::make_shared(); + BMetaDataManager::metaDataManager = metaDataManagerMock; +} + +void AuthHandlerTest::TearDownTestCase() +{ + metaDataManagerMock = nullptr; + BMetaDataManager::metaDataManager = nullptr; +} + /** * @tc.name: UpdateStore * @tc.desc: UpdateStore test the return result of input with different values. @@ -160,12 +198,9 @@ HWTEST_F(UpgradeTest, UpdateStore, TestSize.Level0) }; upgrade.cleaner_ = cleaner; upgrade.exporter_ = nullptr; - upgrade.UpdatePassword(metaData_, password); dbStatus = upgrade.UpdateStore(oldMeta, metaData_, password); EXPECT_EQ(dbStatus, DBStatus::NOT_SUPPORT); - metaData_.isEncrypt = true; - upgrade.UpdatePassword(metaData_, password); EXPECT_TRUE(upgrade.RegisterExporter(oldMeta.version, exporter)); EXPECT_TRUE(upgrade.RegisterCleaner(oldMeta.version, cleaner)); dbStatus = upgrade.UpdateStore(oldMeta, metaData_, password); @@ -176,6 +211,26 @@ HWTEST_F(UpgradeTest, UpdateStore, TestSize.Level0) EXPECT_EQ(dbStatus, DBStatus::OK); } +/** +* @tc.name: UpdateStore002 +* @tc.desc: UpdateStore test the return result of input with different values. +* @tc.type: FUNC +* @tc.author: yl +*/ +HWTEST_F(UpgradeTest, UpdateStore002, TestSize.Level0) +{ + DistributedKv::Upgrade upgrade; + StoreMetaData oldMeta = metaData_; + oldMeta.isNeedUpdateDeviceId = true; + std::vector password; + auto dbStatus = upgrade.UpdateStore(oldMeta, metaData_, password); + EXPECT_EQ(dbStatus, DBStatus::DB_ERROR); + + oldMeta.isEncrypt = true; + dbStatus = upgrade.UpdateStore(oldMeta, metaData_, password); + EXPECT_EQ(dbStatus, DBStatus::OK); +} + /** * @tc.name: ExportStore * @tc.desc: ExportStore test the return result of input with different values. @@ -284,7 +339,7 @@ HWTEST_F(KvStoreSyncManagerTest, GetTimeoutSyncOps, TestSize.Level0) EXPECT_TRUE(syncManager.scheduleSyncOps_.empty()); auto kvStatus = syncManager.GetTimeoutSyncOps(currentTime, syncOps); EXPECT_EQ(kvStatus, false); - syncManager.realtimeSyncingOps_.push_back(syncOp); + syncManager.realtimeSyncingOps_.emplace_back(syncOp); kvStatus = syncManager.GetTimeoutSyncOps(currentTime, syncOps); EXPECT_EQ(kvStatus, false); syncManager.realtimeSyncingOps_ = syncOps; @@ -292,7 +347,7 @@ HWTEST_F(KvStoreSyncManagerTest, GetTimeoutSyncOps, TestSize.Level0) kvStatus = syncManager.GetTimeoutSyncOps(currentTime, syncOps); EXPECT_EQ(kvStatus, false); - syncManager.realtimeSyncingOps_.push_back(syncOp); + syncManager.realtimeSyncingOps_.emplace_back(syncOp); syncManager.scheduleSyncOps_.insert(std::make_pair(syncOp.beginTime, syncOp)); EXPECT_TRUE(!syncManager.realtimeSyncingOps_.empty()); EXPECT_TRUE(!syncManager.scheduleSyncOps_.empty()); @@ -323,6 +378,69 @@ HWTEST_F(KVDBWatcherTest, KVDBWatcher, TestSize.Level0) EXPECT_EQ(result, GeneralError::E_OK); } +/** +* @tc.name: OnChange001 +* @tc.desc: OnChange test function. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(KVDBWatcherTest, OnChange001, TestSize.Level0) +{ + GeneralWatcher::Origin origin; + origin.store = "store"; + GeneralWatcher::PRIFields primaryFields = {{"primaryFields1", "primaryFields2"}}; + GeneralWatcher::ChangeInfo values; + values["store"][OP_INSERT].emplace_back(std::string("values1")); + values["store"][OP_INSERT].emplace_back(11LL); + values["store"][OP_INSERT].emplace_back(1.11); + values["store"][OP_UPDATE].emplace_back(std::string("values2")); + values["store"][OP_UPDATE].emplace_back(22LL); + values["store"][OP_UPDATE].emplace_back(2.22); + values["store"][OP_DELETE].emplace_back(std::string("values3")); + values["store"][OP_DELETE].emplace_back(33LL); + values["store"][OP_DELETE].emplace_back(3.33); + std::shared_ptr watcher = std::make_shared(); + sptr observer = nullptr; + watcher->SetObserver(observer); + EXPECT_EQ(watcher->observer_, nullptr); + auto result = watcher->OnChange(origin, primaryFields, std::move(values)); + EXPECT_EQ(result, GeneralError::E_NOT_INIT); +} + +/** +* @tc.name: OnChange002 +* @tc.desc: OnChange test function. +* @tc.type: FUNC +* @tc.require: +* @tc.author: suoqilong + */ +HWTEST_F(KVDBWatcherTest, OnChange002, TestSize.Level0) +{ + GeneralWatcher::Origin origin; + origin.store = "store"; + GeneralWatcher::Fields fields; + GeneralWatcher::ChangeData datas; + datas["store"][OP_INSERT].push_back({11LL}); + datas["store"][OP_INSERT].push_back({1.11}); + datas["store"][OP_INSERT].push_back({std::string("datas1")}); + datas["store"][OP_INSERT].push_back({Bytes({1, 2, 3})}); + datas["store"][OP_UPDATE].push_back({22LL}); + datas["store"][OP_UPDATE].push_back({2.22}); + datas["store"][OP_UPDATE].push_back({std::string("datas2")}); + datas["store"][OP_UPDATE].push_back({Bytes({4, 5, 6})}); + datas["store"][OP_DELETE].push_back({33LL}); + datas["store"][OP_DELETE].push_back({3.33}); + datas["store"][OP_DELETE].push_back({std::string("datas3")}); + datas["store"][OP_DELETE].push_back({Bytes({7, 8, 9})}); + std::shared_ptr watcher = std::make_shared(); + sptr observer = nullptr; + watcher->SetObserver(observer); + EXPECT_EQ(watcher->observer_, nullptr); + auto result = watcher->OnChange(origin, fields, std::move(datas)); + EXPECT_EQ(result, GeneralError::E_NOT_INIT); +} + /** * @tc.name: ConvertToEntries * @tc.desc: ConvertToEntries test the return result of input with different values. @@ -661,7 +779,7 @@ HWTEST_F(AuthHandlerTest, AuthHandler, TestSize.Level0) aclParams.authType = static_cast(DistributedKv::AuthType::IDENTICAL_ACCOUNT); auto result = AuthDelegate::GetInstance()->CheckAccess(localUserId, peerUserId, peerDeviceId, aclParams); EXPECT_TRUE(result.first); - + aclParams.authType = static_cast(DistributedKv::AuthType::DEFAULT); result = AuthDelegate::GetInstance()->CheckAccess(localUserId, peerUserId, peerDeviceId, aclParams); EXPECT_TRUE(result.first); diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/BUILD.gn b/datamgr_service/services/distributeddataservice/service/test/mock/BUILD.gn index 1f85e3b0..a0833e47 100644 --- a/datamgr_service/services/distributeddataservice/service/test/mock/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/test/mock/BUILD.gn @@ -20,6 +20,7 @@ config("module_private_config") { "${data_service_path}/adapter/include/communicator", "../../../framework/include/", "../../../service/rdb/", + "./", ] defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] @@ -39,8 +40,12 @@ ohos_static_library("distributeddata_mock_static") { "db_store_mock.cpp", "general_store_mock.cpp", "kv_store_nb_delegate_mock.cpp", + "network_delegate_mock.cpp", + "screen_lock_mock.cpp", ] + deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] + external_deps = [ "kv_store:distributeddata_mgr", "kv_store:distributeddb", diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.cpp new file mode 100644 index 00000000..0f70c3f3 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "access_token_mock.h" + +namespace OHOS { +namespace Security { +namespace AccessToken { +ATokenTypeEnum AccessTokenKit::GetTokenTypeFlag(AccessTokenID tokenID) +{ + return BAccessTokenKit::accessTokenkit->GetTokenTypeFlag(tokenID); +} + +int AccessTokenKit::GetHapTokenInfo(AccessTokenID tokenID, HapTokenInfo& hapTokenInfoRes) +{ + return BAccessTokenKit::accessTokenkit->GetHapTokenInfo(tokenID, hapTokenInfoRes); +} + +int AccessTokenKit::VerifyAccessToken(AccessTokenID tokenID, const std::string& permissionName) +{ + return BAccessTokenKit::accessTokenkit->VerifyAccessToken(tokenID, permissionName); +} +} +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.h new file mode 100644 index 00000000..dd6e20e2 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/access_token_mock.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OHOS_ACCESS_TOKEN_KIT_MOCK_H +#define OHOS_ACCESS_TOKEN_KIT_MOCK_H + +#include +#include "accesstoken_kit.h" +#include "access_token.h" + +namespace OHOS { +namespace Security { +namespace AccessToken { +class BAccessTokenKit { +public: + virtual ATokenTypeEnum GetTokenTypeFlag(AccessTokenID) = 0; + virtual int GetHapTokenInfo(AccessTokenID, HapTokenInfo&) = 0; + virtual int VerifyAccessToken(AccessTokenID, const std::string&) = 0; + BAccessTokenKit() = default; + virtual ~BAccessTokenKit() = default; +private: + static inline std::shared_ptr accessTokenkit = nullptr; +}; + +class AccessTokenKitMock : public BAccessTokenKit { +public: + MOCK_METHOD(ATokenTypeEnum, GetTokenTypeFlag, (AccessTokenID)); + MOCK_METHOD(int, GetHapTokenInfo, (AccessTokenID, HapTokenInfo&)); + MOCK_METHOD(int, VerifyAccessToken, (AccessTokenID, const std::string&)); +}; +} +} +} +#endif //OHOS_ACCESS_TOKEN_KIT_MOCK_H diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.cpp new file mode 100644 index 00000000..1f4b6d5e --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "account_delegate_mock.h" + +namespace OHOS { +namespace DistributedData { +AccountDelegate *AccountDelegate::instance_ = nullptr; + +bool AccountDelegate::RegisterAccountInstance(AccountDelegate *instance) +{ + if (instance_ != nullptr) { + return false; + } + instance_ = instance; + return true; +} + +AccountDelegate *AccountDelegate::GetInstance() +{ + return &(AccountDelegateMock::Init()); +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.h new file mode 100644 index 00000000..fe8af6ba --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/account_delegate_mock.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_DISTRIBUTEDDATA_ACCOUNT_DELEGATE_MOCK_H +#define OHOS_DISTRIBUTEDDATA_ACCOUNT_DELEGATE_MOCK_H + +#include +#include +#include "account_delegate.h" +namespace OHOS { +namespace DistributedData { +class AccountDelegateMock : public AccountDelegate { +public: + virtual ~AccountDelegateMock() = default; + static AccountDelegateMock& Init() + { + static AccountDelegateMock instance; + return instance; + } + MOCK_METHOD(int32_t, Subscribe, (std::shared_ptr)); + MOCK_METHOD(int32_t, Unsubscribe, (std::shared_ptr)); + MOCK_METHOD(std::string, GetCurrentAccountId, (), (const)); + MOCK_METHOD(int32_t, GetUserByToken, (uint32_t), (const)); + MOCK_METHOD(void, SubscribeAccountEvent, ()); + MOCK_METHOD(void, UnsubscribeAccountEvent, ()); + MOCK_METHOD(bool, QueryUsers, (std::vector &)); + MOCK_METHOD(bool, QueryForegroundUsers, (std::vector &)); + MOCK_METHOD(bool, IsLoginAccount, ()); + MOCK_METHOD(bool, QueryForegroundUserId, (int &)); + MOCK_METHOD(bool, IsVerified, (int)); + MOCK_METHOD(bool, RegisterHashFunc, (HashFunc)); + MOCK_METHOD(bool, IsDeactivating, (int)); + MOCK_METHOD(void, BindExecutor, (std::shared_ptr)); + MOCK_METHOD(std::string, GetUnencryptedAccountId, (int32_t), (const)); + MOCK_METHOD(bool, IsUserForeground, (int32_t)); +}; +} +} +#endif // OHOS_DISTRIBUTEDDATA_ACCOUNT_DELEGATE_MOCK_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/adapter/permission/src/permission_validator.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/config_factory_mock.cpp similarity index 34% rename from datamgr_service/services/distributeddataservice/adapter/permission/src/permission_validator.cpp rename to datamgr_service/services/distributeddataservice/service/test/mock/config_factory_mock.cpp index c05e483f..aa685cd2 100644 --- a/datamgr_service/services/distributeddataservice/adapter/permission/src/permission_validator.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/mock/config_factory_mock.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2025 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,34 +13,73 @@ * limitations under the License. */ -#define LOG_TAG "PermissionValidator" - -#include "permission_validator.h" -#include "accesstoken_kit.h" -#include "log_print.h" +#include "config_factory.h" +#include namespace OHOS { -namespace DistributedKv { -using namespace Security::AccessToken; -PermissionValidator &PermissionValidator::GetInstance() +namespace DistributedData { +ConfigFactory::ConfigFactory() : file_(std::string(CONF_PATH) + "/config.json") +{ +} + +ConfigFactory::~ConfigFactory() +{ +} + +ConfigFactory &ConfigFactory::GetInstance() +{ + static ConfigFactory factory; + return factory; +} + +int32_t ConfigFactory::Initialize() +{ + return 0; +} + +std::vector *ConfigFactory::GetComponentConfig() +{ + return nullptr; +} + +NetworkConfig *ConfigFactory::GetNetworkConfig() +{ + return nullptr; +} + +CheckerConfig *ConfigFactory::GetCheckerConfig() +{ + return nullptr; +} + +GlobalConfig *ConfigFactory::GetGlobalConfig() +{ + return nullptr; +} + +DirectoryConfig *ConfigFactory::GetDirectoryConfig() +{ + return nullptr; +} + +BackupConfig *ConfigFactory::GetBackupConfig() +{ + return nullptr; +} + +CloudConfig *ConfigFactory::GetCloudConfig() { - static PermissionValidator permissionValidator; - return permissionValidator; + return nullptr; } -// check whether the client process have enough privilege to share data with the other devices. -bool PermissionValidator::CheckSyncPermission(uint32_t tokenId) +std::vector *ConfigFactory::GetAppIdMappingConfig() { - auto permit = AccessTokenKit::VerifyAccessToken(tokenId, DISTRIBUTED_DATASYNC); - ZLOGD("sync permit:%{public}d, token:0x%{public}x", permit, tokenId); - return permit == PERMISSION_GRANTED; + return nullptr; } -bool PermissionValidator::IsCloudConfigPermit(uint32_t tokenId) +ThreadConfig *ConfigFactory::GetThreadConfig() { - auto permit = AccessTokenKit::VerifyAccessToken(tokenId, CLOUD_DATA_CONFIG); - ZLOGD("cloud permit:%{public}d, token:0x%{public}x", permit, tokenId); - return permit == PERMISSION_GRANTED; + return nullptr; } -} // namespace DistributedKv -} // namespace OHOS +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.cpp index 015bec1f..fae2fb0f 100644 --- a/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.cpp @@ -41,4 +41,84 @@ bool OHOS::DistributedData::DeviceManagerAdapter::IsOHOSType(const std::string & } return BDeviceManagerAdapter::deviceManagerAdapter->IsOHOSType(id); } + +bool DeviceManagerAdapter::IsSameAccount(const AccessCaller &accCaller, const AccessCallee &accCallee) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return false; + } + return BDeviceManagerAdapter::deviceManagerAdapter->IsSameAccount(accCaller, accCallee); +} + +bool DeviceManagerAdapter::CheckAccessControl(const AccessCaller &accCaller, const AccessCallee &accCallee) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return false; + } + return BDeviceManagerAdapter::deviceManagerAdapter->CheckAccessControl(accCaller, accCallee); +} + +Status OHOS::DistributedData::DeviceManagerAdapter::StartWatchDeviceChange(const AppDeviceChangeListener *observer, + __attribute__((unused)) const PipeInfo &pipeInfo) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return Status::SUCCESS; + } + return BDeviceManagerAdapter::deviceManagerAdapter->StartWatchDeviceChange(observer, pipeInfo); +} + +Status OHOS::DistributedData::DeviceManagerAdapter::StopWatchDeviceChange(const AppDeviceChangeListener *observer, + __attribute__((unused)) const PipeInfo &pipeInfo) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return Status::SUCCESS; + } + return BDeviceManagerAdapter::deviceManagerAdapter->StopWatchDeviceChange(observer, pipeInfo); +} + +std::vector OHOS::DistributedData::DeviceManagerAdapter::GetRemoteDevices() +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + std::vector info; + return info; + } + return BDeviceManagerAdapter::deviceManagerAdapter->GetRemoteDevices(); +} + +DeviceInfo DeviceManagerAdapter::GetLocalDevice() +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + DeviceInfo info; + return info; + } + return BDeviceManagerAdapter::deviceManagerAdapter->GetLocalDevice(); +} + +std::string OHOS::DistributedData::DeviceManagerAdapter::GetUuidByNetworkId(const std::string &networkId) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return " "; + } + return BDeviceManagerAdapter::deviceManagerAdapter->GetUuidByNetworkId(networkId); +} + +DeviceInfo OHOS::DistributedData::DeviceManagerAdapter::GetDeviceInfo(const std::string &id) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return {}; + } + return BDeviceManagerAdapter::deviceManagerAdapter->GetDeviceInfo(id); +} + +std::string OHOS::DistributedData::DeviceManagerAdapter::ToNetworkID(const std::string &id) +{ + if (BDeviceManagerAdapter::deviceManagerAdapter == nullptr) { + return " "; + } + return BDeviceManagerAdapter::deviceManagerAdapter->ToNetworkID(id); +} + +DeviceManagerAdapter::~DeviceManagerAdapter() +{ +} } \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.h index e302e400..abdef014 100644 --- a/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.h +++ b/datamgr_service/services/distributeddataservice/service/test/mock/device_manager_adapter_mock.h @@ -22,10 +22,22 @@ namespace OHOS { namespace DistributedData { using namespace OHOS::AppDistributedKv; +using AccessCaller = OHOS::AppDistributedKv::AccessCaller; +using AccessCallee = OHOS::AppDistributedKv::AccessCallee; +using DeviceInfo = OHOS::AppDistributedKv::DeviceInfo; class BDeviceManagerAdapter { public: + virtual std::vector GetRemoteDevices() = 0; virtual bool IsOHOSType(const std::string &) = 0; virtual std::vector ToUUID(std::vector) = 0; + virtual Status StartWatchDeviceChange(const AppDeviceChangeListener *, const PipeInfo &) = 0; + virtual Status StopWatchDeviceChange(const AppDeviceChangeListener *, const PipeInfo &) = 0; + virtual bool IsSameAccount(const AccessCaller &, const AccessCallee &) = 0; + virtual std::string GetUuidByNetworkId(const std::string &) = 0; + virtual DeviceInfo GetDeviceInfo(const std::string &) = 0; + virtual std::string ToNetworkID(const std::string &) = 0; + virtual bool CheckAccessControl(const AccessCaller &, const AccessCallee &) = 0; + virtual DeviceInfo GetLocalDevice() = 0; static inline std::shared_ptr deviceManagerAdapter = nullptr; BDeviceManagerAdapter() = default; virtual ~BDeviceManagerAdapter() = default; @@ -33,8 +45,17 @@ public: class DeviceManagerAdapterMock : public BDeviceManagerAdapter { public: - MOCK_METHOD(bool, IsOHOSType, (const std::string &)); - MOCK_METHOD(std::vector, ToUUID, (std::vector)); + MOCK_METHOD0(GetRemoteDevices, std::vector()); + MOCK_METHOD1(IsOHOSType, bool(const std::string &)); + MOCK_METHOD1(ToUUID, std::vector(std::vector)); + MOCK_METHOD2(StartWatchDeviceChange, Status(const AppDeviceChangeListener *, const PipeInfo &)); + MOCK_METHOD2(StopWatchDeviceChange, Status(const AppDeviceChangeListener *, const PipeInfo &)); + MOCK_METHOD2(IsSameAccount, bool(const AccessCaller &, const AccessCallee &)); + MOCK_METHOD1(GetUuidByNetworkId,std::string(const std::string &)); + MOCK_METHOD1(GetDeviceInfo, DeviceInfo(const std::string &)); + MOCK_METHOD1(ToNetworkID, std::string(const std::string &)); + MOCK_METHOD2(CheckAccessControl, bool(const AccessCaller &, const AccessCallee &)); + MOCK_METHOD0(GetLocalDevice, DeviceInfo()); }; } // namespace DistributedData diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.cpp new file mode 100644 index 00000000..5923f7da --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "meta_data_manager_mock.h" + +namespace OHOS::DistributedData { +using namespace std; +OHOS::DistributedData::MetaDataManager &OHOS::DistributedData::MetaDataManager::GetInstance() +{ + static MetaDataManager instance; + return instance; +} + +OHOS::DistributedData::MetaDataManager::MetaDataManager() { } + +OHOS::DistributedData::MetaDataManager::~MetaDataManager() { } + +bool OHOS::DistributedData::MetaDataManager::LoadMeta(const std::string &key, Serializable &value, bool isLocal) +{ + return BMetaDataManager::metaDataManager->LoadMeta(key, value, isLocal); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.h new file mode 100644 index 00000000..28b4562a --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/meta_data_manager_mock.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OHOS_META_DATA_MANAGER_MOCK_H +#define OHOS_META_DATA_MANAGER_MOCK_H + +#include +#include "metadata/meta_data_manager.h" +#include "metadata/strategy_meta_data.h" +#include "metadata/store_meta_data.h" +#include "metadata/appid_meta_data.h" + +namespace OHOS::DistributedData { +class BMetaDataManager { +public: + virtual bool LoadMeta(const std::string &, Serializable &, bool) = 0; + BMetaDataManager() = default; + virtual ~BMetaDataManager() = default; +private: + static inline std::shared_ptr metaDataManager = nullptr; +}; + +class MetaDataManagerMock : public BMetaDataManager { +public: + MOCK_METHOD(bool, LoadMeta, (const std::string &, Serializable &, bool)); +}; +} +#endif //OHOS_META_DATA_MANAGER_MOCK_H diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.cpp new file mode 100644 index 00000000..ce26d253 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "network_delegate_mock.h" + +namespace OHOS { +namespace DistributedData { +bool NetworkDelegateMock::IsNetworkAvailable() +{ + return isNetworkAvailable_; +} + +NetworkDelegate::NetworkType NetworkDelegateMock::GetNetworkType(bool retrieve) +{ + return NetworkDelegate::NONE; +} + +void NetworkDelegateMock::RegOnNetworkChange() +{ + return; +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.h new file mode 100644 index 00000000..d1d65cf9 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/network_delegate_mock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OHOS_NETWORK_DELEGATE_MOCK_H +#define OHOS_NETWORK_DELEGATE_MOCK_H + +#include "network/network_delegate.h" + +namespace OHOS { +namespace DistributedData { +class NetworkDelegateMock : public NetworkDelegate { +public: + bool IsNetworkAvailable() override; + NetworkType GetNetworkType(bool retrieve = false) override; + void RegOnNetworkChange() override; + bool isNetworkAvailable_ = true; + virtual ~NetworkDelegateMock() = default; +}; +} // namespace DistributedData +} // namespace OHOS +#endif //OHOS_NETWORK_DELEGATE_MOCK_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.cpp new file mode 100644 index 00000000..080bb2f4 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "screen_lock_mock.h" + +namespace OHOS { +namespace DistributedData { +bool ScreenLockMock::IsLocked() +{ + return isLocked_; +} + +void ScreenLockMock::Subscribe(std::shared_ptr observer) +{ + return; +} + +void ScreenLockMock::Unsubscribe(std::shared_ptr observer) +{ + return; +} + +void ScreenLockMock::BindExecutor(std::shared_ptr executors) +{ + return; +} + +void ScreenLockMock::SubscribeScreenEvent() +{ + return; +} + +void ScreenLockMock::UnsubscribeScreenEvent() +{ + return; +} +} // namespace DistributedData +} // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.h new file mode 100644 index 00000000..3061d261 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/screen_lock_mock.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OHOS_SCREEN_LOCK_MOCK_H +#define OHOS_SCREEN_LOCK_MOCK_H + +#include "screen/screen_manager.h" + +namespace OHOS { +namespace DistributedData { +class ScreenLockMock : public ScreenManager { +public: + bool IsLocked(); + void Subscribe(std::shared_ptr observer); + void Unsubscribe(std::shared_ptr observer); + void BindExecutor(std::shared_ptr executors); + void SubscribeScreenEvent(); + void UnsubscribeScreenEvent(); + + bool isLocked_ = false; +}; +} // namespace DistributedData +} // namespace OHOS +#endif //OHOS_SCREEN_LOCK_MOCK_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/udmf_types_util.h b/datamgr_service/services/distributeddataservice/service/test/mock/udmf_types_util.h new file mode 100644 index 00000000..3f17b2c8 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/udmf_types_util.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OHOS_TYPES_UTIL_MOCK_H +#define OHOS_TYPES_UTIL_MOCK_H + +#include +#include +#include + +namespace OHOS { +namespace ITypesUtil { + template + static bool Marshal(MessageParcel &parcel, const T &first, const Types &...others); + template + static bool Unmarshal(MessageParcel &parcel, T &first, Types &...others); +} // namespace ITypesUtil + +template +bool ITypesUtil::Marshal(MessageParcel &parcel, const T &first, const Types &...others) +{ + return false; +} + +template +bool ITypesUtil::Unmarshal(MessageParcel &parcel, T &first, Types &...others) +{ + return true; +} +} +#endif //OHOS_TYPES_UTIL_MOCK_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.cpp b/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.cpp new file mode 100644 index 00000000..bc1d1353 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "user_delegate_mock.h" + +namespace OHOS::DistributedData { +using namespace OHOS::DistributedKv; +using DistributedData::UserStatus; +std::string GetLocalDeviceId() +{ + return ""; +} + +UserDelegate &UserDelegate::GetInstance() +{ + static UserDelegate instance; + return instance; +} + +std::vector UserDelegate::GetLocalUserStatus() +{ + if (BUserDelegate::userDelegate == nullptr) { + return {}; + } + return BUserDelegate::userDelegate->GetLocalUserStatus(); +} + +std::vector UserDelegate::GetRemoteUserStatus(const std::string &deviceId) +{ + if (BUserDelegate::userDelegate == nullptr) { + return {}; + } + return BUserDelegate::userDelegate->GetRemoteUserStatus(deviceId); +} + +std::set UserDelegate::GetLocalUsers() +{ + return {}; +} + +std::vector UserDelegate::GetUsers(const std::string &deviceId) +{ + return {}; +} + +void UserDelegate::DeleteUsers(const std::string &deviceId) +{ + return; +} + +void UserDelegate::UpdateUsers(const std::string &deviceId, const std::vector &userStatus) +{ + return; +} + +bool UserDelegate::InitLocalUserMeta() +{ + return true; +} + +void UserDelegate::Init(const std::shared_ptr &executors) +{ + return; +} + +ExecutorPool::Task UserDelegate::GeTask() +{ + return [this] { + return; + }; +} + +bool UserDelegate::NotifyUserEvent(const UserDelegate::UserEvent &userEvent) +{ + return true; +} + +void UserDelegate::LocalUserObserver::OnAccountChanged(const AccountEventInfo &eventInfo, int32_t timeout) +{ + return; +} + +UserDelegate::LocalUserObserver::LocalUserObserver(UserDelegate &userDelegate) : userDelegate_(userDelegate) {} + +std::string UserDelegate::LocalUserObserver::Name() +{ + return "user_delegate"; +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.h b/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.h new file mode 100644 index 00000000..335f8dd9 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/mock/user_delegate_mock.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_DISTRIBUTEDDATA_SERVICE_TEST_USER_DELEGATE_MOCK_H +#define OHOS_DISTRIBUTEDDATA_SERVICE_TEST_USER_DELEGATE_MOCK_H +#include +#include +#include "metadata/user_meta_data.h" + +#include "user_delegate.h" + +namespace OHOS::DistributedData { +using AccountDelegate = DistributedData::AccountDelegate; +using DistributedData::UserStatus; +class BUserDelegate { +public: + virtual std::vector GetLocalUserStatus() = 0; + virtual std::vector GetRemoteUserStatus(const std::string &) = 0; + static inline std::shared_ptr userDelegate = nullptr; + virtual ~BUserDelegate() = default; +}; + +class UserDelegateMock : public BUserDelegate { +public: + MOCK_METHOD((std::vector), GetLocalUserStatus, ()); + MOCK_METHOD((std::vector), GetRemoteUserStatus, (const std::string &)); +}; +} +#endif // OHOS_DISTRIBUTEDDATA_SERVICE_TEST_USER_DELEGATE_MOCK_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/object_asset_loader_test.cpp b/datamgr_service/services/distributeddataservice/service/test/object_asset_loader_test.cpp index 57aa68b6..c25e185c 100644 --- a/datamgr_service/services/distributeddataservice/service/test/object_asset_loader_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/object_asset_loader_test.cpp @@ -20,6 +20,7 @@ #include #include "executor_pool.h" +#include "object_common.h" #include "snapshot/machine_status.h" using namespace testing::ext; @@ -206,4 +207,53 @@ HWTEST_F(ObjectAssetLoaderTest, UpdateDownloaded002, TestSize.Level0) EXPECT_EQ(success, false); EXPECT_EQ(hash, ""); } + +/** +* @tc.name: PushAsset001 +* @tc.desc: PushAsset test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectAssetLoaderTest, PushAsset001, TestSize.Level0) +{ + auto assetLoader = ObjectAssetLoader::GetInstance(); + ASSERT_NE(assetLoader, nullptr); + sptr assetObj = new AssetObj(); + assetObj->dstBundleName_ = bundleName_; + assetObj->srcBundleName_ = bundleName_; + assetObj->dstNetworkId_ = "1"; + assetObj->sessionId_ = "123"; + + sptr sendCallback = new ObjectAssetsSendListener(); + ASSERT_NE(sendCallback, nullptr); + int32_t ret = assetLoader->PushAsset(userId_, assetObj, sendCallback); + EXPECT_NE(ret, DistributedObject::OBJECT_SUCCESS); +} + +/** +* @tc.name: OnSendResult001 +* @tc.desc: OnSendResult test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectAssetLoaderTest, OnSendResult001, TestSize.Level1) +{ + sptr assetObj = nullptr; + + int32_t result = -1; + sptr sendCallback = new ObjectAssetsSendListener(); + int32_t ret = sendCallback->OnSendResult(assetObj, result); + EXPECT_EQ(ret, result); + + assetObj = new AssetObj(); + assetObj->dstBundleName_ = bundleName_; + assetObj->srcBundleName_ = bundleName_; + assetObj->dstNetworkId_ = "1"; + assetObj->sessionId_ = "123"; + + ret = sendCallback->OnSendResult(assetObj, result); + EXPECT_EQ(ret, result); + + result = 0; + ret = sendCallback->OnSendResult(assetObj, result); + EXPECT_EQ(ret, result); +} } // namespace OHOS::Test diff --git a/datamgr_service/services/distributeddataservice/service/test/object_asset_machine_test.cpp b/datamgr_service/services/distributeddataservice/service/test/object_asset_machine_test.cpp index 0405b829..dd3f5b43 100644 --- a/datamgr_service/services/distributeddataservice/service/test/object_asset_machine_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/object_asset_machine_test.cpp @@ -135,7 +135,7 @@ HWTEST_F(ObjectAssetMachineTest, StatusTransfer002, TestSize.Level0) * @tc.desc: Transfer event * @tc.type: FUNC * @tc.require: -* @tc.author: nhj +* @tc.author: whj */ HWTEST_F(ObjectAssetMachineTest, StatusTransfer003, TestSize.Level0) { @@ -154,6 +154,26 @@ HWTEST_F(ObjectAssetMachineTest, StatusTransfer003, TestSize.Level0) ASSERT_EQ(changedAssets_[uri_].asset.hash, asset.hash); } +/** +* @tc.name: DFAPostEvent001 +* @tc.desc: DFAPostEvent invalid eventId test +* @tc.type: FUNC +*/ +HWTEST_F(ObjectAssetMachineTest, DFAPostEvent001, TestSize.Level0) +{ + Asset asset{ + .name = "test_name", + .uri = uri_, + .modifyTime = "modifyTime1", + .size = "size1", + .hash = "modifyTime1_size1", + }; + std::pair changedAsset{ "device_2", asset }; + changedAssets_[uri_].status = STATUS_UPLOADING; + auto ret = machine->DFAPostEvent(EVENT_BUTT, changedAssets_[uri_], asset, changedAsset); + ASSERT_EQ(ret, GeneralError::E_ERROR); +} + /** * @tc.name: StatusUpload001 * @tc.desc: No conflict scenarios: normal cloud sync. @@ -176,6 +196,9 @@ HWTEST_F(ObjectAssetMachineTest, StatusUpload001, TestSize.Level0) machine->DFAPostEvent(UPLOAD_FINISHED, changedAssets_[uri_], asset); ASSERT_EQ(changedAssets_[uri_].status, STATUS_STABLE); + // dotransfer + machine->DFAPostEvent(REMOTE_CHANGED, changedAssets_[uri_], asset, changedAsset); + ASSERT_EQ(changedAssets_[uri_].status, STATUS_TRANSFERRING); } /** @@ -204,11 +227,11 @@ HWTEST_F(ObjectAssetMachineTest, StatusUpload002, TestSize.Level0) .assetName = "asset_" + timestamp + ".jpg", }; StoreInfo storeInfo { - .tokenId = time, + .tokenId = static_cast(time), .bundleName = "bundleName_" + timestamp, .storeName = "store_" + timestamp, - .instanceId = time, - .user = time, + .instanceId = static_cast(time), + .user = static_cast(time), }; ChangedAssetInfo changedAssetInfo(asset, bindInfo, storeInfo); std::pair changedAsset{ "device_" + timestamp, asset }; @@ -229,7 +252,7 @@ HWTEST_F(ObjectAssetMachineTest, StatusUpload002, TestSize.Level0) * @tc.desc: No conflict scenarios: normal cloud sync. * @tc.type: FUNC * @tc.require: -* @tc.author: nhj +* @tc.author: whj */ HWTEST_F(ObjectAssetMachineTest, StatusDownload001, TestSize.Level0) { diff --git a/datamgr_service/services/distributeddataservice/service/test/object_manager_test.cpp b/datamgr_service/services/distributeddataservice/service/test/object_manager_test.cpp index 0a7eca46..835dca5b 100644 --- a/datamgr_service/services/distributeddataservice/service/test/object_manager_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/object_manager_test.cpp @@ -524,26 +524,31 @@ HWTEST_F(ObjectManagerTest, Close001, TestSize.Level0) HWTEST_F(ObjectManagerTest, SyncOnStore001, TestSize.Level0) { auto manager = ObjectStoreManager::GetInstance(); + manager->delegate_ = manager->OpenObjectKvStore(); std::function &results)> func; func = [](const std::map &results) { return results; }; std::string prefix = "ObjectManagerTest"; std::vector deviceList; - deviceList.push_back("local"); + // not local device & syncDevices empty deviceList.push_back("local1"); auto result = manager->SyncOnStore(prefix, deviceList, func); + ASSERT_NE(result, OBJECT_SUCCESS); + // local device + deviceList.push_back("local"); + result = manager->SyncOnStore(prefix, deviceList, func); ASSERT_EQ(result, OBJECT_SUCCESS); } /** -* @tc.name: RevokeSaveToStore001 +* @tc.name: RetrieveFromStore001 * @tc.desc: RetrieveFromStore test. * @tc.type: FUNC * @tc.require: * @tc.author: wangbin */ -HWTEST_F(ObjectManagerTest, RevokeSaveToStore001, TestSize.Level0) +HWTEST_F(ObjectManagerTest, RetrieveFromStore001, TestSize.Level0) { auto manager = ObjectStoreManager::GetInstance(); DistributedDB::KvStoreNbDelegateMock mockDelegate; @@ -683,6 +688,33 @@ HWTEST_F(ObjectManagerTest, ProcessSyncCallback001, TestSize.Level0) manager->ProcessSyncCallback(results, appId_, sessionId_, deviceId_); } +/** +* @tc.name: ProcessSyncCallback002 +* @tc.desc: ProcessSyncCallback test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectManagerTest, ProcessSyncCallback002, TestSize.Level0) +{ + std::string dataDir = "/data/app/el2/100/database"; + auto manager = ObjectStoreManager::GetInstance(); + std::map results; + + results.insert({"remote", 1}); // for testing + ASSERT_EQ(results.empty(), false); + ASSERT_EQ(results.find("local"), results.end()); + + manager->kvStoreDelegateManager_ = nullptr; + // open store failed -> success + manager->ProcessSyncCallback(results, appId_, sessionId_, deviceId_); + + // open store success -> success + manager->SetData(dataDir, userId_); + ASSERT_NE(manager->kvStoreDelegateManager_, nullptr); + manager->delegate_ = manager->OpenObjectKvStore(); + ASSERT_NE(manager->delegate_, nullptr); + manager->ProcessSyncCallback(results, appId_, sessionId_, deviceId_); +} + /** * @tc.name: IsAssetComplete001 * @tc.desc: IsAssetComplete test. @@ -830,7 +862,7 @@ HWTEST_F(ObjectManagerTest, RegisterAssetsLister001, TestSize.Level0) } /** -* @tc.name: RegisterAssetsLister001 +* @tc.name: PushAssets001 * @tc.desc: PushAssets test. * @tc.type: FUNC * @tc.require: @@ -849,6 +881,41 @@ HWTEST_F(ObjectManagerTest, PushAssets001, TestSize.Level0) ASSERT_EQ(result, DistributedObject::OBJECT_SUCCESS); } +/** +* @tc.name: PushAssets002 +* @tc.desc: PushAssets test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectManagerTest, PushAssets002, TestSize.Level0) +{ + auto manager = ObjectStoreManager::GetInstance(); + std::map> data; + std::vector value{0}; + std::string data0 = "[STRING]test"; + value.insert(value.end(), data0.begin(), data0.end()); + + std::string prefix = "bundleName_sessionId_source_target_timestamp"; + std::string dataKey = prefix + "_p_data"; + std::string assetPrefix = prefix + "_p_asset0"; + std::string fieldsPrefix = "p_"; + std::string deviceIdKey = "__deviceId"; + + data.insert({assetPrefix + ObjectStore::NAME_SUFFIX, value}); + data.insert({assetPrefix + ObjectStore::URI_SUFFIX, value}); + data.insert({assetPrefix + ObjectStore::MODIFY_TIME_SUFFIX, value}); + data.insert({assetPrefix + ObjectStore::SIZE_SUFFIX, value}); + data.insert({fieldsPrefix + deviceIdKey, value}); + + manager->objectAssetsSendListener_ = nullptr; + int32_t ret = manager->PushAssets(appId_, appId_, sessionId_, data, deviceId_); + EXPECT_NE(ret, DistributedObject::OBJECT_SUCCESS); + + manager->objectAssetsSendListener_ = new ObjectAssetsSendListener(); + ASSERT_NE(manager->objectAssetsSendListener_, nullptr); + ret = manager->PushAssets(appId_, appId_, sessionId_, data, deviceId_); + EXPECT_NE(ret, DistributedObject::OBJECT_SUCCESS); +} + /** * @tc.name: AddNotifier001 * @tc.desc: AddNotifie and DeleteNotifier test. @@ -897,4 +964,105 @@ HWTEST_F(ObjectManagerTest, BindAsset001, TestSize.Level0) auto result = manager->BindAsset(tokenId, bundleName, sessionId_, assetValue_, assetBindInfo_); ASSERT_EQ(result, DistributedObject::OBJECT_DBSTATUS_ERROR); } + +/** +* @tc.name: OnFinished001 +* @tc.desc: OnFinished test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectManagerTest, OnFinished001, TestSize.Level1) +{ + std::string srcNetworkId = "srcNetworkId"; + sptr assetObj = nullptr; + int32_t result = 100; + ObjectAssetsRecvListener listener; + int32_t ret = listener.OnFinished(srcNetworkId, assetObj, result); + EXPECT_NE(ret, DistributedObject::OBJECT_SUCCESS); + + sptr assetObj_1 = new AssetObj(); + assetObj_1->dstBundleName_ = bundleName_; + assetObj_1->srcBundleName_ = bundleName_; + assetObj_1->dstNetworkId_ = "1"; + assetObj_1->sessionId_ = "123"; + ret = listener.OnFinished(srcNetworkId, assetObj_1, result); + EXPECT_EQ(ret, DistributedObject::OBJECT_SUCCESS); +} + +/** +* @tc.name: GetObjectData001 +* @tc.desc: GetObjectData test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectManagerTest, GetObjectData001, TestSize.Level1) +{ + auto manager = ObjectStoreManager::GetInstance(); + + std::string bundleName = bundleName_; + std::string sessionId = sessionId_; + std::string source = "sourceDeviceId"; + std::string target = "targetDeviceId"; + std::string timestamp = "1234567890"; + ObjectStoreManager::SaveInfo saveInfo(bundleName, sessionId, source, target, timestamp); + std::string prefix = saveInfo.ToPropertyPrefix(); + EXPECT_FALSE(prefix.empty()); + + // p_name not asset key + std::string p_name = "p_namejpg"; + std::string key = bundleName + "_" + sessionId + "_" + source + "_" + target + "_" + timestamp + "_" + p_name; + std::map> changedData = {{ key, data_ }}; + bool hasAsset = false; + auto ret = manager->GetObjectData(changedData, saveInfo, hasAsset); + EXPECT_FALSE(ret.empty()); + EXPECT_FALSE(hasAsset); +} + +/** +* @tc.name: GetObjectData002 +* @tc.desc: GetObjectData test. +* @tc.type: FUNC +*/ +HWTEST_F(ObjectManagerTest, GetObjectData002, TestSize.Level1) +{ + auto manager = ObjectStoreManager::GetInstance(); + + std::string bundleName = ""; + std::string sessionId = ""; + std::string source = ""; + std::string target = ""; + std::string timestamp = ""; + ObjectStoreManager::SaveInfo saveInfo(bundleName, sessionId, source, target, timestamp); + std::string prefix = saveInfo.ToPropertyPrefix(); + EXPECT_TRUE(prefix.empty()); + + // saveInfo.bundleName, sourceDeviceId, targetDeviceId empty + saveInfo.sessionId = sessionId_; + saveInfo.timestamp = "1234567890"; + + bundleName = bundleName_; + sessionId = sessionId_; + source = "sourceDeviceId"; + target = "targetDeviceId"; + timestamp = "1234567890"; + std::string p_name = "p_name.jpg"; + std::string key = bundleName + "_" + sessionId + "_" + source + "_" + target + "_" + timestamp + "_" + p_name; + std::map> changedData = {{ key, data_ }}; + bool hasAsset = false; + auto ret = manager->GetObjectData(changedData, saveInfo, hasAsset); + EXPECT_FALSE(ret.empty()); + EXPECT_EQ(saveInfo.bundleName, bundleName); + EXPECT_TRUE(hasAsset); + + // only targetDeviceId empty + saveInfo.bundleName = "test_bundleName"; + saveInfo.sourceDeviceId = "test_source"; + // p_name not asset key + p_name = "p_namejpg"; + std::string key_1 = bundleName + "_" + sessionId + "_" + source + "_" + target + "_" + timestamp + "_" + p_name; + std::map> changedData_1 = {{ key_1, data_ }}; + hasAsset = false; + ret = manager->GetObjectData(changedData_1, saveInfo, hasAsset); + EXPECT_FALSE(ret.empty()); + EXPECT_NE(saveInfo.bundleName, bundleName); + EXPECT_FALSE(hasAsset); +} } // namespace OHOS::Test diff --git a/datamgr_service/services/distributeddataservice/service/test/object_service_impl_test.cpp b/datamgr_service/services/distributeddataservice/service/test/object_service_impl_test.cpp new file mode 100644 index 00000000..e67f493a --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/object_service_impl_test.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "ObjectServiceImplTest" + +#include "object_service_impl.h" + +#include + +#include "accesstoken_kit.h" +#include "ipc_skeleton.h" +#include "token_setproc.h" + +using namespace testing::ext; +using namespace OHOS::DistributedObject; +namespace OHOS::Test { +class ObjectServiceImplTest : public testing::Test { +public: + void SetUp(); + void TearDown(); +protected: + ObjectStore::Asset asset_; + std::string uri_; + std::string appId_ = "ObjectServiceImplTest_appid_1"; + std::string sessionId_ = "123"; + std::vector data_; + std::string deviceId_ = "7001005458323933328a258f413b3900"; + uint64_t sequenceId_ = 10; + uint64_t sequenceId_2 = 20; + uint64_t sequenceId_3 = 30; + std::string userId_ = "100"; + std::string bundleName_ = "test_bundleName"; + ObjectStore::AssetBindInfo assetBindInfo_; + pid_t pid_ = 10; + uint32_t tokenId_ = 100; +}; + +void ObjectServiceImplTest::SetUp() +{ + uri_ = "file:://com.examples.hmos.notepad/data/storage/el2/distributedfiles/dir/asset1.jpg"; + ObjectStore::Asset asset{ + .id = "test_name", + .name = uri_, + .uri = uri_, + .createTime = "2025.03.05", + .modifyTime = "modifyTime", + .size = "size", + .hash = "modifyTime_size", + .path = "/data/storage/el2", + }; + asset_ = asset; + + data_.push_back(sequenceId_); + data_.push_back(sequenceId_2); + data_.push_back(sequenceId_3); + + ObjectStore::AssetBindInfo AssetBindInfo{ + .storeName = "store_test", + .tableName = "table_test", + .field = "attachment", + .assetName = "asset1.jpg", + }; + assetBindInfo_ = AssetBindInfo; +} + +void ObjectServiceImplTest::TearDown() {} + +/** + * @tc.name: OnAssetChanged001 + * @tc.desc: OnAssetChanged test. + * @tc.type: FUNC + */ +HWTEST_F(ObjectServiceImplTest, OnAssetChanged001, TestSize.Level1) +{ + std::string bundleName = "com.examples.hmos.notepad"; + OHOS::Security::AccessToken::AccessTokenID tokenId = + OHOS::Security::AccessToken::AccessTokenKit::GetHapTokenID(100, bundleName, 0); + SetSelfTokenID(tokenId); + + std::shared_ptr objectServiceImpl = std::make_shared(); + + // bundleName not equal tokenId + auto ret = objectServiceImpl->OnAssetChanged(bundleName_, sessionId_, deviceId_, asset_); + EXPECT_EQ(ret, OBJECT_PERMISSION_DENIED); +} + +/** + * @tc.name: BindAssetStore001 + * @tc.desc: BindAssetStore test. + * @tc.type: FUNC + */ +HWTEST_F(ObjectServiceImplTest, BindAssetStore001, TestSize.Level1) +{ + std::string bundleName = "com.examples.hmos.notepad"; + OHOS::Security::AccessToken::AccessTokenID tokenId = + OHOS::Security::AccessToken::AccessTokenKit::GetHapTokenID(100, bundleName, 0); + SetSelfTokenID(tokenId); + + std::shared_ptr objectServiceImpl = std::make_shared(); + + // bundleName not equal tokenId + auto ret = objectServiceImpl->BindAssetStore(bundleName_, sessionId_, asset_, assetBindInfo_); + EXPECT_EQ(ret, OBJECT_PERMISSION_DENIED); +} + +/** + * @tc.name: DeleteSnapshot001 + * @tc.desc: DeleteSnapshot test. + * @tc.type: FUNC + */ +HWTEST_F(ObjectServiceImplTest, DeleteSnapshot001, TestSize.Level1) +{ + std::string bundleName = "com.examples.hmos.notepad"; + OHOS::Security::AccessToken::AccessTokenID tokenId = + OHOS::Security::AccessToken::AccessTokenKit::GetHapTokenID(100, bundleName, 0); + SetSelfTokenID(tokenId); + + std::shared_ptr objectServiceImpl = std::make_shared(); + + // bundleName not equal tokenId + auto ret = objectServiceImpl->DeleteSnapshot(bundleName_, sessionId_); + EXPECT_EQ(ret, OBJECT_PERMISSION_DENIED); +} + +/** + * @tc.name: ResolveAutoLaunch001 + * @tc.desc: ResolveAutoLaunch test. + * @tc.type: FUNC + */ +HWTEST_F(ObjectServiceImplTest, ResolveAutoLaunch001, TestSize.Level1) +{ + DistributedDB::AutoLaunchParam param { + .userId = userId_, + .appId = appId_, + .storeId = "storeId", + }; + std::string identifier = "identifier"; + std::shared_ptr objectServiceImpl = std::make_shared(); + int32_t ret = objectServiceImpl->ResolveAutoLaunch(identifier, param); + EXPECT_EQ(ret, OBJECT_STORE_NOT_FOUND); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/object_service_stub_test.cpp b/datamgr_service/services/distributeddataservice/service/test/object_service_stub_test.cpp new file mode 100644 index 00000000..1cbb5e1a --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/object_service_stub_test.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "ObjectServiceStubTest" + +#include "object_service_stub.h" + +#include + +#include "ipc_skeleton.h" +#include "object_service_impl.h" + +using namespace testing::ext; +using namespace OHOS::DistributedObject; +namespace OHOS::Test { +const std::u16string INTERFACE_TOKEN = u"OHOS.DistributedObject.IObjectService"; +constexpr uint32_t CODE_MAX = static_cast(ObjectCode::OBJECTSTORE_SERVICE_CMD_MAX) + 1; +constexpr size_t TEST_SIZE = 1; +constexpr uint8_t TEST_DATA = 1; +class ObjectServiceStubTest : public testing::Test { +public: + void SetUp(){}; + void TearDown(){}; +}; +std::shared_ptr objectServiceImpl = std::make_shared(); +std::shared_ptr objectServiceStub = objectServiceImpl; + +/** + * @tc.name: OnRemoteRequest001 + * @tc.desc: OnRemoteRequest data error test + * @tc.type: FUNC + * @tc.require: + * @tc.author: shuxin + */ +HWTEST_F(ObjectServiceStubTest, ObjectServiceTest001, TestSize.Level1) +{ + uint8_t value = TEST_DATA; + uint8_t *data = &value; + size_t size = TEST_SIZE; + uint32_t code = 0; + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + request.WriteBuffer(data, size); + request.RewindRead(0); + MessageParcel reply; + auto result = objectServiceStub->OnRemoteRequest(code, request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + code = CODE_MAX - 1; + result = objectServiceStub->OnRemoteRequest(code, request, reply); + EXPECT_EQ(result, -1); +} + +/** + * @tc.name: OnRemoteRequest002 + * @tc.desc: OnRemoteRequest test + * @tc.type: FUNC + * @tc.require: + * @tc.author: shuxin + */ +HWTEST_F(ObjectServiceStubTest, ObjectServiceTest002, TestSize.Level1) +{ + uint8_t value = TEST_DATA; + uint8_t *data = &value; + size_t size = TEST_SIZE; + MessageParcel request; + request.WriteInterfaceToken(INTERFACE_TOKEN); + request.WriteBuffer(data, size); + request.RewindRead(0); + MessageParcel reply; + + auto result = objectServiceStub->ObjectStoreRevokeSaveOnRemote(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->ObjectStoreRetrieveOnRemote(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->OnSubscribeRequest(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->OnUnsubscribeRequest(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->OnAssetChangedOnRemote(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->ObjectStoreBindAssetOnRemote(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->OnDeleteSnapshot(request, reply); + EXPECT_EQ(result, IPC_STUB_INVALID_DATA_ERR); + + result = objectServiceStub->OnIsContinue(request, reply); + EXPECT_EQ(result, 0); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/permission_validator_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/permission_validator_mock_test.cpp new file mode 100644 index 00000000..33fb38f7 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/permission_validator_mock_test.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "access_token_mock.h" +#include "permission_validator.h" + +namespace OHOS { +namespace DistributedKv { +using namespace std; +using namespace testing; +using namespace OHOS::Security::AccessToken; +class PermissionValidatorMockTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +public: + static inline shared_ptr accessTokenKitMock = nullptr; +}; + +void PermissionValidatorMockTest::SetUpTestCase(void) +{ + accessTokenKitMock = make_shared(); + BAccessTokenKit::accessTokenkit = accessTokenKitMock; +} + +void PermissionValidatorMockTest::TearDownTestCase(void) +{ + BAccessTokenKit::accessTokenkit = nullptr; + accessTokenKitMock = nullptr; +} + +void PermissionValidatorMockTest::SetUp(void) +{} + +void PermissionValidatorMockTest::TearDown(void) +{} + +/** + * @tc.name: CheckSyncPermission + * @tc.desc: check sync premisson. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermissionValidatorMockTest, CheckSyncPermission, testing::ext::TestSize.Level0) +{ + uint32_t tokenId = 0; + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(TypePermissionState::PERMISSION_GRANTED)); + EXPECT_TRUE(PermissionValidator::GetInstance().CheckSyncPermission(tokenId)); + + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(TypePermissionState::PERMISSION_DENIED)); + EXPECT_FALSE(PermissionValidator::GetInstance().CheckSyncPermission(tokenId)); +} + +/** + * @tc.name: IsCloudConfigPermit + * @tc.desc: is loudConfig permit. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermissionValidatorMockTest, IsCloudConfigPermit, testing::ext::TestSize.Level0) +{ + uint32_t tokenId = 0; + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(TypePermissionState::PERMISSION_GRANTED)); + EXPECT_TRUE(PermissionValidator::GetInstance().IsCloudConfigPermit(tokenId)); + + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(TypePermissionState::PERMISSION_DENIED)); + EXPECT_FALSE(PermissionValidator::GetInstance().IsCloudConfigPermit(tokenId)); +} +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/permit_delegate_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/permit_delegate_mock_test.cpp new file mode 100644 index 00000000..e312860b --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/permit_delegate_mock_test.cpp @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "access_token_mock.h" +#include "meta_data_manager_mock.h" +#include "permit_delegate.h" +#include "metadata/store_meta_data.h" +#include "metadata/appid_meta_data.h" + +namespace OHOS::DistributedData { +using namespace OHOS::Security::AccessToken; +using namespace std; +using namespace testing; +using ActiveParam = DistributedDB::ActivationCheckParam; +using CheckParam = DistributedDB::PermissionCheckParam; +class PermitDelegateMockTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +public: + static inline shared_ptr accessTokenKitMock = nullptr; + static inline shared_ptr metaDataMgrMock = nullptr; +}; + +void PermitDelegateMockTest::SetUpTestCase(void) +{ + accessTokenKitMock = make_shared(); + BAccessTokenKit::accessTokenkit = accessTokenKitMock; + metaDataMgrMock = make_shared(); + BMetaDataManager::metaDataManager = metaDataMgrMock; +} + +void PermitDelegateMockTest::TearDownTestCase(void) +{ + BAccessTokenKit::accessTokenkit = nullptr; + accessTokenKitMock = nullptr; + BMetaDataManager::metaDataManager = nullptr; + metaDataMgrMock = nullptr; +} + +void PermitDelegateMockTest::SetUp(void) +{} + +void PermitDelegateMockTest::TearDown(void) +{} + +/** + * @tc.name: SyncActivate001 + * @tc.desc: sync Activate. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, SyncActivate001, testing::ext::TestSize.Level0) +{ + ActiveParam activeParam = { + .userId = "activeparam", + .appId = "appid", + .storeId = "storeid", + .subUserId = "subactiveparam", + .instanceId = 1 + }; + PermitDelegate::GetInstance().Init(); + bool result = PermitDelegate::GetInstance().SyncActivate(activeParam); + EXPECT_FALSE(result); +} + + +/** + * @tc.name: SyncActivate002 + * @tc.desc: sync Activate. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, SyncActivate002, testing::ext::TestSize.Level0) +{ + ActiveParam activeParam = { + .userId = "1", + .appId = "", + .storeId = "", + .subUserId = "subuserid", + .instanceId = 0 + }; + EXPECT_CALL(*accessTokenKitMock, GetTokenTypeFlag(_)).Times(AnyNumber()); + bool result = PermitDelegate::GetInstance().SyncActivate(activeParam); + EXPECT_FALSE(result); +} + +/** + * @tc.name: VerifyPermission_001 + * @tc.desc: verify permission. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, VerifyPermission_001, testing::ext::TestSize.Level0) +{ + std::string permission = ""; + uint32_t tokenId = 1; + bool result = PermitDelegate::GetInstance().VerifyPermission(permission, tokenId); + EXPECT_TRUE(result); +} + +/** + * @tc.name: VerifyPermission_002 + * @tc.desc: verify permission. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, VerifyPermission_002, testing::ext::TestSize.Level0) +{ + std::string permission = "premmit002"; + uint32_t tokenId = 1; + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(PermissionState::PERMISSION_GRANTED)); + bool result = PermitDelegate::GetInstance().VerifyPermission(permission, tokenId); + EXPECT_TRUE(result); +} + +/** + * @tc.name: VerifyPermission_003 + * @tc.desc: verify permission. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, VerifyPermission_003, testing::ext::TestSize.Level0) +{ + std::string permission = "premmit003"; + uint32_t tokenId = 0; + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(PermissionState::PERMISSION_DENIED)); + bool result = PermitDelegate::GetInstance().VerifyPermission(permission, tokenId); + EXPECT_FALSE(result); +} + +/** + * @tc.name: VerifyPermission001 + * @tc.desc: verify permission. + * @tc.type: OVERRIDE FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, VerifyPermission001, testing::ext::TestSize.Level0) +{ + bool result = PermitDelegate::GetInstance().appId2BundleNameMap_.Insert("Permission001", ""); + ASSERT_TRUE(result); + AppIDMetaData appMeta("permitdelegatemocktestId", "com.permitdelegatetest.app"); + EXPECT_CALL(*metaDataMgrMock, LoadMeta(_, _, _)).WillOnce(DoAll(SetArgReferee<1>(appMeta), Return(true))); + CheckParam checkParam; + checkParam.appId = "permitdelegatemocktestId"; + checkParam.userId = "userid"; + checkParam.storeId = "storeid"; + checkParam.deviceId = "deviceid"; + checkParam.instanceId = 1; + uint8_t flag = 1; + EXPECT_CALL(*metaDataMgrMock, LoadMeta(_, _, false)).WillOnce(Return(false)); + result = PermitDelegate::GetInstance().VerifyPermission(checkParam, flag); + EXPECT_FALSE(result); +} + +/** + * @tc.name: VerifyPermission002 + * @tc.desc: verify permission. + * @tc.type: OVERRIDE FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(PermitDelegateMockTest, VerifyPermission002, testing::ext::TestSize.Level0) +{ + std::string key = "Permission002"; + std::string value = "com.permitDelegateUnitTest.app"; + PermitDelegate::GetInstance().appId2BundleNameMap_.Insert(key, value); + auto ret = PermitDelegate::GetInstance().appId2BundleNameMap_.Find(key); + ASSERT_TRUE(ret.second == value); + CheckParam checkParam = { + .userId = "userid2", + .appId = "permitdelegatemocktestId2", + .storeId = "storeid2", + .deviceId = "deviceid2", + .instanceId = 0 + }; + uint8_t flag = 1; + EXPECT_CALL(*metaDataMgrMock, LoadMeta(_, _, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*accessTokenKitMock, VerifyAccessToken(_, _)) + .WillOnce(Return(PermissionState::PERMISSION_GRANTED)); + bool result = PermitDelegate::GetInstance().VerifyPermission(checkParam, flag); + EXPECT_TRUE(result); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/query_helper_test.cpp b/datamgr_service/services/distributeddataservice/service/test/query_helper_test.cpp new file mode 100644 index 00000000..8ad4a0f5 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/query_helper_test.cpp @@ -0,0 +1,590 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "query_helper.h" + +namespace OHOS::DistributedKv { +using namespace testing; +using namespace std; + +class QueryHelperUnitTest : public testing::Test { +public: + static void SetUpTestCase(void) {} + static void TearDownTestCase(void) {} + void SetUp() {} + void TearDown() {} +}; + +/** + * @tc.name: StringToDbQuery001 + * @tc.desc: in order to test HandleEqualTo. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery001, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^EQUAL INTEGER TYPE_INTEGER 0"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^EQUAL LONG TYPE_LONG 1l"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^EQUAL DOUBLE TYPE_DOUBLE 2.1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^EQUAL BOOL TYPE_BOOLEAN 3"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^EQUAL STRING TYPE_STRING dafafg"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^EQUAL CHAR STRING 4"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery002 + * @tc.desc: in order to test HandleNotEqualTo. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery002, testing::ext::TestSize.Level0) +{ + bool isOk = false; + string query = "^NOT_EQUAL INTEGER TYPE_INTEGER 0"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_TRUE(isOk); + + isOk = false; + query = "^NOT_EQUAL LONG TYPE_LONG 1l"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_TRUE(isOk); + + isOk = false; + query = "^NOT_EQUAL DOUBLE TYPE_DOUBLE 2.1"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_TRUE(isOk); + + isOk = false; + query = "^NOT_EQUAL BOOL TYPE_BOOLEAN 3"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_TRUE(isOk); + + isOk = false; + query = "^NOT_EQUAL STRING TYPE_STRING dafafg"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_TRUE(isOk); + + query = "^NOT_EQUAL CHAR STRING 4"; + (void)QueryHelper::StringToDbQuery(query, isOk); + EXPECT_FALSE(isOk); +} + +/** + * @tc.name: StringToDbQuery003 + * @tc.desc: in order to test HandleGreaterThan. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery003, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^GREATER INTEGER TYPE_INTEGER 2"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER LONG TYPE_LONG 3l"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER DOUBLE TYPE_DOUBLE 4.1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER STRING TYPE_STRING gohkhkhk"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^GREATER BOOL STRING 5"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery004 + * @tc.desc: in order to test HandleGreaterThanOrEqualTo. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery004, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^GREATER_EQUAL INTEGER TYPE_INTEGER 0"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER_EQUAL LONG TYPE_LONG 2l"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER_EQUAL DOUBLE TYPE_DOUBLE 4.1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^GREATER_EQUAL STRING TYPE_STRING lofhfgh"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^GREATER_EQUAL CHAR STRING 6"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery005 + * @tc.desc: in order to test HandleLessThanOrEqualTo. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery005, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^LESS_EQUAL INTEGER TYPE_INTEGER 1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS_EQUAL LONG TYPE_LONG 3l"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS_EQUAL DOUBLE TYPE_DOUBLE 5.1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS_EQUAL STRING TYPE_STRING lofhfgh"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^LESS_EQUAL CHAR STRING a"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery006 + * @tc.desc: in order to test Handle's HandleIsNull. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery006, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^IS_NULL"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + isSuccess = false; + query = "^IS_NULL field1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery007 + * @tc.desc: in order to test Handle's HandleIn. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery007, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^IN"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + isSuccess = true; + query = "^IN INTEGER grade other ^START"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^IN INTEGER old ^START ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN INTEGER old ^START 95 ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN INTEGER grade ^START 123"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^IN LONG salary ^START ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN LONG grade ^START 650 ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN LONG range ^START 123478"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^IN DOUBLE 1234 ^START ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN DOUBLE salary ^START 129456 ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN DOUBLE salary ^START 16478"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery008 + * @tc.desc: in order to test Handle's HandleIn. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery008, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^IN STRING fieldname ^START ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN STRING fieldname ^START xadada ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^IN STRING fieldname ^START other"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^IN CHAR fieldname ^START other"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery009 + * @tc.desc: in order to test Handle's HandleNotIn. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery009, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^NOT_IN"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + isSuccess = true; + query = "^IN INTEGER 12 other ^START"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + isSuccess = false; + query = "^NOT_IN INTEGER name ^START ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^NOT_IN LONG salary ^START 10000 ^END"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + isSuccess = false; + query = "^NOT_IN DOUBLE height ^START 175"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^NOT_IN STRING 14578 ^START fieldname"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^NOT_IN CHAR department ^START other"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery010 + * @tc.desc: in order to test Handle's HandleLike. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery010, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^LIKE"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^LIKE Name Mr*"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery011 + * @tc.desc: in order to test Handle's HandleNotLike. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery011, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^NOT_LIKE"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^NOT_LIKE Name Wan*"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery012 + * @tc.desc: in order to test Handle's HandleAnd. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery012, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^AND"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery013 + * @tc.desc: in order to test Handle's HandleOr. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery013, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^OR"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery014 + * @tc.desc: in order to test Handle's HandleOrderByAsc. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery014, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^ASC"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^ASC old"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery015 + * @tc.desc: in order to test Handle's HandleOrderByDesc. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery015, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^DESC"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^DESC grade"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery016 + * @tc.desc: in order to test Handle's HandleOrderByWriteTime. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery016, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^OrderByWriteTime"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^OrderByWriteTime salary"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery017 + * @tc.desc: in order to test Handle's HandleLimit. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery017, testing::ext::TestSize.Level0) +{ + bool isSuccess = true; + string query = "^LIMIT"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^LIMIT 0 5"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery018 + * @tc.desc: in order to test Handle's HandleExtra. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery018, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^BEGIN_GROUP"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^END_GROUP"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^KEY_PREFIX"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + query = "^KEY_PREFIX _XXX"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^IS_NOT_NULL"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + query = "^IS_NOT_NULL NAME"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^DEVICE_ID"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + isSuccess = true; + query = "^DEVICE_ID 4DAAAJFGKAGNGM ^KEY_PREFIX"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + + query = "^DEVICE_ID FJGKGNMGJM4DAFA78"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^SUGGEST_INDEX"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); + query = "^SUGGEST_INDEX xadada"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^IN_KEYS xxxx instart"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} + +/** + * @tc.name: StringToDbQuery019 + * @tc.desc: in order to test HandleLessThan. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caozhijun + */ +HWTEST_F(QueryHelperUnitTest, StringToDbQuery019, testing::ext::TestSize.Level0) +{ + bool isSuccess = false; + string query = "^LESS INTEGER TYPE_INTEGER 1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS LONG TYPE_LONG 2l"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS DOUBLE TYPE_DOUBLE 3.1"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + isSuccess = false; + query = "^LESS STRING TYPE_STRING bnnjjjh"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_TRUE(isSuccess); + + query = "^LESS BOOL STRING 5"; + (void)QueryHelper::StringToDbQuery(query, isSuccess); + EXPECT_FALSE(isSuccess); +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/rdb_asset_loader_test.cpp b/datamgr_service/services/distributeddataservice/service/test/rdb_asset_loader_test.cpp index 0e746a73..f8d56ca2 100644 --- a/datamgr_service/services/distributeddataservice/service/test/rdb_asset_loader_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/rdb_asset_loader_test.cpp @@ -62,6 +62,11 @@ public: { return GeneralError::E_OK; } + + int32_t CancelDownload() override + { + return GeneralError::E_OK; + } }; class RdbWatcherTest : public testing::Test { @@ -179,6 +184,26 @@ HWTEST_F(RdbAssetLoaderTest, RemoveLocalAssets, TestSize.Level0) EXPECT_EQ(result, DistributedDB::DBStatus::OK); } +/** +* @tc.name: CancelDownloadTest +* @tc.desc: RdbAssetLoader CancelDownload test. +* @tc.type: FUNC +* @tc.require: +* @tc.author: Hollokin +*/ +HWTEST_F(RdbAssetLoaderTest, CancelDownloadTest, TestSize.Level0) +{ + BindAssets bindAssets; + DistributedRdb::RdbAssetLoader rdbAssetLoader1(nullptr, &bindAssets); + auto result = rdbAssetLoader1.CancelDownload(); + EXPECT_EQ(result, DistributedDB::DBStatus::DB_ERROR); + + std::shared_ptr cloudAssetLoader = std::make_shared(); + DistributedRdb::RdbAssetLoader rdbAssetLoader2(cloudAssetLoader, &bindAssets); + result = rdbAssetLoader2.CancelDownload(); + EXPECT_EQ(result, DistributedDB::DBStatus::OK); +} + /** * @tc.name: PostEvent001 * @tc.desc: RdbAssetLoader PostEvent001 test @@ -246,6 +271,8 @@ HWTEST_F(RdbAssetLoaderTest, ConvertStatus, TestSize.Level0) EXPECT_EQ(status, DistributedDB::DBStatus::OK); status = RdbAssetLoader::ConvertStatus(DistributedRdb::RdbAssetLoader::AssetStatus::STATUS_BUTT); EXPECT_EQ(status, DistributedDB::DBStatus::CLOUD_ERROR); + status = RdbAssetLoader::ConvertStatus(DistributedRdb::RdbAssetLoader::AssetStatus::STATUS_SKIP_ASSET); + EXPECT_EQ(status, DistributedDB::DBStatus::SKIP_ASSET); } /** diff --git a/datamgr_service/services/distributeddataservice/service/test/rdb_cloud_test.cpp b/datamgr_service/services/distributeddataservice/service/test/rdb_cloud_test.cpp index 1cfe316a..cca78a80 100644 --- a/datamgr_service/services/distributeddataservice/service/test/rdb_cloud_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/rdb_cloud_test.cpp @@ -195,6 +195,8 @@ HWTEST_F(RdbCloudTest, ConvertStatus, TestSize.Level1) EXPECT_EQ(result, DBStatus::TIME_OUT); result = rdbCloud.ConvertStatus(GeneralError::E_CLOUD_DISABLED); EXPECT_EQ(result, DBStatus::CLOUD_DISABLED); + result = rdbCloud.ConvertStatus(GeneralError::E_SKIP_ASSET); + EXPECT_EQ(result, DBStatus::SKIP_ASSET); } /** diff --git a/datamgr_service/services/distributeddataservice/service/test/rdb_general_store_test.cpp b/datamgr_service/services/distributeddataservice/service/test/rdb_general_store_test.cpp index 11ce2486..70e914dd 100644 --- a/datamgr_service/services/distributeddataservice/service/test/rdb_general_store_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/rdb_general_store_test.cpp @@ -998,6 +998,8 @@ HWTEST_F(RdbGeneralStoreTest, ConvertStatus, TestSize.Level1) EXPECT_EQ(result, GeneralError::E_ERROR); result = store->ConvertStatus(DBStatus::CLOUD_DISABLED); EXPECT_EQ(result, GeneralError::E_CLOUD_DISABLED); + result = store->ConvertStatus(DBStatus::CLOUD_SYNC_TASK_MERGED); + EXPECT_EQ(result, GeneralError::E_SYNC_TASK_MERGED); } /** diff --git a/datamgr_service/services/distributeddataservice/service/test/udmf_run_time_store_test.cpp b/datamgr_service/services/distributeddataservice/service/test/udmf_run_time_store_test.cpp index ec7fb210..d8d440cf 100644 --- a/datamgr_service/services/distributeddataservice/service/test/udmf_run_time_store_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/udmf_run_time_store_test.cpp @@ -25,11 +25,10 @@ #include "kvstore_meta_manager.h" #include "metadata/meta_data_manager.h" #include "nativetoken_kit.h" +#include "preprocess_utils.h" +#include "runtime_store.h" #include "text.h" #include "token_setproc.h" -#define private public -#include "store/runtime_store.h" -#undef private using namespace testing::ext; using namespace OHOS::DistributedData; @@ -501,7 +500,7 @@ HWTEST_F(UdmfRunTimeStoreTest, GetDetailsFromUData, TestSize.Level1) details1.insert({ "udmf_key", "udmf_value" }); auto records = data1.GetRecords(); EXPECT_EQ(records.size(), 0); - status = store->GetDetailsFromUData(data1, details1); + status = PreProcessUtils::GetDetailsFromUData(data1, details1); EXPECT_FALSE(status); status = store->Delete(KEY_PREFIX); @@ -548,7 +547,7 @@ HWTEST_F(UdmfRunTimeStoreTest, GetDetailsFromUData01, TestSize.Level1) auto outputRecords = outputData.GetRecords(); ASSERT_EQ(inputRecords.size(), 512); ASSERT_EQ(0, outputRecords.size()); - status = store->GetDetailsFromUData(inputData, details1); + status = PreProcessUtils::GetDetailsFromUData(inputData, details1); EXPECT_FALSE(status); } @@ -572,7 +571,8 @@ HWTEST_F(UdmfRunTimeStoreTest, GetSummary, TestSize.Level1) data.AddRecord(text); Summary summary; - auto status = store->GetSummary(KEY_PREFIX, summary); + UnifiedKey key(KEY_PREFIX); + auto status = store->GetSummary(key, summary); ASSERT_EQ(status, E_DB_ERROR); } }; // namespace DistributedDataTest diff --git a/datamgr_service/services/distributeddataservice/service/test/udmf_service_impl_test.cpp b/datamgr_service/services/distributeddataservice/service/test/udmf_service_impl_test.cpp index 2b06a261..d93a5590 100644 --- a/datamgr_service/services/distributeddataservice/service/test/udmf_service_impl_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/udmf_service_impl_test.cpp @@ -197,4 +197,47 @@ HWTEST_F(UdmfServiceImplTest, SetAppShareOption004, TestSize.Level1) int32_t ret = udmfServiceImpl.SetAppShareOption(intention, shareOption); EXPECT_EQ(ret, E_INVALID_PARAMETERS); } + +/** +* @tc.name: OnUserChangeTest001 +* @tc.desc: OnUserChange test +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceImplTest, OnUserChangeTest001, TestSize.Level1) +{ + uint32_t code = 4; + std::string user = "OH_USER_test"; + std::string account = "OH_ACCOUNT_test"; + UdmfServiceImpl udmfServiceImpl; + auto status = udmfServiceImpl.OnUserChange(code, user, account); + ASSERT_EQ(status, E_OK); + auto sizeAfter = StoreCache::GetInstance().stores_.Size(); + ASSERT_EQ(sizeAfter, 0); +} + +/** +* @tc.name: TransferToEntriesIfNeedTest001 +* @tc.desc: TransferToEntriesIfNeed test +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceImplTest, TransferToEntriesIfNeedTest001, TestSize.Level1) +{ + UnifiedData data; + QueryOption query; + auto record1 = std::make_shared(); + auto record2 = std::make_shared(); + data.AddRecord(record1); + data.AddRecord(record2); + auto properties = std::make_shared(); + properties->tag = "records_to_entries_data_format"; + data.SetProperties(properties); + query.tokenId = 1; + UdmfServiceImpl udmfServiceImpl; + udmfServiceImpl.TransferToEntriesIfNeed(query, data); + EXPECT_TRUE(data.IsNeedTransferToEntries()); + int recordSize = 2; + EXPECT_EQ(data.GetRecords().size(), recordSize); +} }; // namespace UDMF \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_mock_test.cpp new file mode 100644 index 00000000..e9935eab --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_mock_test.cpp @@ -0,0 +1,258 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#define LOG_TAG "UdmfServiceStubMockTest" +#include "udmf_service_stub.h" +#include "udmf_service_impl.h" +#include "udmf_types_util.h" +#include "gtest/gtest.h" + +using namespace OHOS::DistributedData; +namespace OHOS::UDMF { +using namespace testing::ext; +class UdmfServiceStubMockTest : public testing::Test { +public: + static void SetUpTestCase(void) {} + static void TearDownTestCase(void) {} + void SetUp() {} + void TearDown() {} +}; + +/** +* @tc.name: OnRemoteRequest001 +* @tc.desc: Abnormal test of OnRemoteRequest, code is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnRemoteRequest001, TestSize.Level1) +{ + uint32_t code = 17; // invalid code + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnRemoteRequest(code, data, reply); + EXPECT_EQ(ret, -1); +} + +/** +* @tc.name: OnSetData001 +* @tc.desc: Abnormal test of OnSetData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnSetData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnSetData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetData001 +* @tc.desc: Abnormal test of OnGetData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnGetData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetBatchData001 +* @tc.desc: Abnormal test of OnGetBatchData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnGetBatchData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetBatchData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnUpdateData001 +* @tc.desc: Abnormal test of OnUpdateData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnUpdateData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnUpdateData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnDeleteData001 +* @tc.desc: Abnormal test of OnDeleteData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnDeleteData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnDeleteData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetSummary001 +* @tc.desc: Abnormal test of OnGetSummary, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnGetSummary001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetSummary(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnAddPrivilege001 +* @tc.desc: Abnormal test of OnAddPrivilege, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnAddPrivilege001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnAddPrivilege(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnIsRemoteData001 +* @tc.desc: Abnormal test of OnIsRemoteData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnIsRemoteData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnIsRemoteData(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnSetAppShareOption001 +* @tc.desc: Abnormal test of OnSetAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnSetAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnSetAppShareOption(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetAppShareOption001 +* @tc.desc: Abnormal test of OnGetAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnGetAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetAppShareOption(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name: OnRemoveAppShareOption001 +* @tc.desc: Abnormal test of OnRemoveAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnRemoveAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnRemoveAppShareOption(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name:OnObtainAsynProcess001 +* @tc.desc: Abnormal test of OnObtainAsynProcess, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnObtainAsynProcess001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnObtainAsynProcess(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name:OnClearAsynProcessByKey001 +* @tc.desc: Abnormal test of OnClearAsynProcessByKey, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnClearAsynProcessByKey001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnClearAsynProcessByKey(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} + +/** +* @tc.name:OnSync001 +* @tc.desc: Abnormal test of OnSync, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubMockTest, OnSync001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnSync(data, reply); + EXPECT_EQ(ret, E_WRITE_PARCEL_ERROR); +} +}; // namespace UDMF \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_test.cpp b/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_test.cpp new file mode 100644 index 00000000..867b8d0a --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/udmf_service_stub_test.cpp @@ -0,0 +1,243 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#define LOG_TAG "UdmfServiceStubTest" +#include "udmf_service_stub.h" +#include "udmf_service_impl.h" +#include "udmf_types_util.h" +#include "gtest/gtest.h" + +using namespace OHOS::DistributedData; +namespace OHOS::UDMF { +using namespace testing::ext; +class UdmfServiceStubTest : public testing::Test { +public: + static void SetUpTestCase(void) {} + static void TearDownTestCase(void) {} + void SetUp() {} + void TearDown() {} +}; + +/** +* @tc.name: OnRemoteRequest001 +* @tc.desc: Abnormal test of OnRemoteRequest, code is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnRemoteRequest001, TestSize.Level1) +{ + uint32_t code = 17; // invalid code + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnRemoteRequest(code, data, reply); + EXPECT_EQ(ret, -1); +} + +/** +* @tc.name: OnSetData001 +* @tc.desc: Abnormal test of OnSetData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnSetData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnSetData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetData001 +* @tc.desc: Abnormal test of OnGetData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnGetData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetBatchData001 +* @tc.desc: Abnormal test of OnGetBatchData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnGetBatchData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetBatchData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnUpdateData001 +* @tc.desc: Abnormal test of OnUpdateData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnUpdateData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnUpdateData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnDeleteData001 +* @tc.desc: Abnormal test of OnDeleteData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnDeleteData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnDeleteData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetSummary001 +* @tc.desc: Abnormal test of OnGetSummary, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnGetSummary001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetSummary(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnAddPrivilege001 +* @tc.desc: Abnormal test of OnAddPrivilege, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnAddPrivilege001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnAddPrivilege(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnIsRemoteData001 +* @tc.desc: Abnormal test of OnIsRemoteData, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnIsRemoteData001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnIsRemoteData(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnSetAppShareOption001 +* @tc.desc: Abnormal test of OnSetAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnSetAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnSetAppShareOption(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnGetAppShareOption001 +* @tc.desc: Abnormal test of OnGetAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnGetAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnGetAppShareOption(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name: OnRemoveAppShareOption001 +* @tc.desc: Abnormal test of OnRemoveAppShareOption, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnRemoveAppShareOption001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnRemoveAppShareOption(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name:OnObtainAsynProcess001 +* @tc.desc: Abnormal test of OnObtainAsynProcess, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnObtainAsynProcess001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnObtainAsynProcess(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} + +/** +* @tc.name:OnClearAsynProcessByKey001 +* @tc.desc: Abnormal test of OnClearAsynProcessByKey, data is invalid +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UdmfServiceStubTest, OnClearAsynProcessByKey001, TestSize.Level1) +{ + MessageParcel data; + MessageParcel reply; + UdmfServiceImpl udmfServiceImpl; + int ret = udmfServiceImpl.OnClearAsynProcessByKey(data, reply); + EXPECT_EQ(ret, E_READ_PARCEL_ERROR); +} +}; // namespace UDMF \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/upgrade_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/upgrade_mock_test.cpp new file mode 100644 index 00000000..2a45df9c --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/upgrade_mock_test.cpp @@ -0,0 +1,132 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include +#include +#include "upgrade.h" +#include "access_token_mock.h" +#include "kv_store_delegate_manager.h" +#include "kv_store_nb_delegate_mock.h" + +using namespace OHOS::DistributedKv; +using namespace testing::ext; +using namespace testing; +using namespace std; +using namespace OHOS::Security::AccessToken; +using namespace DistributedDB; + +KvStoreNbDelegateMock g_nbDelegateMock; + +DBStatus KvStoreDelegateManager::CloseKvStore(KvStoreDelegate *kvStore) +{ + return DBStatus::NOT_SUPPORT; +} + +void KvStoreDelegateManager::GetKvStore(const std::string &storeId, const KvStoreNbDelegate::Option &option, + const std::function &callback) +{ + DBStatus status = DBStatus::OK; + if (callback) { + callback(status, &g_nbDelegateMock); + } + return; +} + +namespace OHOS::Test { +namespace DistributedDataTest { +using DBStatus = DistributedDB::DBStatus; +namespace { +std::string ExporterPwd(const DistributedKv::Upgrade::StoreMeta &storeMeta, + DistributedKv::Upgrade::DBPassword &pwd) +{ + if (storeMeta.user.empty()) { + return ""; + } + return "testIt"; +} + +Status cleanData(const DistributedKv::Upgrade::StoreMeta &) +{ + return Status::SUCCESS; +} +} + +class UpgradeMockTest : public testing::Test { +public: + using StoreMeta = OHOS::DistributedData::StoreMetaData; + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp() {} + void TearDown() {} + static inline shared_ptr accessTokenKitMock = nullptr; +}; + +void UpgradeMockTest::SetUpTestCase(void) +{ + accessTokenKitMock = make_shared(); + BAccessTokenKit::accessTokenkit = accessTokenKitMock; +} + +void UpgradeMockTest::TearDownTestCase(void) +{ + BAccessTokenKit::accessTokenkit = nullptr; + accessTokenKitMock = nullptr; +} + +/** +* @tc.name: UpgradeMockTest001 +* @tc.desc: Get encrypted uuid by Meta. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UpgradeMockTest, UpgradeMockTest001, TestSize.Level0) +{ + StoreMeta meta; + meta.appId = "UpgradeMockTest"; + meta.deviceId = ""; + EXPECT_CALL(*accessTokenKitMock, GetTokenTypeFlag(_)).WillOnce(Return(TOKEN_INVALID)); + std::string uuid = Upgrade::GetInstance().GetEncryptedUuidByMeta(meta); + EXPECT_TRUE(uuid.empty()); + + meta.appId = "UpgradeTest"; + meta.deviceId = ""; + EXPECT_CALL(*accessTokenKitMock, GetTokenTypeFlag(_)).WillRepeatedly(Return(TOKEN_HAP)); + uuid = Upgrade::GetInstance().GetEncryptedUuidByMeta(meta); + EXPECT_TRUE(uuid.empty()); +} + +/** +* @tc.name: UpgradeMockTest002 +* @tc.desc: Get encrypted uuid by Meta. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UpgradeMockTest, UpgradeMockTest002, TestSize.Level0) +{ + DistributedKv::Upgrade::StoreMeta old; + old.version = StoreMeta::UUID_CHANGED_TAG + 1, + old.dataDir = "/data/user/test"; + old.user = ""; + DistributedKv::Upgrade::StoreMeta meta; + meta.dataDir = "/data/user"; + std::vector pwd; + bool ret = Upgrade::GetInstance().RegisterExporter(0, ExporterPwd); + ret = Upgrade::GetInstance().RegisterCleaner(0, cleanData) && ret; + EXPECT_TRUE(ret); + Upgrade::DBStatus status = Upgrade::GetInstance().UpdateStore(old, meta, pwd); + EXPECT_TRUE(status == Upgrade::DBStatus::NOT_FOUND); +} +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/user_delegate_mock_test.cpp b/datamgr_service/services/distributeddataservice/service/test/user_delegate_mock_test.cpp new file mode 100644 index 00000000..373782b7 --- /dev/null +++ b/datamgr_service/services/distributeddataservice/service/test/user_delegate_mock_test.cpp @@ -0,0 +1,151 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include +#include +#include "user_delegate.h" +#include "device_manager_adapter_mock.h" +#include "account_delegate_mock.h" +#include "metadata/meta_data_manager.h" + +using namespace OHOS::DistributedData; +using namespace testing::ext; +using namespace testing; +using namespace std; +bool MetaDataManager::Subscribe(std::string prefix, Observer observer, bool isLocal) +{ + if (observer) { + for (int i = 0; i <= (MetaDataManager::DELETE + 1); ++i) { + static int32_t flag = MetaDataManager::INSERT; + static std::string key = prefix + "1"; + static std::string value = ""; + observer(key, value, flag); + ++flag; + } + return true; + } + return false; +} + +namespace OHOS::Test { +namespace DistributedDataTest { +class UserDelegateMockTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp() {} + void TearDown() {} + static inline shared_ptr devMgrAdapterMock = nullptr; +}; + +void UserDelegateMockTest::SetUpTestCase(void) +{ + devMgrAdapterMock = make_shared(); + BDeviceManagerAdapter::deviceManagerAdapter = devMgrAdapterMock; +} + +void UserDelegateMockTest::TearDownTestCase(void) +{ + BDeviceManagerAdapter::deviceManagerAdapter = nullptr; + devMgrAdapterMock = nullptr; +} + +/** +* @tc.name: GetLocalUserStatus +* @tc.desc: test GetLocalUserStatus normal branch. +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UserDelegateMockTest, GetLocalUserStatus, TestSize.Level0) +{ + DeviceInfo devInfo = { .uuid = "AFAGA45WF3663FAGA" }; + EXPECT_CALL(*devMgrAdapterMock, GetLocalDevice()).WillOnce(Return(devInfo)); + std::vector vec = UserDelegate::GetInstance().GetLocalUserStatus(); + EXPECT_TRUE(vec.empty()); +} + +/** +* @tc.name: InitLocalUserMeta +* @tc.desc: test InitLocalUserMeta +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UserDelegateMockTest, InitLocalUserMeta, TestSize.Level0) +{ + EXPECT_CALL(AccountDelegateMock::Init(), QueryUsers(_)).WillOnce(Return(false)); + bool ret = UserDelegate::GetInstance().InitLocalUserMeta(); + EXPECT_FALSE(ret); + + std::vector users; + EXPECT_CALL(AccountDelegateMock::Init(), QueryUsers(_)) + .WillRepeatedly(DoAll(SetArgReferee<0>(users), Return(true))); + ret = UserDelegate::GetInstance().InitLocalUserMeta(); + UserDelegate::GetInstance().DeleteUsers("users"); + EXPECT_FALSE(ret); +} + +/** +* @tc.name: GetLocalUsers +* @tc.desc: test GetLocalUsers +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UserDelegateMockTest, GetLocalUsers, TestSize.Level0) +{ + DeviceInfo devInfo = { .uuid = "TESTGHJJ46785FAGA9323FGAGATEST" }; + EXPECT_CALL(*devMgrAdapterMock, GetLocalDevice()).WillOnce(Return(devInfo)); + std::map value; + bool ret = UserDelegate::GetInstance().deviceUser_.Insert(devInfo.uuid, value); + EXPECT_TRUE(ret); + std::set user = UserDelegate::GetInstance().GetLocalUsers(); + EXPECT_TRUE(user.empty()); +} + +/** +* @tc.name: GetUsers +* @tc.desc: test GetUsers +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UserDelegateMockTest, GetUsers, TestSize.Level0) +{ + std::string deviceId = "45677afatghfrttWISKKM"; + std::map val; + bool ret = UserDelegate::GetInstance().deviceUser_.Insert(deviceId, val); + EXPECT_TRUE(ret); + std::vector statusVec = UserDelegate::GetInstance().GetUsers(deviceId); + EXPECT_TRUE(statusVec.empty()); +} + +/** +* @tc.name: Init +* @tc.desc: test Init +* @tc.type: FUNC +* @tc.require: +*/ +HWTEST_F(UserDelegateMockTest, Init, TestSize.Level0) +{ + EXPECT_CALL(AccountDelegateMock::Init(), Subscribe(_)).WillRepeatedly(Return(0)); + DeviceInfo devInfo = { .uuid = "HJJ4FGAGAAGA45WF3663FAGA" }; + EXPECT_CALL(*devMgrAdapterMock, GetLocalDevice()).WillRepeatedly(Return(devInfo)); + std::shared_ptr poolPtr = std::make_shared(0, 1); + ASSERT_NE(poolPtr, nullptr); + UserDelegate::GetInstance().executors_ = poolPtr; + ASSERT_NE(UserDelegate::GetInstance().executors_, nullptr); + UserDelegate::GetInstance().Init(poolPtr); + ASSERT_TRUE(UserDelegate::GetInstance().executors_ != nullptr); +} +} +} \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/test/value_proxy_test.cpp b/datamgr_service/services/distributeddataservice/service/test/value_proxy_test.cpp index 8ada4d4c..66deaa95 100644 --- a/datamgr_service/services/distributeddataservice/service/test/value_proxy_test.cpp +++ b/datamgr_service/services/distributeddataservice/service/test/value_proxy_test.cpp @@ -362,4 +362,103 @@ HWTEST_F(ValueProxyServiceTest, TempAssetConvertToDataStatus, TestSize.Level0) result = ValueProxy::TempAsset::ConvertToDataStatus(status); EXPECT_NE(result, DistributedData::Asset::STATUS_NORMAL); } + +/** +* @tc.name: AssetsOperator001 +* @tc.desc: AssetsOperator test. +* @tc.type: FUNC +*/ +HWTEST_F(ValueProxyServiceTest, AssetsOperator001, TestSize.Level0) +{ + ValueProxy::Assets proxy1; + ValueProxy::Assets proxy2; + ValueProxy::Assets proxy3; + DistributedDB::Asset asset; + asset.version = 1; + asset.status = DistributedData::Asset::Status::STATUS_INSERT; + asset.name = "Asset1"; + asset.uri = "uri1"; + asset.size = "1"; + asset.hash = "hash1"; + proxy1.assets_.push_back(asset); + + asset.version = 2; + asset.status = DistributedData::Asset::Status::STATUS_NORMAL; + asset.name = "Asset2"; + asset.uri = "uri2"; + asset.size = "2"; + asset.hash = "hash2"; + proxy2.assets_.push_back(asset); + + asset.version = 3; + asset.status = DistributedData::Asset::Status::STATUS_NORMAL; + asset.name = "Asset3"; + asset.uri = "uri3"; + asset.size = "3"; + asset.hash = "hash3"; + proxy2.assets_.push_back(asset); + // operator "=" same asset case + proxy3 = proxy1; + proxy1 = proxy3; + + // operator "=" different aseet case + proxy1 = proxy2; + EXPECT_EQ(proxy1.assets_.size(), 2); + + // operator Distributeddata Asset() + asset = proxy1.assets_[0]; + EXPECT_EQ(asset.version, 2); + + // operator "=" noexcept same asset case + proxy2 = std::move(proxy1); + // operator "=" noexcept different asset case + proxy1 = std::move(proxy3); + EXPECT_EQ(proxy1.assets_.size(), 1); + asset = proxy1.assets_[0]; + EXPECT_EQ(asset.version, 1); +} + +/** +* @tc.name: AssetOperator001 +* @tc.desc: AssetOperator test. +* @tc.type: FUNC +*/ +HWTEST_F(ValueProxyServiceTest, AssetOperator001, TestSize.Level0) +{ + ValueProxy::Asset asset1 = DistributedDB::Asset { + .version = 1, + .name = "Asset1", + .uri = "uri1", + .size = "1", + .hash = "hash1", + .status = DistributedData::Asset::Status::STATUS_INSERT, + }; + ValueProxy::Asset asset2 = DistributedDB::Asset { + .version = 2, + .name = "Asset2", + .uri = "uri2", + .size = "2", + .hash = "hash2", + .status = DistributedData::Asset::Status::STATUS_NORMAL, + }; + ValueProxy::Asset asset3 = asset1; + DistributedData::Asset asset = asset3; + EXPECT_EQ(asset.version, 1); + + // operator "=" same asset case + asset1 = asset3; + EXPECT_EQ(asset.version, 1); + // operator "=" different aseet case + asset1 = asset2; + asset = asset1; + EXPECT_EQ(asset.version, 2); + + // operator "=" noexcept same asset case + asset2 = std::move(asset1); + EXPECT_EQ(asset.version, 2); + // operator "=" noexcept different asset case + asset1 = std::move(asset3); + asset = asset1; + EXPECT_EQ(asset.version, 1); +} } // namespace OHOS::Test \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/BUILD.gn b/datamgr_service/services/distributeddataservice/service/udmf/BUILD.gn index 15ae2788..971ab272 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/BUILD.gn +++ b/datamgr_service/services/distributeddataservice/service/udmf/BUILD.gn @@ -18,9 +18,9 @@ config("module_public_config") { include_dirs = [ "${data_service_path}/framework/include", + "${data_service_path}/framework/include/dfx", "${data_service_path}/adapter/include/communicator", "${data_service_path}/adapter/include/account", - "${data_service_path}/adapter/include/dfx", "${data_service_path}/service/udmf/lifecycle", "${data_service_path}/service/udmf/permission", "${data_service_path}/service/udmf/preprocess", @@ -57,7 +57,6 @@ ohos_source_set("udmf_server") { "permission/uri_permission_manager.cpp", "preprocess/data_handler.cpp", "preprocess/preprocess_utils.cpp", - "preprocess/udmf_dialog.cpp", "store/runtime_store.cpp", "store/store_account_observer.cpp", "store/store_cache.cpp", @@ -71,25 +70,18 @@ ohos_source_set("udmf_server") { "-D_LIBCPP_HAS_COND_CLOCKWAIT", "-Werror", "-Oz", + "-fstack-protector-strong", ] - deps = [ - "${data_service_path}/framework:distributeddatasvcfwk", - "${data_service_path}/service/bootstrap:distributeddata_bootstrap", - ] - defines = [] + deps = [ "${data_service_path}/framework:distributeddatasvcfwk" ] external_deps = [ "ability_base:zuri", - "ability_runtime:ability_manager", "ability_runtime:uri_permission_mgr", - "ability_runtime:wantagent_innerkits", - "access_token:libaccesstoken_sdk", "access_token:libtokenid_sdk", "app_file_service:remote_file_share_native", "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", - "c_utils:utils", "device_manager:devicemanagersdk", "hilog:libhilog", "hisysevent:libhisysevent", @@ -97,23 +89,15 @@ ohos_source_set("udmf_server") { "hitrace:libhitracechain", "ipc:ipc_core", "kv_store:distributeddb", - "safwk:system_ability_fwk", - "samgr:samgr_proxy", "udmf:udmf_client", "udmf:utd_client", ] cflags_cc = [ "-fvisibility=hidden", "-Oz", + "-fstack-protector-strong", ] subsystem_name = "distributeddatamgr" part_name = "datamgr_service" - - if (window_manager_use_sceneboard) { - external_deps += [ "window_manager:libwm_lite" ] - defines += [ "SCENE_BOARD_ENABLE" ] - } else { - external_deps += [ "window_manager:libwm" ] - } } diff --git a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.cpp b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.cpp index c7560122..cbfdca07 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.cpp @@ -16,9 +16,6 @@ #include "lifecycle_manager.h" -#include -#include - #include "log_print.h" namespace OHOS { @@ -39,7 +36,7 @@ Status LifeCycleManager::OnGot(const UnifiedKey &key) { auto findPolicy = intentionPolicy_.find(key.intention); if (findPolicy == intentionPolicy_.end()) { - ZLOGE("Invalid intention, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Invalid intention:%{public}s", key.intention.c_str()); return E_INVALID_PARAMETERS; } auto policy = findPolicy->second; @@ -47,7 +44,7 @@ Status LifeCycleManager::OnGot(const UnifiedKey &key) policy->OnGot(key); }); if (taskId == ExecutorPool::INVALID_TASK_ID) { - ZLOGE("OnGot task execute failed."); + ZLOGE("Task execution failed"); return E_ERROR; } return E_OK; diff --git a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.h b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.h index b1420ae2..1f477cd3 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_manager.h @@ -15,15 +15,7 @@ #ifndef UDMF_LIFECYCLE_MANAGER_H #define UDMF_LIFECYCLE_MANAGER_H -#include -#include -#include -#include -#include - -#include "clean_on_startup.h" #include "clean_on_timeout.h" -#include "executor_pool.h" namespace OHOS { namespace UDMF { diff --git a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.cpp b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.cpp index 2749c3e5..cc3d8865 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.cpp @@ -27,11 +27,11 @@ Status LifeCyclePolicy::OnGot(const UnifiedKey &key) { auto store = StoreCache::GetInstance().GetStore(key.intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Get store failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } if (store->Delete(key.key) != E_OK) { - ZLOGE("Remove data failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Remove data failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -41,11 +41,11 @@ Status LifeCyclePolicy::OnStart(const std::string &intention) { auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Store get failed:%{public}s", intention.c_str()); return E_DB_ERROR; } if (store->Clear() != E_OK) { - ZLOGE("Remove data failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Data removal failed:%{public}s", intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -55,17 +55,17 @@ Status LifeCyclePolicy::OnTimeout(const std::string &intention) { auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Store get failed:%{public}s", intention.c_str()); return E_DB_ERROR; } std::vector timeoutKeys; auto status = GetTimeoutKeys(store, INTERVAL, timeoutKeys); if (status != E_OK) { - ZLOGE("Get timeout keys failed."); + ZLOGE("Timeout keys get failed"); return E_DB_ERROR; } if (store->DeleteBatch(timeoutKeys) != E_OK) { - ZLOGE("Remove data failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Data removal failed:%{public}s", intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -87,7 +87,7 @@ Status LifeCyclePolicy::GetTimeoutKeys( auto curTime = PreProcessUtils::GetTimestamp(); for (const auto &data : datas) { if (data.GetRuntime() == nullptr) { - ZLOGD("Runtime data is null."); + ZLOGD("Runtime is null"); return E_DB_ERROR; } if (curTime > data.GetRuntime()->createTime + duration_cast(interval).count() diff --git a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.h b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.h index ed3fd63f..f7703e56 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/lifecycle/lifecycle_policy.h @@ -15,12 +15,7 @@ #ifndef UDMF_LIFECYCLE_POLICY_H #define UDMF_LIFECYCLE_POLICY_H -#include -#include -#include - #include "store_cache.h" -#include "unified_key.h" namespace OHOS { namespace UDMF { diff --git a/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.cpp b/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.cpp index cfb61bf0..a90966a6 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.cpp @@ -17,7 +17,7 @@ namespace OHOS { namespace UDMF { -const std::string DATA_CHECKER = "DataChecker"; +constexpr const char *DATA_CHECKER = "DataChecker"; CheckerManager &CheckerManager::GetInstance() { static CheckerManager instance; diff --git a/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.h b/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.h index 5be1f2ca..bb892e15 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/permission/checker_manager.h @@ -16,8 +16,6 @@ #ifndef UDMF_CHECKER_MANAGER_H #define UDMF_CHECKER_MANAGER_H -#include - #include "concurrent_map.h" #include "unified_types.h" diff --git a/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.cpp b/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.cpp index 29dc7132..1e7e1e90 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.cpp @@ -19,7 +19,6 @@ #include "log_print.h" #include "preprocess_utils.h" #include "uri_permission_manager_client.h" -#include "want.h" namespace OHOS { namespace UDMF { @@ -35,12 +34,12 @@ Status UriPermissionManager::GrantUriPermission(const std::vector &allUri, { std::string bundleName; if (!PreProcessUtils::GetHapBundleNameByToken(tokenId, bundleName)) { - ZLOGE("Get BundleName fail, key:%{public}s, tokenId:%{public}u.", queryKey.c_str(), tokenId); + ZLOGE("BundleName get failed:%{public}s,tokenId:%{public}u", queryKey.c_str(), tokenId); return E_ERROR; } int32_t instIndex = -1; if (!PreProcessUtils::GetInstIndex(tokenId, instIndex)) { - ZLOGE("Get InstIndex fail, key:%{public}s, tokenId:%{public}u.", queryKey.c_str(), tokenId); + ZLOGE("InstIndex get failed:%{public}s,tokenId:%{public}u", queryKey.c_str(), tokenId); return E_ERROR; } diff --git a/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.h b/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.h index 4cbc7812..edc1758c 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/permission/uri_permission_manager.h @@ -16,12 +16,7 @@ #ifndef UDMF_URI_PERMISSION_MANAGER_H #define UDMF_URI_PERMISSION_MANAGER_H -#include -#include - -#include "concurrent_map.h" #include "error_code.h" -#include "executor_pool.h" #include "uri.h" namespace OHOS { diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.cpp b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.cpp index 273ae73d..d772d999 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.cpp @@ -16,12 +16,12 @@ #include "data_handler.h" #include "log_print.h" -#include "tlv_object.h" #include "tlv_util.h" namespace OHOS::UDMF { constexpr const char *UD_KEY_SEPARATOR = "/"; constexpr const char *UD_KEY_ENTRY_SEPARATOR = "#"; +constexpr const char *UD_KEY_PROPERTIES_SEPARATOR = "#properties"; Status DataHandler::MarshalToEntries(const UnifiedData &unifiedData, std::vector &entries) { @@ -29,12 +29,22 @@ Status DataHandler::MarshalToEntries(const UnifiedData &unifiedData, std::vector std::vector runtimeBytes; auto runtimeTlv = TLVObject(runtimeBytes); if (!TLVUtil::Writing(*unifiedData.GetRuntime(), runtimeTlv, TAG::TAG_RUNTIME)) { - ZLOGE("Marshall runtime info failed, dataPrefix: %{public}s.", unifiedKey.c_str()); + ZLOGE("Runtime info marshalling failed:%{public}s", unifiedKey.c_str()); return E_WRITE_PARCEL_ERROR; } std::vector udKeyBytes = { unifiedKey.begin(), unifiedKey.end() }; Entry entry = { udKeyBytes, runtimeBytes }; entries.emplace_back(entry); + std::string propsKey = unifiedData.GetRuntime()->key.GetPropertyKey() + UD_KEY_PROPERTIES_SEPARATOR; + std::vector propsBytes; + auto propsTlv = TLVObject(propsBytes); + if (!TLVUtil::Writing(*unifiedData.GetProperties(), propsTlv, TAG::TAG_PROPERTIES)) { + ZLOGE("Properties info marshalling failed:%{public}s", propsKey.c_str()); + return E_WRITE_PARCEL_ERROR; + } + std::vector propsKeyBytes = { propsKey.begin(), propsKey.end() }; + Entry propsEntry = { propsKeyBytes, propsBytes }; + entries.emplace_back(std::move(propsEntry)); return BuildEntries(unifiedData.GetRecords(), unifiedKey, entries); } @@ -76,7 +86,14 @@ Status DataHandler::UnmarshalEntryItem(UnifiedData &unifiedData, const std::vect continue; } auto isStartWithKey = keyStr.find(key) == 0; - if (!isStartWithKey) { + std::string propsKey = UnifiedKey(key).GetPropertyKey() + UD_KEY_PROPERTIES_SEPARATOR; + if (!isStartWithKey && (keyStr == propsKey)) { + std::shared_ptr properties; + if (!TLVUtil::ReadTlv(properties, data, TAG::TAG_PROPERTIES)) { + ZLOGE("Unmarshall unified properties failed."); + return E_READ_PARCEL_ERROR; + } + unifiedData.SetProperties(std::move(properties)); continue; } auto isEntryItem = keyStr.rfind(UD_KEY_ENTRY_SEPARATOR) != std::string::npos; diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.h b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.h index c6b3c94e..b3e89721 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/data_handler.h @@ -15,8 +15,8 @@ #ifndef DATA_HANDLER_H #define DATA_HANDLER_H -#include "error_code.h" #include "unified_data.h" +#include "tlv_util.h" #include "types_export.h" namespace OHOS::UDMF { @@ -27,6 +27,10 @@ public: static Status MarshalToEntries(const UnifiedData &unifiedData, std::vector &entries); static Status UnmarshalEntries(const std::string &key, const std::vector &entries, UnifiedData &unifiedData); + template + static Status MarshalToEntries(const T &data, Value &value, TAG tag); + template + static Status UnmarshalEntries(const Value &value, T &data, TAG tag); private: static Status BuildEntries(const std::vector> &records, @@ -35,5 +39,25 @@ private: const std::string &key, std::map> &records, std::map> &innerEntries); }; + +template +Status DataHandler::MarshalToEntries(const T &data, Value &value, TAG tag) +{ + auto tlvObject = TLVObject(value); + if (!TLVUtil::Writing(data, tlvObject, tag)) { + return E_WRITE_PARCEL_ERROR; + } + return E_OK; +} + +template +Status DataHandler::UnmarshalEntries(const Value &value, T &data, TAG tag) +{ + auto tlvObject = TLVObject(const_cast &>(value)); + if (!TLVUtil::ReadTlv(data, tlvObject, tag)) { + return E_READ_PARCEL_ERROR; + } + return E_OK; +} } // namespace UDMF::OHOS #endif // DATA_HANDLER_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.cpp b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.cpp index 4ce5d175..ab49f959 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.cpp @@ -16,23 +16,18 @@ #include "preprocess_utils.h" -#include +#include #include #include "dds_trace.h" #include "udmf_radar_reporter.h" #include "accesstoken_kit.h" -#include "bundlemgr/bundle_mgr_client_impl.h" #include "device_manager_adapter.h" -#include "error_code.h" -#include "ipc_skeleton.h" #include "log_print.h" #include "udmf_radar_reporter.h" #include "udmf_utils.h" #include "remote_file_share.h" -#include "uri.h" #include "utils/crypto.h" -#include "want.h" #include "uri_permission_manager_client.h" namespace OHOS { namespace UDMF { @@ -42,7 +37,15 @@ static constexpr int MAXIMUM = 121; constexpr char SPECIAL = '^'; constexpr const char *FILE_SCHEME = "file"; constexpr const char *TAG = "PreProcessUtils::"; +constexpr const char *FILE_SCHEME_PREFIX = "file://"; +constexpr const char *DOCS_LOCAL_TAG = "/docs/"; +static constexpr uint32_t DOCS_LOCAL_PATH_SUBSTR_START_INDEX = 1; static constexpr uint32_t VERIFY_URI_PERMISSION_MAX_SIZE = 500; +constexpr const char *TEMP_UNIFIED_DATA_FLAG = "temp_udmf_file_flag"; +static constexpr size_t TEMP_UDATA_RECORD_SIZE = 1; +static constexpr uint32_t PREFIX_LEN = 24; +static constexpr uint32_t INDEX_LEN = 8; +static constexpr const char PLACE_HOLDER = '0'; using namespace OHOS::DistributedDataDfx; using namespace Security::AccessToken; using namespace OHOS::AppFileService::ModuleRemoteFileShare; @@ -69,6 +72,7 @@ int32_t PreProcessUtils::RuntimeDataImputation(UnifiedData &data, CustomOption & runtime.deviceId = GetLocalDeviceId(); runtime.recordTotalNum = static_cast(data.GetRecords().size()); runtime.tokenId = option.tokenId; + runtime.sdkVersion = data.GetSdkVersion(); data.SetRuntime(runtime); return E_OK; } @@ -121,7 +125,7 @@ bool PreProcessUtils::GetHapBundleNameByToken(int tokenId, std::string &bundleNa return true; } } - ZLOGE("GetHapBundleNameByToken faild"); + ZLOGE("Get bundle name faild"); return false; } @@ -160,7 +164,7 @@ void PreProcessUtils::SetRemoteData(UnifiedData &data) std::shared_ptr detailObj; obj->GetValue(DETAILS, detailObj); if (detailObj == nullptr) { - ZLOGE("Not contain details for object!"); + ZLOGE("No details for object"); return false; } UDDetails details = ObjectUtils::ConvertToUDDetails(detailObj); @@ -189,23 +193,24 @@ int32_t PreProcessUtils::SetRemoteUri(uint32_t tokenId, UnifiedData &data) std::string oriUri; obj->GetValue(ORI_URI, oriUri); if (oriUri.empty()) { - ZLOGW("Get uri empty, plase check the uri."); + ZLOGW("URI is empty, please check"); return false; } Uri uri(oriUri); std::string scheme = uri.GetScheme(); std::transform(scheme.begin(), scheme.end(), scheme.begin(), ::tolower); if (uri.GetAuthority().empty() || scheme != FILE_SCHEME) { - ZLOGW("Get uri authority empty or uri scheme not equals to file."); + ZLOGW("Empty URI authority or scheme not file"); return false; } uris.push_back(oriUri); return true; }); + GetHtmlFileUris(tokenId, data, true, uris); if (!uris.empty()) { ZLOGI("Read to check uri authorization"); if (!CheckUriAuthorization(uris, tokenId)) { - ZLOGE("CheckUriAuthorization failed, bundleName:%{public}s, tokenId: %{public}d, uris size:%{public}zu.", + ZLOGE("UriAuth check failed:bundleName:%{public}s,tokenId:%{public}d,uris size:%{public}zu", data.GetRuntime()->createPackage.c_str(), tokenId, uris.size()); RadarReporterAdapter::ReportFail(std::string(__FUNCTION__), BizScene::SET_DATA, SetDataStage::VERIFY_SHARE_PERMISSIONS, StageRes::FAILED, E_NO_PERMISSION); @@ -248,6 +253,18 @@ int32_t PreProcessUtils::GetDfsUrisFromLocal(const std::vector &uri } return true; }); + for (auto &record : data.GetRecords()) { + if (record == nullptr) { + continue; + } + record->ComputeUris([&dfsUris] (UriInfo &uriInfo) { + auto iter = dfsUris.find(uriInfo.authUri); + if (iter != dfsUris.end()) { + uriInfo.dfsUri = (iter->second).uriStr; + } + return true; + }); + } RadarReporterAdapter::ReportNormal(std::string(__FUNCTION__), BizScene::SET_DATA, SetDataStage::GERERATE_DFS_URI, StageRes::SUCCESS, BizState::DFX_END); return E_OK; @@ -288,7 +305,7 @@ bool PreProcessUtils::IsNetworkingEnabled() { std::vector devInfos = DistributedData::DeviceManagerAdapter::GetInstance().GetRemoteDevices(); - ZLOGI("DM remote devices count is %{public}u.", static_cast(devInfos.size())); + ZLOGI("Remote devices count:%{public}u", static_cast(devInfos.size())); if (devInfos.empty()) { return false; } @@ -315,5 +332,144 @@ void PreProcessUtils::ProcessFileType(std::vector } } } + +void PreProcessUtils::ProcessRecord(std::shared_ptr record, uint32_t tokenId, + bool isLocal, std::vector &uris) +{ + record->ComputeUris([&uris, &isLocal, &tokenId] (UriInfo &uriInfo) { + std::string newUriStr = ""; + if (isLocal) { + Uri tmpUri(uriInfo.oriUri); + std::string path = tmpUri.GetPath(); + std::string bundleName; + if (!GetHapBundleNameByToken(tokenId, bundleName)) { + return true; + } + if (path.substr(0, strlen(DOCS_LOCAL_TAG)) == DOCS_LOCAL_TAG) { + newUriStr = FILE_SCHEME_PREFIX; + newUriStr += path.substr(DOCS_LOCAL_PATH_SUBSTR_START_INDEX); + } else { + newUriStr = FILE_SCHEME_PREFIX; + newUriStr += bundleName + path; + } + uriInfo.authUri = newUriStr; + } else { + newUriStr = uriInfo.dfsUri; + } + Uri uri(newUriStr); + if (uri.GetAuthority().empty()) { + return true; + } + std::string scheme = uri.GetScheme(); + std::transform(scheme.begin(), scheme.end(), scheme.begin(), ::tolower); + if (scheme != FILE_SCHEME) { + return true; + } + auto iter = std::find(uris.begin(), uris.end(), newUriStr); + if (iter == uris.end()) { + uris.push_back(std::move(newUriStr)); + } + return true; + }); +} + +void PreProcessUtils::GetHtmlFileUris(uint32_t tokenId, UnifiedData &data, + bool isLocal, std::vector &uris) +{ + for (auto &record : data.GetRecords()) { + if (record == nullptr) { + continue; + } + PreProcessUtils::ProcessRecord(record, tokenId, isLocal, uris); + } +} + +void PreProcessUtils::ClearHtmlDfsUris(UnifiedData &data) +{ + for (auto &record : data.GetRecords()) { + if (record == nullptr) { + continue; + } + record->ComputeUris([] (UriInfo &uriInfo) { + uriInfo.dfsUri = ""; + return true; + }); + } +} + +void PreProcessUtils::ProcessHtmlFileUris(uint32_t tokenId, UnifiedData &data, bool isLocal, std::vector &uris) +{ + std::vector strUris; + PreProcessUtils::GetHtmlFileUris(tokenId, data, isLocal, strUris); + for (auto &uri : strUris) { + uris.emplace_back(Uri(uri)); + } + if (isLocal) { + PreProcessUtils::ClearHtmlDfsUris(data); + } +} + +void PreProcessUtils::SetRecordUid(UnifiedData &data) +{ + uint32_t index = 0; + auto prefix = PreProcessUtils::GenerateId().substr(0, PREFIX_LEN); + for (const auto &record : data.GetRecords()) { + std::ostringstream oss; + oss << std::setw(INDEX_LEN) << std::setfill(PLACE_HOLDER) << index; + record->SetUid(prefix + oss.str()); + index++; + } +} + +bool PreProcessUtils::GetDetailsFromUData(const UnifiedData &data, UDDetails &details) +{ + auto records = data.GetRecords(); + if (records.size() != TEMP_UDATA_RECORD_SIZE) { + ZLOGE("Records size error.size:%{public}zu", records.size()); + return false; + } + if (records[0] == nullptr) { + ZLOGE("First record is null."); + return false; + } + if (records[0]->GetType() != UDType::FILE) { + ZLOGE("First record is not file."); + return false; + } + auto value = records[0]->GetOriginValue(); + auto obj = std::get_if>(&value); + if (obj == nullptr || *obj == nullptr) { + ZLOGE("ValueType is not Object!"); + return false; + } + std::shared_ptr detailObj; + (*obj)->GetValue(DETAILS, detailObj); + if (detailObj == nullptr) { + ZLOGE("Not contain details for object!"); + return false; + } + auto result = ObjectUtils::ConvertToUDDetails(detailObj); + if (result.find(TEMP_UNIFIED_DATA_FLAG) == result.end()) { + ZLOGE("Not find temp file."); + return false; + } + details = std::move(result); + return true; +} + +Status PreProcessUtils::GetSummaryFromDetails(const UDDetails &details, Summary &summary) +{ + for (auto &item : details) { + if (item.first == TEMP_UNIFIED_DATA_FLAG) { + continue; + } + auto int64Value = std::get_if(&item.second); + if (int64Value != nullptr) { + summary.summary[item.first] = *int64Value; + summary.totalSize += *int64Value; + } + } + return E_OK; +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.h b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.h index 9d3770b6..6d613007 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/preprocess_utils.h @@ -15,13 +15,7 @@ #ifndef UDMF_PREPROCESS_UTILS_H #define UDMF_PREPROCESS_UTILS_H -#include -#include -#include -#include - #include "unified_data.h" -#include "unified_meta.h" namespace OHOS { namespace UDMF { @@ -40,6 +34,14 @@ public: static bool IsNetworkingEnabled(); static void ProcessFileType(std::vector> records, std::function)> callback); + static void GetHtmlFileUris(uint32_t tokenId, UnifiedData &data, bool isLocal, std::vector &uris); + static void ClearHtmlDfsUris(UnifiedData &data); + static void ProcessHtmlFileUris(uint32_t tokenId, UnifiedData &data, bool isLocal, std::vector &uris); + static void ProcessRecord(std::shared_ptr record, uint32_t tokenId, + bool isLocal, std::vector &uris); + static void SetRecordUid(UnifiedData &data); + static bool GetDetailsFromUData(const UnifiedData &data, UDDetails &details); + static Status GetSummaryFromDetails(const UDDetails &details, Summary &summary); private: static bool CheckUriAuthorization(const std::vector& uris, uint32_t tokenId); static int32_t GetDfsUrisFromLocal(const std::vector &uris, int32_t userId, UnifiedData &data); diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.cpp b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.cpp deleted file mode 100644 index c95f78eb..00000000 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2025 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#define LOG_TAG "UdmfDialog" - -#include "udmf_dialog.h" - -#include "ability_connect_callback_stub.h" -#include "error_code.h" -#include "in_process_call_wrapper.h" -#include "iservice_registry.h" -#include "log_print.h" -#include "system_ability_definition.h" -#ifdef SCENE_BOARD_ENABLE -#include "window_manager_lite.h" -#else -#include "window_manager.h" -#endif - -namespace OHOS::UDMF { -using namespace OHOS::AAFwk; - -ProgressDialog &ProgressDialog::GetInstance() -{ - static ProgressDialog instance; - return instance; -} - -ProgressDialog::FocusedAppInfo ProgressDialog::GetFocusedAppInfo(void) const -{ - FocusedAppInfo appInfo = { 0 }; - Rosen::FocusChangeInfo info; -#ifdef SCENE_BOARD_ENABLE - Rosen::WindowManagerLite::GetInstance().GetFocusWindowInfo(info); -#else - Rosen::WindowManager::GetInstance().GetFocusWindowInfo(info); -#endif - appInfo.windowId = info.windowId_; - appInfo.abilityToken = info.abilityToken_; - return appInfo; -} - -sptr ProgressDialog::GetAbilityManagerService() -{ - auto systemAbilityManager = SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager(); - if (systemAbilityManager == nullptr) { - ZLOGE("Failed to get ability manager."); - return nullptr; - } - sptr remoteObject = systemAbilityManager->GetSystemAbility(ABILITY_MGR_SERVICE_ID); - if (!remoteObject) { - ZLOGE("Failed to get ability manager service."); - return nullptr; - } - return iface_cast(remoteObject); -} - -int32_t ProgressDialog::ShowProgress(const ProgressMessageInfo &message) -{ - ZLOGD("Begin."); - auto abilityManager = GetAbilityManagerService(); - if (abilityManager == nullptr) { - ZLOGE("Get ability manager failed."); - return E_ERROR; - } - - Want want; - want.SetElementName(PASTEBOARD_DIALOG_APP, PASTEBOARD_PROGRESS_ABILITY); - want.SetAction(PASTEBOARD_PROGRESS_ABILITY); - want.SetParam("promptText", message.promptText); - want.SetParam("remoteDeviceName", message.remoteDeviceName); - want.SetParam("progressKey", message.progressKey); - want.SetParam("isRemote", message.isRemote); - want.SetParam("windowId", message.windowId); - want.SetParam("ipcCallback", message.clientCallback); - if (message.callerToken != nullptr) { - want.SetParam("tokenKey", message.callerToken); - } else { - ZLOGW("CallerToken is nullptr."); - } - - int32_t status = IN_PROCESS_CALL(abilityManager->StartAbility(want)); - if (status != 0) { - ZLOGE("Start progress failed, status:%{public}d.", status); - } - return status; -} -} // namespace OHOS::UDMF \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.h b/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.h deleted file mode 100644 index 94a302fe..00000000 --- a/datamgr_service/services/distributeddataservice/service/udmf/preprocess/udmf_dialog.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2024 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef UDMF_DIALOG_H -#define UDMF_DIALOG_H - -#include "error_code.h" -#include "ability_manager_interface.h" -#include "refbase.h" - -namespace OHOS::UDMF { - -class ProgressDialog { -public: - struct FocusedAppInfo { - int32_t windowId = 0; - sptr abilityToken = nullptr; - }; - struct ProgressMessageInfo { - std::string promptText { DEFAULT_LABEL }; - std::string remoteDeviceName { DEFAULT_LABEL }; - std::string progressKey { DEFAULT_LABEL }; - bool isRemote { false }; - int32_t windowId { 0 }; - sptr callerToken { nullptr }; - sptr clientCallback { nullptr }; - }; - - static ProgressDialog &GetInstance(); - int32_t ShowProgress(const ProgressMessageInfo &message); - FocusedAppInfo GetFocusedAppInfo(void) const; - -private: - static sptr GetAbilityManagerService(); - - static constexpr const char *DEFAULT_LABEL = "unknown"; - static constexpr const char *PASTEBOARD_DIALOG_APP = "com.ohos.pasteboarddialog"; - static constexpr const char *PASTEBOARD_PROGRESS_ABILITY = "PasteboardProgressAbility"; -}; -} // namespace UDMF::OHOS -#endif // UDMF_DIALOG_H \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.cpp b/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.cpp index 40a580d9..d8742dc7 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.cpp @@ -16,26 +16,21 @@ #include "runtime_store.h" -#include -#include +#include #include -#include - #include "data_handler.h" #include "log_print.h" #include "ipc_skeleton.h" #include "unified_data_helper.h" #include "udmf_radar_reporter.h" -#include "unified_meta.h" -#include "tlv_util.h" #include "account/account_delegate.h" -#include "metadata/store_meta_data.h" #include "metadata/meta_data_manager.h" #include "metadata/appid_meta_data.h" #include "device_manager_adapter.h" #include "bootstrap.h" #include "directory/directory_manager.h" #include "utils/anonymous.h" +#include "preprocess_utils.h" namespace OHOS { namespace UDMF { @@ -43,9 +38,7 @@ using namespace RadarReporter; using namespace DistributedDB; using Anonymous = OHOS::DistributedData::Anonymous; using DmAdapter = OHOS::DistributedData::DeviceManagerAdapter; - -constexpr const char *TEMP_UNIFIED_DATA_FLAG = "temp_udmf_file_flag"; -static constexpr size_t TEMP_UDATA_RECORD_SIZE = 1; +constexpr const char *SUMMARY_SUFIX = "#summary"; RuntimeStore::RuntimeStore(const std::string &storeId) : storeId_(storeId) { @@ -107,6 +100,10 @@ Status RuntimeStore::Put(const UnifiedData &unifiedData) { UpdateTime(); std::vector entries; + std::string intention = unifiedData.GetRuntime()->key.intention; + if (intention == UD_INTENTION_MAP.at(UD_INTENTION_DRAG)) { + PutSummary(unifiedData, entries); + } auto status = DataHandler::MarshalToEntries(unifiedData, entries); if (status != E_OK) { return status; @@ -118,7 +115,7 @@ Status RuntimeStore::Get(const std::string &key, UnifiedData &unifiedData) { UpdateTime(); std::vector entries; - if (GetEntries(key, entries) != E_OK) { + if (GetEntries(UnifiedKey(key).GetPropertyKey(), entries) != E_OK) { ZLOGE("GetEntries failed, dataPrefix: %{public}s.", key.c_str()); return E_DB_ERROR; } @@ -129,63 +126,89 @@ Status RuntimeStore::Get(const std::string &key, UnifiedData &unifiedData) return DataHandler::UnmarshalEntries(key, entries, unifiedData); } -bool RuntimeStore::GetDetailsFromUData(UnifiedData &data, UDDetails &details) +Status RuntimeStore::PutSummary(const UnifiedData &data, std::vector &entries) { - auto records = data.GetRecords(); - if (records.size() != TEMP_UDATA_RECORD_SIZE) { - return false; - } - if (records[0] == nullptr || records[0]->GetType() != UDType::FILE) { - return false; - } - auto obj = std::get>(records[0]->GetOriginValue()); - if (obj == nullptr) { - ZLOGE("ValueType is not Object!"); - return false; + UpdateTime(); + UDDetails details {}; + Summary summary; + if (PreProcessUtils::GetDetailsFromUData(data, details)) { + PreProcessUtils::GetSummaryFromDetails(details, summary); + } else { + UnifiedDataHelper::GetSummary(data, summary); } - std::shared_ptr detailObj; - obj->GetValue(DETAILS, detailObj); - if (detailObj == nullptr) { - ZLOGE("Not contain details for object!"); - return false; - } - auto result = ObjectUtils::ConvertToUDDetails(detailObj); - if (result.find(TEMP_UNIFIED_DATA_FLAG) == result.end()) { - return false; + + auto propertyKey = data.GetRuntime()->key.GetPropertyKey(); + Value value; + auto status = DataHandler::MarshalToEntries(summary, value, TAG::TAG_SUMMARY); + if (status != E_OK) { + ZLOGE("Marshal summary failed, key: %{public}s, status:%{public}d", propertyKey.c_str(), status); + return status; } - details = result; - return true; + auto summaryKey = propertyKey + SUMMARY_SUFIX; + entries.push_back({{summaryKey.begin(), summaryKey.end()}, value}); + return E_OK; } -Status RuntimeStore::GetSummaryFromDetails(const UDDetails &details, Summary &summary) +Status RuntimeStore::GetSummary(UnifiedKey &key, Summary &summary) { - for (auto &item : details) { - if (item.first != TEMP_UNIFIED_DATA_FLAG) { - auto int64Value = std::get_if(&item.second); - if (int64Value != nullptr) { - auto size = std::get(item.second); - summary.summary[item.first] = size; - summary.totalSize += size; - } + UpdateTime(); + Value value; + auto summaryKey = key.GetPropertyKey() + SUMMARY_SUFIX; + auto res = kvStore_->Get({summaryKey.begin(), summaryKey.end()}, value); + if (res != OK || value.empty()) { + ZLOGW("Get stored summary failed, key: %{public}s, status:%{public}d", summaryKey.c_str(), res); + UnifiedData unifiedData; + auto udKey = key.GetUnifiedKey(); + if (Get(udKey, unifiedData) != E_OK) { + ZLOGE("Get unified data failed, key: %{public}s", udKey.c_str()); + return E_DB_ERROR; + } + UDDetails details {}; + if (PreProcessUtils::GetDetailsFromUData(unifiedData, details)) { + return PreProcessUtils::GetSummaryFromDetails(details, summary); } + UnifiedDataHelper::GetSummary(unifiedData, summary); + return E_OK; + } + auto status = DataHandler::UnmarshalEntries(value, summary, TAG::TAG_SUMMARY); + if (status != E_OK) { + ZLOGE("Unmarshal summary failed, key: %{public}s, status:%{public}d", summaryKey.c_str(), status); + return status; } return E_OK; } -Status RuntimeStore::GetSummary(const std::string &key, Summary &summary) +Status RuntimeStore::PutRuntime(const std::string &key, const Runtime &runtime) { UpdateTime(); - UnifiedData unifiedData; - if (Get(key, unifiedData) != E_OK) { - ZLOGE("Get unified data failed, dataPrefix: %{public}s", key.c_str()); + Value value; + auto status = DataHandler::MarshalToEntries(runtime, value, TAG::TAG_RUNTIME); + if (status != E_OK) { + ZLOGE("Marshal runtime failed, key: %{public}s, status:%{public}d", key.c_str(), status); + return status; + } + auto res = kvStore_->Put({key.begin(), key.end()}, value); + if (res != OK) { + ZLOGE("Put failed, key:%{public}s, status:%{public}d", key.c_str(), res); return E_DB_ERROR; } + return E_OK; +} - UDDetails details {}; - if (GetDetailsFromUData(unifiedData, details)) { - return GetSummaryFromDetails(details, summary); +Status RuntimeStore::GetRuntime(const std::string &key, Runtime &runtime) +{ + UpdateTime(); + Value value; + auto res = kvStore_->Get({key.begin(), key.end()}, value); + if (res != OK || value.empty()) { + ZLOGE("Get failed, key: %{public}s, status:%{public}d", key.c_str(), res); + return E_DB_ERROR; + } + auto status = DataHandler::UnmarshalEntries(value, runtime, TAG::TAG_RUNTIME); + if (status != E_OK) { + ZLOGE("Unmarshal runtime failed, key: %{public}s, status:%{public}d", key.c_str(), status); + return status; } - UnifiedDataHelper::GetSummary(unifiedData, summary); return E_OK; } @@ -350,7 +373,8 @@ Status RuntimeStore::GetBatchData(const std::string &dataPrefix, std::vector keySet; for (const auto &entry : entries) { std::string keyStr = {entry.key.begin(), entry.key.end()}; - if (std::count(keyStr.begin(), keyStr.end(), '/') == SLASH_COUNT_IN_KEY) { + if (std::count(keyStr.begin(), keyStr.end(), '/') == SLASH_COUNT_IN_KEY && + std::count(keyStr.begin(), keyStr.end(), '#') == 0) { keySet.emplace_back(keyStr); } } diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.h b/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.h index 68f32a6b..f195eba3 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/runtime_store.h @@ -19,7 +19,6 @@ #include "store.h" #include "kv_store_delegate_manager.h" #include "metadata/store_meta_data.h" -#include "visibility.h" namespace OHOS { namespace UDMF { @@ -31,7 +30,7 @@ public: ~RuntimeStore(); Status Put(const UnifiedData &unifiedData) override; Status Get(const std::string &key, UnifiedData &unifiedData) override; - Status GetSummary(const std::string &key, Summary &summary) override; + Status GetSummary(UnifiedKey &key, Summary &summary) override; Status Update(const UnifiedData &unifiedData) override; Status Delete(const std::string &key) override; Status DeleteBatch(const std::vector &unifiedKeys) override; @@ -42,6 +41,8 @@ public: Status PutLocal(const std::string &key, const std::string &value) override; Status GetLocal(const std::string &key, std::string &value) override; Status DeleteLocal(const std::string &key) override; + Status PutRuntime(const std::string &key, const Runtime &runtime) override; + Status GetRuntime(const std::string &key, Runtime &runtime) override; void Close() override; bool Init() override; @@ -60,11 +61,10 @@ private: Status DeleteEntries(const std::vector &keys); Status UnmarshalEntries( const std::string &key, std::vector &entries, UnifiedData &unifiedData); - bool GetDetailsFromUData(UnifiedData &data, UDDetails &details); - Status GetSummaryFromDetails(const UDDetails &details, Summary &summary); bool BuildMetaDataParam(DistributedData::StoreMetaData &metaData); void NotifySyncProcss(const DevSyncProcessMap &processMap, ProcessCallback callback, const DevNameMap &deviceNameMap); + Status PutSummary(const UnifiedData &data, std::vector &entries); }; } // namespace UDMF } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/store.h b/datamgr_service/services/distributeddataservice/service/udmf/store/store.h index 76471ef5..1c54e65d 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/store.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/store.h @@ -20,10 +20,7 @@ #include #include -#include "error_code.h" #include "unified_data.h" -#include "unified_key.h" -#include "unified_types.h" namespace OHOS { namespace UDMF { @@ -33,7 +30,7 @@ public: using Time = std::chrono::steady_clock::time_point; virtual Status Put(const UnifiedData &unifiedData) = 0; virtual Status Get(const std::string &key, UnifiedData &unifiedData) = 0; - virtual Status GetSummary(const std::string &key, Summary &summary) = 0; + virtual Status GetSummary(UnifiedKey &key, Summary &summary) = 0; virtual Status Update(const UnifiedData &unifiedData) = 0; virtual Status Delete(const std::string &key) = 0; virtual Status DeleteBatch(const std::vector &unifiedKeys) = 0; @@ -44,6 +41,8 @@ public: virtual Status PutLocal(const std::string &key, const std::string &value) = 0; virtual Status GetLocal(const std::string &key, std::string &value) = 0; virtual Status DeleteLocal(const std::string &key) = 0; + virtual Status PutRuntime(const std::string &key, const Runtime &runtime) = 0; + virtual Status GetRuntime(const std::string &key, Runtime &runtime) = 0; virtual bool Init() = 0; virtual void Close() = 0; diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/store_account_observer.cpp b/datamgr_service/services/distributeddataservice/service/udmf/store/store_account_observer.cpp index f2ba9c72..9085a9b5 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/store_account_observer.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/store_account_observer.cpp @@ -15,10 +15,8 @@ #define LOG_TAG "RuntimeStoreAccountObserver" #include "store_account_observer.h" -#include "runtime_store.h" #include "log_print.h" #include "directory/directory_manager.h" -#include "account/account_delegate.h" #include "bootstrap.h" #include "ipc_skeleton.h" diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.cpp b/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.cpp index aa3c0ec2..f5395918 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.cpp @@ -14,12 +14,9 @@ */ #define LOG_TAG "StoreCache" #include "store_cache.h" -#include -#include #include "log_print.h" #include "runtime_store.h" -#include "unified_meta.h" #include "account/account_delegate.h" namespace OHOS { @@ -91,5 +88,11 @@ void StoreCache::SetThreadPool(std::shared_ptr executors) { executorPool_ = executors; } + +void StoreCache::CloseStores() +{ + ZLOGI("CloseStores, stores size:%{public}zu", stores_.Size()); + stores_.Clear(); +} } // namespace UDMF } // namespace OHOS diff --git a/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.h b/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.h index aa9ad330..fd052949 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/store/store_cache.h @@ -16,13 +16,9 @@ #ifndef UDMF_STORE_CACHE_H #define UDMF_STORE_CACHE_H -#include -#include - #include "concurrent_map.h" #include "executor_pool.h" #include "store.h" -#include "unified_meta.h" namespace OHOS { namespace UDMF { @@ -31,6 +27,7 @@ public: std::shared_ptr GetStore(std::string intention); static StoreCache &GetInstance(); void SetThreadPool(std::shared_ptr executors); + void CloseStores(); private: StoreCache() {} diff --git a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.cpp b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.cpp index 30bb840e..acd9857b 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.cpp @@ -17,31 +17,28 @@ #include "udmf_service_impl.h" -#include "iservice_registry.h" #include "ipc_skeleton.h" #include "tokenid_kit.h" #include "accesstoken_kit.h" +#include "bootstrap.h" +#include "bundle_info.h" +#include "bundlemgr/bundle_mgr_proxy.h" #include "checker_manager.h" -#include "dfx_types.h" -#include "distributed_kv_data_manager.h" +#include "device_manager_adapter.h" +#include "iservice_registry.h" #include "lifecycle/lifecycle_manager.h" #include "log_print.h" +#include "metadata/store_meta_data.h" +#include "metadata/meta_data_manager.h" #include "preprocess_utils.h" -#include "reporter.h" +#include "dfx/reporter.h" +#include "store_account_observer.h" +#include "system_ability_definition.h" #include "uri_permission_manager.h" -#include "uri.h" -#include "udmf_conversion.h" #include "udmf_radar_reporter.h" -#include "securec.h" -#include "unified_types.h" -#include "device_manager_adapter.h" -#include "store_account_observer.h" +#include "unified_data_helper.h" #include "utils/anonymous.h" -#include "bootstrap.h" -#include "metadata/store_meta_data.h" -#include "metadata/meta_data_manager.h" -#include "udmf_dialog.h" namespace OHOS { namespace UDMF { @@ -58,6 +55,9 @@ constexpr const char *DATA_PREFIX = "udmf://"; constexpr const char *FILE_SCHEME = "file"; constexpr const char *PRIVILEGE_READ_AND_KEEP = "readAndKeep"; constexpr const char *MANAGE_UDMF_APP_SHARE_OPTION = "ohos.permission.MANAGE_UDMF_APP_SHARE_OPTION"; +constexpr const char *DEVICE_2IN1_TAG = "2in1"; +constexpr const char *DEVICE_PHONE_TAG = "phone"; +constexpr const char *DEVICE_DEFAULT_TAG = "default"; constexpr const char *HAP_LIST[] = {"com.ohos.pasteboarddialog"}; __attribute__((used)) UdmfServiceImpl::Factory UdmfServiceImpl::factory_; UdmfServiceImpl::Factory::Factory() @@ -110,7 +110,7 @@ int32_t UdmfServiceImpl::SetData(CustomOption &option, UnifiedData &unifiedData, } msg.dataType = types; msg.dataSize = unifiedData.GetSize(); - Reporter::GetInstance()->BehaviourReporter()->UDMFReport(msg); + Reporter::GetInstance()->GetBehaviourReporter()->UDMFReport(msg); return res; } @@ -141,19 +141,16 @@ int32_t UdmfServiceImpl::SaveData(CustomOption &option, UnifiedData &unifiedData return ret; } } - - for (const auto &record : unifiedData.GetRecords()) { - record->SetUid(PreProcessUtils::GenerateId()); - } + PreProcessUtils::SetRecordUid(unifiedData); auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Get store failed:%{public}s", intention.c_str()); return E_DB_ERROR; } if (store->Put(unifiedData) != E_OK) { - ZLOGE("Put unified data failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Put unified data failed:%{public}s", intention.c_str()); return E_DB_ERROR; } key = unifiedData.GetRuntime()->key.GetUnifiedKey(); @@ -178,6 +175,7 @@ int32_t UdmfServiceImpl::GetData(const QueryOption &query, UnifiedData &unifiedD msg.appId = bundleName; res = RetrieveData(query, unifiedData); } + TransferToEntriesIfNeed(query, unifiedData); auto errFind = ERROR_MAP.find(res); msg.result = errFind == ERROR_MAP.end() ? "E_ERROR" : errFind->second; for (const auto &record : unifiedData.GetRecords()) { @@ -187,7 +185,7 @@ int32_t UdmfServiceImpl::GetData(const QueryOption &query, UnifiedData &unifiedD } msg.dataType = types; msg.dataSize = unifiedData.GetSize(); - Reporter::GetInstance()->BehaviourReporter()->UDMFReport(msg); + Reporter::GetInstance()->GetBehaviourReporter()->UDMFReport(msg); return res; } @@ -200,17 +198,17 @@ int32_t UdmfServiceImpl::RetrieveData(const QueryOption &query, UnifiedData &uni } auto store = StoreCache::GetInstance().GetStore(key.intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Get store failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } int32_t res = store->Get(query.key, unifiedData); if (res != E_OK) { - ZLOGE("Get data from store failed, res: %{public}d, key: %{public}s.", res, query.key.c_str()); + ZLOGE("Get data failed,res:%{public}d,key:%{public}s", res, query.key.c_str()); return res; } if (!unifiedData.IsComplete()) { - ZLOGE("Get data from DB is incomplete, key: %{public}s.", query.key.c_str()); + ZLOGE("Get data incomplete,key:%{public}s", query.key.c_str()); return E_NOT_FOUND; } @@ -231,13 +229,13 @@ int32_t UdmfServiceImpl::RetrieveData(const QueryOption &query, UnifiedData &uni if (ret != E_OK) { RadarReporterAdapter::ReportFail(std::string(__FUNCTION__), BizScene::GET_DATA, GetDataStage::GRANT_URI_PERMISSION, StageRes::FAILED, ret); - ZLOGE("ProcessUri failed. ret=%{public}d", ret); + ZLOGE("ProcessUri failed:%{public}d", ret); return E_NO_PERMISSION; } } if (!IsReadAndKeep(runtime->privileges, query)) { if (LifeCycleManager::GetInstance().OnGot(key) != E_OK) { - ZLOGE("Remove data failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Remove data failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } } @@ -264,7 +262,7 @@ bool UdmfServiceImpl::IsReadAndKeep(const std::vector &privileges, co return true; } } - + auto iter = privilegeCache_.find(query.key); if (iter != privilegeCache_.end() && iter->second.tokenId == query.tokenId && iter->second.readPermission == PRIVILEGE_READ_AND_KEEP) { @@ -277,19 +275,19 @@ int32_t UdmfServiceImpl::ProcessUri(const QueryOption &query, UnifiedData &unifi { std::string localDeviceId = PreProcessUtils::GetLocalDeviceId(); std::vector allUri; - int32_t verifyRes = ProcessCrossDeviceData(unifiedData, allUri); + int32_t verifyRes = ProcessCrossDeviceData(query.tokenId, unifiedData, allUri); if (verifyRes != E_OK) { ZLOGE("verify unifieddata fail, key=%{public}s, stauts=%{public}d", query.key.c_str(), verifyRes); return verifyRes; } std::string bundleName; if (!PreProcessUtils::GetHapBundleNameByToken(query.tokenId, bundleName)) { - ZLOGE("GetHapBundleNameByToken fail, key=%{public}s, tokenId=%{private}d.", query.key.c_str(), query.tokenId); + ZLOGE("Get bundleName fail,key=%{public}s,tokenId=%d", query.key.c_str(), query.tokenId); return E_ERROR; } std::string sourceDeviceId = unifiedData.GetRuntime()->deviceId; if (localDeviceId == sourceDeviceId && query.tokenId == unifiedData.GetRuntime()->tokenId) { - ZLOGW("No need to grant uri permissions, queryKey=%{public}s.", query.key.c_str()); + ZLOGW("No uri permissions needed,queryKey=%{public}s", query.key.c_str()); return E_OK; } if (UriPermissionManager::GetInstance().GrantUriPermission(allUri, query.tokenId, query.key) != E_OK) { @@ -300,14 +298,13 @@ int32_t UdmfServiceImpl::ProcessUri(const QueryOption &query, UnifiedData &unifi return E_OK; } -int32_t UdmfServiceImpl::ProcessCrossDeviceData(UnifiedData &unifiedData, std::vector &uris) +int32_t UdmfServiceImpl::ProcessCrossDeviceData(uint32_t tokenId, UnifiedData &unifiedData, std::vector &uris) { if (unifiedData.GetRuntime() == nullptr) { ZLOGE("Get runtime empty!"); return E_DB_ERROR; } - std::string localDeviceId = PreProcessUtils::GetLocalDeviceId(); - std::string sourceDeviceId = unifiedData.GetRuntime()->deviceId; + bool isLocal = PreProcessUtils::GetLocalDeviceId() == unifiedData.GetRuntime()->deviceId; auto records = unifiedData.GetRecords(); bool hasError = false; PreProcessUtils::ProcessFileType(records, [&] (std::shared_ptr obj) { @@ -323,11 +320,11 @@ int32_t UdmfServiceImpl::ProcessCrossDeviceData(UnifiedData &unifiedData, std::v Uri uri(oriUri); std::string scheme = uri.GetScheme(); std::transform(scheme.begin(), scheme.end(), scheme.begin(), ::tolower); - if (localDeviceId != sourceDeviceId) { + if (!isLocal) { std::string remoteUri; obj->GetValue(REMOTE_URI, remoteUri); if (remoteUri.empty() && scheme == FILE_SCHEME) { - ZLOGE("when cross devices, remote uri is required!"); + ZLOGE("Remote URI required for cross-device"); hasError = true; return false; } @@ -339,12 +336,13 @@ int32_t UdmfServiceImpl::ProcessCrossDeviceData(UnifiedData &unifiedData, std::v } } if (uri.GetAuthority().empty() || scheme != FILE_SCHEME) { - ZLOGW("Get authority is empty or uri scheme not equals to file."); + ZLOGW("Empty authority or scheme not file"); return false; } uris.push_back(uri); return true; }); + PreProcessUtils::ProcessHtmlFileUris(tokenId, unifiedData, isLocal, uris); return hasError ? E_ERROR : E_OK; } @@ -359,7 +357,7 @@ int32_t UdmfServiceImpl::GetBatchData(const QueryOption &query, std::vectorGet(query.key, data); if (res != E_OK) { - ZLOGE("Get data from store failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Get data failed:%{public}s", key.intention.c_str()); return res; } if (data.IsEmpty()) { @@ -402,16 +400,14 @@ int32_t UdmfServiceImpl::UpdateData(const QueryOption &query, UnifiedData &unifi return E_DB_ERROR; } if (runtime->tokenId != query.tokenId && !HasDatahubPriviledge(bundleName)) { - ZLOGE("update data failed, query option tokenId not equals data's tokenId"); + ZLOGE("Update failed: tokenId mismatch"); return E_INVALID_PARAMETERS; } runtime->lastModifiedTime = PreProcessUtils::GetTimestamp(); unifiedData.SetRuntime(*runtime); - for (auto &record : unifiedData.GetRecords()) { - record->SetUid(PreProcessUtils::GenerateId()); - } + PreProcessUtils::SetRecordUid(unifiedData); if (store->Update(unifiedData) != E_OK) { - ZLOGE("Update unified data failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Unified data update failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -444,7 +440,7 @@ int32_t UdmfServiceImpl::DeleteData(const QueryOption &query, std::vectorGetSummary(query.key, summary) != E_OK) { - ZLOGE("Store get summary failed, intention: %{public}s.", key.intention.c_str()); + if (store->GetSummary(key, summary) != E_OK) { + ZLOGE("Store get summary failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -482,7 +478,7 @@ int32_t UdmfServiceImpl::AddPrivilege(const QueryOption &query, Privilege &privi ZLOGD("start"); UnifiedKey key(query.key); if (!key.IsValid()) { - ZLOGE("Unified key: %{public}s is invalid.", query.key.c_str()); + ZLOGE("Invalid unified key:%{public}s", query.key.c_str()); return E_INVALID_PARAMETERS; } @@ -494,7 +490,7 @@ int32_t UdmfServiceImpl::AddPrivilege(const QueryOption &query, Privilege &privi if (key.intention == UD_INTENTION_MAP.at(UD_INTENTION_DRAG)) { if (find(DRAG_AUTHORIZED_PROCESSES, std::end(DRAG_AUTHORIZED_PROCESSES), processName) == std::end(DRAG_AUTHORIZED_PROCESSES)) { - ZLOGE("Process: %{public}s has no permission to intention: drag", processName.c_str()); + ZLOGE("Process:%{public}s lacks permission for intention:drag", processName.c_str()); return E_NO_PERMISSION; } } else { @@ -504,30 +500,27 @@ int32_t UdmfServiceImpl::AddPrivilege(const QueryOption &query, Privilege &privi auto store = StoreCache::GetInstance().GetStore(key.intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Get store failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } - UnifiedData data; - int32_t res = store->Get(query.key, data); + Runtime runtime; + auto res = store->GetRuntime(query.key, runtime); if (res == E_NOT_FOUND) { privilegeCache_[query.key] = privilege; ZLOGW("Add privilege in cache, key: %{public}s.", query.key.c_str()); return E_OK; } if (res != E_OK) { - ZLOGE("Get data from store failed, res:%{public}d,intention: %{public}s.", res, key.intention.c_str()); + ZLOGE("Get runtime failed, res:%{public}d, key:%{public}s.", res, query.key.c_str()); return res; } - if (data.GetRuntime() == nullptr) { - return E_DB_ERROR; - } - data.GetRuntime()->privileges.emplace_back(privilege); - if (store->Update(data) != E_OK) { - ZLOGE("Update unified data failed, intention: %{public}s.", key.intention.c_str()); - return E_DB_ERROR; + runtime.privileges.emplace_back(privilege); + res = store->PutRuntime(query.key, runtime); + if (res != E_OK) { + ZLOGE("Update runtime failed, res:%{public}d, key:%{public}s", res, query.key.c_str()); } - return E_OK; + return res; } int32_t UdmfServiceImpl::Sync(const QueryOption &query, const std::vector &devices) @@ -547,7 +540,7 @@ int32_t UdmfServiceImpl::Sync(const QueryOption &query, const std::vectorSync(devices, callback) != E_OK) { - ZLOGE("Store sync failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Store sync failed:%{public}s", key.intention.c_str()); RadarReporterAdapter::ReportFail(std::string(__FUNCTION__), BizScene::SYNC_DATA, SyncDataStage::SYNC_END, StageRes::FAILED, E_DB_ERROR, BizState::DFX_END); return E_DB_ERROR; @@ -576,19 +569,19 @@ int32_t UdmfServiceImpl::IsRemoteData(const QueryOption &query, bool &result) { UnifiedKey key(query.key); if (!key.IsValid()) { - ZLOGE("Unified key: %{public}s is invalid.", query.key.c_str()); + ZLOGE("Invalid unified key:%{public}s", query.key.c_str()); return E_INVALID_PARAMETERS; } auto store = StoreCache::GetInstance().GetStore(key.intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Get store failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } UnifiedData unifiedData; if (store->Get(query.key, unifiedData) != E_OK) { - ZLOGE("Store get unifiedData failed, intention: %{public}s.", key.intention.c_str()); + ZLOGE("Store get unifiedData failed:%{public}s", key.intention.c_str()); return E_DB_ERROR; } std::shared_ptr runtime = unifiedData.GetRuntime(); @@ -607,7 +600,7 @@ int32_t UdmfServiceImpl::IsRemoteData(const QueryOption &query, bool &result) int32_t UdmfServiceImpl::SetAppShareOption(const std::string &intention, int32_t shareOption) { if (intention.empty() || shareOption >= SHARE_OPTIONS_BUTT || shareOption < IN_APP) { - ZLOGE("SetAppShareOption : para is invalid, intention: %{public}s, shareOption:%{public}d.", + ZLOGE("para is invalid,intention:%{public}s,shareOption:%{public}d", intention.c_str(), shareOption); return E_INVALID_PARAMETERS; } @@ -616,23 +609,23 @@ int32_t UdmfServiceImpl::SetAppShareOption(const std::string &intention, int32_t bool isSystemApp = TokenIdKit::IsSystemAppByFullTokenID(accessTokenIDEx); bool hasSharePermission = VerifyPermission(MANAGE_UDMF_APP_SHARE_OPTION, IPCSkeleton::GetCallingTokenID()); if (!isSystemApp && !hasSharePermission) { - ZLOGE("No system permission and no shareOption permission, intention: %{public}s.", intention.c_str()); + ZLOGE("No system or shareOption permission,intention:%{public}s", intention.c_str()); return E_NO_PERMISSION; } auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Get store failed:%{public}s", intention.c_str()); return E_DB_ERROR; } std::string shareOptionTmp; if (store->GetLocal(std::to_string(accessTokenIDEx), shareOptionTmp) == E_OK) { - ZLOGE("SetAppShareOption failed, shareOption has already been set, %{public}s.", shareOptionTmp.c_str()); + ZLOGE("SetAppShareOption failed,shareOption already set:%{public}s", shareOptionTmp.c_str()); return E_SETTINGS_EXISTED; } if (store->PutLocal(std::to_string(accessTokenIDEx), ShareOptionsUtil::GetEnumStr(shareOption)) != E_OK) { - ZLOGE("Store get unifiedData failed, intention: %{public}d.", shareOption); + ZLOGE("Store get unifiedData failed:%{public}d", shareOption); return E_DB_ERROR; } return E_OK; @@ -641,23 +634,22 @@ int32_t UdmfServiceImpl::SetAppShareOption(const std::string &intention, int32_t int32_t UdmfServiceImpl::GetAppShareOption(const std::string &intention, int32_t &shareOption) { if (intention.empty()) { - ZLOGE("GetAppShareOption : para is invalid, %{public}s is invalid.", intention.c_str()); + ZLOGE("intention is empty:%{public}s", intention.c_str()); return E_INVALID_PARAMETERS; } uint64_t accessTokenIDEx = IPCSkeleton::GetCallingFullTokenID(); auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Get store failed:%{public}s", intention.c_str()); return E_DB_ERROR; } std::string appShareOption; int32_t ret = store->GetLocal(std::to_string(accessTokenIDEx), appShareOption); if (ret != E_OK) { - ZLOGW("GetAppShareOption empty, intention: %{public}s.", intention.c_str()); + ZLOGW("GetLocal failed:%{public}s", intention.c_str()); return ret; } - ZLOGI("GetAppShareOption, intention: %{public}s, appShareOption:%{public}s.", - intention.c_str(), appShareOption.c_str()); + ZLOGI("GetLocal ok intention:%{public}s,appShareOption:%{public}s", intention.c_str(), appShareOption.c_str()); shareOption = ShareOptionsUtil::GetEnumNum(appShareOption); return E_OK; } @@ -672,18 +664,18 @@ int32_t UdmfServiceImpl::RemoveAppShareOption(const std::string &intention) bool isSystemApp = TokenIdKit::IsSystemAppByFullTokenID(accessTokenIDEx); bool hasSharePermission = VerifyPermission(MANAGE_UDMF_APP_SHARE_OPTION, IPCSkeleton::GetCallingTokenID()); if (!isSystemApp && !hasSharePermission) { - ZLOGE("No system permission and no shareOption permission, intention: %{public}s.", intention.c_str()); + ZLOGE("No system or shareOption permission:%{public}s", intention.c_str()); return E_NO_PERMISSION; } auto store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Get store failed:%{public}s", intention.c_str()); return E_DB_ERROR; } UnifiedData unifiedData; if (store->DeleteLocal(std::to_string(accessTokenIDEx)) != E_OK) { - ZLOGE("Store DeleteLocal failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Store DeleteLocal failed:%{public}s", intention.c_str()); return E_DB_ERROR; } return E_OK; @@ -716,13 +708,13 @@ int32_t UdmfServiceImpl::QueryDataCommon( UnifiedKey key(query.key); key.IsValid(); if (intention.empty()) { - dataPrefix = key.key; + dataPrefix = UnifiedKey(key.key).GetPropertyKey(); intention = key.intention; } ZLOGD("dataPrefix = %{public}s, intention: %{public}s.", dataPrefix.c_str(), intention.c_str()); store = StoreCache::GetInstance().GetStore(intention); if (store == nullptr) { - ZLOGE("Get store failed, intention: %{public}s.", intention.c_str()); + ZLOGE("Get store failed:%{public}s", intention.c_str()); return E_DB_ERROR; } if (store->GetBatchData(dataPrefix, dataSet) != E_OK) { @@ -839,24 +831,57 @@ void UdmfServiceImpl::RegisterAsyncProcessInfo(const std::string &businessUdKey) asyncProcessInfoMap_.insert_or_assign(businessUdKey, std::move(info)); } -int32_t UdmfServiceImpl::InvokeHap(const std::string &progressKey, const sptr &observer) +int32_t UdmfServiceImpl::OnUserChange(uint32_t code, const std::string &user, const std::string &account) { - ProgressDialog::ProgressMessageInfo message; - message.promptText = "PromptText_PasteBoard_Local"; - message.remoteDeviceName = ""; - message.isRemote = false; - message.progressKey = progressKey; - message.clientCallback = observer; - - ProgressDialog::FocusedAppInfo appInfo = ProgressDialog::GetInstance().GetFocusedAppInfo(); - message.windowId = appInfo.windowId; - message.callerToken = appInfo.abilityToken; - auto status = ProgressDialog::GetInstance().ShowProgress(message); - if (status != E_OK) { - ZLOGE("ShowProgress fail, status:%{public}d", status); + ZLOGI("user change, code:%{public}u, user:%{public}s, account:%{public}s", code, user.c_str(), account.c_str()); + if (code == static_cast(DistributedData::AccountStatus::DEVICE_ACCOUNT_SWITCHED)) { + StoreCache::GetInstance().CloseStores(); } - return E_OK; + return Feature::OnUserChange(code, user, account); } +void UdmfServiceImpl::TransferToEntriesIfNeed(const QueryOption &query, UnifiedData &unifiedData) +{ + if (unifiedData.IsNeedTransferToEntries() && IsNeedTransferDeviceType(query)) { + unifiedData.TransferToEntries(unifiedData); + } +} + +bool UdmfServiceImpl::IsNeedTransferDeviceType(const QueryOption &query) +{ + auto samgrProxy = SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager(); + if (samgrProxy == nullptr) { + ZLOGE("Failed to get system ability mgr."); + return false; + } + auto bundleMgrProxy = samgrProxy->GetSystemAbility(BUNDLE_MGR_SERVICE_SYS_ABILITY_ID); + if (bundleMgrProxy == nullptr) { + ZLOGE("Failed to Get BMS SA."); + return false; + } + auto bundleManager = iface_cast(bundleMgrProxy); + if (bundleManager == nullptr) { + ZLOGE("Failed to get bundle manager"); + return false; + } + std::string bundleName; + PreProcessUtils::GetHapBundleNameByToken(query.tokenId, bundleName); + int32_t userId = DistributedData::AccountDelegate::GetInstance()->GetUserByToken( + IPCSkeleton::GetCallingFullTokenID()); + AppExecFwk::BundleInfo bundleInfo; + bundleManager->GetBundleInfoV9(bundleName, static_cast( + AppExecFwk::GetBundleInfoFlag::GET_BUNDLE_INFO_WITH_HAP_MODULE), bundleInfo, userId); + for (const auto &hapModuleInfo : bundleInfo.hapModuleInfos) { + if (std::find(hapModuleInfo.deviceTypes.begin(), hapModuleInfo.deviceTypes.end(), + DEVICE_PHONE_TAG) == hapModuleInfo.deviceTypes.end() + && std::find(hapModuleInfo.deviceTypes.begin(), hapModuleInfo.deviceTypes.end(), + DEVICE_DEFAULT_TAG) == hapModuleInfo.deviceTypes.end() + && std::find(hapModuleInfo.deviceTypes.begin(), hapModuleInfo.deviceTypes.end(), + DEVICE_2IN1_TAG) != hapModuleInfo.deviceTypes.end()) { + return true; + } + } + return false; +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.h b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.h index 012de04f..48a80b44 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_impl.h @@ -16,17 +16,8 @@ #ifndef UDMF_SERVICE_IMPL_H #define UDMF_SERVICE_IMPL_H -#include -#include -#include -#include - -#include "error_code.h" #include "store_cache.h" #include "udmf_service_stub.h" -#include "unified_data.h" -#include "unified_types.h" -#include "visibility.h" #include "kv_store_delegate_manager.h" namespace OHOS { namespace UDMF { @@ -55,7 +46,7 @@ public: int32_t ObtainAsynProcess(AsyncProcessInfo &processInfo) override; int32_t ClearAsynProcessByKey(const std::string &businessUdKey) override; int32_t ResolveAutoLaunch(const std::string &identifier, DBLaunchParam ¶m) override; - int32_t InvokeHap(const std::string &progressKey, const sptr &observer) override; + int32_t OnUserChange(uint32_t code, const std::string &user, const std::string &account) override; private: int32_t SaveData(CustomOption &option, UnifiedData &unifiedData, std::string &key); int32_t RetrieveData(const QueryOption &query, UnifiedData &unifiedData); @@ -63,10 +54,12 @@ private: int32_t ProcessUri(const QueryOption &query, UnifiedData &unifiedData); bool IsPermissionInCache(const QueryOption &query); bool IsReadAndKeep(const std::vector &privileges, const QueryOption &query); - int32_t ProcessCrossDeviceData(UnifiedData &unifiedData, std::vector &uris); + int32_t ProcessCrossDeviceData(uint32_t tokenId, UnifiedData &unifiedData, std::vector &uris); bool VerifyPermission(const std::string &permission, uint32_t callerTokenId); bool HasDatahubPriviledge(const std::string &bundleName); void RegisterAsyncProcessInfo(const std::string &businessUdKey); + void TransferToEntriesIfNeed(const QueryOption &query, UnifiedData &unifiedData); + bool IsNeedTransferDeviceType(const QueryOption &query); class Factory { public: diff --git a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.cpp b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.cpp index 32651fa7..f433413b 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.cpp +++ b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.cpp @@ -16,17 +16,9 @@ #define LOG_TAG "UdmfServiceStub" #include "udmf_service_stub.h" - -#include -#include - -#include "accesstoken_kit.h" #include "ipc_skeleton.h" #include "log_print.h" -#include "udmf_conversion.h" #include "udmf_types_util.h" -#include "unified_data.h" -#include "unified_meta.h" namespace OHOS { namespace UDMF { @@ -83,7 +75,7 @@ int32_t UdmfServiceStub::OnGetData(MessageParcel &data, MessageParcel &reply) UnifiedData unifiedData; int32_t status = GetData(query, unifiedData); if (!ITypesUtil::Marshal(reply, status, unifiedData)) { - ZLOGE("Marshal status or unifiedData failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -102,7 +94,7 @@ int32_t UdmfServiceStub::OnGetBatchData(MessageParcel &data, MessageParcel &repl std::vector unifiedDataSet; int32_t status = GetBatchData(query, unifiedDataSet); if (!ITypesUtil::Marshal(reply, status, unifiedDataSet)) { - ZLOGE("Marshal status or unifiedDataSet failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -121,7 +113,7 @@ int32_t UdmfServiceStub::OnUpdateData(MessageParcel &data, MessageParcel &reply) query.tokenId = token; int32_t status = UpdateData(query, unifiedData); if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal status failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -140,7 +132,7 @@ int32_t UdmfServiceStub::OnDeleteData(MessageParcel &data, MessageParcel &reply) std::vector unifiedDataSet; int32_t status = DeleteData(query, unifiedDataSet); if (!ITypesUtil::Marshal(reply, status, unifiedDataSet)) { - ZLOGE("Marshal status or unifiedDataSet failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -229,7 +221,7 @@ int32_t UdmfServiceStub::OnSetAppShareOption(MessageParcel &data, MessageParcel } int32_t status = SetAppShareOption(intention, shareOption); if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal OnSetAppShareOption status failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -245,7 +237,7 @@ int32_t UdmfServiceStub::OnGetAppShareOption(MessageParcel &data, MessageParcel } int32_t status = GetAppShareOption(intention, shareOption); if (!ITypesUtil::Marshal(reply, status, shareOption)) { - ZLOGE("Marshal OnGetAppShareOption status failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -260,7 +252,7 @@ int32_t UdmfServiceStub::OnRemoveAppShareOption(MessageParcel &data, MessageParc } int32_t status = RemoveAppShareOption(intention); if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal OnRemoveAppShareOption status failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; @@ -292,32 +284,10 @@ int32_t UdmfServiceStub::OnClearAsynProcessByKey(MessageParcel &data, MessagePar } int32_t status = ClearAsynProcessByKey(businessUdKey); if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal status failed, status: %{public}d", status); - return E_WRITE_PARCEL_ERROR; - } - return E_OK; -} - -int32_t UdmfServiceStub::OnInvokeHap(MessageParcel &data, MessageParcel &reply) -{ - ZLOGD("start"); - std::string progressKey; - sptr callback = nullptr; - if (!ITypesUtil::Unmarshal(data, progressKey, callback)) { - ZLOGE("Unmarshal failed"); - return E_READ_PARCEL_ERROR; - } - if (callback == nullptr) { - ZLOGE("Callback is null"); - return E_ERROR; - } - int32_t status = InvokeHap(progressKey, callback); - if (!ITypesUtil::Marshal(reply, status)) { - ZLOGE("Marshal status failed, status: %{public}d", status); + ZLOGE("Marshal failed:%{public}d", status); return E_WRITE_PARCEL_ERROR; } return E_OK; } - } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.h b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.h index fddcaba3..793a13d9 100644 --- a/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.h +++ b/datamgr_service/services/distributeddataservice/service/udmf/udmf_service_stub.h @@ -16,14 +16,9 @@ #ifndef UDMF_SERVICE_STUB_H #define UDMF_SERVICE_STUB_H -#include -#include - #include "feature/feature_system.h" -#include "message_parcel.h" #include "distributeddata_udmf_ipc_interface_code.h" -#include "error_code.h" #include "udmf_service.h" namespace OHOS { @@ -67,8 +62,7 @@ private: &UdmfServiceStub::OnGetAppShareOption, &UdmfServiceStub::OnRemoveAppShareOption, &UdmfServiceStub::OnObtainAsynProcess, - &UdmfServiceStub::OnClearAsynProcessByKey, - &UdmfServiceStub::OnInvokeHap + &UdmfServiceStub::OnClearAsynProcessByKey }; }; } // namespace UDMF diff --git a/googletest/BUILD.gn b/googletest/BUILD.gn index ec835ee7..079a0f2e 100644 --- a/googletest/BUILD.gn +++ b/googletest/BUILD.gn @@ -100,6 +100,13 @@ static_library("gtest_rtti") { configs += [ ":gtest_private_config_rtti" ] configs -= [ "//build/config/coverage:default_coverage" ] } +static_library("gtest_rtti_main") { #Add gtest entry with RTTI compilation + # option + testonly = true + sources = [ "googletest/src/gtest_main.cc" ] + public_deps = [ ":gtest_rtti" ] + configs -= [ "//build/config/coverage:default_coverage" ] +} static_library("gtest_main") { testonly = true diff --git a/googletest/bundle.json b/googletest/bundle.json index a82e37e2..c952d26b 100644 --- a/googletest/bundle.json +++ b/googletest/bundle.json @@ -33,6 +33,9 @@ { "name": "//third_party/googletest:gmock" }, + { + "name": "//third_party/googletest:gtest_rtti_main" + }, { "name": "//third_party/googletest:gtest_main" }, @@ -41,6 +44,9 @@ }, { "name": "//third_party/googletest:gtest_rtti" + }, + { + "name": "//third_party/googletest:gmock_rtti" } ], "test": [] diff --git a/interface_sdk/api/@ohos.data.relationalStore.d.ts b/interface_sdk/api/@ohos.data.relationalStore.d.ts index 336a72d5..1c9b5a9c 100644 --- a/interface_sdk/api/@ohos.data.relationalStore.d.ts +++ b/interface_sdk/api/@ohos.data.relationalStore.d.ts @@ -435,6 +435,16 @@ declare namespace relationalStore { */ tokenizer?: Tokenizer; + + /** + * Specifies whether the database need persistence. + * + * @type { ?boolean } + * @syscap SystemCapability.DistributedDataManager.RelationalStore.Core + * @since 18 + */ + + persist?: boolean; } /** diff --git a/kv_store/BUILD.gn b/kv_store/BUILD.gn index 61726f98..dd401f98 100644 --- a/kv_store/BUILD.gn +++ b/kv_store/BUILD.gn @@ -27,7 +27,7 @@ group("build_native_test") { "frameworks/libs/distributeddb/test:unittest", "frameworks/libs/distributeddb/test/moduletest:moduletest", ] - if (!defined(global_parts_info.distributeddatamgr_kv_store_lite)) { + if (!defined(global_parts_info.distributeddatamgr_arkdata_database_core)) { deps += [ "frameworks/libs/distributeddb/gaussdb_rd/test/unittest:unittest" ] } diff --git a/kv_store/databaseutils/CMakeLists.txt b/kv_store/databaseutils/CMakeLists.txt index 997988f6..10c48493 100644 --- a/kv_store/databaseutils/CMakeLists.txt +++ b/kv_store/databaseutils/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.2) project(kvdb) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") diff --git a/kv_store/databaseutils/test/BUILD.gn b/kv_store/databaseutils/test/BUILD.gn index 6fc794d0..2899a28f 100644 --- a/kv_store/databaseutils/test/BUILD.gn +++ b/kv_store/databaseutils/test/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/kv_store/kv_store.gni") -module_output_path = "kv_store/databaseutils" +module_output_path = "kv_store/kv_store/databaseutils" config("database_utils_config") { visibility = [ ":*" ] diff --git a/kv_store/frameworks/CMakeLists.txt b/kv_store/frameworks/CMakeLists.txt index 424d737e..47ef5436 100644 --- a/kv_store/frameworks/CMakeLists.txt +++ b/kv_store/frameworks/CMakeLists.txt @@ -1,11 +1,11 @@ cmake_minimum_required(VERSION 3.11.2) project(kvdb) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -Wattributes") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -Wattributes -Wno-error=deprecated-declarations -Wno-deprecated-declarations") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") set(MOCK_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../mock) diff --git a/kv_store/frameworks/common/BUILD.gn b/kv_store/frameworks/common/BUILD.gn index d296ef28..9b2388b7 100644 --- a/kv_store/frameworks/common/BUILD.gn +++ b/kv_store/frameworks/common/BUILD.gn @@ -30,7 +30,7 @@ ohos_static_library("datamgr_common") { cfi_cross_dso = true debug = false } - + cflags = [ "-Wno-c99-designator" ] public_configs = [ ":datamgr_common_config" ] subsystem_name = "distributeddatamgr" diff --git a/kv_store/frameworks/common/test/BUILD.gn b/kv_store/frameworks/common/test/BUILD.gn index eedbe5d6..cb733120 100644 --- a/kv_store/frameworks/common/test/BUILD.gn +++ b/kv_store/frameworks/common/test/BUILD.gn @@ -13,12 +13,14 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/kv_store/kv_store.gni") -module_output_path = "kv_store/common" +module_output_path = "kv_store/kv_store/common" ############################################################################### config("module_private_config") { visibility = [ ":*" ] + cflags = [ "-Wno-c99-designator" ] + include_dirs = [ "../", "${kv_store_base_path}/interfaces/innerkits/distributeddata/include", diff --git a/kv_store/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp b/kv_store/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp index b0c5d8c0..8ee73b95 100644 --- a/kv_store/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp +++ b/kv_store/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp @@ -71,15 +71,15 @@ Status DistributedKvDataManager::GetSingleKvStore(const Options &options, const return status; } -Status DistributedKvDataManager::GetAllKvStoreId(const AppId &appId, std::vector &storeIds) +Status DistributedKvDataManager::GetAllKvStoreId(const AppId &appId, std::vector &storeIds, int32_t subUser) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__)); KvStoreServiceDeathNotifier::SetAppId(appId); - return StoreManager::GetInstance().GetStoreIds(appId, storeIds); + return StoreManager::GetInstance().GetStoreIds(appId, storeIds, subUser); } -Status DistributedKvDataManager::CloseKvStore(const AppId &appId, const StoreId &storeId) +Status DistributedKvDataManager::CloseKvStore(const AppId &appId, const StoreId &storeId, int32_t subUser) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__), TraceSwitch::BYTRACE_ON | TraceSwitch::TRACE_CHAIN_ON); @@ -90,7 +90,7 @@ Status DistributedKvDataManager::CloseKvStore(const AppId &appId, const StoreId return Status::INVALID_ARGUMENT; } - return StoreManager::GetInstance().CloseKVStore(appId, storeId); + return StoreManager::GetInstance().CloseKVStore(appId, storeId, subUser); } Status DistributedKvDataManager::CloseKvStore(const AppId &appId, std::shared_ptr &kvStorePtr) @@ -104,21 +104,23 @@ Status DistributedKvDataManager::CloseKvStore(const AppId &appId, std::shared_pt } KvStoreServiceDeathNotifier::SetAppId(appId); StoreId storeId = kvStorePtr->GetStoreId(); + int32_t subUser = kvStorePtr->GetSubUser(); kvStorePtr = nullptr; - return StoreManager::GetInstance().CloseKVStore(appId, storeId); + return StoreManager::GetInstance().CloseKVStore(appId, storeId, subUser); } -Status DistributedKvDataManager::CloseAllKvStore(const AppId &appId) +Status DistributedKvDataManager::CloseAllKvStore(const AppId &appId, int32_t subUser) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__), TraceSwitch::BYTRACE_ON | TraceSwitch::TRACE_CHAIN_ON); KvStoreServiceDeathNotifier::SetAppId(appId); - return StoreManager::GetInstance().CloseAllKVStore(appId); + return StoreManager::GetInstance().CloseAllKVStore(appId, subUser); } -Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path) +Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId &storeId, + const std::string &path, int32_t subUser) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__), TraceSwitch::BYTRACE_ON | TraceSwitch::TRACE_CHAIN_ON); @@ -132,10 +134,10 @@ Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId } KvStoreServiceDeathNotifier::SetAppId(appId); - return StoreManager::GetInstance().Delete(appId, storeId, path); + return StoreManager::GetInstance().Delete(appId, storeId, path, subUser); } -Status DistributedKvDataManager::DeleteAllKvStore(const AppId &appId, const std::string &path) +Status DistributedKvDataManager::DeleteAllKvStore(const AppId &appId, const std::string &path, int32_t subUser) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__), TraceSwitch::BYTRACE_ON | TraceSwitch::TRACE_CHAIN_ON); @@ -146,12 +148,12 @@ Status DistributedKvDataManager::DeleteAllKvStore(const AppId &appId, const std: KvStoreServiceDeathNotifier::SetAppId(appId); std::vector storeIds; - Status status = GetAllKvStoreId(appId, storeIds); + Status status = GetAllKvStoreId(appId, storeIds, subUser); if (status != SUCCESS) { return status; } for (auto &storeId : storeIds) { - status = StoreManager::GetInstance().Delete(appId, storeId, path); + status = StoreManager::GetInstance().Delete(appId, storeId, path, subUser); if (status != SUCCESS) { return status; } diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/auto_sync_timer.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/auto_sync_timer.h index 860f6518..27e88459 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/auto_sync_timer.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/auto_sync_timer.h @@ -31,6 +31,7 @@ public: private: static constexpr size_t TIME_TASK_NUM = 5; static constexpr size_t SYNC_STORE_NUM = 10; + static constexpr int32_t DEFAULT_USER_ID = 0; AutoSyncTimer() = default; ~AutoSyncTimer() = default; std::map> GetStoreIds(); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/backup_manager.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/backup_manager.h index 12a4afaa..387768dd 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/backup_manager.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/backup_manager.h @@ -34,6 +34,8 @@ public: std::string appId; std::string storeId; bool encrypt = false; + bool isCheckIntegrity = false; + int32_t subUser = 0; }; struct ResidueInfo { size_t tmpBackupSize; @@ -53,12 +55,12 @@ public: static BackupManager &GetInstance(); void Init(const std::string &baseDir); void Prepare(const std::string &path, const std::string &storeId); - Status Backup(const BackupInfo &info, std::shared_ptr dbStore, bool isCheckIntegrity); - Status Restore(const BackupInfo &info, std::shared_ptr dbStore, bool isCheckIntegrity); + Status Backup(const BackupInfo &info, std::shared_ptr dbStore); + Status Restore(const BackupInfo &info, std::shared_ptr dbStore); Status DeleteBackup(std::map &deleteList, const std::string &baseDir, const std::string &storeId); - Status GetSecretKeyFromService(const AppId &appId, const StoreId &storeId, - std::vector> &keys); + Status GetSecretKeyFromService(const AppId &appId, const StoreId &storeId, std::vector> &keys, + int32_t subUser = 0); private: BackupManager(); ~BackupManager(); @@ -68,8 +70,7 @@ private: void CleanTmpData(const std::string &name); StoreUtil::FileInfo GetBackupFileInfo(const std::string &name, const std::string &baseDir, const std::string &storeId); - DBPassword GetRestorePassword(const std::string &name, const std::string &baseDir, - const std::string &appId, const std::string &storeId); + DBPassword GetRestorePassword(const std::string &name, const BackupInfo &info); bool HaveResidueFile(const std::vector &files); bool HaveResidueKey(const std::vector &files, std::string storeId); std::string GetBackupName(const std::string &fileName); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/kv_hiview_reporter.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/kv_hiview_reporter.h index e48c5a2a..34045f2d 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/kv_hiview_reporter.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/kv_hiview_reporter.h @@ -33,12 +33,6 @@ enum BusinessType { GAUSSPD, }; -enum DFXEvent { - FAULT = 0x1, // 001 - CORRUPTED = 0x2, // 010 - REBUILD = 0x4, // 100 -}; - struct ReportInfo { Options options; uint32_t errorCode; @@ -63,7 +57,8 @@ private: static std::string GetCurrentMicrosecondTimeFormat(); - static bool IsReportedCorruptedFault(const std::string &dbPath, const std::string &storeId); + static bool IsReportedCorruptedFault(const std::string &appId, const std::string &storeId, + const std::string &dbPath); static void CreateCorruptedFlag(const std::string &dbPath, const std::string &storeId); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service.h index b8ffd218..cf438e9e 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service.h @@ -46,26 +46,32 @@ public: API_EXPORT virtual ~KVDBService() = default; - virtual Status GetStoreIds(const AppId &appId, std::vector &storeIds) = 0; + virtual Status GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) = 0; virtual Status BeforeCreate(const AppId &appId, const StoreId &storeId, const Options &options) = 0; virtual Status AfterCreate( const AppId &appId, const StoreId &storeId, const Options &options, const std::vector &password) = 0; - virtual Status Delete(const AppId &appId, const StoreId &storeId) = 0; - virtual Status Close(const AppId &appId, const StoreId &storeId) = 0; - virtual Status Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) = 0; + virtual Status Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) = 0; + virtual Status Close(const AppId &appId, const StoreId &storeId, int32_t subUser) = 0; + virtual Status Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) = 0; virtual Status RegServiceNotifier(const AppId &appId, sptr notifier) = 0; virtual Status UnregServiceNotifier(const AppId &appId) = 0; - virtual Status SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) = 0; - virtual Status GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) = 0; - virtual Status EnableCapability(const AppId &appId, const StoreId &storeId) = 0; - virtual Status DisableCapability(const AppId &appId, const StoreId &storeId) = 0; - virtual Status SetCapability(const AppId &appId, const StoreId &storeId, const std::vector &local, - const std::vector &remote) = 0; - virtual Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) = 0; - virtual Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) = 0; - virtual Status Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) = 0; - virtual Status Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) = 0; - virtual Status GetBackupPassword(const AppId &appId, const StoreId &storeId, + virtual Status SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) = 0; + virtual Status GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + KvSyncParam &syncParam) = 0; + virtual Status EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) = 0; + virtual Status DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) = 0; + virtual Status SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::vector &local, const std::vector &remote) = 0; + virtual Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) = 0; + virtual Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) = 0; + virtual Status Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) = 0; + virtual Status Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) = 0; + virtual Status GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, std::vector> &passwords, int32_t passwordType) = 0; virtual Status CloudSync(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) = 0; virtual Status NotifyDataChange(const AppId &appId, const StoreId &storeId, uint64_t delay) = 0; @@ -74,7 +80,8 @@ public: virtual Status SubscribeSwitchData(const AppId &appId) = 0; virtual Status UnsubscribeSwitchData(const AppId &appId) = 0; virtual Status SetConfig(const AppId &appId, const StoreId &storeId, const StoreConfig &storeConfig) = 0; - virtual Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) = 0; + virtual Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) = 0; }; } // namespace OHOS::DistributedKv #endif // OHOS_DISTRIBUTED_DATA_FRAMEWORKS_KVDB_SERVICE_H \ No newline at end of file diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service_client.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service_client.h index 9ff96260..cacb9283 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service_client.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/kvdb_service_client.h @@ -30,27 +30,32 @@ public: class KVDBServiceClient : public IRemoteProxy { public: static std::shared_ptr GetInstance(); - Status GetStoreIds(const AppId &appId, std::vector &storeIds) override; + Status GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) override; Status BeforeCreate(const AppId &appId, const StoreId &storeId, const Options &options) override; Status AfterCreate(const AppId &appId, const StoreId &storeId, const Options &options, const std::vector &password) override; - Status Delete(const AppId &appId, const StoreId &storeId) override; - Status Close(const AppId &appId, const StoreId &storeId) override; - Status Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) override; + Status Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status Close(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) override; Status RegServiceNotifier(const AppId &appId, sptr notifier) override; Status UnregServiceNotifier(const AppId &appIdd) override; - Status SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) override; - Status GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) override; - Status EnableCapability(const AppId &appId, const StoreId &storeId) override; - Status DisableCapability(const AppId &appId, const StoreId &storeId) override; - Status SetCapability(const AppId &appId, const StoreId &storeId, const std::vector &local, - const std::vector &remote) override; - Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; - Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; - Status Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) override; - Status Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) override; - Status GetBackupPassword(const AppId &appId, const StoreId &storeId, std::vector> &passwords, - int32_t passwordType) override; + Status SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) override; + Status GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, KvSyncParam &syncParam) override; + Status EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) override; + Status SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::vector &local, const std::vector &remote) override; + Status AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) override; + Status RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) override; + Status Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) override; + Status Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) override; + Status GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, + std::vector> &passwords, int32_t passwordType) override; Status CloudSync(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) override; Status NotifyDataChange(const AppId &appId, const StoreId &storeId, uint64_t delay) override; Status PutSwitch(const AppId &appId, const SwitchData &data) override; @@ -58,7 +63,8 @@ public: Status SubscribeSwitchData(const AppId &appId) override; Status UnsubscribeSwitchData(const AppId &appId) override; Status SetConfig(const AppId &appId, const StoreId &storeId, const StoreConfig &storeConfig) override; - Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) override; + Status RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) override; sptr GetServiceAgent(const AppId &appId); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/observer_bridge.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/observer_bridge.h index 0450ba91..56eceb16 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/observer_bridge.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/observer_bridge.h @@ -28,7 +28,8 @@ public: using DBKey = DistributedDB::Key; using DBChangedData = DistributedDB::KvStoreChangedData; - ObserverBridge(AppId appId, StoreId storeId, std::shared_ptr observer, const Convertor &cvt); + ObserverBridge(AppId appId, StoreId storeId, int32_t subUser, std::shared_ptr observer, + const Convertor &cvt); ~ObserverBridge(); Status RegisterRemoteObserver(uint32_t realType); Status UnregisterRemoteObserver(uint32_t realType); @@ -52,6 +53,7 @@ private: static std::vector ConvertDB(const T &dbEntries, std::string &deviceId, const Convertor &convert); AppId appId_; StoreId storeId_; + int32_t subUser_; std::shared_ptr observer_; sptr remote_; const Convertor &convert_; diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h index a44a77fa..4315696a 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h @@ -49,6 +49,7 @@ public: SingleStoreImpl(std::shared_ptr dbStore, const AppId &appId, const Options &options, const Convertor &cvt); ~SingleStoreImpl(); StoreId GetStoreId() const override; + int32_t GetSubUser() override; Status Put(const Key &key, const Value &value) override; Status PutBatch(const std::vector &entries) override; Status Delete(const Key &key) override; @@ -116,7 +117,7 @@ private: (std::chrono::steady_clock::now() + std::chrono::milliseconds(offset)).time_since_epoch()) .count(); } - static constexpr int32_t INTEGRITY_CHECK_API_VERSION = 16; + static constexpr int32_t INTEGRITY_CHECK_API_VERSION = 20; static constexpr uint32_t NOTIFY_INTERVAL = 200; // ms static constexpr size_t MAX_VALUE_LENGTH = 4 * 1024 * 1024; static constexpr size_t MAX_OBSERVER_SIZE = 8; @@ -135,6 +136,7 @@ private: void Register(); void ReportDBFaultEvent(Status status, const std::string &functionName) const; + int32_t subUser_ = 0; int32_t apiVersion_ = -1; bool isApplication_ = false; bool autoSync_ = false; diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/store_factory.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/store_factory.h index f39131e2..c8ab684c 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/store_factory.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/store_factory.h @@ -22,27 +22,31 @@ #include "security_manager.h" #include "single_store_impl.h" namespace OHOS::DistributedKv { +struct StoreParams { + bool isCreate = false; + SecurityManager::DBPassword password; +}; class StoreFactory { public: + using DBPassword = DistributedKv::SecurityManager::DBPassword; static StoreFactory &GetInstance(); std::shared_ptr GetOrOpenStore(const AppId &appId, const StoreId &storeId, const Options &options, - Status &status, bool &isCreate); - Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path); - Status Close(const AppId &appId, const StoreId &storeId, bool isForce = false); + Status &status, StoreParams &storeParams); + Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser = 0); + Status Close(const AppId &appId, const StoreId &storeId, int32_t subUser = 0, bool isForce = false); private: using DBManager = DistributedDB::KvStoreDelegateManager; using DBOption = DistributedDB::KvStoreNbDelegate::Option; using DBStore = DistributedDB::KvStoreNbDelegate; using DBStatus = DistributedDB::DBStatus; - using DBPassword = DistributedKv::SecurityManager::DBPassword; static constexpr int REKEY_TIMES = 3; static constexpr const char *REKEY_NEW = ".new"; static constexpr uint64_t MAX_WAL_SIZE = 200 * 1024 * 1024; // the max size of WAL is 200MB StoreFactory(); - std::shared_ptr GetDBManager(const std::string &path, const AppId &appId); + std::shared_ptr GetDBManager(const std::string &path, const AppId &appId, int32_t subUser = 0); DBOption GetDBOption(const Options &options, const DBPassword &dbPassword) const; void ReKey(const std::string &storeId, const std::string &path, DBPassword &dbPassword, std::shared_ptr dbManager, const Options &options); @@ -52,6 +56,7 @@ private: Status IsPwdValid(const std::string &storeId, std::shared_ptr dbManager, const Options &options, DBPassword &dbPassword); Status SetDbConfig(std::shared_ptr dbStore); + std::string GenerateKey(const std::string &userId, const std::string &storeId) const; ConcurrentMap> dbManagers_; ConcurrentMap>> stores_; Convertor *convertors_[INVALID_TYPE]; diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/include/store_manager.h b/kv_store/frameworks/innerkitsimpl/kvdb/include/store_manager.h index 88891c77..025895b3 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/include/store_manager.h +++ b/kv_store/frameworks/innerkitsimpl/kvdb/include/store_manager.h @@ -16,16 +16,17 @@ #define OHOS_DISTRIBUTED_DATA_FRAMEWORKS_KVDB_STORE_MANAGER_H #include "single_kvstore.h" #include "kv_hiview_reporter.h" +#include "store_factory.h" namespace OHOS::DistributedKv { class StoreManager { public: static StoreManager &GetInstance(); std::shared_ptr GetKVStore(const AppId &appId, const StoreId &storeId, const Options &options, Status &status); - Status CloseKVStore(const AppId &appId, const StoreId &storeId); - Status CloseAllKVStore(const AppId &appId); - Status GetStoreIds(const AppId &appId, std::vector &storeIds); - Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path); + Status CloseKVStore(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); + Status CloseAllKVStore(const AppId &appId, int32_t subUser = 0); + Status GetStoreIds(const AppId &appId, std::vector &storeIds, int32_t subUser = 0); + Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser = 0); Status PutSwitch(const AppId &appId, const SwitchData &data); std::pair GetSwitch(const AppId &appId, const std::string &networkId); Status SubscribeSwitchData(const AppId &appId, std::shared_ptr observer); @@ -33,7 +34,7 @@ public: private: std::shared_ptr OpenWithSecretKeyFromService(const AppId &appId, const StoreId &storeId, - const Options &options, Status &status, bool &isCreate); + const Options &options, Status &status, StoreParams &storeParams); }; } #endif // OHOS_DISTRIBUTED_DATA_FRAMEWORKS_KVDB_STORE_MANAGER_H diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer.cpp index 0c5236b3..2ffd4028 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer.cpp @@ -110,7 +110,7 @@ std::function AutoSyncTimer::ProcessTask() syncInfo.devices.push_back(res.second); ZLOGD("DoSync appId:%{public}s store size:%{public}zu", id.first.c_str(), id.second.size()); for (const auto &storeId : id.second) { - service->Sync({ id.first }, storeId, syncInfo); + service->Sync({ id.first }, storeId, DEFAULT_USER_ID, syncInfo); } } if (HasSyncStores()) { diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer_mock.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer_mock.cpp index c6212b22..e7c45dac 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer_mock.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/auto_sync_timer_mock.cpp @@ -107,7 +107,7 @@ std::function AutoSyncTimer::ProcessTask() ZLOGD("DoSync appId:%{public}s store size:%{public}zu", id.first.c_str(), id.second.size()); for (const auto &storeId : id.second) { KVDBService::SyncInfo syncInfo; - service->Sync({ id.first }, storeId, syncInfo); + service->Sync({ id.first }, storeId, DEFAULT_USER_ID, syncInfo); } } if (HasSyncStores()) { diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/backup_manager.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/backup_manager.cpp index a557d373..3e795d12 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/backup_manager.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/backup_manager.cpp @@ -106,12 +106,12 @@ void BackupManager::CleanTmpData(const std::string &name) StoreUtil::Remove(tmpName); } -Status BackupManager::Backup(const BackupInfo &info, std::shared_ptr dbStore, bool isCheckIntegrity) +Status BackupManager::Backup(const BackupInfo &info, std::shared_ptr dbStore) { if (dbStore == nullptr) { return ALREADY_CLOSED; } - if (isCheckIntegrity) { + if (info.isCheckIntegrity) { auto integrityStatus = dbStore->CheckIntegrity(); if (integrityStatus != DistributedDB::DBStatus::OK) { return StoreUtil::ConvertStatus(integrityStatus); @@ -179,7 +179,7 @@ StoreUtil::FileInfo BackupManager::GetBackupFileInfo( return backupFile; } -Status BackupManager::Restore(const BackupInfo &info, std::shared_ptr dbStore, bool isCheckIntegrity) +Status BackupManager::Restore(const BackupInfo &info, std::shared_ptr dbStore) { if (dbStore == nullptr) { return ALREADY_CLOSED; @@ -192,21 +192,20 @@ Status BackupManager::Restore(const BackupInfo &info, std::shared_ptr d return INVALID_ARGUMENT; } auto fullName = info.baseDir + BACKUP_TOP_PATH + "/" + info.storeId + "/" + backupFile.name; - auto password = GetRestorePassword(backupFile.name, info.baseDir, info.appId, info.storeId).password; - auto dbStatus = dbStore->Import(fullName, password, isCheckIntegrity); + auto password = GetRestorePassword(backupFile.name, info).password; + auto dbStatus = dbStore->Import(fullName, password, info.isCheckIntegrity); if (dbStatus == DistributedDB::DBStatus::INVALID_FILE && info.encrypt) { ZLOGI("Use the key from server to restore"); - auto retryStatus = ImportWithSecretKeyFromService(info, dbStore, fullName, isCheckIntegrity); + auto retryStatus = ImportWithSecretKeyFromService(info, dbStore, fullName, info.isCheckIntegrity); return retryStatus == SUCCESS ? SUCCESS : CRYPT_ERROR; } return StoreUtil::ConvertStatus(dbStatus); } -BackupManager::DBPassword BackupManager::GetRestorePassword( - const std::string &name, const std::string &baseDir, const std::string &appId, const std::string &storeId) +BackupManager::DBPassword BackupManager::GetRestorePassword(const std::string &name, const BackupInfo &info) { auto backupName = name.substr(0, name.length() - BACKUP_POSTFIX_SIZE); - auto keyName = BACKUP_KEY_PREFIX + storeId + "_" + backupName; + auto keyName = BACKUP_KEY_PREFIX + info.storeId + "_" + backupName; DBPassword dbPassword; if (backupName == AUTO_BACKUP_NAME) { auto service = KVDBServiceClient::GetInstance(); @@ -214,7 +213,8 @@ BackupManager::DBPassword BackupManager::GetRestorePassword( return dbPassword; } std::vector> pwds; - service->GetBackupPassword({ appId }, { storeId }, pwds, KVDBService::PasswordType::BACKUP_SECRET_KEY); + service->GetBackupPassword({ info.appId }, { info.storeId }, info.subUser, pwds, + KVDBService::PasswordType::BACKUP_SECRET_KEY); if (pwds.size() != 0) { // When obtaining the key for automatic backup, there is only one element in the list dbPassword.SetValue(pwds[0].data(), pwds[0].size()); @@ -223,13 +223,13 @@ BackupManager::DBPassword BackupManager::GetRestorePassword( pwd.assign(pwd.size(), 0); } } else { - dbPassword = SecurityManager::GetInstance().GetDBPassword(keyName, baseDir); + dbPassword = SecurityManager::GetInstance().GetDBPassword(keyName, info.baseDir); } return dbPassword; } Status BackupManager::GetSecretKeyFromService(const AppId &appId, const StoreId &storeId, - std::vector> &keys) + std::vector> &keys, int32_t subUser) { auto service = KVDBServiceClient::GetInstance(); if (service == nullptr) { @@ -237,7 +237,7 @@ Status BackupManager::GetSecretKeyFromService(const AppId &appId, const StoreId appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); return Status::SERVER_UNAVAILABLE; } - auto status = service->GetBackupPassword(appId, storeId, keys, KVDBService::PasswordType::SECRET_KEY); + auto status = service->GetBackupPassword(appId, storeId, subUser, keys, KVDBService::PasswordType::SECRET_KEY); if (status != Status::SUCCESS) { ZLOGE("Get password from service failed! status:%{public}d, appId:%{public}s storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -256,7 +256,7 @@ Status BackupManager::ImportWithSecretKeyFromService(const BackupInfo &info, std { Status status = NOT_FOUND; std::vector> keys; - if (GetSecretKeyFromService({ info.appId }, { info.storeId }, keys) != Status::SUCCESS) { + if (GetSecretKeyFromService({ info.appId }, { info.storeId }, keys, info.subUser) != Status::SUCCESS) { for (auto &key : keys) { key.assign(key.size(), 0); } diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter.cpp index b5246f30..abeacad5 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter.cpp @@ -23,6 +23,7 @@ #include #include #include +#include "concurrent_map.h" #include "hisysevent_c.h" #include "log_print.h" #include "types.h" @@ -56,6 +57,8 @@ static constexpr const char *BUSINESS_TYPE[] = { "gausspd" }; +static ConcurrentMap> stores_; + struct KVDBFaultEvent { std::string bundleName; std::string moduleName; @@ -104,7 +107,7 @@ void KVDBFaultHiViewReporter::ReportKVFaultEvent(const ReportInfo &reportInfo) void KVDBFaultHiViewReporter::ReportKVRebuildEvent(const ReportInfo &reportInfo) { auto reportDir = GetDBPath(reportInfo.options.GetDatabaseDir(), reportInfo.storeId); - if (!IsReportedCorruptedFault(reportDir, reportInfo.storeId)) { + if (!IsReportedCorruptedFault(reportInfo.appId, reportInfo.storeId, reportDir)) { return; } KVDBFaultEvent eventInfo(reportInfo.options); @@ -121,6 +124,11 @@ void KVDBFaultHiViewReporter::ReportKVRebuildEvent(const ReportInfo &reportInfo) eventInfo.appendix = GenerateAppendix(eventInfo); eventInfo.appendix += "\n" + std::string(DATABASE_REBUILD); ReportCommonFault(eventInfo); + auto storeName = eventInfo.storeName; + stores_.Compute(eventInfo.bundleName, [&storeName](auto &key, auto &value) { + value.erase(storeName); + return true; + }); } } @@ -156,13 +164,18 @@ void KVDBFaultHiViewReporter::ReportCurruptedEvent(KVDBFaultEvent eventInfo) StoreUtil::Anonymous(eventInfo.storeName).c_str()); return; } - if (IsReportedCorruptedFault(eventInfo.appendix, eventInfo.storeName)) { + if (IsReportedCorruptedFault(eventInfo.bundleName, eventInfo.storeName, eventInfo.appendix)) { return; } CreateCorruptedFlag(eventInfo.appendix, eventInfo.storeName); eventInfo.appendix = GenerateAppendix(eventInfo); ZLOGI("Db corrupted report:storeId:%{public}s", StoreUtil::Anonymous(eventInfo.storeName).c_str()); ReportCommonFault(eventInfo); + auto storeName = eventInfo.storeName; + stores_.Compute(eventInfo.bundleName, [&storeName](auto &key, auto &value) { + value.insert(storeName); + return true; + }); } std::string KVDBFaultHiViewReporter::GetCurrentMicrosecondTimeFormat() @@ -264,13 +277,15 @@ bool KVDBFaultHiViewReporter::IsReportedFault(const KVDBFaultEvent& eventInfo) return false; } -bool KVDBFaultHiViewReporter::IsReportedCorruptedFault(const std::string &dbPath, const std::string &storeId) +bool KVDBFaultHiViewReporter::IsReportedCorruptedFault(const std::string &appId, const std::string &storeId, + const std::string &dbPath) { - std::string flagFilename = dbPath + storeId + DB_CORRUPTED_POSTFIX; - if (access(flagFilename.c_str(), F_OK) == 0) { + if (stores_.ContainIf(appId, [&storeId]( + const std::set &stores) -> bool { return stores.count(storeId) != 0; })) { return true; } - return false; + std::string flagFilename = dbPath + storeId + DB_CORRUPTED_POSTFIX; + return access(flagFilename.c_str(), F_OK) == 0; } void KVDBFaultHiViewReporter::CreateCorruptedFlag(const std::string &dbPath, const std::string &storeId) diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter_mock.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter_mock.cpp index b105283e..8ea740bf 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter_mock.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/kv_hiview_reporter_mock.cpp @@ -65,7 +65,8 @@ void KVDBFaultHiViewReporter::ReportCommonFault(__attribute__((unused)) return; } -bool KVDBFaultHiViewReporter::IsReportedCorruptedFault(const std::string &dbPath, const std::string &storeId) +bool KVDBFaultHiViewReporter::IsReportedCorruptedFault(const std::string &appId, const std::string &storeId, + const std::string &dbPath) { return false; } diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/kvdb_service_client.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/kvdb_service_client.cpp index 4ed771e4..5604fd76 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/kvdb_service_client.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/kvdb_service_client.cpp @@ -103,11 +103,11 @@ KVDBServiceClient::KVDBServiceClient(const sptr &handle) : IRemot remote_ = Remote(); } -Status KVDBServiceClient::GetStoreIds(const AppId &appId, std::vector &storeIds) +Status KVDBServiceClient::GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_GET_STORE_IDS), - reply, appId, StoreId(), storeIds); + reply, appId, StoreId(), subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s", status, appId.appId.c_str()); } @@ -140,11 +140,11 @@ Status KVDBServiceClient::AfterCreate( return static_cast(status); } -Status KVDBServiceClient::Delete(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_DELETE), - reply, appId, storeId); + reply, appId, storeId, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -152,11 +152,11 @@ Status KVDBServiceClient::Delete(const AppId &appId, const StoreId &storeId) return static_cast(status); } -Status KVDBServiceClient::Close(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::Close(const AppId &appId, const StoreId &storeId, int32_t subUser) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_CLOSE), - reply, appId, storeId); + reply, appId, storeId, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -164,11 +164,11 @@ Status KVDBServiceClient::Close(const AppId &appId, const StoreId &storeId) return static_cast(status); } -Status KVDBServiceClient::Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) +Status KVDBServiceClient::Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_SYNC), reply, appId, storeId, - syncInfo.seqId, syncInfo.mode, syncInfo.devices, syncInfo.delay, syncInfo.query); + syncInfo.seqId, syncInfo.mode, syncInfo.devices, syncInfo.delay, syncInfo.query, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s, sequenceId:%{public}" PRIu64, status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), syncInfo.seqId); @@ -223,11 +223,12 @@ Status KVDBServiceClient::UnregServiceNotifier(const AppId &appId) return static_cast(status); } -Status KVDBServiceClient::SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) +Status KVDBServiceClient::SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_SET_SYNC_PARAM), reply, - appId, storeId, syncParam.allowedDelayMs); + appId, storeId, syncParam.allowedDelayMs, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -235,11 +236,12 @@ Status KVDBServiceClient::SetSyncParam(const AppId &appId, const StoreId &storeI return static_cast(status); } -Status KVDBServiceClient::GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) +Status KVDBServiceClient::GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + KvSyncParam &syncParam) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_GET_SYNC_PARAM), - reply, appId, storeId); + reply, appId, storeId, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -249,11 +251,11 @@ Status KVDBServiceClient::GetSyncParam(const AppId &appId, const StoreId &storeI return static_cast(status); } -Status KVDBServiceClient::EnableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_ENABLE_CAP), - reply, appId, storeId); + reply, appId, storeId, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -261,11 +263,11 @@ Status KVDBServiceClient::EnableCapability(const AppId &appId, const StoreId &st return static_cast(status); } -Status KVDBServiceClient::DisableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_DISABLE_CAP), - reply, appId, storeId); + reply, appId, storeId, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -273,12 +275,12 @@ Status KVDBServiceClient::DisableCapability(const AppId &appId, const StoreId &s return static_cast(status); } -Status KVDBServiceClient::SetCapability(const AppId &appId, const StoreId &storeId, +Status KVDBServiceClient::SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, const std::vector &local, const std::vector &remote) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_SET_CAP), - reply, appId, storeId, local, remote); + reply, appId, storeId, local, remote, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -286,11 +288,12 @@ Status KVDBServiceClient::SetCapability(const AppId &appId, const StoreId &store return static_cast(status); } -Status KVDBServiceClient::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceClient::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_ADD_SUB), - reply, appId, storeId, syncInfo.seqId, syncInfo.devices, syncInfo.query); + reply, appId, storeId, syncInfo.seqId, syncInfo.devices, syncInfo.query, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s, query:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), @@ -299,11 +302,12 @@ Status KVDBServiceClient::AddSubscribeInfo(const AppId &appId, const StoreId &st return static_cast(status); } -Status KVDBServiceClient::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceClient::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_RMV_SUB), - reply, appId, storeId, syncInfo.seqId, syncInfo.devices, syncInfo.query); + reply, appId, storeId, syncInfo.seqId, syncInfo.devices, syncInfo.query, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s, query:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), @@ -312,11 +316,12 @@ Status KVDBServiceClient::RmvSubscribeInfo(const AppId &appId, const StoreId &st return static_cast(status); } -Status KVDBServiceClient::Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceClient::Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_SUB), - reply, appId, storeId, observer->AsObject()); + reply, appId, storeId, observer->AsObject(), subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s, observer:0x%{public}x", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), @@ -325,11 +330,12 @@ Status KVDBServiceClient::Subscribe(const AppId &appId, const StoreId &storeId, return static_cast(status); } -Status KVDBServiceClient::Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceClient::Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_UNSUB), - reply, appId, storeId, observer->AsObject().GetRefPtr()); + reply, appId, storeId, observer->AsObject().GetRefPtr(), subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x, appId:%{public}s, storeId:%{public}s, observer:0x%{public}x", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), @@ -338,12 +344,12 @@ Status KVDBServiceClient::Unsubscribe(const AppId &appId, const StoreId &storeId return static_cast(status); } -Status KVDBServiceClient::GetBackupPassword( - const AppId &appId, const StoreId &storeId, std::vector> &passwords, int32_t passwordType) +Status KVDBServiceClient::GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, + std::vector> &passwords, int32_t passwordType) { MessageParcel reply; int32_t status = IPC_SEND(static_cast(KVDBServiceInterfaceCode::TRANS_GET_PASSWORD), - reply, appId, storeId, passwordType); + reply, appId, storeId, passwordType, subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); @@ -360,6 +366,10 @@ sptr KVDBServiceClient::GetServiceAgent(const AppId &appId) } sptr serviceAgent = new (std::nothrow) KVDBNotifierClient(); + if (serviceAgent == nullptr) { + ZLOGE("New KVDBNotifierClient failed, appId:%{public}s", appId.appId.c_str()); + return nullptr; + } auto status = RegServiceNotifier(appId, serviceAgent); if (status == SUCCESS) { serviceAgent_ = std::move(serviceAgent); @@ -425,11 +435,13 @@ Status KVDBServiceClient::SetConfig(const AppId &appId, const StoreId &storeId, return static_cast(status); } -Status KVDBServiceClient::RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) +Status KVDBServiceClient::RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) { MessageParcel reply; int32_t status = IPC_SEND( - static_cast(KVDBServiceInterfaceCode::TRANS_REMOVE_DEVICE_DATA), reply, appId, storeId, device); + static_cast(KVDBServiceInterfaceCode::TRANS_REMOVE_DEVICE_DATA), reply, appId, storeId, device, + subUser); if (status != SUCCESS) { ZLOGE("status:0x%{public}x appId:%{public}s, storeId:%{public}s", status, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/observer_bridge.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/observer_bridge.cpp index 8a466243..f85d8042 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/observer_bridge.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/observer_bridge.cpp @@ -12,13 +12,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#define LOG_TAG "ObserverBridge" #include "observer_bridge.h" #include "kvdb_service_client.h" #include "kvstore_observer_client.h" +#include "log_print.h" namespace OHOS::DistributedKv { constexpr uint32_t INVALID_SUBSCRIBE_TYPE = 0; -ObserverBridge::ObserverBridge(AppId appId, StoreId store, std::shared_ptr observer, const Convertor &cvt) - : appId_(std::move(appId)), storeId_(std::move(store)), observer_(std::move(observer)), convert_(cvt) +ObserverBridge::ObserverBridge(AppId appId, StoreId storeId, int32_t subUser, std::shared_ptr observer, + const Convertor &cvt) : appId_(std::move(appId)), storeId_(std::move(storeId)), subUser_(subUser), + observer_(std::move(observer)), convert_(cvt) { } @@ -31,7 +34,7 @@ ObserverBridge::~ObserverBridge() if (service == nullptr) { return; } - service->Unsubscribe(appId_, storeId_, remote_); + service->Unsubscribe(appId_, storeId_, subUser_, remote_); } Status ObserverBridge::RegisterRemoteObserver(uint32_t realType) @@ -48,7 +51,11 @@ Status ObserverBridge::RegisterRemoteObserver(uint32_t realType) } remote_ = new (std::nothrow) ObserverClient(observer_, convert_); - auto status = service->Subscribe(appId_, storeId_, remote_); + if (remote_ == nullptr) { + ZLOGE("New ObserverClient failed, appId:%{public}s", appId_.appId.c_str()); + return ERROR; + } + auto status = service->Subscribe(appId_, storeId_, subUser_, remote_); if (status != SUCCESS) { remote_ = nullptr; } else { @@ -73,7 +80,7 @@ Status ObserverBridge::UnregisterRemoteObserver(uint32_t realType) remote_->realType_ &= ~SUBSCRIBE_TYPE_LOCAL; remote_->realType_ &= ~realType; if (remote_->realType_ == 0) { - status = service->Unsubscribe(appId_, storeId_, remote_); + status = service->Unsubscribe(appId_, storeId_, subUser_, remote_); remote_ = nullptr; } return status; diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp index cde4083a..928b1459 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp @@ -51,6 +51,7 @@ SingleStoreImpl::SingleStoreImpl( securityLevel_ = options.securityLevel; area_ = options.area; hapName_ = options.hapName; + subUser_ = options.subUser; if (options.backup) { BackupManager::GetInstance().Prepare(path, storeId_); } @@ -76,6 +77,11 @@ StoreId SingleStoreImpl::GetStoreId() const return { storeId_ }; } +int32_t SingleStoreImpl::GetSubUser() +{ + return subUser_; +} + Status SingleStoreImpl::RetryWithCheckPoint(std::function lambda) { auto dbStatus = lambda(); @@ -572,7 +578,7 @@ Status SingleStoreImpl::RemoveDeviceData(const std::string &device) return SERVER_UNAVAILABLE; } - Status status = service->RemoveDeviceData({ appId_ }, { storeId_ }, device); + Status status = service->RemoveDeviceData({ appId_ }, { storeId_ }, subUser_, device); if (status != SUCCESS) { ReportDBFaultEvent(status, std::string(__FUNCTION__)); ZLOGE("status:%{public}d device:%{public}s", status, StoreUtil::Anonymous(device).c_str()); @@ -653,7 +659,7 @@ Status SingleStoreImpl::SetSyncParam(const KvSyncParam &syncParam) if (service == nullptr) { return SERVER_UNAVAILABLE; } - return service->SetSyncParam({ appId_ }, { storeId_ }, syncParam); + return service->SetSyncParam({ appId_ }, { storeId_ }, subUser_, syncParam); } Status SingleStoreImpl::GetSyncParam(KvSyncParam &syncParam) @@ -663,7 +669,7 @@ Status SingleStoreImpl::GetSyncParam(KvSyncParam &syncParam) if (service == nullptr) { return SERVER_UNAVAILABLE; } - return service->GetSyncParam({ appId_ }, { storeId_ }, syncParam); + return service->GetSyncParam({ appId_ }, { storeId_ }, subUser_, syncParam); } Status SingleStoreImpl::SetCapabilityEnabled(bool enabled) const @@ -674,9 +680,9 @@ Status SingleStoreImpl::SetCapabilityEnabled(bool enabled) const return SERVER_UNAVAILABLE; } if (enabled) { - return service->EnableCapability({ appId_ }, { storeId_ }); + return service->EnableCapability({ appId_ }, { storeId_ }, subUser_); } - return service->DisableCapability({ appId_ }, { storeId_ }); + return service->DisableCapability({ appId_ }, { storeId_ }, subUser_); } Status SingleStoreImpl::SetCapabilityRange( @@ -687,7 +693,7 @@ Status SingleStoreImpl::SetCapabilityRange( if (service == nullptr) { return SERVER_UNAVAILABLE; } - return service->SetCapability({ appId_ }, { storeId_ }, localLabels, remoteLabels); + return service->SetCapability({ appId_ }, { storeId_ }, subUser_, localLabels, remoteLabels); } Status SingleStoreImpl::SubscribeWithQuery(const std::vector &devices, const DataQuery &query) @@ -709,7 +715,7 @@ Status SingleStoreImpl::SubscribeWithQuery(const std::vector &devic } serviceAgent->AddSyncCallback(syncObserver_, syncInfo.seqId); - return service->AddSubscribeInfo({ appId_ }, { storeId_ }, syncInfo); + return service->AddSubscribeInfo({ appId_ }, { storeId_ }, subUser_, syncInfo); } Status SingleStoreImpl::UnsubscribeWithQuery(const std::vector &devices, const DataQuery &query) @@ -731,7 +737,7 @@ Status SingleStoreImpl::UnsubscribeWithQuery(const std::vector &dev } serviceAgent->AddSyncCallback(syncObserver_, syncInfo.seqId); - return service->RmvSubscribeInfo({ appId_ }, { storeId_ }, syncInfo); + return service->RmvSubscribeInfo({ appId_ }, { storeId_ }, subUser_, syncInfo); } int32_t SingleStoreImpl::AddRef() @@ -761,8 +767,9 @@ int32_t SingleStoreImpl::Close(bool isForce) Status SingleStoreImpl::Backup(const std::string &file, const std::string &baseDir) { DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__)); - BackupManager::BackupInfo info = { .name = file, .baseDir = baseDir, .storeId = storeId_ }; - auto status = BackupManager::GetInstance().Backup(info, dbStore_, isCheckIntegrity_); + BackupManager::BackupInfo info = { .name = file, .baseDir = baseDir, .storeId = storeId_, + .isCheckIntegrity = isCheckIntegrity_ }; + auto status = BackupManager::GetInstance().Backup(info, dbStore_); if (status != SUCCESS) { ZLOGE("status:0x%{public}x storeId:%{public}s backup:%{public}s ", status, StoreUtil::Anonymous(storeId_).c_str(), file.c_str()); @@ -775,11 +782,11 @@ Status SingleStoreImpl::Restore(const std::string &file, const std::string &base DdsTrace trace(std::string(LOG_TAG "::") + std::string(__FUNCTION__)); auto service = KVDBServiceClient::GetInstance(); if (service != nullptr) { - service->Close({ appId_ }, { storeId_ }); + service->Close({ appId_ }, { storeId_ }, subUser_); } BackupManager::BackupInfo info = { .name = file, .baseDir = baseDir, .appId = appId_, .storeId = storeId_, - .encrypt = encrypt_ }; - auto status = BackupManager::GetInstance().Restore(info, dbStore_, isCheckIntegrity_); + .encrypt = encrypt_, .isCheckIntegrity = isCheckIntegrity_, .subUser = subUser_ }; + auto status = BackupManager::GetInstance().Restore(info, dbStore_); if (status != SUCCESS) { ZLOGE("status:0x%{public}x storeId:%{public}s backup:%{public}s ", status, StoreUtil::Anonymous(storeId_).c_str(), file.c_str()); @@ -849,7 +856,7 @@ std::shared_ptr SingleStoreImpl::PutIn(uint32_t &realType, std:: auto release = BridgeReleaser(); StoreId storeId{ storeId_ }; AppId appId{ appId_ }; - pair.second = { new ObserverBridge(appId, storeId, observer, convertor_), release }; + pair.second = { new ObserverBridge(appId, storeId, subUser_, observer, convertor_), release }; } bridge = pair.second; realType = (realType & (~pair.first)); @@ -954,9 +961,8 @@ Status SingleStoreImpl::DoClientSync(SyncInfo &syncInfo, std::shared_ptr observer) { - Status cStatus = Status::SUCCESS; if (isClientSync_) { - cStatus = DoClientSync(syncInfo, observer); + return DoClientSync(syncInfo, observer); } auto service = KVDBServiceClient::GetInstance(); @@ -972,20 +978,13 @@ Status SingleStoreImpl::DoSync(SyncInfo &syncInfo, std::shared_ptr } serviceAgent->AddSyncCallback(observer, syncInfo.seqId); - auto status = service->Sync({ appId_ }, { storeId_ }, syncInfo); + auto status = service->Sync({ appId_ }, { storeId_ }, subUser_, syncInfo); if (status != Status::SUCCESS) { + ZLOGE("Service sync failed! app:%{public}s store:%{public}s status:%{public}d", appId_.c_str(), + StoreUtil::Anonymous(storeId_).c_str(), status); serviceAgent->DeleteSyncCallback(syncInfo.seqId); } - - if (!isClientSync_) { - return status; - } - if (cStatus == SUCCESS || status == SUCCESS) { - return SUCCESS; - } else { - ZLOGE("Sync failed!: %{public}d, %{public}d", cStatus, status); - return ERROR; - } + return status; } Status SingleStoreImpl::SetConfig(const StoreConfig &storeConfig) diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp index 40a468ef..e15c248c 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp @@ -28,6 +28,7 @@ #include "runtime_config.h" namespace OHOS::DistributedKv { using namespace DistributedDB; +static constexpr const char *KEY_SPLIT = "###"; StoreFactory &StoreFactory::GetInstance() { static StoreFactory instance; @@ -57,20 +58,29 @@ Status StoreFactory::SetDbConfig(std::shared_ptr dbStore) return StoreUtil::ConvertStatus(status); } +std::string StoreFactory::GenerateKey(const std::string &userId, const std::string &storeId) const +{ + std::string key = ""; + if (storeId.empty() || userId.empty()) { + return key; + } + return key.append(userId).append(KEY_SPLIT).append(storeId); +} + std::shared_ptr StoreFactory::GetOrOpenStore(const AppId &appId, const StoreId &storeId, - const Options &options, Status &status, bool &isCreate) + const Options &options, Status &status, StoreParams &storeParams) { std::shared_ptr kvStore; - isCreate = false; + auto key = GenerateKey(std::to_string(options.subUser), storeId.storeId); stores_.Compute(appId, [&](auto &, auto &stores) { - if (stores.find(storeId) != stores.end()) { - kvStore = stores[storeId]; + if (stores.find(key) != stores.end()) { + kvStore = stores[key]; kvStore->AddRef(); status = SUCCESS; return !stores.empty(); } std::string path = options.GetDatabaseDir(); - auto dbManager = GetDBManager(path, appId); + auto dbManager = GetDBManager(path, appId, options.subUser); auto dbPassword = SecurityManager::GetInstance().GetDBPassword(storeId.storeId, path, options.encrypt); if (options.encrypt && !dbPassword.IsValid()) { @@ -109,29 +119,32 @@ std::shared_ptr StoreFactory::GetOrOpenStore(const AppId &appId, appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), path.c_str()); return !stores.empty(); } - isCreate = true; - stores[storeId] = kvStore; + stores[key] = kvStore; KvStoreServiceDeathNotifier::AddServiceDeathWatcher(kvStore); + storeParams.isCreate = true; + storeParams.password = dbPassword; + dbPassword.Clear(); return !stores.empty(); }); return kvStore; } -Status StoreFactory::Delete(const AppId &appId, const StoreId &storeId, const std::string &path) +Status StoreFactory::Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser) { - Close(appId, storeId, true); - auto dbManager = GetDBManager(path, appId); + Close(appId, storeId, subUser, true); + auto dbManager = GetDBManager(path, appId, subUser); auto status = dbManager->DeleteKvStore(storeId); SecurityManager::GetInstance().DelDBPassword(storeId.storeId, path); return StoreUtil::ConvertStatus(status); } -Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, bool isForce) +Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, int32_t subUser, bool isForce) { Status status = STORE_NOT_OPEN; - stores_.ComputeIfPresent(appId, [&storeId, &status, isForce](auto &, auto &values) { + auto key = GenerateKey(std::to_string(subUser), storeId.storeId); + stores_.ComputeIfPresent(appId, [&key, &status, isForce](auto &, auto &values) { for (auto it = values.begin(); it != values.end();) { - if (!storeId.storeId.empty() && (it->first != storeId.storeId)) { + if (!key.empty() && (it->first != key)) { ++it; continue; } @@ -150,17 +163,18 @@ Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, bool isFo return status; } -std::shared_ptr StoreFactory::GetDBManager(const std::string &path, const AppId &appId) +std::shared_ptr StoreFactory::GetDBManager(const std::string &path, const AppId &appId, + int32_t subUser) { std::shared_ptr dbManager; - dbManagers_.Compute(path, [&dbManager, &appId](const auto &path, std::shared_ptr &manager) { + dbManagers_.Compute(path, [&dbManager, &appId, &subUser](const auto &path, std::shared_ptr &manager) { std::string fullPath = path + "/kvdb"; auto result = StoreUtil::InitPath(fullPath); if (manager != nullptr && result) { dbManager = manager; return true; } - dbManager = std::make_shared(appId.appId, "default"); + dbManager = std::make_shared(appId.appId, std::to_string(subUser)); dbManager->SetKvStoreConfig({ fullPath }); manager = dbManager; BackupManager::GetInstance().Init(path); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp index 650a30a3..5a8230b4 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp @@ -20,7 +20,7 @@ #include "kvdb_service_client.h" #include "log_print.h" #include "security_manager.h" -#include "store_factory.h" + #include "store_util.h" namespace OHOS::DistributedKv { StoreManager &StoreManager::GetInstance() @@ -46,50 +46,48 @@ std::shared_ptr StoreManager::GetKVStore(const AppId &appId, cons status = service->BeforeCreate(appId, storeId, options); } if (status == STORE_META_CHANGED) { -// ReportInfo reportInfo = { .options = options, -// .errorCode = static_cast(status), -// .systemErrorNo = errno, -// .appId = appId.appId, -// .storeId = storeId.storeId, -// .functionName = std::string(__FUNCTION__) }; -// KVDBFaultHiViewReporter::ReportKVFaultEvent(reportInfo); + ReportInfo reportInfo = { .options = options, .errorCode = status, .systemErrorNo = errno, + .appId = appId.appId, .storeId = storeId.storeId, .functionName = std::string(__FUNCTION__) }; + KVDBFaultHiViewReporter::ReportKVFaultEvent(reportInfo); ZLOGE("appId:%{public}s, storeId:%{public}s type:%{public}d encrypt:%{public}d", appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), options.kvStoreType, options.encrypt); return nullptr; } - bool isCreate = false; - auto kvStore = StoreFactory::GetInstance().GetOrOpenStore(appId, storeId, options, status, isCreate); + StoreParams storeParams; + auto kvStore = StoreFactory::GetInstance().GetOrOpenStore(appId, storeId, options, status, storeParams); if ((status == DATA_CORRUPTED || status == CRYPT_ERROR) && options.encrypt) { - kvStore = OpenWithSecretKeyFromService(appId, storeId, options, status, isCreate); + kvStore = OpenWithSecretKeyFromService(appId, storeId, options, status, storeParams); } if (status != SUCCESS) { -// ReportInfo reportInfo = { .options = options, .appId = appId.appId, .storeId = storeId.storeId, -// .errorCode = static_cast(status), .systemErrorNo = errno, .functionName = std::string(__FUNCTION__) }; -// KVDBFaultHiViewReporter::ReportKVFaultEvent(reportInfo); + ReportInfo reportInfo = { .options = options, .errorCode = status, .systemErrorNo = errno, + .appId = appId.appId, .storeId = storeId.storeId, .functionName = std::string(__FUNCTION__) }; + KVDBFaultHiViewReporter::ReportKVFaultEvent(reportInfo); } else if (kvStore != nullptr && kvStore->IsRebuild()) { -// ReportInfo reportInfo = { .options = options, .appId = appId.appId, .storeId = storeId.storeId, -// .errorCode = static_cast(status), .systemErrorNo = errno, .functionName = std::string(__FUNCTION__) }; + ReportInfo reportInfo = { .options = options, .errorCode = status, .systemErrorNo = errno, + .appId = appId.appId, .storeId = storeId.storeId, .functionName = std::string(__FUNCTION__) }; ZLOGI("Rebuild store success, storeId:%{public}s", StoreUtil::Anonymous(storeId.storeId).c_str()); -// KVDBFaultHiViewReporter::ReportKVRebuildEvent(reportInfo); + KVDBFaultHiViewReporter::ReportKVRebuildEvent(reportInfo); } - if (isCreate && options.persistent) { - auto dbPassword = SecurityManager::GetInstance().GetDBPassword(storeId.storeId, path, options.encrypt); - std::vector pwd(dbPassword.GetData(), dbPassword.GetData() + dbPassword.GetSize()); + if (storeParams.isCreate && options.persistent) { + std::vector pwd(storeParams.password.GetData(), storeParams.password.GetData() + + storeParams.password.GetSize()); if (service != nullptr) { // delay notify service->AfterCreate(appId, storeId, options, pwd); } pwd.assign(pwd.size(), 0); + storeParams.password.Clear(); } return kvStore; } std::shared_ptr StoreManager::OpenWithSecretKeyFromService(const AppId &appId, const StoreId &storeId, - const Options &options, Status &status, bool &isCreate) + const Options &options, Status &status, StoreParams &storeParams) { std::shared_ptr kvStore; std::vector> keys; - if (BackupManager::GetInstance().GetSecretKeyFromService(appId, storeId, keys) != Status::SUCCESS) { + if (BackupManager::GetInstance().GetSecretKeyFromService(appId, storeId, keys, options.subUser) != + Status::SUCCESS) { for (auto &key : keys) { key.assign(key.size(), 0); } @@ -100,7 +98,7 @@ std::shared_ptr StoreManager::OpenWithSecretKeyFromService(const SecurityManager::DBPassword dbPassword; dbPassword.SetValue(key.data(), key.size()); SecurityManager::GetInstance().SaveDBPassword(storeId.storeId, path, dbPassword.password); - kvStore = StoreFactory::GetInstance().GetOrOpenStore(appId, storeId, options, status, isCreate); + kvStore = StoreFactory::GetInstance().GetOrOpenStore(appId, storeId, options, status, storeParams); if (status == SUCCESS) { break; } @@ -111,27 +109,27 @@ std::shared_ptr StoreManager::OpenWithSecretKeyFromService(const return kvStore; } -Status StoreManager::CloseKVStore(const AppId &appId, const StoreId &storeId) +Status StoreManager::CloseKVStore(const AppId &appId, const StoreId &storeId, int32_t subUser) { ZLOGD("appId:%{public}s, storeId:%{public}s", appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); if (!appId.IsValid() || !storeId.IsValid()) { return INVALID_ARGUMENT; } - return StoreFactory::GetInstance().Close(appId, storeId); + return StoreFactory::GetInstance().Close(appId, storeId, subUser); } -Status StoreManager::CloseAllKVStore(const AppId &appId) +Status StoreManager::CloseAllKVStore(const AppId &appId, int32_t subUser) { ZLOGD("appId:%{public}s", appId.appId.c_str()); if (!appId.IsValid()) { return INVALID_ARGUMENT; } - return StoreFactory::GetInstance().Close(appId, { "" }, true); + return StoreFactory::GetInstance().Close(appId, { "" }, subUser, true); } -Status StoreManager::GetStoreIds(const AppId &appId, std::vector &storeIds) +Status StoreManager::GetStoreIds(const AppId &appId, std::vector &storeIds, int32_t subUser) { ZLOGD("appId:%{public}s", appId.appId.c_str()); if (!appId.IsValid()) { @@ -142,10 +140,10 @@ Status StoreManager::GetStoreIds(const AppId &appId, std::vector &store if (service == nullptr) { return SERVER_UNAVAILABLE; } - return service->GetStoreIds(appId, storeIds); + return service->GetStoreIds(appId, subUser, storeIds); } -Status StoreManager::Delete(const AppId &appId, const StoreId &storeId, const std::string &path) +Status StoreManager::Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser) { ZLOGD("appId:%{public}s, storeId:%{public}s dir:%{public}s", appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str(), path.c_str()); @@ -155,11 +153,11 @@ Status StoreManager::Delete(const AppId &appId, const StoreId &storeId, const st auto service = KVDBServiceClient::GetInstance(); if (service != nullptr) { - service->Delete(appId, storeId); + service->Delete(appId, storeId, subUser); } - auto status = StoreFactory::GetInstance().Delete(appId, storeId, path); + auto status = StoreFactory::GetInstance().Delete(appId, storeId, path, subUser); Options options = { .baseDir = path }; - ReportInfo reportInfo = { .options = options, .errorCode = static_cast(status), .systemErrorNo = errno, + ReportInfo reportInfo = { .options = options, .errorCode = status, .systemErrorNo = errno, .appId = appId.appId, .storeId = storeId.storeId, .functionName = std::string(__FUNCTION__) }; if (status != SUCCESS) { KVDBFaultHiViewReporter::ReportKVFaultEvent(reportInfo); diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/test/BUILD.gn b/kv_store/frameworks/innerkitsimpl/kvdb/test/BUILD.gn index eed77320..e05c20b2 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/test/BUILD.gn +++ b/kv_store/frameworks/innerkitsimpl/kvdb/test/BUILD.gn @@ -13,12 +13,14 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/kv_store/kv_store.gni") -module_output_path = "kv_store/distributeddatafwk" +module_output_path = "kv_store/kv_store/distributeddatafwk" ############################################################################### config("module_private_config") { visibility = [ ":*" ] + cflags = [ "-Wno-c99-designator" ] + include_dirs = [ "${kv_store_base_path}/frameworks/innerkitsimpl/distributeddatasvc/include", "../../../../interfaces/innerkits/distributeddatamgr/include/", @@ -527,91 +529,6 @@ ohos_unittest("SingleStoreImplMockTest") { use_exceptions = true } -ohos_unittest("BackupManagerUnitTest") { - module_out_path = module_output_path - - sources = [ - "../../kvdb/src/kv_hiview_reporter_mock.cpp", - "backup_manager_unittest.cpp", - ] - - configs = [ ":module_private_config" ] - - external_deps = [ - "hilog:libhilog", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", - "huks:libhukssdk", - ] - - deps = [ - ":kvdb_src_file", - "../../../libs/distributeddb/:distributeddb", - ] - - public_external_deps = [ - "googletest:gmock_main", - "googletest:gtest_main", - ] -} - -ohos_unittest("DevManagerUnitTest") { - module_out_path = module_output_path - - sources = [ - "../../kvdb/src/kv_hiview_reporter_mock.cpp", - "dev_manager_unittest.cpp", - ] - - configs = [ ":module_private_config" ] - - external_deps = [ - "hilog:libhilog", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", - "huks:libhukssdk", - "ipc:ipc_single", - "safwk:system_ability_fwk", - ] - - deps = [ - ":kvdb_src_file", - "../../../libs/distributeddb/:distributeddb", - ] - - public_external_deps = [ - "googletest:gmock_main", - "googletest:gtest_main", - ] -} - -ohos_unittest("SecurityManagerUnitTest") { - module_out_path = module_output_path - - sources = [ "security_manager_unittest.cpp" ] - - configs = [ ":module_private_config" ] - - external_deps = [ - "hilog:libhilog", - "hitrace:hitrace_meter", - "hitrace:libhitracechain", - "huks:libhukssdk", - "ipc:ipc_single", - "safwk:system_ability_fwk", - ] - - deps = [ - ":kvdb_src_file", - "../../../libs/distributeddb/:distributeddb", - ] - - public_external_deps = [ - "googletest:gmock_main", - "googletest:gtest_main", - ] -} - ############################################################################### group("unittest") { testonly = true diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/test/backup_manager_test.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/test/backup_manager_test.cpp index 1934f578..fba0ad58 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/test/backup_manager_test.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/test/backup_manager_test.cpp @@ -18,6 +18,7 @@ #include #include +#include "backup_manager.h" #include "dev_manager.h" #include "file_ex.h" #include "store_manager.h" @@ -472,4 +473,66 @@ HWTEST_F(BackupManagerTest, BackUpEntry, TestSize.Level0) status = StoreManager::GetInstance().Delete(appId, storeId, baseDir); ASSERT_EQ(status, SUCCESS); } + +/** + * @tc.name: RestoreEncryptWithKeyFromService + * @tc.desc: Restore encrypt store with key from service + * @tc.type: FUNC + * @tc.require: + * @tc.author: yanhui + */ +HWTEST_F(BackupManagerTest, RestoreEncryptWithKeyFromService, TestSize.Level0) +{ + AppId appId = { "BackupManagerTest" }; + StoreId storeId = { "SingleKVStoreEncrypt" }; + std::string baseDir = "/data/service/el1/public/database/BackupManagerTest"; + + // put and Backup + auto kvStoreEncrypt = CreateKVStore(storeId.storeId, "BackupManagerTest", baseDir, SINGLE_VERSION, true); + ASSERT_NE(kvStoreEncrypt, nullptr); + auto status = kvStoreEncrypt->Put({ "Put Test" }, { "Put Value" }); + ASSERT_EQ(status, SUCCESS); + status = kvStoreEncrypt->Backup("testbackup", baseDir); + ASSERT_EQ(status, SUCCESS); + + // delete backup key + auto ret = remove((baseDir + "/key/Prefix_backup_SingleKVStoreEncrypt_testbackup.key").c_str()); + ASSERT_EQ(ret, 0); + status = kvStoreEncrypt->Delete("Put Test"); + ASSERT_EQ(status, SUCCESS); + + // restore + status = kvStoreEncrypt->Restore("testbackup", baseDir); + ASSERT_EQ(status, SUCCESS); + Value value; + status = kvStoreEncrypt->Get({ "Put Test" }, value); + ASSERT_EQ(status, SUCCESS); + ASSERT_EQ(std::string("Put Value"), value.ToString()); + + status = DeleteBackUpFiles(kvStoreEncrypt, baseDir, storeId); + ASSERT_EQ(status, SUCCESS); + status = StoreManager::GetInstance().CloseKVStore(appId, storeId); + ASSERT_EQ(status, SUCCESS); + status = StoreManager::GetInstance().Delete(appId, storeId, baseDir); + ASSERT_EQ(status, SUCCESS); +} + +/** + * @tc.name: GetSecretKeyFromServiceInvalid + * @tc.desc: GetSecretKeyFromService with invalid arguments + * @tc.type: FUNC + * @tc.require: + * @tc.author: yanhui + */ +HWTEST_F(BackupManagerTest, GetSecretKeyFromServiceInvalid, TestSize.Level0) +{ + AppId appId = { "BackupManagerTest" }; + StoreId storeId = { "SingleKVStoreEncryptNotFound" }; + std::vector> keys; + auto status = BackupManager::GetInstance().GetSecretKeyFromService({}, {}, keys); + ASSERT_NE(status, SUCCESS); + + status = BackupManager::GetInstance().GetSecretKeyFromService(appId, storeId, keys); + ASSERT_EQ(keys.size(), 0); +} } // namespace OHOS::Test \ No newline at end of file diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/kvdb_service_client_mock.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/kvdb_service_client_mock.cpp index 52e3877c..b2f462f8 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/kvdb_service_client_mock.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/kvdb_service_client_mock.cpp @@ -38,7 +38,7 @@ void KVDBServiceClient::ServiceDeath::OnRemoteDied() { } -Status KVDBServiceClient::GetStoreIds(const AppId &appId, std::vector &storeIds) +Status KVDBServiceClient::GetStoreIds(const AppId &appId, int32_t subUser, std::vector &storeIds) { return KVDBServiceClientMock::status; } @@ -54,17 +54,17 @@ Status KVDBServiceClient::AfterCreate( return KVDBServiceClientMock::status; } -Status KVDBServiceClient::Delete(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::Delete(const AppId &appId, const StoreId &storeId, int32_t subUser) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::Close(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::Close(const AppId &appId, const StoreId &storeId, int32_t subUser) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::Sync(const AppId &appId, const StoreId &storeId, SyncInfo &syncInfo) +Status KVDBServiceClient::Sync(const AppId &appId, const StoreId &storeId, int32_t subUser, SyncInfo &syncInfo) { return KVDBServiceClientMock::status; } @@ -89,54 +89,60 @@ Status KVDBServiceClient::UnregServiceNotifier(const AppId &appId) return KVDBServiceClientMock::status; } -Status KVDBServiceClient::SetSyncParam(const AppId &appId, const StoreId &storeId, const KvSyncParam &syncParam) +Status KVDBServiceClient::SetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + const KvSyncParam &syncParam) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::GetSyncParam(const AppId &appId, const StoreId &storeId, KvSyncParam &syncParam) +Status KVDBServiceClient::GetSyncParam(const AppId &appId, const StoreId &storeId, int32_t subUser, + KvSyncParam &syncParam) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::EnableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::EnableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::DisableCapability(const AppId &appId, const StoreId &storeId) +Status KVDBServiceClient::DisableCapability(const AppId &appId, const StoreId &storeId, int32_t subUser) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::SetCapability(const AppId &appId, const StoreId &storeId, +Status KVDBServiceClient::SetCapability(const AppId &appId, const StoreId &storeId, int32_t subUser, const std::vector &local, const std::vector &remote) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceClient::AddSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, const SyncInfo &syncInfo) +Status KVDBServiceClient::RmvSubscribeInfo(const AppId &appId, const StoreId &storeId, int32_t subUser, + const SyncInfo &syncInfo) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::Subscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceClient::Subscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::Unsubscribe(const AppId &appId, const StoreId &storeId, sptr observer) +Status KVDBServiceClient::Unsubscribe(const AppId &appId, const StoreId &storeId, int32_t subUser, + sptr observer) { return KVDBServiceClientMock::status; } -Status KVDBServiceClient::GetBackupPassword( - const AppId &appId, const StoreId &storeId, std::vector> &password, int32_t passwordType) +Status KVDBServiceClient::GetBackupPassword(const AppId &appId, const StoreId &storeId, int32_t subUser, + std::vector> &passwords, int32_t passwordType) { return KVDBServiceClientMock::status; } @@ -166,7 +172,8 @@ Status KVDBServiceClient::SetConfig(const AppId &appId, const StoreId &storeId, return KVDBServiceClientMock::status; } -Status KVDBServiceClient::RemoveDeviceData(const AppId &appId, const StoreId &storeId, const std::string &device) +Status KVDBServiceClient::RemoveDeviceData(const AppId &appId, const StoreId &storeId, int32_t subUser, + const std::string &device) { return KVDBServiceClientMock::status; } diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/observer_bridge_mock.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/observer_bridge_mock.cpp index f5d1e85e..9328e3a8 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/observer_bridge_mock.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/test/mock/src/observer_bridge_mock.cpp @@ -15,8 +15,9 @@ #include "include/observer_bridge_mock.h" namespace OHOS::DistributedKv { -ObserverBridge::ObserverBridge(AppId appId, StoreId store, std::shared_ptr observer, const Convertor &cvt) - : appId_(std::move(appId)), storeId_(std::move(store)), observer_(std::move(observer)), convert_(cvt) +ObserverBridge::ObserverBridge(AppId appId, StoreId storeId, int32_t subUser, std::shared_ptr observer, + const Convertor &cvt) : appId_(std::move(appId)), storeId_(std::move(storeId)), subUser_(subUser), + observer_(std::move(observer)), convert_(cvt) { } diff --git a/kv_store/frameworks/innerkitsimpl/kvdb/test/single_store_impl_test.cpp b/kv_store/frameworks/innerkitsimpl/kvdb/test/single_store_impl_test.cpp index 36e46397..42188eb7 100644 --- a/kv_store/frameworks/innerkitsimpl/kvdb/test/single_store_impl_test.cpp +++ b/kv_store/frameworks/innerkitsimpl/kvdb/test/single_store_impl_test.cpp @@ -179,6 +179,18 @@ HWTEST_F(SingleStoreImplTest, GetStoreId, TestSize.Level0) ASSERT_EQ(storeId.storeId, "SingleKVStore"); } +/** + * @tc.name: GetSubUser + * @tc.desc: get the subUser of the kv store + * @tc.type: FUNC + */ +HWTEST_F(SingleStoreImplTest, GetSubUser, TestSize.Level0) +{ + ASSERT_NE(kvStore_, nullptr); + auto subUser = kvStore_->GetSubUser(); + ASSERT_EQ(subUser, 0); +} + /** * @tc.name: Put * @tc.desc: put key-value data to the kv store @@ -1956,7 +1968,7 @@ HWTEST_F(SingleStoreImplTest, DoSync001, TestSize.Level1) kvStore->isClientSync_ = true; kvStore->syncObserver_ = nullptr; syncStatus = kvStore->Sync(deviceIds, SyncMode::PUSH, allowedDelayMs); - EXPECT_EQ(syncStatus, Status::SUCCESS) << "sync device should return success"; + EXPECT_NE(syncStatus, Status::SUCCESS) << "sync device should return error"; } /** @@ -2050,17 +2062,17 @@ HWTEST_F(SingleStoreImplTest, IsRemoteChanged, TestSize.Level0) } /** - * @tc.name: ReportDBCorruptedFault + * @tc.name: ReportDBFaultEvent * @tc.desc: report DB corrupted fault * @tc.type: FUNC */ -HWTEST_F(SingleStoreImplTest, ReportDBCorruptedFault, TestSize.Level0) +HWTEST_F(SingleStoreImplTest, ReportDBFaultEvent, TestSize.Level0) { std::shared_ptr kvStore; kvStore = CreateKVStore(); ASSERT_NE(kvStore, nullptr); Status status = DATA_CORRUPTED; -// kvStore->ReportDBCorruptedFault(status); + kvStore->ReportDBFaultEvent(status, std::string(__FUNCTION__)); EXPECT_TRUE(status == DATA_CORRUPTED); } } // namespace OHOS::Test \ No newline at end of file diff --git a/kv_store/frameworks/jskitsimpl/distributeddata/include/uv_queue.h b/kv_store/frameworks/jskitsimpl/distributeddata/include/uv_queue.h index 43f6ff6c..88759682 100644 --- a/kv_store/frameworks/jskitsimpl/distributeddata/include/uv_queue.h +++ b/kv_store/frameworks/jskitsimpl/distributeddata/include/uv_queue.h @@ -31,14 +31,7 @@ public: napi_env GetEnv(); void AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs = NapiArgsGenerator()); private: - static void Work(uv_work_t* work, int uvStatus); - struct UvEntry { - napi_env env; - NapiCallbackGetter callback; - NapiArgsGenerator args; - }; napi_env env_ = nullptr; - uv_loop_s* loop_ = nullptr; }; } // namespace OHOS::DistributedData #endif // OHOS_UV_QUEUE_H diff --git a/kv_store/frameworks/jskitsimpl/distributeddata/src/js_kv_store.cpp b/kv_store/frameworks/jskitsimpl/distributeddata/src/js_kv_store.cpp index 41e96699..0e76225f 100644 --- a/kv_store/frameworks/jskitsimpl/distributeddata/src/js_kv_store.cpp +++ b/kv_store/frameworks/jskitsimpl/distributeddata/src/js_kv_store.cpp @@ -15,13 +15,11 @@ #define LOG_TAG "JsKVStore" #include "js_kv_store.h" #include "js_util.h" -#include "js_kv_store_resultset.h" #include "log_print.h" #include "napi_queue.h" #include "datashare_values_bucket.h" #include "datashare_predicates.h" #include "single_kvstore.h" -#include "kv_utils.h" using namespace OHOS::DistributedKv; using namespace OHOS::DataShare; @@ -563,6 +561,7 @@ void JsKVStore::OffSyncComplete(napi_env env, size_t argc, napi_value* argv, std proxy->syncObservers_.erase(it); break; } + ++it; } ctxt->status = napi_ok; } diff --git a/kv_store/frameworks/jskitsimpl/distributeddata/src/napi_queue.cpp b/kv_store/frameworks/jskitsimpl/distributeddata/src/napi_queue.cpp index 975e4a01..4107111a 100644 --- a/kv_store/frameworks/jskitsimpl/distributeddata/src/napi_queue.cpp +++ b/kv_store/frameworks/jskitsimpl/distributeddata/src/napi_queue.cpp @@ -82,18 +82,18 @@ napi_value NapiQueue::AsyncWork(napi_env env, std::shared_ptr ctxt, aCtx->complete = std::move(complete); napi_value promise = nullptr; if (aCtx->ctx->callbackRef == nullptr) { - auto ret = napi_create_promise(env, &aCtx->deferred, &promise); - CHECK_RETURN(ret == napi_ok, "napi_create_promise fail", nullptr); - ZLOGD("Create deferred promise"); + if (napi_create_promise(env, &aCtx->deferred, &promise) != napi_ok) { + ZLOGE("Create deferred promise fail"); + delete aCtx; + return nullptr; + } } else { napi_get_undefined(env, &promise); } napi_value resource = nullptr; napi_create_string_utf8(env, name.c_str(), NAPI_AUTO_LENGTH, &resource); - napi_create_async_work( - env, nullptr, resource, - [](napi_env env, void* data) { + napi_create_async_work(env, nullptr, resource, [](napi_env env, void* data) { CHECK_RETURN_VOID(data != nullptr, "napi_async_execute_callback nullptr"); auto actx = reinterpret_cast(data); ZLOGD("napi_async_execute_callback ctxt->status=%{public}d", actx->ctx->status); diff --git a/kv_store/frameworks/jskitsimpl/distributeddata/src/uv_queue.cpp b/kv_store/frameworks/jskitsimpl/distributeddata/src/uv_queue.cpp index 0c5d17ee..7a5480f2 100644 --- a/kv_store/frameworks/jskitsimpl/distributeddata/src/uv_queue.cpp +++ b/kv_store/frameworks/jskitsimpl/distributeddata/src/uv_queue.cpp @@ -14,17 +14,14 @@ */ #define LOG_TAG "UvQueue" -#include "uv_queue.h" #include "log_print.h" #include "napi_queue.h" +#include "uv_queue.h" namespace OHOS::DistributedData { UvQueue::UvQueue(napi_env env) : env_(env) { - if (env != nullptr) { - napi_get_uv_event_loop(env, &loop_); - } } UvQueue::~UvQueue() @@ -35,67 +32,45 @@ UvQueue::~UvQueue() void UvQueue::AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs) { - if (loop_ == nullptr || !getter) { - ZLOGE("This loop_ or callback is nullptr"); - return; - } - - uv_work_t* work = new (std::nothrow) uv_work_t; - if (work == nullptr) { - ZLOGE("No memory for uv_work_t"); - return; - } - work->data = new UvEntry{ env_, getter, std::move(genArgs) }; - if (work->data == nullptr) { - ZLOGE("No memory for UvEntry"); - delete work; - work = nullptr; + if (!getter) { + ZLOGE("This callback is nullptr"); return; } - int retVal = uv_queue_work( - loop_, work, [](uv_work_t* work) { - ZLOGD("AsyncCall callback"); - }, UvQueue::Work); - if (retVal != 0) { - delete (reinterpret_cast(work->data)); - work->data = nullptr; - delete work; - work = nullptr; - } -} - -void UvQueue::Work(uv_work_t* work, int uvStatus) -{ - std::shared_ptr entry(static_cast(work->data), [work](UvEntry* data) { - delete data; - delete work; - }); - napi_handle_scope scope = nullptr; - napi_open_handle_scope(entry->env, &scope); - napi_value method = entry->callback(entry->env); - if (method == nullptr) { - ZLOGE("The callback is invalid, maybe is cleared!"); - if (scope != nullptr) { - napi_close_handle_scope(entry->env, scope); + auto task = [env = env_, getter, genArgs]() { + napi_handle_scope scope = nullptr; + napi_open_handle_scope(env, &scope); + if (scope == nullptr) { + return; } - return; - } - int argc = 0; - napi_value argv[ARGC_MAX] = { nullptr }; - if (entry->args) { - argc = ARGC_MAX; - entry->args(entry->env, argc, argv); - } - ZLOGD("queue uv_after_work_cb"); - napi_value global = nullptr; - napi_get_global(entry->env, &global); - napi_value result; - napi_status status = napi_call_function(entry->env, global, method, argc, argv, &result); + napi_value method = getter(env); + if (method == nullptr) { + ZLOGE("The callback is invalid, maybe is cleared!"); + napi_close_handle_scope(env, scope); + return; + } + int argc = 0; + napi_value argv[ARGC_MAX] = { nullptr }; + if (genArgs) { + argc = ARGC_MAX; + genArgs(env, argc, argv); + } + napi_value global = nullptr; + napi_status status = napi_get_global(env, &global); + if (status != napi_ok) { + ZLOGE("Get napi global failed. status: %{public}d.", status); + napi_close_handle_scope(env, scope); + return; + } + napi_value result; + status = napi_call_function(env, global, method, argc, argv, &result); + if (status != napi_ok) { + ZLOGE("Notify data change failed. status:%{public}d.", status); + } + napi_close_handle_scope(env, scope); + }; + napi_status status = napi_send_event(env_, task, napi_eprio_immediate); if (status != napi_ok) { - ZLOGE("Notify data change failed status:%{public}d.", status); - } - if (scope != nullptr) { - napi_close_handle_scope(entry->env, scope); + ZLOGE("Failed to napi_send_event. status:%{public}d", status); } } @@ -103,4 +78,4 @@ napi_env UvQueue::GetEnv() { return env_; } -} // namespace OHOS::DistributedData +} // namespace OHOS::DistributedData \ No newline at end of file diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_field_node.h b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_field_node.h index 600e2f27..1a132e6e 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_field_node.h +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_field_node.h @@ -23,8 +23,8 @@ namespace OHOS::DistributedKVStore { class JsFieldNode { public: using json = nlohmann::json; - explicit JsFieldNode(const std::string& fName); - ~JsFieldNode() = default; + explicit JsFieldNode(const std::string& fName, napi_env env); + ~JsFieldNode(); std::string GetFieldName(); std::string Dump(); @@ -57,6 +57,8 @@ private: JSUtil::KvStoreVariant defaultValue_; bool isWithDefaultValue_ = false; bool isNullable_ = false; + napi_env env_ = nullptr; // manage the root. set/get. + std::list refs_; }; } // namespace OHOS::DistributedKVStore #endif // OHOS_FIELD_NODE_H diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_kv_manager.h b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_kv_manager.h index 8b8b00d5..61c5c3e0 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_kv_manager.h +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/js_kv_manager.h @@ -63,7 +63,7 @@ private: std::shared_ptr uvQueue_; std::shared_ptr param_; static constexpr int MAX_APP_ID_LEN = 256; - static constexpr int API_16_VERSION = 16; + static constexpr int API_20_VERSION = 20; }; } // namespace OHOS::DistributedKVStore #endif // OHOS_KV_MANAGER_H diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/uv_queue.h b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/uv_queue.h index 67da9bd6..bd2c0e7d 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/include/uv_queue.h +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/include/uv_queue.h @@ -31,14 +31,7 @@ public: napi_env GetEnv(); void AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs = NapiArgsGenerator()); private: - static void Work(uv_work_t* work, int uvStatus); - struct UvEntry { - napi_env env; - NapiCallbackGetter callback; - NapiArgsGenerator args; - }; napi_env env_ = nullptr; - uv_loop_s* loop_ = nullptr; }; } // namespace OHOS::DistributedKVStore #endif // OHOS_UV_QUEUE_H diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_field_node.cpp b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_field_node.cpp index 10b7d423..d62f08c3 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_field_node.cpp +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_field_node.cpp @@ -38,11 +38,21 @@ std::map JsFieldNode::valueTypeToString_ = { { JSUtil::DOUBLE, std::string("DOUBLE") } }; -JsFieldNode::JsFieldNode(const std::string& fName) - : fieldName_(fName) +JsFieldNode::JsFieldNode(const std::string& fName, napi_env env) + : fieldName_(fName), env_(env) { } +JsFieldNode::~JsFieldNode() +{ + ZLOGI("no memory leak for JsFieldNode"); + for (auto ref : refs_) { + if (ref != nullptr) { + napi_delete_reference(env_, ref); + } + } +} + std::string JsFieldNode::GetFieldName() { return fieldName_; @@ -96,7 +106,7 @@ napi_value JsFieldNode::New(napi_env env, napi_callback_info info) ctxt->GetCbInfoSync(env, info, input); ASSERT_NULL(!ctxt->isThrowError, "JsFieldNode New exit"); - JsFieldNode* fieldNode = new (std::nothrow) JsFieldNode(fieldName); + JsFieldNode* fieldNode = new (std::nothrow) JsFieldNode(fieldName, env); ASSERT_ERR(env, fieldNode != nullptr, Status::INVALID_ARGUMENT, "Parameter error:fieldNode is nullptr"); auto finalize = [](napi_env env, void* data, void* hint) { @@ -112,22 +122,25 @@ napi_value JsFieldNode::New(napi_env env, napi_callback_info info) napi_value JsFieldNode::AppendChild(napi_env env, napi_callback_info info) { ZLOGD("FieldNode::AppendChild"); - JsFieldNode* child = nullptr; auto ctxt = std::make_shared(); - auto input = [env, ctxt, &child](size_t argc, napi_value* argv) { + auto input = [env, ctxt](size_t argc, napi_value* argv) { // required 1 arguments :: ASSERT_BUSINESS_ERR(ctxt, argc >= 1, Status::INVALID_ARGUMENT, "Parameter error:Mandatory parameters are left unspecified"); + JsFieldNode* child = nullptr; ctxt->status = JSUtil::Unwrap(env, argv[0], reinterpret_cast(&child), JsFieldNode::Constructor(env)); ASSERT_BUSINESS_ERR(ctxt, ((ctxt->status == napi_ok) && (child != nullptr)), Status::INVALID_ARGUMENT, "Parameter error:child is nullptr"); + + auto fieldNode = reinterpret_cast(ctxt->native); + napi_ref ref = nullptr; + ctxt->status = napi_create_reference(env, argv[0], 1, &ref); + ASSERT_STATUS(ctxt, "napi_create_reference to FieldNode failed"); + fieldNode->fields_.push_back(child); + fieldNode->refs_.push_back(ref); }; ctxt->GetCbInfoSync(env, info, input); ASSERT_NULL(!ctxt->isThrowError, "AppendChild exit"); - - auto fieldNode = reinterpret_cast(ctxt->native); - fieldNode->fields_.push_back(child); - napi_get_boolean(env, true, &ctxt->output); return ctxt->output; } diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_kv_manager.cpp b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_kv_manager.cpp index 823948ff..b2917179 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_kv_manager.cpp +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_kv_manager.cpp @@ -152,7 +152,7 @@ napi_value JsKVManager::GetKVStore(napi_env env, napi_callback_info info) status = kvm->kvDataManager_.GetSingleKvStore(ctxt->options, appId, storeId, kvStore); ZLOGE("Data has corrupted, rebuild db"); } - if (status == CRYPT_ERROR && kvm->param_->apiVersion < API_16_VERSION) { + if (status == CRYPT_ERROR && kvm->param_->apiVersion < API_20_VERSION) { status = DATA_CORRUPTED; } ctxt->status = (GenerateNapiError(status, ctxt->jsCode, ctxt->error) == Status::SUCCESS) ? diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_single_kv_store.cpp b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_single_kv_store.cpp index 117dbb6f..1e3ffe49 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_single_kv_store.cpp +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/js_single_kv_store.cpp @@ -787,6 +787,7 @@ void JsSingleKVStore::OffSyncComplete(napi_env env, size_t argc, napi_value* arg proxy->syncObservers_.erase(it); break; } + ++it; } ctxt->status = napi_ok; } diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/napi_queue.cpp b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/napi_queue.cpp index 5c75659c..29bb4cf1 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/napi_queue.cpp +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/napi_queue.cpp @@ -82,18 +82,18 @@ napi_value NapiQueue::AsyncWork(napi_env env, std::shared_ptr ctxt, aCtx->complete = std::move(complete); napi_value promise = nullptr; if (aCtx->ctx->callbackRef == nullptr) { - auto ret = napi_create_promise(env, &aCtx->deferred, &promise); - ASSERT(ret == napi_ok, "napi_create_promise fail", nullptr); - ZLOGD("Create deferred promise"); + if (napi_create_promise(env, &aCtx->deferred, &promise) != napi_ok) { + ZLOGE("Create deferred promise fail"); + delete aCtx; + return nullptr; + } } else { napi_get_undefined(env, &promise); } napi_value resource = nullptr; napi_create_string_utf8(env, name.c_str(), NAPI_AUTO_LENGTH, &resource); - napi_create_async_work( - env, nullptr, resource, - [](napi_env env, void* data) { + napi_create_async_work(env, nullptr, resource, [](napi_env env, void* data) { ASSERT_VOID(data != nullptr, "napi_async_execute_callback nullptr"); auto actx = reinterpret_cast(data); ZLOGD("napi_async_execute_callback ctxt->status=%{public}d", actx->ctx->status); diff --git a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/uv_queue.cpp b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/uv_queue.cpp index 3bf0d286..5df050e5 100644 --- a/kv_store/frameworks/jskitsimpl/distributedkvstore/src/uv_queue.cpp +++ b/kv_store/frameworks/jskitsimpl/distributedkvstore/src/uv_queue.cpp @@ -14,17 +14,14 @@ */ #define LOG_TAG "UvQueue" -#include "uv_queue.h" #include "log_print.h" #include "napi_queue.h" +#include "uv_queue.h" namespace OHOS::DistributedKVStore { UvQueue::UvQueue(napi_env env) : env_(env) { - if (env != nullptr) { - napi_get_uv_event_loop(env, &loop_); - } } UvQueue::~UvQueue() @@ -35,67 +32,45 @@ UvQueue::~UvQueue() void UvQueue::AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs) { - if (loop_ == nullptr || !getter) { - ZLOGE("loop_ or callback is nullptr"); - return; - } - - uv_work_t* work = new (std::nothrow) uv_work_t; - if (work == nullptr) { - ZLOGE("No memory for uv_work_t"); - return; - } - work->data = new UvEntry{ env_, getter, std::move(genArgs) }; - if (work->data == nullptr) { - ZLOGE("No memory for UvEntry"); - delete work; - work = nullptr; + if (!getter) { + ZLOGE("This callback is nullptr"); return; } - int retVal = uv_queue_work( - loop_, work, [](uv_work_t* work) { - ZLOGD("AsyncCall callback"); - }, UvQueue::Work); - if (retVal != 0) { - delete (reinterpret_cast(work->data)); - work->data = nullptr; - delete work; - work = nullptr; - } -} - -void UvQueue::Work(uv_work_t* work, int uvStatus) -{ - std::shared_ptr entry(static_cast(work->data), [work](UvEntry* data) { - delete data; - delete work; - }); - napi_handle_scope scope = nullptr; - napi_open_handle_scope(entry->env, &scope); - napi_value method = entry->callback(entry->env); - if (method == nullptr) { - ZLOGE("The callback is invalid, maybe is cleared!"); - if (scope != nullptr) { - napi_close_handle_scope(entry->env, scope); + auto task = [env = env_, getter, genArgs]() { + napi_handle_scope scope = nullptr; + napi_open_handle_scope(env, &scope); + if (scope == nullptr) { + return; } - return; - } - int argc = 0; - napi_value argv[ARGC_MAX] = { nullptr }; - if (entry->args) { - argc = ARGC_MAX; - entry->args(entry->env, argc, argv); - } - ZLOGD("queue uv_after_work_cb"); - napi_value global = nullptr; - napi_get_global(entry->env, &global); - napi_value result; - napi_status status = napi_call_function(entry->env, global, method, argc, argv, &result); + napi_value method = getter(env); + if (method == nullptr) { + ZLOGE("The callback is invalid, maybe is cleared!"); + napi_close_handle_scope(env, scope); + return; + } + int argc = 0; + napi_value argv[ARGC_MAX] = { nullptr }; + if (genArgs) { + argc = ARGC_MAX; + genArgs(env, argc, argv); + } + napi_value global = nullptr; + napi_status status = napi_get_global(env, &global); + if (status != napi_ok) { + ZLOGE("Get napi global failed. status: %{public}d.", status); + napi_close_handle_scope(env, scope); + return; + } + napi_value result; + status = napi_call_function(env, global, method, argc, argv, &result); + if (status != napi_ok) { + ZLOGE("Notify data change failed. status:%{public}d.", status); + } + napi_close_handle_scope(env, scope); + }; + napi_status status = napi_send_event(env_, task, napi_eprio_immediate); if (status != napi_ok) { - ZLOGE("Notify data change failed status:%{public}d.", status); - } - if (scope != nullptr) { - napi_close_handle_scope(entry->env, scope); + ZLOGE("Failed to napi_send_event. status:%{public}d", status); } } @@ -103,4 +78,4 @@ napi_env UvQueue::GetEnv() { return env_; } -} // namespace OHOS::DistributedKVStore +} // namespace OHOS::DistributedKVStore \ No newline at end of file diff --git a/kv_store/frameworks/libs/CMakeLists.txt b/kv_store/frameworks/libs/CMakeLists.txt index b2bfe55f..b2d1b46f 100644 --- a/kv_store/frameworks/libs/CMakeLists.txt +++ b/kv_store/frameworks/libs/CMakeLists.txt @@ -1,10 +1,10 @@ cmake_minimum_required(VERSION 3.10.2) project(distributeddb) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -Wattributes") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -Wattributes -Wno-error=deprecated-declarations -Wno-deprecated-declarations") set(MOCK_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../mock) add_definitions(-DUSE_HARMONY_SQLITE -DUSING_HILOG_LOGGER -DOMIT_FLATBUFFER -DOMIT_MULTI_VER -DEVLOOP_TIMER_ONLY) #-DOMIT_JSON add_definitions(-DUSING_DB_JSON_EXTRACT_AUTOMATICALLY -DSQLITE_ENABLE_JSON1 -DRELATIONAL_STORE -DOMIT_ZLIB) diff --git a/kv_store/frameworks/libs/distributeddb/BUILD.gn b/kv_store/frameworks/libs/distributeddb/BUILD.gn index da27de45..345756ce 100644 --- a/kv_store/frameworks/libs/distributeddb/BUILD.gn +++ b/kv_store/frameworks/libs/distributeddb/BUILD.gn @@ -114,6 +114,7 @@ ohos_shared_library("distributeddb") { cflags_cc = [ "-fvisibility=hidden", "-Os", + "-D_FORTIFY_SOURCE=2", ] deps = [ "gaussdb_rd:gaussdb_rd" ] diff --git a/kv_store/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h b/kv_store/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h index 1b1a815c..464bcc61 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/cloud/cloud_db_constant.h @@ -35,7 +35,6 @@ public: static constexpr const char *ERROR_FIELD = "#_error"; static constexpr const char *SHARING_RESOURCE_FIELD = "#_sharing_resource"; static constexpr const char *HASH_KEY_FIELD = "#_hash_key"; - static constexpr const char *ROW_ID_FIELD_NAME = "rowid"; static constexpr const char *ASSET = "asset"; static constexpr const char *ASSETS = "assets"; static constexpr const char *SHARED = "_shared"; @@ -44,7 +43,6 @@ public: static constexpr const char *DEFAULT_CLOUD_DEV = "cloud"; // use for inner - static constexpr const char *ROWID = "rowid"; static constexpr const char *FLAG = "flag"; static constexpr const char *TIMESTAMP = "timestamp"; static constexpr const char *HASH_KEY = "hash_key"; diff --git a/kv_store/frameworks/libs/distributeddb/common/include/db_common.h b/kv_store/frameworks/libs/distributeddb/common/include/db_common.h index ce92782f..5f056954 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/db_common.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/db_common.h @@ -159,6 +159,9 @@ public: static std::set TransformToCaseInsensitive( const std::vector &origin); + + static std::string GetStoreIdentifier(const StoreInfo &info, const std::string &subUser, bool syncDualTupleMode, + bool allowStoreIdWithDot); private: static void InsertNodesByScore(const std::map> &graph, const std::vector &generateNodes, const std::map &scoreGraph, diff --git a/kv_store/frameworks/libs/distributeddb/common/include/db_constant.h b/kv_store/frameworks/libs/distributeddb/common/include/db_constant.h index a3f03e08..acbc03a8 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/db_constant.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/db_constant.h @@ -182,7 +182,7 @@ public: static constexpr const int HASH_KEY_SIZE = 32; // size of SHA256_DIGEST_LENGTH - static constexpr const char *TABLE_WAS_DROPPED = "table_was_dropped_"; + static constexpr const char *TABLE_IS_DROPPED = "table_is_dropped_"; static constexpr const char *SQLITE_INNER_ROWID = "_rowid_"; static constexpr const int32_t DEFAULT_ROW_ID = -1; @@ -198,6 +198,8 @@ public: static constexpr const char *KV_SYNC_TABLE_NAME = "sync_data"; static constexpr const char *KV_LOCAL_TABLE_NAME = "local_data"; + + static constexpr const char *ROWID = "rowid"; }; } // namespace DistributedDB #endif // DISTRIBUTEDDB_CONSTANT_H diff --git a/kv_store/frameworks/libs/distributeddb/common/include/db_errno.h b/kv_store/frameworks/libs/distributeddb/common/include/db_errno.h index c5a5cf2d..ceb9841b 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/db_errno.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/db_errno.h @@ -137,7 +137,7 @@ constexpr const int E_CLOUD_NETWORK_ERROR = (E_BASE + 113); // network error in constexpr const int E_CLOUD_SYNC_UNSET = (E_BASE + 114); // not set sync option in cloud constexpr const int E_CLOUD_FULL_RECORDS = (E_BASE + 115); // cloud's record is full constexpr const int E_CLOUD_LOCK_ERROR = (E_BASE + 116); // cloud failed to get sync lock -constexpr const int E_CLOUD_ASSET_SPACE_INSUFFICIENT = (E_BASE + 117); // cloud failed to download asset +constexpr const int E_CLOUD_ASSET_SPACE_INSUFFICIENT = (E_BASE + 117); // cloud asset space is insufficient constexpr const int E_CLOUD_INVALID_ASSET = (E_BASE + 118); // the asset is invalid constexpr const int E_TASK_PAUSED = (E_BASE + 119); // the task was paused, don't finished it constexpr const int E_CLOUD_VERSION_CONFLICT = (E_BASE + 120); // cloud failed to update version diff --git a/kv_store/frameworks/libs/distributeddb/common/include/relational/relational_schema_object.h b/kv_store/frameworks/libs/distributeddb/common/include/relational/relational_schema_object.h index b47ffa2d..3e664718 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -105,6 +105,7 @@ private: int ParseCheckReferenceColumns(const JsonObject &inJsonObject, TableReferenceProperty &tableReferenceProperty); // parse one reference column pair int ParseCheckReferenceColumn(const JsonObject &inJsonObject, TableReferenceProperty &tableReferenceProperty); + int ParseDistributedVersion(const JsonObject &inJsonObject); // parse distributed version if need int ParseDistributedSchema(const JsonObject &inJsonObject); // parse distributed schema if need int ParseDistributedTables(const JsonObject &inJsonObject); // parse distributed tables if need int ParseDistributedTable(const JsonObject &inJsonObject); // parse distributed table if need diff --git a/kv_store/frameworks/libs/distributeddb/common/include/relational/table_info.h b/kv_store/frameworks/libs/distributeddb/common/include/relational/table_info.h index 4b914602..363cd74e 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/relational/table_info.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/relational/table_info.h @@ -84,6 +84,7 @@ public: const IndexInfoMap &GetIndexDefine() const; const std::map &GetPrimaryKey() const; bool IsPrimaryKey(const FieldName &fieldName) const; + bool IsUniqueField(const std::string &colName) const; const std::vector &GetUniqueDefine() const; void SetTableName(const std::string &tableName); @@ -127,6 +128,7 @@ public: bool Empty() const; bool IsNoPkTable() const; + bool IsMultiPkTable() const; bool IsFieldExist(const std::string &fieldName) const; @@ -135,6 +137,8 @@ public: std::vector GetSyncField() const; std::vector GetSyncDistributedPk() const; + + const std::vector GetUniqueAndPkDefine() const; private: void AddFieldDefineString(std::string &attrStr) const; void AddIndexDefineString(std::string &attrStr) const; diff --git a/kv_store/frameworks/libs/distributeddb/common/include/runtime_context.h b/kv_store/frameworks/libs/distributeddb/common/include/runtime_context.h index 0d184668..42a9f99e 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/runtime_context.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/runtime_context.h @@ -208,6 +208,8 @@ public: virtual void SetBatchDownloadAssets(bool isBatchDownload) = 0; virtual std::shared_ptr GetAssetsDownloadManager() = 0; + + virtual void ClearOnlineLabel() = 0; protected: RuntimeContext() = default; virtual ~RuntimeContext() {} diff --git a/kv_store/frameworks/libs/distributeddb/common/include/schema_negotiate.h b/kv_store/frameworks/libs/distributeddb/common/include/schema_negotiate.h index 2908c6dd..39b7f133 100644 --- a/kv_store/frameworks/libs/distributeddb/common/include/schema_negotiate.h +++ b/kv_store/frameworks/libs/distributeddb/common/include/schema_negotiate.h @@ -46,7 +46,7 @@ public: static SyncStrategy ConcludeSyncStrategy(const SyncOpinion &localOpinion, const SyncOpinion &remoteOpinion); static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, - const std::string &remoteSchema, uint8_t remoteSchemaType); + const std::string &remoteSchema, uint8_t remoteSchemaType, uint32_t remoteSoftwareVersion); // The remoteOpinion.checkOnReceive is ignored static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, @@ -61,6 +61,12 @@ private: static RelationalSyncOpinion MakeOpinionEachTable(const RelationalSchemaObject &localSchema, const RelationalSchemaObject &remoteSchema); + + static bool IsDistributedSchemaInvalid(const RelationalSchemaObject &localSchema, + const RelationalSchemaObject &remoteSchema); + + static bool IsDistributedTableInvalid(const DistributedTable &local, const DistributedTable &remote, + const TableInfo &localTable, const TableInfo &remoteTable); }; } diff --git a/kv_store/frameworks/libs/distributeddb/common/src/cloud/assets_download_manager.cpp b/kv_store/frameworks/libs/distributeddb/common/src/cloud/assets_download_manager.cpp index 1f3ad794..c608ef2f 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/cloud/assets_download_manager.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/cloud/assets_download_manager.cpp @@ -15,6 +15,7 @@ #include "cloud/assets_download_manager.h" #include "cloud/cloud_db_constant.h" + #include "db_errno.h" #include "log_print.h" namespace DistributedDB { diff --git a/kv_store/frameworks/libs/distributeddb/common/src/concurrent_adapter.cpp b/kv_store/frameworks/libs/distributeddb/common/src/concurrent_adapter.cpp index 5f5223c6..19198891 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/concurrent_adapter.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/concurrent_adapter.cpp @@ -65,11 +65,12 @@ void ConcurrentAdapter::AdapterAutoUnLock(ffrt::mutex &mutex) #else void ConcurrentAdapter::AdapterAutoLock(std::mutex &mutex) { - std::lock_guard lock(mutex); + mutex.lock(); } -void ConcurrentAdapter::AdapterAutoUnLock([[gnu::unused]] std::mutex &mutex) +void ConcurrentAdapter::AdapterAutoUnLock(std::mutex &mutex) { + mutex.unlock(); } #endif } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/common/src/db_common.cpp b/kv_store/frameworks/libs/distributeddb/common/src/db_common.cpp index 4f5c51f5..e735fed7 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/db_common.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/db_common.cpp @@ -28,6 +28,7 @@ #include "cloud/cloud_db_constant.h" #include "cloud/cloud_db_types.h" #include "db_errno.h" +#include "param_check_utils.h" #include "platform_specific.h" #include "query_sync_object.h" #include "hash.h" @@ -492,11 +493,6 @@ bool DBCommon::CheckIsAlnumOrUnderscore(const std::string &text) bool DBCommon::CheckQueryWithoutMultiTable(const Query &query) { - QuerySyncObject syncObject(query); - if (!syncObject.GetRelationTableNames().empty()) { - LOGE("check query table names from tables failed!"); - return false; - } if (!QuerySyncObject::GetQuerySyncObject(query).empty()) { LOGE("check query object from table failed!"); return false; @@ -885,4 +881,19 @@ std::set DBCommon::TransformToCaseInsens } return res; } + +std::string DBCommon::GetStoreIdentifier(const StoreInfo &info, const std::string &subUser, bool syncDualTupleMode, + bool allowStoreIdWithDot) +{ + if (!ParamCheckUtils::CheckStoreParameter(info, syncDualTupleMode, subUser, allowStoreIdWithDot)) { + return ""; + } + if (syncDualTupleMode) { + return DBCommon::TransferHashString(info.appId + "-" + info.storeId); + } + if (subUser.empty()) { + return DBCommon::TransferHashString(info.userId + "-" + info.appId + "-" + info.storeId); + } + return DBCommon::TransferHashString(info.userId + "-" + info.appId + "-" + info.storeId + "-" + subUser); +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/common/src/evloop/src/event_impl.cpp b/kv_store/frameworks/libs/distributeddb/common/src/evloop/src/event_impl.cpp index e17b3bc2..2976967e 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/evloop/src/event_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/evloop/src/event_impl.cpp @@ -199,6 +199,7 @@ int EventImpl::Detach(bool wait) int errCode = loop->Remove(this); if (errCode == -E_OBJ_IS_KILLED) { + LOGW("[EventImpl] [Detach] obj is killed."); errCode = E_OK; } diff --git a/kv_store/frameworks/libs/distributeddb/common/src/query_expression.cpp b/kv_store/frameworks/libs/distributeddb/common/src/query_expression.cpp index 773e010c..a6b01f47 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/query_expression.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/query_expression.cpp @@ -220,39 +220,47 @@ void QueryExpression::QueryByKeyRange(const std::vector &keyBegin, cons endKey_ = keyEnd; } +void QueryExpression::SetAssetsOnlyValidStatusIfNeed(int status) +{ + if (validStatusForAssetsOnly_ != E_OK && validStatusForAssetsOnly_ == -E_INVALID_ARGS) { + return; + } + validStatusForAssetsOnly_ = status; +} + void QueryExpression::QueryAssetsOnly(const AssetsMap &assets) { isAssetsOnly_ = true; if (useFromTable_) { expressions_[fromTable_].QueryAssetsOnly(assets); - validStatusForAssetsOnly_ = expressions_[fromTable_].GetExpressionStatusForAssetsOnly(); + SetAssetsOnlyValidStatusIfNeed(expressions_[fromTable_].GetExpressionStatusForAssetsOnly()); return; } if (queryInfo_.empty()) { LOGE("[QueryExpression] the QueryAssetsOnly option must be connected with And."); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_INVALID_ARGS); return; } else if (queryInfo_.back().operFlag != QueryObjType::AND) { LOGE("[QueryExpression] the QueryAssetsOnly option must be connected with And."); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_INVALID_ARGS); return; } else { queryInfo_.pop_back(); } if (assetsGroupMap_.find(groupNum_) != assetsGroupMap_.end()) { LOGE("[QueryExpression]assets only already set!"); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_NOT_SUPPORT); return; } if (assets.empty()) { LOGE("[QueryExpression]assets map can not be empty!"); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_NOT_SUPPORT); return; } for (const auto &item : assets) { if (item.second.empty() && item.first.empty()) { LOGE("[QueryExpression]assets filed or asset name can not be empty!"); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_NOT_SUPPORT); return; } } @@ -260,7 +268,7 @@ void QueryExpression::QueryAssetsOnly(const AssetsMap &assets) for (uint32_t i = 0; i <= groupNum_; i++) { if (assetsGroupMap_.find(i) == assetsGroupMap_.end()) { LOGE("[QueryExpression]asset group " PRIu32 " not found, may be AssetsOnly interface use in wrong way.", i); - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + SetAssetsOnlyValidStatusIfNeed(-E_NOT_SUPPORT); return; } } @@ -352,7 +360,7 @@ void QueryExpression::BeginGroup() if (isAssetsOnly_) { auto iter = queryInfo_.rbegin(); if (iter != queryInfo_.rend() && (*iter).operFlag != QueryObjType::OR) { - validStatusForAssetsOnly_ = -E_INVALID_ARGS; + validStatusForAssetsOnly_ = -E_NOT_SUPPORT; } } SetNotSupportIfFromTables(); @@ -426,6 +434,7 @@ void QueryExpression::SetTables(const std::vector &tableNames) } } tables_ = syncTable; + isUseFromTables_ = true; if (useFromTable_) { validStatus_ = validStatus_ != E_OK ? validStatus_ : -E_NOT_SUPPORT; } @@ -547,4 +556,9 @@ uint32_t QueryExpression::GetGroupNum() const { return groupNum_; } + +bool QueryExpression::IsUseFromTables() const +{ + return isUseFromTables_; +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/kv_store/frameworks/libs/distributeddb/common/src/relational/relational_schema_object.cpp index ac642890..f555ad2d 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -1213,6 +1213,9 @@ bool RelationalSchemaObject::CheckDistributedFieldChange(const std::vector(fieldValue.integerValue); return ParseDistributedTables(schemaObj); } +int RelationalSchemaObject::ParseDistributedVersion(const JsonObject &inJsonObject) +{ + const std::string fieldName = SchemaConstant::KEYWORD_DISTRIBUTED_VERSION; + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + LOGE("[RelationalSchema][ParseDistributedVersion] Distributed schema has no version"); + return -E_SCHEMA_PARSE_FAIL; + } + + FieldType fieldType; + int errCode = inJsonObject.GetFieldTypeByFieldPath(FieldPath {fieldName}, fieldType); + if (errCode != E_OK) { + LOGE("[RelationalSchema][ParseDistributedVersion] Get fieldType of version failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + if (fieldType != FieldType::LEAF_FIELD_INTEGER && fieldType != FieldType::LEAF_FIELD_LONG) { + LOGE("[RelationalSchema][ParseDistributedVersion] Get fieldType of version failed. Found type: %d.", fieldType); + return -E_SCHEMA_PARSE_FAIL; + } + + FieldValue fieldValue; + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {fieldName}, fieldValue); + if (errCode != E_OK) { + LOGE("[RelationalSchema][ParseDistributedVersion] Get version value failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + + int64_t versionValue = fieldType == FieldType::LEAF_FIELD_INTEGER ? + static_cast(fieldValue.integerValue) : fieldValue.longValue; + if (versionValue < 0 || versionValue > UINT32_MAX) { + LOGE("[RelationalSchema][ParseDistributedVersion] Version value out of range, value is: %" PRId64, + versionValue); + return -E_SCHEMA_PARSE_FAIL; + } + dbSchema_.version = static_cast(versionValue); + return E_OK; +} + int RelationalSchemaObject::ParseDistributedTables(const JsonObject &inJsonObject) { if (!inJsonObject.IsFieldPathExist(FieldPath {SchemaConstant::KEYWORD_DISTRIBUTED_TABLE})) { diff --git a/kv_store/frameworks/libs/distributeddb/common/src/relational/table_info.cpp b/kv_store/frameworks/libs/distributeddb/common/src/relational/table_info.cpp index e1091c33..6a644e60 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/relational/table_info.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/relational/table_info.cpp @@ -360,6 +360,18 @@ bool TableInfo::IsPrimaryKey(const std::string &colName) const return false; } +bool TableInfo::IsUniqueField(const std::string &colName) const +{ + for (const auto &unique : uniqueDefines_) { + for (const auto &it : unique) { + if (colName == it) { + return true; + } + } + } + return false; +} + CompositeFields TableInfo::GetIdentifyKey() const { if (primaryKey_.size() == 1 && primaryKey_.at(0) == ROW_ID) { @@ -817,6 +829,11 @@ bool TableInfo::IsNoPkTable() const return false; } +bool TableInfo::IsMultiPkTable() const +{ + return primaryKey_.size() > 1; +} + bool TableInfo::IsFieldExist(const std::string &fieldName) const { if (fields_.find(fieldName) != fields_.end()) { @@ -854,4 +871,11 @@ std::vector TableInfo::GetSyncDistributedPk() const } return res; } + +const std::vector TableInfo::GetUniqueAndPkDefine() const +{ + std::vector res = uniqueDefines_; + res.push_back(GetIdentifyKey()); + return res; +} } // namespace DistributeDB \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.cpp b/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.cpp index d768614b..acc9e291 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -1273,4 +1273,14 @@ std::shared_ptr RuntimeContextImpl::GetAssetsDownloadMana assetsDownloadManager_ = std::make_shared(); return assetsDownloadManager_; } + +void RuntimeContextImpl::ClearOnlineLabel() +{ + std::lock_guard autoLock(communicatorLock_); + if (communicatorAggregator_ == nullptr) { + LOGE("[Runtime] clear online label with null aggregator"); + return; + } + communicatorAggregator_->ClearOnlineLabel(); +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.h b/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.h index 99cb7e51..d2e87d0c 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.h +++ b/kv_store/frameworks/libs/distributeddb/common/src/runtime_context_impl.h @@ -190,6 +190,8 @@ public: void SetBatchDownloadAssets(bool isBatchDownload) override; std::shared_ptr GetAssetsDownloadManager() override; + + void ClearOnlineLabel() override; private: static constexpr int MAX_TP_THREADS = 10; // max threads of the task pool. static constexpr int MIN_TP_THREADS = 1; // min threads of the task pool. diff --git a/kv_store/frameworks/libs/distributeddb/common/src/schema_negotiate.cpp b/kv_store/frameworks/libs/distributeddb/common/src/schema_negotiate.cpp index 3c9d0fb3..2efd5f39 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/schema_negotiate.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/schema_negotiate.cpp @@ -17,6 +17,7 @@ #include "db_common.h" #include "log_print.h" #include "schema_utils.h" +#include "version.h" namespace DistributedDB { // Some principle in current version describe below. (Relative-type will be introduced in future but not involved now) @@ -124,7 +125,7 @@ RelationalSyncOpinion SchemaNegotiate::MakeOpinionEachTable(const RelationalSche } RelationalSyncOpinion SchemaNegotiate::MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, - const std::string &remoteSchema, uint8_t remoteSchemaType) + const std::string &remoteSchema, uint8_t remoteSchemaType, uint32_t remoteSoftwareVersion) { SchemaType localType = localSchema.GetSchemaType(); SchemaType remoteType = ReadSchemaType(remoteSchemaType); @@ -165,6 +166,16 @@ RelationalSyncOpinion SchemaNegotiate::MakeLocalSyncOpinion(const RelationalSche return {}; } + if ((remoteSoftwareVersion >= SOFTWARE_VERSION_RELEASE_11_0) && + (localSchema.GetTableMode() == DistributedTableMode::COLLABORATION) && + (remoteSchemaObj.GetDistributedSchema().tables.empty() || localSchema.GetDistributedSchema().tables.empty())) { + LOGW("[RelationalSchema][opinion] Distributed schema is empty, local %zu, remote %zu", + localSchema.GetDistributedSchema().tables.size(), remoteSchemaObj.GetDistributedSchema().tables.size()); + return {}; + } + if (IsDistributedSchemaInvalid(localSchema, remoteSchemaObj)) { + return {}; + } return MakeOpinionEachTable(localSchema, remoteSchemaObj); } @@ -263,4 +274,48 @@ int SchemaNegotiate::DeserializeData(Parcel &parcel, RelationalSyncOpinion &opin } return parcel.IsError() ? -E_INVALID_ARGS : E_OK; } + +bool SchemaNegotiate::IsDistributedSchemaInvalid(const RelationalSchemaObject &localSchema, + const RelationalSchemaObject &remoteSchema) +{ + auto localTables = localSchema.GetTableNames(); + for (const auto &localTable : localTables) { + auto localDistributedTable = localSchema.GetDistributedTable(localTable); + if (localDistributedTable.tableName.empty()) { + continue; + } + auto remoteDistributedTable = remoteSchema.GetDistributedTable(localTable); + if (remoteDistributedTable.tableName.empty()) { + continue; + } + if (IsDistributedTableInvalid(localDistributedTable, remoteDistributedTable, + localSchema.GetTable(localTable), remoteSchema.GetTable(localTable))) { + return true; + } + } + return false; +} + +bool SchemaNegotiate::IsDistributedTableInvalid(const DistributedTable &local, const DistributedTable &remote, + const TableInfo &localTable, const TableInfo &remoteTable) +{ + std::set localSpecified; + for (const auto &item : local.fields) { + if (item.isP2pSync && item.isSpecified && !localTable.IsPrimaryKey(item.colName)) { + localSpecified.insert(item.colName); + } + } + std::set remoteSpecified; + for (const auto &item : remote.fields) { + if (item.isP2pSync && item.isSpecified && !remoteTable.IsPrimaryKey(item.colName)) { + remoteSpecified.insert(item.colName); + } + } + if (localSpecified != remoteSpecified) { + LOGE("[SchemaNegotiate] specified field is diff local size %zu remote size %zu", + localSpecified.size(), remoteSpecified.size()); + return true; + } + return false; +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.cpp b/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.cpp index 13ed16c2..a499ad52 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.cpp @@ -28,7 +28,8 @@ TaskPoolImpl::TaskPoolImpl(int maxThreads, int minThreads) maxThreads_(maxThreads), minThreads_(minThreads), curThreads_(0), - idleThreads_(0) + idleThreads_(0), + exitingThreads_(0) {} TaskPoolImpl::~TaskPoolImpl() @@ -70,7 +71,7 @@ void TaskPoolImpl::Stop() isStopping_ = true; hasTasks_.notify_all(); allThreadsExited_.wait(lock, [this]() { - return this->curThreads_ <= 0; + return this->curThreads_ <= 0 && this->exitingThreads_ <= 0; }); isStarted_ = false; } @@ -204,6 +205,7 @@ void TaskPoolImpl::GetTask(Task &task, TaskQueue *&queue) genericThread_ = std::thread::id(); } --curThreads_; + ++exitingThreads_; } } @@ -247,6 +249,7 @@ void TaskPoolImpl::ExitWorker() { std::lock_guard guard(tasksMutex_); allThreadsExited_.notify_all(); + --exitingThreads_; LOGI("Task pool thread exit, cur:%d idle:%d, genericTaskCount:%d, queuedTaskCount:%d.", curThreads_, idleThreads_, genericTaskCount_, queuedTaskCount_); } diff --git a/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.h b/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.h index a530f3fa..c9173585 100644 --- a/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.h +++ b/kv_store/frameworks/libs/distributeddb/common/src/task_pool_impl.h @@ -81,6 +81,7 @@ private: int minThreads_; int curThreads_; int idleThreads_; + int exitingThreads_; }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/communicator/include/communicator_aggregator.h b/kv_store/frameworks/libs/distributeddb/communicator/include/communicator_aggregator.h index a552ccb4..6034cb15 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/include/communicator_aggregator.h +++ b/kv_store/frameworks/libs/distributeddb/communicator/include/communicator_aggregator.h @@ -92,6 +92,7 @@ public: std::shared_ptr GetExtendHeaderHandle(const ExtendInfo ¶mInfo); + void ClearOnlineLabel() override; private: // Working in a dedicated thread void SendDataRoutine(); @@ -104,18 +105,18 @@ private: // Call from Adapter by register these function void OnBytesReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, - const std::string &userId); + const DataUserInfoProc &userInfoProc); void OnTargetChange(const std::string &target, bool isConnect); void OnSendable(const std::string &target); void OnFragmentReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, - const ParseResult &inResult, const std::string &userId); + const ParseResult &inResult, const DataUserInfoProc &userInfoProc); int OnCommLayerFrameReceive(const std::string &srcTarget, const ParseResult &inResult); int OnAppLayerFrameReceive(const std::string &srcTarget, const uint8_t *bytes, - uint32_t length, const ParseResult &inResult, const std::string &userId); + uint32_t length, const ParseResult &inResult, const DataUserInfoProc &userInfoProc); int OnAppLayerFrameReceive(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, - const ParseResult &inResult, const std::string &userId); + const ParseResult &inResult, const DataUserInfoProc &userInfoProc); // Function with suffix NoMutex should be called with mutex in the caller int TryDeliverAppLayerFrameToCommunicatorNoMutex(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, @@ -162,6 +163,9 @@ private: uint64_t IncreaseSendSequenceId(const std::string &target); + int GetDataUserId(const ParseResult &inResult, const LabelType &toLabel, const DataUserInfoProc &userInfoProc, + std::string &userId); + DECLARE_OBJECT_TAG(CommunicatorAggregator); static std::atomic isCommunicatorNotFoundFeedbackEnable_; diff --git a/kv_store/frameworks/libs/distributeddb/communicator/include/iadapter.h b/kv_store/frameworks/libs/distributeddb/communicator/include/iadapter.h index 4ded51af..d9273d82 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/include/iadapter.h +++ b/kv_store/frameworks/libs/distributeddb/communicator/include/iadapter.h @@ -24,9 +24,14 @@ #include "iprocess_communicator.h" namespace DistributedDB { +struct DataUserInfoProc { + const uint8_t *data = nullptr; + uint32_t length = 0; + std::shared_ptr processCommunicator = nullptr; +}; // SendableCallback only notify when status changed from unsendable to sendable using BytesReceiveCallback = std::function; + const DataUserInfoProc &userInfoProc)>; using TargetChangeCallback = std::function; using SendableCallback = std::function; diff --git a/kv_store/frameworks/libs/distributeddb/communicator/include/icommunicator_aggregator.h b/kv_store/frameworks/libs/distributeddb/communicator/include/icommunicator_aggregator.h index 656e165b..b3c74020 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/include/icommunicator_aggregator.h +++ b/kv_store/frameworks/libs/distributeddb/communicator/include/icommunicator_aggregator.h @@ -49,6 +49,8 @@ public: virtual int RegCommunicatorLackCallback(const CommunicatorLackCallback &onCommLack, const Finalizer &inOper) = 0; virtual int RegOnConnectCallback(const OnConnectCallback &onConnect, const Finalizer &inOper) = 0; virtual int GetLocalIdentity(std::string &outTarget) const = 0; + virtual void ClearOnlineLabel() = 0; + virtual ~ICommunicatorAggregator() {}; }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_aggregator.cpp b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_aggregator.cpp index b825143b..ea724de9 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_aggregator.cpp +++ b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_aggregator.cpp @@ -484,7 +484,7 @@ void CommunicatorAggregator::NotifySendableToAllCommunicator() } void CommunicatorAggregator::OnBytesReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, - const std::string &userId) + const DataUserInfoProc &userInfoProc) { ProtocolProto::DisplayPacketInformation(bytes, length); ParseResult packetResult; @@ -508,14 +508,14 @@ void CommunicatorAggregator::OnBytesReceive(const std::string &srcTarget, const } if (packetResult.IsFragment()) { - OnFragmentReceive(srcTarget, bytes, length, packetResult, userId); + OnFragmentReceive(srcTarget, bytes, length, packetResult, userInfoProc); } else if (packetResult.GetFrameTypeInfo() != FrameType::APPLICATION_MESSAGE) { errCode = OnCommLayerFrameReceive(srcTarget, packetResult); if (errCode != E_OK) { LOGE("[CommAggr][Receive] CommLayer receive fail, errCode=%d.", errCode); } } else { - errCode = OnAppLayerFrameReceive(srcTarget, bytes, length, packetResult, userId); + errCode = OnAppLayerFrameReceive(srcTarget, bytes, length, packetResult, userInfoProc); if (errCode != E_OK) { LOGE("[CommAggr][Receive] AppLayer receive fail, errCode=%d.", errCode); } @@ -571,7 +571,7 @@ void CommunicatorAggregator::OnSendable(const std::string &target) } void CommunicatorAggregator::OnFragmentReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, - const ParseResult &inResult, const std::string &userId) + const ParseResult &inResult, const DataUserInfoProc &userInfoProc) { int errorNo = E_OK; ParseResult frameResult; @@ -604,7 +604,7 @@ void CommunicatorAggregator::OnFragmentReceive(const std::string &srcTarget, con delete frameBuffer; frameBuffer = nullptr; } else { - errCode = OnAppLayerFrameReceive(srcTarget, frameBuffer, frameResult, userId); + errCode = OnAppLayerFrameReceive(srcTarget, frameBuffer, frameResult, userInfoProc); if (errCode != E_OK) { LOGE("[CommAggr][Receive] AppLayer receive fail after combination, errCode=%d.", errCode); } @@ -634,7 +634,7 @@ int CommunicatorAggregator::OnCommLayerFrameReceive(const std::string &srcTarget } int CommunicatorAggregator::OnAppLayerFrameReceive(const std::string &srcTarget, const uint8_t *bytes, - uint32_t length, const ParseResult &inResult, const std::string &userId) + uint32_t length, const ParseResult &inResult, const DataUserInfoProc &userInfoProc) { SerialBuffer *buffer = new (std::nothrow) SerialBuffer(); if (buffer == nullptr) { @@ -649,7 +649,7 @@ int CommunicatorAggregator::OnAppLayerFrameReceive(const std::string &srcTarget, buffer = nullptr; return -E_INTERNAL_ERROR; } - return OnAppLayerFrameReceive(srcTarget, buffer, inResult, userId); + return OnAppLayerFrameReceive(srcTarget, buffer, inResult, userInfoProc); } // In early time, we cover "OnAppLayerFrameReceive" totally by commMapMutex_, then search communicator, if not found, @@ -671,9 +671,17 @@ int CommunicatorAggregator::OnAppLayerFrameReceive(const std::string &srcTarget, // 3:Search communicator under commMapMutex_ again, if found then deliver frame to that communicator and end. // 4:If still not found, retain this frame if need or otherwise send CommunicatorNotFound feedback. int CommunicatorAggregator::OnAppLayerFrameReceive(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, - const ParseResult &inResult, const std::string &userId) + const ParseResult &inResult, const DataUserInfoProc &userInfoProc) { LabelType toLabel = inResult.GetCommLabel(); + std::string userId; + int ret = GetDataUserId(inResult, toLabel, userInfoProc, userId); + if (ret != E_OK) { + LOGE("[CommAggr][AppReceive] get data user id err, ret=%d", ret); + delete inFrameBuffer; + inFrameBuffer = nullptr; + return ret; + } { std::lock_guard commMapLockGuard(commMapMutex_); int errCode = TryDeliverAppLayerFrameToCommunicatorNoMutex(srcTarget, inFrameBuffer, toLabel, userId); @@ -712,6 +720,30 @@ int CommunicatorAggregator::OnAppLayerFrameReceive(const std::string &srcTarget, return E_OK; } +int CommunicatorAggregator::GetDataUserId(const ParseResult &inResult, const LabelType &toLabel, + const DataUserInfoProc &userInfoProc, std::string &userId) +{ + if (userInfoProc.processCommunicator == nullptr) { + LOGE("[CommAggr][GetDataUserId] processCommunicator is nullptr"); + return E_INVALID_ARGS; + } + std::string label(toLabel.begin(), toLabel.end()); + std::vector userInfos; + DBStatus ret = userInfoProc.processCommunicator->GetDataUserInfo(userInfoProc.data, userInfoProc.length, label, + userInfos); + LOGI("[CommAggr][GetDataUserId] get data user info, ret=%d", ret); + if (ret == NO_PERMISSION) { + LOGE("[CommAggr][GetDataUserId] userId dismatched, drop packet"); + return ret; + } + if (userInfos.size() >= 1) { + userId = userInfos[0].receiveUser; + } else { + LOGW("[CommAggr][GetDataUserId] userInfos is empty"); + } + return E_OK; +} + int CommunicatorAggregator::TryDeliverAppLayerFrameToCommunicatorNoMutex(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, const LabelType &toLabel, const std::string &userId) { @@ -751,8 +783,9 @@ int CommunicatorAggregator::RegCallbackToAdapter() { RefObject::IncObjRef(this); // Reference to be hold by adapter int errCode = adapterHandle_->RegBytesReceiveCallback( - [this](const std::string &srcTarget, const uint8_t *bytes, uint32_t length, const std::string &userId) { - OnBytesReceive(srcTarget, bytes, length, userId); + [this](const std::string &srcTarget, const uint8_t *bytes, uint32_t length, + const DataUserInfoProc &userInfoProc) { + OnBytesReceive(srcTarget, bytes, length, userInfoProc); }, [this]() { RefObject::DecObjRef(this); }); if (errCode != E_OK) { RefObject::DecObjRef(this); // Rollback in case reg failed @@ -1127,5 +1160,15 @@ uint64_t CommunicatorAggregator::IncreaseSendSequenceId(const std::string &targe std::lock_guard autoLock(sendSequenceMutex_); return ++sendSequence_[target]; } + +void CommunicatorAggregator::ClearOnlineLabel() +{ + std::lock_guard autoLock(commMapMutex_); + if (commLinker_ == nullptr) { + LOGE("[CommAggr] clear online label with null linker"); + return; + } + commLinker_->ClearOnlineLabel(); +} DEFINE_OBJECT_TAG_FACILITIES(CommunicatorAggregator) } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.cpp b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.cpp index 37f61b2c..3bf8832b 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.cpp +++ b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.cpp @@ -496,5 +496,14 @@ bool CommunicatorLinker::TriggerLabelExchangeEvent(bool checkAdapter) } return everFail; } + +void CommunicatorLinker::ClearOnlineLabel() +{ + std::lock_guard autoLock(entireInfoMutex_); + for (const auto &[dev, labels] : targetMapOnlineLabels_) { + LOGD("[Linker] clear remote %s online label %zu", DBCommon::StringMiddleMasking(dev).c_str(), labels.size()); + } + targetMapOnlineLabels_.clear(); +} DEFINE_OBJECT_TAG_FACILITIES(CommunicatorLinker) } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.h b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.h index 10733a16..f02630ac 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.h +++ b/kv_store/frameworks/libs/distributeddb/communicator/src/communicator_linker.h @@ -77,6 +77,8 @@ public: void UpdateOnlineLabels(const std::string &device, const std::map &labels); bool TriggerLabelExchangeEvent(bool checkAdapter = true); + + void ClearOnlineLabel(); private: DECLARE_OBJECT_TAG(CommunicatorLinker); diff --git a/kv_store/frameworks/libs/distributeddb/communicator/src/network_adapter.cpp b/kv_store/frameworks/libs/distributeddb/communicator/src/network_adapter.cpp index 688ea394..d3071c96 100644 --- a/kv_store/frameworks/libs/distributeddb/communicator/src/network_adapter.cpp +++ b/kv_store/frameworks/libs/distributeddb/communicator/src/network_adapter.cpp @@ -155,6 +155,10 @@ uint32_t NetworkAdapter::GetMtuSize() { std::lock_guard mtuSizeLockGuard(mtuSizeMutex_); if (!isMtuSizeValid_) { + if (processCommunicator_ == nullptr) { + LOGE("[NAdapt][GetMtu] processCommunicator_ is nullptr."); + return DBConstant::MAX_MTU_SIZE; + } mtuSize_ = processCommunicator_->GetMtuSize(); LOGD("[NAdapt][GetMtu] mtuSize=%" PRIu32 ".", mtuSize_); mtuSize_ = CheckAndAdjustMtuSize(mtuSize_); @@ -165,6 +169,10 @@ uint32_t NetworkAdapter::GetMtuSize() uint32_t NetworkAdapter::GetMtuSize(const std::string &target) { + if (processCommunicator_ == nullptr) { + LOGE("[NAdapt][GetMtu] processCommunicator_ is nullptr and target is %s{private}.", target.c_str()); + return DBConstant::MAX_MTU_SIZE; + } #ifndef OMIT_MTU_CACHE DeviceInfos devInfo; devInfo.identifier = target; @@ -185,6 +193,10 @@ uint32_t NetworkAdapter::GetMtuSize(const std::string &target) uint32_t NetworkAdapter::GetTimeout() { + if (processCommunicator_ == nullptr) { + LOGE("[NAdapt][GetTimeout] processCommunicator_ is nullptr."); + return DBConstant::MAX_TIMEOUT; + } uint32_t timeout = processCommunicator_->GetTimeout(); LOGD("[NAdapt][GetTimeout] timeout_=%" PRIu32 " ms.", timeout); return CheckAndAdjustTimeout(timeout); @@ -192,6 +204,10 @@ uint32_t NetworkAdapter::GetTimeout() uint32_t NetworkAdapter::GetTimeout(const std::string &target) { + if (processCommunicator_ == nullptr) { + LOGE("[NAdapt][GetTimeout] processCommunicator_ is nullptr and target is %s{private}.", target.c_str()); + return DBConstant::MAX_TIMEOUT; + } DeviceInfos devInfos; devInfos.identifier = target; uint32_t timeout = processCommunicator_->GetTimeout(devInfos); @@ -268,13 +284,11 @@ void NetworkAdapter::OnDataReceiveHandler(const DeviceInfos &srcDevInfo, const u return; } uint32_t headLength = 0; - std::vector userId; - DBStatus errCode = processCommunicator_->CheckAndGetDataHeadInfo(data, length, headLength, userId); + DBStatus errCode = processCommunicator_->GetDataHeadInfo(data, length, headLength); LOGI("[NAdapt][OnDataRecv] Enter, from=%s{private}, extendHeadLength=%u, totalLength=%u", srcDevInfo.identifier.c_str(), headLength, length); - if (errCode == NO_PERMISSION) { - LOGI("[NAdapt][OnDataRecv] userId dismatched, drop packet"); - return; + if (errCode != OK) { + LOGW("[NAdapt][OnDataRecv] get data head info err, drop packet, errCode=%d", errCode); } if (headLength >= length) { LOGW("[NAdapt][OnDataRecv] head len is too big, drop packet"); @@ -286,11 +300,8 @@ void NetworkAdapter::OnDataReceiveHandler(const DeviceInfos &srcDevInfo, const u LOGE("[NAdapt][OnDataRecv] onReceiveHandle invalid."); return; } - std::string currentUserId; - if (userId.size() >= 1) { - currentUserId = userId[0]; - } - onReceiveHandle_(srcDevInfo.identifier, data + headLength, length - headLength, currentUserId); + DataUserInfoProc userInfoProc = {data, length, processCommunicator_}; + onReceiveHandle_(srcDevInfo.identifier, data + headLength, length - headLength, userInfoProc); } // These code is compensation for the probable defect of IProcessCommunicator implementation. // As described in the agreement, for the missed online situation, we check the source dev when received. diff --git a/kv_store/frameworks/libs/distributeddb/gaussdb_rd/src/common/src/json_common.cpp b/kv_store/frameworks/libs/distributeddb/gaussdb_rd/src/common/src/json_common.cpp index 40b92e3f..ba0881a8 100644 --- a/kv_store/frameworks/libs/distributeddb/gaussdb_rd/src/common/src/json_common.cpp +++ b/kv_store/frameworks/libs/distributeddb/gaussdb_rd/src/common/src/json_common.cpp @@ -467,6 +467,9 @@ bool JsonNodeReplace(const JsonObject &src, const JsonFieldPath &itemPath, const { int errCode = E_OK; JsonFieldPath fatherPath = itemPath; + if (fatherPath.empty()) { + return false; + } fatherPath.pop_back(); if (!fatherPath.empty()) { JsonObject fatherItem = src.FindItem(fatherPath, errCode); diff --git a/kv_store/frameworks/libs/distributeddb/gaussdb_rd/test/unittest/BUILD.gn b/kv_store/frameworks/libs/distributeddb/gaussdb_rd/test/unittest/BUILD.gn index a23631f3..8979be7d 100644 --- a/kv_store/frameworks/libs/distributeddb/gaussdb_rd/test/unittest/BUILD.gn +++ b/kv_store/frameworks/libs/distributeddb/gaussdb_rd/test/unittest/BUILD.gn @@ -12,7 +12,7 @@ # limitations under the License. import("//build/test.gni") -module_output_path = "kv_store/gaussdb_rd" +module_output_path = "kv_store/kv_store/gaussdb_rd" ############################################################################### config("module_private_config") { diff --git a/kv_store/frameworks/libs/distributeddb/include/query_expression.h b/kv_store/frameworks/libs/distributeddb/include/query_expression.h index e0763cc4..57889e67 100644 --- a/kv_store/frameworks/libs/distributeddb/include/query_expression.h +++ b/kv_store/frameworks/libs/distributeddb/include/query_expression.h @@ -163,6 +163,8 @@ public: uint32_t GetGroupNum() const; int GetExpressionStatusForAssetsOnly() const; + bool IsUseFromTables() const; + private: void AssemblyQueryInfo(const QueryObjType queryOperType, const std::string &field, const QueryValueType type, const std::vector &value, bool isNeedFieldPath); @@ -173,6 +175,8 @@ private: void SetNotSupportIfNeed(QueryObjType type); + void SetAssetsOnlyValidStatusIfNeed(int status); + std::list queryInfo_; bool errFlag_ = true; std::vector prefixKey_; @@ -186,6 +190,7 @@ private: std::vector tables_; bool useFromTable_ = false; + bool isUseFromTables_ = false; std::string fromTable_; std::list tableSequence_; std::map expressions_; diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/cloud/cloud_store_types.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/cloud/cloud_store_types.h index f9858e21..06f39763 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/cloud/cloud_store_types.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/cloud/cloud_store_types.h @@ -39,6 +39,16 @@ enum ClearMode { BUTT = 4, }; +enum class ClearMetaDataMode : uint64_t { + CLOUD_WATERMARK = 0x01, // clear watermark of device to cloud sync + BUTT, +}; + +struct ClearMetaDataOption { + ClearMetaDataMode mode = ClearMetaDataMode::CLOUD_WATERMARK; + std::set tableNameList; // an empty set means clearing meta data on all tables +}; + enum class AssetOpType { NO_CHANGE = 0, INSERT, diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/iprocess_communicator.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/iprocess_communicator.h index e1ac4b17..0fdb757a 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/iprocess_communicator.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/iprocess_communicator.h @@ -37,6 +37,10 @@ struct ExtendInfo { std::string subUserId; }; +struct UserInfo { + std::string receiveUser; +}; + class ExtendHeaderHandle { public: ExtendHeaderHandle() {}; @@ -157,7 +161,6 @@ public: return nullptr; } // called after OnDataReceive - // return NO_PERMISSION while no need to handle the dataBuff if remote device userId is not mate with local userId // return INVALID_FORMAT and headLength = 0 if data service can not deSerialize the buff // return OK if deSerialize ok and get HeadLength/localUserId successfully virtual DBStatus CheckAndGetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength, @@ -166,6 +169,17 @@ public: headLength = 0; return OK; } + virtual DBStatus GetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength) + { + headLength = 0; + return OK; + } + // return NO_PERMISSION while no need to handle the dataBuff if remote device userId is not mate with local userId + virtual DBStatus GetDataUserInfo(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) + { + return OK; + } virtual void RegOnSendAble([[gnu::unused]] const OnSendAble &sendAbleCallback) { diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h index c70d32fc..e9b3f73a 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h @@ -307,6 +307,11 @@ public: DB_API virtual DBStatus GetDeviceEntries(const std::string &device, std::vector &entries) const = 0; DB_API virtual DatabaseStatus GetDatabaseStatus() const = 0; + + DB_API virtual DBStatus OperateDataStatus([[gnu::unused]] uint32_t dataOperator) + { + return OK; + } }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_client.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_client.h index 6a785f50..c2cd1a65 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_client.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_client.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -51,4 +52,10 @@ DB_API DistributedDB::DBStatus Lock(const std::string &tableName, const std::vec DB_API DistributedDB::DBStatus UnLock(const std::string &tableName, const std::vector> &hashKey, sqlite3 *db); + +DB_API void RegisterDbHook(sqlite3 *db); + +DB_API void UnregisterDbHook(sqlite3 *db); + +DB_API DistributedDB::DBStatus CreateDataChangeTempTrigger(sqlite3 *db); #endif // RELATIONAL_STORE_CLIENT_H diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 65720ca2..d005d85a 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -18,6 +18,7 @@ #include #include +#include #include "distributeddb/result_set.h" #include "cloud/cloud_store_types.h" #include "cloud/icloud_db.h" @@ -43,6 +44,15 @@ public: DistributedTableMode tableMode = DistributedTableMode::SPLIT_BY_DEVICE; }; + struct StoreConfig { + std::optional tableMode; + }; + + DB_API virtual DBStatus SetStoreConfig(const StoreConfig &config) + { + return OK; + } + DB_API DBStatus CreateDistributedTable(const std::string &tableName, TableSyncType type = DEVICE_COOPERATION) { return CreateDistributedTableInner(tableName, type); @@ -56,6 +66,11 @@ public: return 0; } + DB_API virtual DBStatus ClearMetaData([[gnu::unused]] const ClearMetaDataOption &option) + { + return OK; + } + DB_API DBStatus RemoveDeviceData(const std::string &device, ClearMode mode = DEFAULT) { return RemoveDeviceDataInner(device, mode); @@ -145,6 +160,11 @@ public: { return {OK, 0}; } + + DB_API virtual DBStatus OperateDataStatus([[gnu::unused]] uint32_t dataOperator) + { + return OK; + } protected: virtual DBStatus RemoveDeviceDataInner(const std::string &device, ClearMode mode) = 0; virtual DBStatus CreateDistributedTableInner(const std::string &tableName, TableSyncType type) = 0; diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/include/store_types.h b/kv_store/frameworks/libs/distributeddb/interfaces/include/store_types.h index 5fff893a..a75a5b42 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/include/store_types.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/include/store_types.h @@ -78,7 +78,7 @@ enum DBStatus { CLOUD_SYNC_UNSET, // not set sync option in cloud CLOUD_FULL_RECORDS, // cloud's record is full CLOUD_LOCK_ERROR, // cloud failed to get sync lock - CLOUD_ASSET_SPACE_INSUFFICIENT, // cloud failed to download asset + CLOUD_ASSET_SPACE_INSUFFICIENT, // cloud asset space is insufficient PROPERTY_CHANGED, // reference property changed CLOUD_VERSION_CONFLICT, // cloud failed to update version CLOUD_RECORD_EXIST_CONFLICT, // this error happen in Download/BatchInsert/BatchUpdate @@ -313,5 +313,10 @@ enum class DistributedTableMode : int { COLLABORATION = 0, // Save all devices data in user table SPLIT_BY_DEVICE // Save device data in each table split by device }; + +enum class DataOperator : uint32_t { + UPDATE_TIME = 0x01, + RESET_UPLOAD_CLOUD = 0x02 +}; } // namespace DistributedDB #endif // KV_STORE_TYPE_H diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index 8a6399c2..008c33e8 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -371,6 +371,15 @@ DBStatus KvStoreNbDelegateImpl::RegisterObserver(const Key &key, unsigned int mo if (key.size() > DBConstant::MAX_KEY_SIZE) { return INVALID_ARGS; } + if (observer == nullptr) { + LOGE("[KvStoreNbDelegate][RegisterObserver] Observer is null"); + return INVALID_ARGS; + } + if (conn_ == nullptr) { + LOGE("[RegisterObserver]%s", INVALID_CONNECTION); + return DB_ERROR; + } + uint64_t rawMode = DBCommon::EraseBit(mode, DBConstant::OBSERVER_CHANGES_MASK); if (rawMode == static_cast(ObserverMode::OBSERVER_CHANGES_CLOUD)) { return RegisterCloudObserver(key, mode, observer); @@ -378,35 +387,37 @@ DBStatus KvStoreNbDelegateImpl::RegisterObserver(const Key &key, unsigned int mo return RegisterDeviceObserver(key, static_cast(rawMode), observer); } -DBStatus KvStoreNbDelegateImpl::RegisterDeviceObserver(const Key &key, unsigned int mode, KvStoreObserver *observer) +DBStatus KvStoreNbDelegateImpl::CheckDeviceObserver(const Key &key, unsigned int mode, KvStoreObserver *observer) { if (!ParamCheckUtils::CheckObserver(key, mode)) { - LOGE("Register nb observer by illegal mode or key size!"); - return INVALID_ARGS; - } - - if (observer == nullptr) { + LOGE("[KvStoreNbDelegate][CheckDeviceObserver] Register nb observer by illegal mode or key size!"); return INVALID_ARGS; } std::lock_guard lockGuard(observerMapLock_); if (observerMap_.size() >= DBConstant::MAX_OBSERVER_COUNT) { - LOGE("[KvStoreNbDelegate] The number of kv observers has been over limit, storeId[%.3s]", storeId_.c_str()); + LOGE("[KvStoreNbDelegate][CheckDeviceObserver] The number of kv observers has been over limit, storeId[%.3s]", + storeId_.c_str()); return OVER_MAX_LIMITS; } if (observerMap_.find(observer) != observerMap_.end()) { - LOGE("[KvStoreNbDelegate] Observer has been already registered!"); + LOGE("[KvStoreNbDelegate][CheckDeviceObserver] Observer has been already registered!"); return ALREADY_SET; } + return OK; +} - if (conn_ == nullptr) { - LOGE("%s", INVALID_CONNECTION); - return DB_ERROR; - } - +DBStatus KvStoreNbDelegateImpl::RegisterDeviceObserver(const Key &key, unsigned int mode, KvStoreObserver *observer) +{ if (conn_->IsTransactionStarted()) { + LOGE("[KvStoreNbDelegate][RegisterDeviceObserver] Transaction unfinished"); return BUSY; } + DBStatus status = CheckDeviceObserver(key, mode, observer); + if (status != OK) { + LOGE("[KvStoreNbDelegate][RegisterDeviceObserver] Observer map cannot be registered, status:%d", status); + return status; + } int errCode = E_OK; auto storeId = storeId_; @@ -414,55 +425,59 @@ DBStatus KvStoreNbDelegateImpl::RegisterDeviceObserver(const Key &key, unsigned mode, key, [observer, storeId](const KvDBCommitNotifyData ¬ifyData) { KvStoreChangedDataImpl data(¬ifyData); - LOGD("[KvStoreNbDelegate] Trigger [%s] on change", storeId.c_str()); + LOGD("[KvStoreNbDelegate][RegisterDeviceObserver] Trigger [%s] on change", storeId.c_str()); observer->OnChange(data); }, errCode); if (errCode != E_OK || observerHandle == nullptr) { - LOGE("[KvStoreNbDelegate] RegisterListener failed:%d!", errCode); + LOGE("[KvStoreNbDelegate][RegisterDeviceObserver] Register device observer failed:%d!", errCode); return DB_ERROR; } observerMap_.insert(std::pair(observer, observerHandle)); - LOGI("[KvStoreNbDelegate] RegisterDeviceObserver ok mode:%u", mode); + LOGI("[KvStoreNbDelegate][RegisterDeviceObserver] Register device observer ok mode:%u", mode); return OK; } -DBStatus KvStoreNbDelegateImpl::RegisterCloudObserver(const Key &key, unsigned int mode, - KvStoreObserver *observer) +DBStatus KvStoreNbDelegateImpl::CheckCloudObserver(KvStoreObserver *observer) { - if (conn_ == nullptr) { - LOGE("%s", INVALID_CONNECTION); - return DB_ERROR; - } - std::lock_guard lockGuard(observerMapLock_); if (cloudObserverMap_.size() >= DBConstant::MAX_OBSERVER_COUNT) { - LOGE("[KvStoreNbDelegate] The number of kv cloud observers has been over limit, storeId[%.3s]", + LOGE("[KvStoreNbDelegate][CheckCloudObserver] The number of kv cloud observers over limit, storeId[%.3s]", storeId_.c_str()); return OVER_MAX_LIMITS; } - if (cloudObserverMap_.find(observer) != cloudObserverMap_.end() && cloudObserverMap_[observer] == mode) { - LOGE("[KvStoreNbDelegate] Cloud observer has been already registered!"); + if (cloudObserverMap_.find(observer) != cloudObserverMap_.end()) { + LOGE("[KvStoreNbDelegate][CheckCloudObserver] Cloud observer has been already registered!"); return ALREADY_SET; } + return OK; +} + +DBStatus KvStoreNbDelegateImpl::RegisterCloudObserver(const Key &key, unsigned int mode, KvStoreObserver *observer) +{ + DBStatus status = CheckCloudObserver(observer); + if (status != OK) { + LOGE("[KvStoreNbDelegate][RegisterCloudObserver] Cloud observer map cannot be registered, status:%d", status); + return status; + } auto storeId = storeId_; ObserverAction action = [observer, storeId]( const std::string &device, ChangedData &&changedData, bool isChangedData) { if (isChangedData) { - LOGD("[KvStoreNbDelegate] Trigger [%s] on change", storeId.c_str()); + LOGD("[KvStoreNbDelegate][RegisterCloudObserver] Trigger [%s] on change", storeId.c_str()); observer->OnChange(Origin::ORIGIN_CLOUD, device, std::move(changedData)); } }; int errCode = conn_->RegisterObserverAction(observer, action); if (errCode != E_OK) { - LOGE("[KvStoreNbDelegate] RegisterCloudObserver failed:%d!", errCode); + LOGE("[KvStoreNbDelegate][RegisterCloudObserver] Register cloud observer failed:%d!", errCode); return DB_ERROR; } cloudObserverMap_[observer] = mode; - LOGI("[KvStoreNbDelegate] RegisterCloudObserver ok mode:%u", mode); + LOGI("[KvStoreNbDelegate][RegisterCloudObserver] Register cloud observer ok mode:%u", mode); return OK; } @@ -583,11 +598,16 @@ DBStatus KvStoreNbDelegateImpl::Sync(const std::vector &devices, Sy return NOT_SUPPORT; } + QuerySyncObject querySyncObj(query); + if (!querySyncObj.GetRelationTableNames().empty()) { + LOGE("check query table names from tables failed!"); + return NOT_SUPPORT; + } + if (!DBCommon::CheckQueryWithoutMultiTable(query)) { LOGE("not support for invalid query"); return NOT_SUPPORT; } - QuerySyncObject querySyncObj(query); if (querySyncObj.GetSortType() != SortType::NONE || querySyncObj.IsQueryByRange()) { LOGE("not support order by timestamp and query by range"); return NOT_SUPPORT; @@ -636,6 +656,10 @@ DBStatus KvStoreNbDelegateImpl::Sync(const DeviceSyncOption &option, const Devic int errCode = E_OK; if (option.isQuery) { QuerySyncObject querySyncObj(option.query); + if (!querySyncObj.GetRelationTableNames().empty()) { + LOGE("Sync with option and check query table names from tables failed!"); + return NOT_SUPPORT; + } if (!DBCommon::CheckQueryWithoutMultiTable(option.query)) { LOGE("Not support for invalid query"); return NOT_SUPPORT; @@ -1349,4 +1373,19 @@ std::pair> KvStoreNbDelegateImpl::G return res; } #endif + +DBStatus KvStoreNbDelegateImpl::OperateDataStatus(uint32_t dataOperator) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION); + return DB_ERROR; + } + int errCode = conn_->OperateDataStatus(dataOperator); + if (errCode != E_OK) { + LOGE("[KvStoreNbDelegate] Operate data status errCode:%d", errCode); + } else { + LOGI("[KvStoreNbDelegate] Operate data status success"); + } + return TransferDBErrno(errCode); +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h b/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h index 2a4c9b0d..6ec08032 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h @@ -193,6 +193,8 @@ public: std::pair> GetCloudVersion(const std::string &device) override; #endif + + DBStatus OperateDataStatus(uint32_t dataOperator) override; private: DBStatus GetInner(const IOption &option, const Key &key, Value &value) const; DBStatus PutInner(const IOption &option, const Key &key, const Value &value); @@ -213,6 +215,10 @@ private: DBStatus UnRegisterCloudObserver(const KvStoreObserver *observer); + DBStatus CheckDeviceObserver(const Key &key, unsigned int mode, KvStoreObserver *observer); + + DBStatus CheckCloudObserver(KvStoreObserver *observer); + IKvDBConnection *conn_; std::string storeId_; bool releaseFlag_; diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 99741c8b..cf405a1e 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -530,6 +530,70 @@ int32_t RelationalStoreDelegateImpl::GetCloudSyncTaskCount() } return count; } + +DBStatus RelationalStoreDelegateImpl::ClearMetaData(const ClearMetaDataOption &option) +{ + if (option.mode >= ClearMetaDataMode::BUTT) { + LOGE("[RelationalStore Delegate] Invalid mode for clear meta data."); + return INVALID_ARGS; + } + + if (option.mode == ClearMetaDataMode::CLOUD_WATERMARK) { + return ClearWatermark(option); + } + return OK; +} + +DBStatus RelationalStoreDelegateImpl::ClearWatermark(const ClearMetaDataOption &option) +{ + if (!option.tableNameList.empty()) { + LOGE("[RelationalStore Delegate] Clearing watermark of specific tables is not supported yet!"); + return NOT_SUPPORT; + } + + if (conn_ == nullptr) { + LOGE("[RelationalStore Delegate] Invalid connection for clear water mark!"); + return DB_ERROR; + } + + int errCode = conn_->ClearCloudWatermark(option.tableNameList); + if (errCode != E_OK) { + LOGE("[RelationalStore Delegate] clear cloud water mark failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} #endif + +DBStatus RelationalStoreDelegateImpl::SetStoreConfig(const StoreConfig &config) +{ + if (!config.tableMode.has_value()) { + return OK; + } + if (conn_ == nullptr) { + LOGE("[RelationalStore Delegate] Invalid connection for set store config."); + return DB_ERROR; + } + int errCode = conn_->SetTableMode(config.tableMode.value()); + if (errCode != E_OK) { + LOGE("[RelationalStore Delegate] set store config failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + +DBStatus RelationalStoreDelegateImpl::OperateDataStatus(uint32_t dataOperator) +{ + if (conn_ == nullptr) { + LOGE("[RelationalStore Delegate] Invalid connection for operate data status."); + return DB_ERROR; + } + int errCode = conn_->OperateDataStatus(dataOperator); + if (errCode != E_OK) { + LOGE("[RelationalStore Delegate] operate data failed:%d op:%" PRIu32, errCode, dataOperator); + return TransferDBErrno(errCode); + } + return OK; +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 3ec59811..caea7eb5 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -29,6 +29,8 @@ public: DISABLE_COPY_ASSIGN_MOVE(RelationalStoreDelegateImpl); + DBStatus SetStoreConfig(const StoreConfig &config) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, const Query &query, const SyncStatusCallback &onComplete, bool wait) override; @@ -90,11 +92,17 @@ public: SyncProcess GetCloudTaskStatus(uint64_t taskId) override; DBStatus SetIAssetLoader(const std::shared_ptr &loader) override; + + DBStatus ClearMetaData(const ClearMetaDataOption &option) override; #endif private: static void OnSyncComplete(const std::map> &devicesStatus, const SyncStatusCallback &onComplete); +#ifdef USE_DISTRIBUTEDDB_CLOUD + DBStatus ClearWatermark(const ClearMetaDataOption &option); +#endif + DBStatus OperateDataStatus(uint32_t dataOperator) override; RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 3ebb2e62..f72eeb1e 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -238,13 +238,17 @@ void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchReques std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId, bool syncDualTupleMode) { - return RuntimeConfig::GetStoreIdentifier(userId, appId, storeId, syncDualTupleMode); + return RelationalStoreManager::GetRelationalStoreIdentifier(userId, "", appId, storeId, syncDualTupleMode); } std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &subUserId, const std::string &appId, const std::string &storeId, bool syncDualTupleMode) { - return RuntimeConfig::GetStoreIdentifier(userId, subUserId, appId, storeId, syncDualTupleMode); + StoreInfo info; + info.storeId = storeId; + info.appId = appId; + info.userId = userId; + return DBCommon::GetStoreIdentifier(info, subUserId, syncDualTupleMode, true); } std::vector RelationalStoreManager::ParserQueryNodes(const Bytes &queryBytes, diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp index e314e3e2..54cd766c 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp @@ -401,6 +401,8 @@ std::map> g_storeChangedDataMap; std::mutex g_clientCreateTableMutex; std::set g_clientCreateTable; +std::mutex g_registerSqliteHookMutex; + int RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) { if (db == nullptr) { @@ -451,6 +453,7 @@ void CalcHashKey(sqlite3_context *ctx, int argc, sqlite3_value **argv) if (collateType == DistributedDB::CollateType::COLLATE_NOCASE) { auto colChar = reinterpret_cast(sqlite3_value_text(argv[0])); if (colChar == nullptr) { + sqlite3_result_error(ctx, "Parameters is invalid while collateType is NOCASE.", -1); return; } std::string colStr(colChar); @@ -461,6 +464,7 @@ void CalcHashKey(sqlite3_context *ctx, int argc, sqlite3_value **argv) } else if (collateType == DistributedDB::CollateType::COLLATE_RTRIM) { auto colChar = reinterpret_cast(sqlite3_value_text(argv[0])); if (colChar == nullptr) { + sqlite3_result_error(ctx, "Parameters is invalid while collateType is RTRIM.", -1); return; } std::string colStr(colChar); @@ -494,9 +498,10 @@ int RegisterCalcHash(sqlite3 *db) return RegisterFunction(db, "calc_hash", 2, nullptr, func); // 2 is params count } -int GetLocalTimeOffsetFromMeta(sqlite3 *db, TimeOffset &offset) +int GetNumValueFromMeta(sqlite3 *db, std::string keyStr, int64_t &numResult) { if (db == nullptr) { + LOGE("[GetNumValueFromMeta] db is null"); return -E_ERROR; } @@ -507,7 +512,6 @@ int GetLocalTimeOffsetFromMeta(sqlite3 *db, TimeOffset &offset) return -E_ERROR; } - std::string keyStr = "localTimeOffset"; std::vector key(keyStr.begin(), keyStr.end()); errCode = BindBlobToStatement(stmt, 1, key); if (errCode != E_OK) { @@ -533,10 +537,37 @@ int GetLocalTimeOffsetFromMeta(sqlite3 *db, TimeOffset &offset) std::string valueStr(value.begin(), value.end()); int64_t result = std::strtoll(valueStr.c_str(), nullptr, STR_TO_LL_BY_DEVALUE); if (errno != ERANGE && result != LLONG_MIN && result != LLONG_MAX) { - offset = result; + numResult = result; } (void)ResetStatement(stmt); - LOGD("Get local time offset from meta: %" PRId64, offset); + return E_OK; +} + +int GetLocalTimeOffsetFromMeta(sqlite3 *db, TimeOffset &offset) +{ + std::string keyStr(DBConstant::LOCALTIME_OFFSET_KEY); + int64_t dbVal = 0; + int errCode = GetNumValueFromMeta(db, keyStr, dbVal); + if (errCode != E_OK) { + LOGE("[GetLocalTimeOffsetFromMeta] Failed to get localTimeOffset. %d", errCode); + return errCode; + } + offset = dbVal; + LOGD("[GetLocalTimeOffsetFromMeta] Get local time offset: %" PRId64, offset); + return E_OK; +} + +int GetTableModeFromMeta(sqlite3 *db, DistributedTableMode &mode) +{ + std::string keyStr = RelationalDBProperties::DISTRIBUTED_TABLE_MODE; + int64_t dbVal = 0; + int errCode = GetNumValueFromMeta(db, keyStr, dbVal); + if (errCode != E_OK) { + LOGE("[GetTableModeFromMeta] Failed to get distributed table mode. %d", errCode); + return errCode; + } + mode = static_cast(dbVal); + LOGD("[GetTableModeFromMeta] Get distributed table mode: %d", mode); return E_OK; } @@ -1057,8 +1088,6 @@ int RegisterDataChangeObserver(sqlite3 *db) void RegisterCommitAndRollbackHook(sqlite3 *db) { sqlite3_set_authorizer(db, AuthorizerCallback, nullptr); - sqlite3_wal_hook(db, LogCommitHookCallback, db); - sqlite3_rollback_hook(db, RollbackHookCallback, db); } int ResetStatement(sqlite3_stmt *&stmt) @@ -1252,8 +1281,9 @@ int GetTableSyncType(sqlite3 *db, const std::string &tableName, std::string &tab return -E_ERROR; } (void)sqlite3_finalize(statement); - tableType = DEVICE_TYPE; - return E_OK; + LOGW("[GetTableSyncType] Not found sync type for %s[%zu]", DBCommon::StringMiddleMasking(tableName).c_str(), + tableName.size()); + return -E_NOT_FOUND; } void HandleDropCloudSyncTable(sqlite3 *db, const std::string &tableName) @@ -1317,7 +1347,7 @@ int HandleDropLogicDeleteData(sqlite3 *db, const std::string &tableName, uint64_ int SaveDeleteFlagToDB(sqlite3 *db, const std::string &tableName) { - std::string keyStr = DBConstant::TABLE_WAS_DROPPED + tableName; + std::string keyStr = DBConstant::TABLE_IS_DROPPED + tableName; Key key; DBCommon::StringToVector(keyStr, key); Value value; @@ -1329,7 +1359,7 @@ int SaveDeleteFlagToDB(sqlite3 *db, const std::string &tableName) LOGE("[SaveDeleteFlagToDB] prepare statement failed, %d", errCode); return -E_ERROR; } - + if (sqlite3_bind_blob(statement, 1, static_cast(key.data()), key.size(), SQLITE_TRANSIENT) != SQLITE_OK) { // LCOV_EXCL_BR_LINE (void)sqlite3_finalize(statement); @@ -1380,8 +1410,17 @@ void ClearTheLogAfterDropTable(sqlite3 *db, const char *tableName, const char *s if (tableType == DEVICE_TYPE) { RegisterGetSysTime(db); RegisterGetLastTime(db); - std::string sql = "UPDATE " + logTblName + " SET data_key=-1, flag=0x03, timestamp=get_sys_time(0) " - "WHERE flag&0x03=0x02 AND timestamp<" + std::to_string(dropTimeStamp); + std::string targetFlag = "flag|0x01"; + std::string originalFlag = "flag&0x01=0x0"; + auto tableMode = DistributedTableMode::COLLABORATION; + (void)GetTableModeFromMeta(db, tableMode); + if (tableMode == DistributedTableMode::SPLIT_BY_DEVICE) { + targetFlag = "0x03"; + originalFlag = "flag&0x03=0x02"; + } + std::string sql = "UPDATE " + logTblName + " SET data_key=-1, flag=" + targetFlag + + ", timestamp=get_sys_time(0) WHERE " + originalFlag + + " AND timestamp<" + std::to_string(dropTimeStamp); (void)sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); } else { HandleDropCloudSyncTable(db, tableStr); @@ -1668,8 +1707,8 @@ DB_API DistributedDB::DBStatus RegisterStoreObserver(sqlite3 *db, const std::sha std::lock_guard lock(g_storeObserverMutex); if (std::find(g_storeObserverMap[hashFileName].begin(), g_storeObserverMap[hashFileName].end(), storeObserver) != g_storeObserverMap[hashFileName].end()) { - LOGE("[RegisterStoreObserver] Duplicate observer."); - return DistributedDB::INVALID_ARGS; + LOGW("[RegisterStoreObserver] Duplicate observer."); + return DistributedDB::OK; } g_storeObserverMap[hashFileName].push_back(storeObserver); return DistributedDB::OK; @@ -1712,6 +1751,7 @@ DB_API DistributedDB::DBStatus UnregisterStoreObserver(sqlite3 *db) std::string fileName; if (!GetDbFileName(db, fileName)) { LOGD("[UnregisterAllStoreObserver] StoreObserver is invalid."); + return DistributedDB::INVALID_ARGS; } @@ -1773,6 +1813,24 @@ DB_API DistributedDB::DBStatus UnLock(const std::string &tableName, const std::v return HandleDataLock(tableName, hashKey, db, false); } +DB_API void RegisterDbHook(sqlite3 *db) +{ + std::lock_guard lock(g_registerSqliteHookMutex); + sqlite3_wal_hook(db, LogCommitHookCallback, db); + sqlite3_rollback_hook(db, RollbackHookCallback, db); +} + +DB_API void UnregisterDbHook(sqlite3 *db) +{ + std::lock_guard lock(g_registerSqliteHookMutex); + sqlite3_wal_hook(db, nullptr, db); + sqlite3_rollback_hook(db, nullptr, db); +} + +DB_API DistributedDB::DBStatus CreateDataChangeTempTrigger(sqlite3 *db) +{ + return DistributedDB::TransferDBErrno(CreateTempTrigger(db)); +} // hw export the symbols #ifdef SQLITE_DISTRIBUTE_RELATIONAL #if defined(__GNUC__) diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 8dacecdf..012ba84c 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -179,6 +179,8 @@ public: int CleanCloudData(ClearMode mode, const std::vector &tableNameList, const RelationalSchemaObject &localSchema, std::vector &assets) override; + int ClearCloudLogVersion(const std::vector &tableNameList) override; + int FillCloudAssetForDownload(const std::string &tableName, VBucket &asset, bool isDownloadSuccess) override; int FillCloudAssetForAsyncDownload(const std::string &tableName, VBucket &asset, bool isDownloadSuccess) override; @@ -266,6 +268,8 @@ public: void PrintCursorChange(const std::string &tableName) override; int GetLockStatusByGid(const std::string &tableName, const std::string &gid, LockStatus &status) override; + + bool IsExistTableContainAssets() override; protected: int FillReferenceData(CloudSyncData &syncData); @@ -287,6 +291,8 @@ protected: int ReviseLocalModTime(const std::string &tableName, const std::vector &revisedData) override; + bool IsSetDistributedSchema(const std::string &tableName, RelationalSchemaObject &schemaObj); + private: SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode, OperatePerm perm = OperatePerm::NORMAL_PERM) const; @@ -374,6 +380,8 @@ private: CloudUploadRecorder uploadRecorder_; std::map> cursorChangeMap_; + + std::mutex cursorChangeMutex_; }; } // namespace DistributedDB #endif diff --git a/kv_store/frameworks/libs/distributeddb/interfaces/src/runtime_config.cpp b/kv_store/frameworks/libs/distributeddb/interfaces/src/runtime_config.cpp index 5e5bf4c0..9d27786c 100644 --- a/kv_store/frameworks/libs/distributeddb/interfaces/src/runtime_config.cpp +++ b/kv_store/frameworks/libs/distributeddb/interfaces/src/runtime_config.cpp @@ -166,16 +166,11 @@ std::string RuntimeConfig::GetStoreIdentifier(const std::string &userId, const s std::string RuntimeConfig::GetStoreIdentifier(const std::string &userId, const std::string &subUserId, const std::string &appId, const std::string &storeId, bool syncDualTupleMode) { - if (!ParamCheckUtils::CheckStoreParameter(storeId, appId, userId, syncDualTupleMode, subUserId)) { - return ""; - } - if (syncDualTupleMode) { - return DBCommon::TransferHashString(appId + "-" + storeId); - } - if (subUserId.empty()) { - return DBCommon::TransferHashString(userId + "-" + appId + "-" + storeId); - } - return DBCommon::TransferHashString(userId + "-" + appId + "-" + storeId + "-" + subUserId); + StoreInfo info; + info.storeId = storeId; + info.appId = appId; + info.userId = userId; + return DBCommon::GetStoreIdentifier(info, subUserId, syncDualTupleMode, false); } void RuntimeConfig::ReleaseAutoLaunch(const std::string &userId, const std::string &appId, const std::string &storeId, diff --git a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/include/tokenizer_sqlite.h b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/include/tokenizer_sqlite.h index 975d4810..dd31e4d0 100644 --- a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/include/tokenizer_sqlite.h +++ b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/include/tokenizer_sqlite.h @@ -33,8 +33,8 @@ extern "C" { typedef int (*XTokenFn)(void *, int, const char *, int, int, int); GRD_API int fts5_customtokenizer_xCreate(void *sqlite3, const char **azArg, int nArg, Fts5Tokenizer **ppOut); -GRD_API int fts5_customtokenizer_xTokenize(Fts5Tokenizer *tokenizer_ptr, void *pCtx, int flags, const char *pText, - int nText, XTokenFn xToken); +GRD_API int fts5_customtokenizer_xTokenize( + Fts5Tokenizer *tokenizer_ptr, void *pCtx, int flags, const char *pText, int nText, XTokenFn xToken); GRD_API void fts5_customtokenizer_xDelete(Fts5Tokenizer *tokenizer_ptr); GRD_API int sqlite3_customtokenizer_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_api.h b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_api.h index df04fa4e..5710a0cd 100644 --- a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_api.h +++ b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_api.h @@ -1,16 +1,18 @@ /* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -Copyright (c) 2025 Huawei Device Co., Ltd. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ #ifndef GRD_TOKENIZER_API_H #define GRD_TOKENIZER_API_H @@ -33,4 +35,4 @@ int32_t GRD_TokenizerDestroy(void *ptr); } -#endif // GRD_TOKENIZER_API_H +#endif // GRD_TOKENIZER_API_H \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_export_type.h b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_export_type.h index 366c75e3..97b04188 100644 --- a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_export_type.h +++ b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_export_type.h @@ -35,6 +35,8 @@ extern "C" { #endif #endif // GRD_API +#define GRD_TOKENIZER_POS_LEN 4 + typedef enum GRD_CutMode { CUT_MMSEG = 0, CUT_BUTT // INVALID TokenizeMode @@ -63,6 +65,8 @@ typedef struct GRD_WordEntryList GRD_WordEntryListT; typedef struct GRD_WordEntry { const char *word; uint32_t length; + char partOfSpeech[GRD_TOKENIZER_POS_LEN + 1]; + double idf; } GRD_WordEntryT; #ifdef __cplusplus diff --git a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_sqlite.cpp b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_sqlite.cpp index afaf739b..c2d6b4e6 100644 --- a/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_sqlite.cpp +++ b/kv_store/frameworks/libs/distributeddb/sqlite_adapter/src/tokenizer_sqlite.cpp @@ -34,7 +34,7 @@ int fts5_customtokenizer_xCreate(void *sqlite3, const char **azArg, int nArg, Ft (void)sqlite3; std::lock_guard lock(g_mtx); g_refCount++; - if (g_refCount != 1) { + if (g_refCount != 1) { // 说明已经初始化过了，直接返回 *ppOut = (Fts5Tokenizer *)&g_magicCode; return SQLITE_OK; } @@ -45,7 +45,7 @@ int fts5_customtokenizer_xCreate(void *sqlite3, const char **azArg, int nArg, Ft sqlite3_log(ret, "GRD_TokenizerInit wrong"); return ret; } - *ppOut = (Fts5Tokenizer *)&g_magicCode; + *ppOut = (Fts5Tokenizer *)&g_magicCode; // 需要保证*ppOut不为NULL，否则会使用默认分词器而不是自定的 return SQLITE_OK; } @@ -60,14 +60,15 @@ static char *CpyStr(const char *pText, int nText) } errno_t err = memcpy_s(ptr, nText + 1, pText, nText); if (err != EOK) { + free(ptr); return nullptr; } ptr[nText] = '\0'; return ptr; } -int fts5_customtokenizer_xTokenize(Fts5Tokenizer *tokenizer_ptr, void *pCtx, int flags, const char *pText, int nText, - XTokenFn xToken) +int fts5_customtokenizer_xTokenize( + Fts5Tokenizer *tokenizer_ptr, void *pCtx, int flags, const char *pText, int nText, XTokenFn xToken) { if (nText == 0) { return SQLITE_OK; @@ -82,14 +83,15 @@ int fts5_customtokenizer_xTokenize(Fts5Tokenizer *tokenizer_ptr, void *pCtx, int int ret = GRD_TokenizerCut(ptr, option, &entryList); if (ret != GRD_OK) { sqlite3_log(ret, "GRD_TokenizerCut wrong"); + free(ptr); return ret; } GRD_WordEntryT entry; - int start = 0; - int end = 0; + int start = 0; // 词在句子中的起始位置 + int end = 0; // 词在句子中的结束位置 while ((ret = GRD_TokenizerNext(entryList, &entry)) == GRD_OK) { start = entry.word - ptr; - end = start + entry.length; + end = start + static_cast(entry.length); ret = xToken(pCtx, 0, entry.word, entry.length, start, end); if (ret != SQLITE_OK) { sqlite3_log(ret, "xToken wrong"); @@ -108,7 +110,7 @@ void fts5_customtokenizer_xDelete(Fts5Tokenizer *tokenizer_ptr) { std::lock_guard lock(g_mtx); g_refCount--; - if (g_refCount != 0) { + if (g_refCount != 0) { // 说明还有其他的地方在使用，不能释放资源 return; } (void)GRD_TokenizerDestroy(tokenizer_ptr); @@ -142,10 +144,7 @@ int sqlite3_customtokenizer_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api SQLITE_EXTENSION_INIT2(pApi) fts5_tokenizer tokenizer = { - fts5_customtokenizer_xCreate, - fts5_customtokenizer_xDelete, - fts5_customtokenizer_xTokenize - }; + fts5_customtokenizer_xCreate, fts5_customtokenizer_xDelete, fts5_customtokenizer_xTokenize}; fts5_api *fts5api; rc = fts5_api_from_db(db, &fts5api); if (rc != SQLITE_OK) { diff --git a/kv_store/frameworks/libs/distributeddb/storage/include/cloud/cloud_storage_utils.h b/kv_store/frameworks/libs/distributeddb/storage/include/cloud/cloud_storage_utils.h index 58c21972..addeeb3c 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/include/cloud/cloud_storage_utils.h +++ b/kv_store/frameworks/libs/distributeddb/storage/include/cloud/cloud_storage_utils.h @@ -175,7 +175,7 @@ public: static int IdentifyCloudType(const CloudUploadRecorder &recorder, CloudSyncData &cloudSyncData, VBucket &data, VBucket &log, VBucket &flags); - static int CheckAbnormalData(CloudSyncData &cloudSyncData, const VBucket &data, bool isInsert); + static int CheckAbnormalData(CloudSyncData &cloudSyncData, VBucket &data, bool isInsert, bool isAsyncDownloading); static std::pair GetDataItemFromCloudData(VBucket &data); @@ -210,14 +210,21 @@ public: static int FillCloudQueryToExtend(QuerySyncObject &obj, VBucket &extend); + static void SaveChangedDataByType(const DataValue &dataValue, Type &value); + private: static int IdentifyCloudTypeInner(CloudSyncData &cloudSyncData, VBucket &data, VBucket &log, VBucket &flags); static int FillQueryByPK(const std::string &tableName, bool isKv, std::map dataIndex, - std::vector>> syncPkVec, std::vector &syncQuery); + std::vector>> &syncPkVec, std::vector &syncQuery); static void PutSyncPkVec(const std::string &col, std::map> &syncPk, std::vector>> &syncPkVec); + + static bool IsAssetNotDownload(const uint32_t &status); + + static void CheckAbnormalDataInner(const bool isAsyncDownloading, VBucket &data, bool &isSyncAssetAbnormal, + bool &isAsyncAssetAbnormal); }; } #endif // CLOUD_STORAGE_UTILS_H diff --git a/kv_store/frameworks/libs/distributeddb/storage/include/icloud_sync_storage_interface.h b/kv_store/frameworks/libs/distributeddb/storage/include/icloud_sync_storage_interface.h index 738fbd8a..3f56d255 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/include/icloud_sync_storage_interface.h +++ b/kv_store/frameworks/libs/distributeddb/storage/include/icloud_sync_storage_interface.h @@ -151,6 +151,11 @@ public: return E_OK; } + virtual int ClearCloudLogVersion(const std::vector &tableNameList) + { + return E_OK; + } + virtual void TriggerObserverAction(const std::string &deviceName, ChangedData &&changedData, bool isChangedData) = 0; @@ -311,6 +316,11 @@ public: { return E_OK; } + + virtual bool IsExistTableContainAssets() + { + return false; + } }; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/include/ikvdb_connection.h b/kv_store/frameworks/libs/distributeddb/storage/include/ikvdb_connection.h index ea3fda63..d9c8a60e 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/include/ikvdb_connection.h +++ b/kv_store/frameworks/libs/distributeddb/storage/include/ikvdb_connection.h @@ -170,6 +170,8 @@ public: virtual void MarkRebuild() = 0; virtual bool IsRebuild() const = 0; + + virtual int OperateDataStatus(uint32_t dataOperator) = 0; }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/include/relational_store_connection.h b/kv_store/frameworks/libs/distributeddb/storage/include/relational_store_connection.h index 0de01196..13652f3b 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/include/relational_store_connection.h +++ b/kv_store/frameworks/libs/distributeddb/storage/include/relational_store_connection.h @@ -78,11 +78,15 @@ public: virtual int SetDistributedDbSchema(const DistributedSchema &schema) = 0; + virtual int SetTableMode(DistributedTableMode tableMode) = 0; + #ifdef USE_DISTRIBUTEDDB_CLOUD virtual int32_t GetCloudSyncTaskCount() = 0; virtual int DoClean(ClearMode mode) = 0; + virtual int ClearCloudWatermark(const std::set &tableNames) = 0; + virtual int SetCloudDB(const std::shared_ptr &cloudDb) = 0; virtual int PrepareAndSetCloudDbSchema(const DataBaseSchema &schema) = 0; @@ -95,6 +99,7 @@ public: virtual SyncProcess GetCloudTaskStatus(uint64_t taskId) = 0; #endif + virtual int OperateDataStatus(uint32_t dataOperator) = 0; protected: // Get the stashed 'RelationalDB_ pointer' without ref. template diff --git a/kv_store/frameworks/libs/distributeddb/storage/include/storage_proxy.h b/kv_store/frameworks/libs/distributeddb/storage/include/storage_proxy.h index 36d2a991..2ccc6c7c 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/include/storage_proxy.h +++ b/kv_store/frameworks/libs/distributeddb/storage/include/storage_proxy.h @@ -87,6 +87,8 @@ public: int CleanCloudData(ClearMode mode, const std::vector &tableNameList, const RelationalSchemaObject &localSchema, std::vector &assets); + int ClearCloudLogVersion(const std::vector &tableNameList); + int CheckSchema(const TableName &tableName) const; int CheckSchema(std::vector &tables); @@ -185,7 +187,7 @@ public: int GetLockStatusByGid(const std::string &tableName, const std::string &gid, LockStatus &status); - bool IsContainAssetsTable(); + bool IsExistTableContainAssets(); protected: void Init(); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/cloud/cloud_storage_utils.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/cloud/cloud_storage_utils.cpp index 899e4ca1..ee7540a2 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/cloud/cloud_storage_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/cloud/cloud_storage_utils.cpp @@ -1273,7 +1273,7 @@ int CloudStorageUtils::IdentifyCloudType(const CloudUploadRecorder &recorder, Cl int CloudStorageUtils::IdentifyCloudTypeInner(CloudSyncData &cloudSyncData, VBucket &data, VBucket &log, VBucket &flags) { - int64_t *rowid = std::get_if(&flags[CloudDbConstant::ROWID]); + int64_t *rowid = std::get_if(&flags[DBConstant::ROWID]); int64_t *flag = std::get_if(&flags[CloudDbConstant::FLAG]); int64_t *timeStamp = std::get_if(&flags[CloudDbConstant::TIMESTAMP]); Bytes *hashKey = std::get_if(&flags[CloudDbConstant::HASH_KEY]); @@ -1298,7 +1298,9 @@ int CloudStorageUtils::IdentifyCloudTypeInner(CloudSyncData &cloudSyncData, VBuc cloudSyncData.delData.timestamp.push_back(*timeStamp); cloudSyncData.delData.rowid.push_back(*rowid); } else { - int errCode = CheckAbnormalData(cloudSyncData, data, isInsert); + bool isAsyncDownloading = ((static_cast(*flag) & + static_cast(LogInfoFlag::FLAG_ASSET_DOWNLOADING_FOR_ASYNC)) != 0); + int errCode = CheckAbnormalData(cloudSyncData, data, isInsert, isAsyncDownloading); if (errCode != E_OK) { return errCode; } @@ -1318,43 +1320,79 @@ int CloudStorageUtils::IdentifyCloudTypeInner(CloudSyncData &cloudSyncData, VBuc return E_OK; } -int CloudStorageUtils::CheckAbnormalData(CloudSyncData &cloudSyncData, const VBucket &data, bool isInsert) +bool CloudStorageUtils::IsAssetNotDownload(const uint32_t &status) { - if (data.empty()) { - LOGE("The cloud data is empty, isInsert:%d", static_cast(isInsert)); - return -E_INVALID_DATA; - } - bool isDataAbnormal = false; - for (const auto &item : data) { + return status == static_cast(AssetStatus::ABNORMAL) || + status == static_cast(AssetStatus::DOWNLOADING) || + (status & static_cast(AssetStatus::DOWNLOAD_WITH_NULL)) != 0; +} + +void CloudStorageUtils::CheckAbnormalDataInner(const bool isAsyncDownloading, VBucket &data, + bool &isSyncAssetAbnormal, bool &isAsyncAssetAbnormal) +{ + std::vector abnormalAssetFields; + for (auto &item : data) { const Asset *asset = std::get_if>(&item.second); if (asset != nullptr) { - if (asset->status == static_cast(AssetStatus::ABNORMAL) || - (asset->status & static_cast(AssetStatus::DOWNLOAD_WITH_NULL)) != 0) { - isDataAbnormal = true; - break; + bool isAssetNotDownload = IsAssetNotDownload(asset->status); + if (!isAssetNotDownload) { + continue; + } + if (!isAsyncDownloading) { + isSyncAssetAbnormal = true; + return; } + isAsyncAssetAbnormal = true; + abnormalAssetFields.push_back(item.first); continue; } - const Assets *assets = std::get_if>(&item.second); + auto *assets = std::get_if>(&item.second); if (assets == nullptr) { continue; } - for (const auto &oneAsset : *assets) { - if (oneAsset.status == static_cast(AssetStatus::ABNORMAL) || - (oneAsset.status & static_cast(AssetStatus::DOWNLOAD_WITH_NULL)) != 0) { - isDataAbnormal = true; - break; + for (auto it = assets->begin(); it != assets->end();) { + const auto &oneAsset = *it; + bool isOneAssetNotDownload = IsAssetNotDownload(oneAsset.status); + if (!isOneAssetNotDownload) { + ++it; + continue; } + if (!isAsyncDownloading) { + isSyncAssetAbnormal = true; + return; + } + isAsyncAssetAbnormal = true; + it = assets->erase(it); } } - if (isDataAbnormal) { + for (const auto &item : abnormalAssetFields) { + data.erase(item); + } +} + +int CloudStorageUtils::CheckAbnormalData(CloudSyncData &cloudSyncData, VBucket &data, bool isInsert, + bool isAsyncDownloading) +{ + if (data.empty()) { + LOGE("The cloud data is empty, isInsert:%d", static_cast(isInsert)); + return -E_INVALID_DATA; + } + bool isSyncAssetAbnormal = false; + bool isAsyncAssetAbnormal = false; + CheckAbnormalDataInner(isAsyncDownloading, data, isSyncAssetAbnormal, isAsyncAssetAbnormal); + if (isSyncAssetAbnormal) { std::string gid; (void)GetValueFromVBucket(CloudDbConstant::GID_FIELD, data, gid); - LOGW("This data is abnormal, ignore it when upload, isInsert:%d, gid:%s", static_cast(isInsert), - gid.c_str()); + LOGW("This data is abnormal, ignore it when upload, isInsert:%d, gid:%s", + static_cast(isInsert), gid.c_str()); cloudSyncData.ignoredCount++; return -E_IGNORE_DATA; } + if (isAsyncAssetAbnormal) { + std::string gid; + (void)GetValueFromVBucket(CloudDbConstant::GID_FIELD, data, gid); + LOGW("This data has assets that are not downloaded, upload data only, gid:%s", gid.c_str()); + } return E_OK; } @@ -1492,9 +1530,9 @@ std::string CloudStorageUtils::GetCursorUpgradeSql(const std::string &tableName) } int CloudStorageUtils::FillQueryByPK(const std::string &tableName, bool isKv, std::map dataIndex, - std::vector>> syncPkVec, std::vector &syncQuery) + std::vector>> &syncPkVec, std::vector &syncQuery) { - for (const auto syncPk : syncPkVec) { + for (const auto &syncPk : syncPkVec) { Query query = Query::Select().From(tableName); if (isKv) { QueryUtils::FillQueryInKeys(syncPk, dataIndex, query); @@ -1643,4 +1681,54 @@ int CloudStorageUtils::FillCloudQueryToExtend(QuerySyncObject &obj, VBucket &ext extend[CloudDbConstant::QUERY_FIELD] = bytes; return E_OK; } + +void CloudStorageUtils::SaveChangedDataByType(const DataValue &dataValue, Type &value) +{ + int ret = E_OK; + switch (dataValue.GetType()) { + case StorageType::STORAGE_TYPE_TEXT: + { + std::string sValue; + ret = dataValue.GetText(sValue); + if (ret != E_OK) { + LOGE("[CloudStorageUtils] save changed string data failed %d", ret); + return; + } + value = sValue; + } break; + case StorageType::STORAGE_TYPE_BLOB: + { + Blob blob; + (void)dataValue.GetBlob(blob); + if (blob.GetSize() == 0u) { + LOGE("[CloudStorageUtils] save changed Blob data failed"); + return; + } + value = std::vector(blob.GetData(), blob.GetData() + blob.GetSize()); + } break; + case StorageType::STORAGE_TYPE_INTEGER: + { + int64_t iValue; + ret = dataValue.GetInt64(iValue); + if (ret != E_OK) { + LOGE("[CloudStorageUtils] save changed int64 data failed %d", ret); + return; + } + value = iValue; + } break; + case StorageType::STORAGE_TYPE_REAL: + { + double dValue; + ret = dataValue.GetDouble(dValue); + if (ret != E_OK) { + LOGE("[CloudStorageUtils] save changed double data failed %d", ret); + return; + } + value = dValue; + } break; + default: + LOGE("[CloudStorageUtils] save changed failed, wrong storage type :%" PRIu32, dataValue.GetType()); + return; + } +} } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/cloud/schema_mgr.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/cloud/schema_mgr.cpp index 4b6865ce..4f5eca04 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/cloud/schema_mgr.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/cloud/schema_mgr.cpp @@ -84,7 +84,7 @@ int SchemaMgr::CompareFieldSchema(std::map &primaryKeys, FieldIn } cloudColNames.emplace(cloudField.colName); } - if (!primaryKeys.empty() && !(primaryKeys.size() == 1 && primaryKeys[0] == CloudDbConstant::ROW_ID_FIELD_NAME)) { + if (!primaryKeys.empty() && !(primaryKeys.size() == 1 && primaryKeys[0] == DBConstant::ROWID)) { LOGE("Local schema contain extra primary key:%d", -E_SCHEMA_MISMATCH); return -E_SCHEMA_MISMATCH; } @@ -165,7 +165,8 @@ void SchemaMgr::SetCloudDbSchema(const DataBaseSchema &schema, RelationalSchemaO // remove the fields that are not found in local schema from cloud schema for (auto it = table.fields.begin(); it != table.fields.end();) { if (localFields.find((*it).colName) == localFields.end()) { - LOGW("Column mismatch, colName: %s", (*it).colName.c_str()); + LOGW("Column mismatch, colName: %s, length: %zu", DBCommon::StringMiddleMasking((*it).colName).c_str(), + (*it).colName.length()); it = table.fields.erase(it); } else { ++it; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/data_transformer.h b/kv_store/frameworks/libs/distributeddb/storage/src/data_transformer.h index c77d4cc7..e6847b34 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/data_transformer.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/data_transformer.h @@ -59,6 +59,7 @@ enum class LogInfoFlag : uint32_t { FLAG_LOGIC_DELETE_FOR_LOGOUT = 0x800, FLAG_ASSET_DOWNLOADING_FOR_ASYNC = 0x1000, FLAG_LOGIN_USER = 0x2000, // same hash key, login user's data + FLAG_CLOUD_UPDATE_LOCAL = 0x4000, }; struct RowDataWithLog { diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/gaussdb_rd/rd_utils.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/gaussdb_rd/rd_utils.cpp index c1116e9b..0cfce6b0 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/gaussdb_rd/rd_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/gaussdb_rd/rd_utils.cpp @@ -193,9 +193,16 @@ int RdKVRangeScan(GRD_DB *db, const char *collectionName, const Key &beginKey, c LOGE("[rdUtils][RdKVScan] invalid db"); return -E_INVALID_DB; } - GRD_KVItemT beginInnerKey{(void *)&beginKey[0], (uint32_t)beginKey.size()}; - GRD_KVItemT endInnerKey{(void *)&endKey[0], (uint32_t)endKey.size()}; - GRD_FilterOption filterOpt{KV_SCAN_RANGE, beginInnerKey, endInnerKey}; + + GRD_FilterOptionT filterOpt = {}; + filterOpt.mode = KV_SCAN_RANGE; + if (!beginKey.empty()) { + filterOpt.begin = {(void *)&beginKey[0], (uint32_t)beginKey.size()}; + } + if (!endKey.empty()) { + filterOpt.end = {(void *)&endKey[0], (uint32_t)endKey.size()}; + } + return TransferGrdErrno(GRD_KVFilter(db, collectionName, &filterOpt, resultSet)); } @@ -235,7 +242,13 @@ int RdKVBatchPrepare(uint16_t itemNum, GRD_KVBatchT **batch) int RdKVBatchPushback(GRD_KVBatchT *batch, const Key &key, const Value &value) { GRD_KVItemT innerKey{(void *)&key[0], (uint32_t)key.size()}; - GRD_KVItemT innerVal{(void *)&value[0], (uint32_t)value.size()}; + GRD_KVItemT innerVal{nullptr, 0}; + + if (!value.empty()) { + innerVal.data = (void *)&value[0]; + innerVal.dataLen = (uint32_t)value.size(); + } + int ret = TransferGrdErrno( GRD_KVBatchPushback(innerKey.data, innerKey.dataLen, innerVal.data, innerVal.dataLen, batch)); if (ret != E_OK) { diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb.cpp index 03c826ea..13c4cf90 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb.cpp @@ -93,6 +93,9 @@ std::string GenericKvDB::GetStoreId() const void GenericKvDB::DelConnection(GenericKvDBConnection *connection) { + if (connection == nullptr) { + return; + } delete connection; connection = nullptr; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.cpp index b260e793..74fabd6e 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.cpp @@ -456,4 +456,9 @@ bool GenericKvDBConnection::IsRebuild() const { return isRebuild_; } + +int GenericKvDBConnection::OperateDataStatus([[gnu::unused]] uint32_t dataOperator) +{ + return -E_NOT_SUPPORT; +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.h b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.h index b9ebe914..e752a7dd 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/kv/generic_kvdb_connection.h @@ -112,6 +112,8 @@ public: void MarkRebuild() override; bool IsRebuild() const override; + + int OperateDataStatus(uint32_t dataOperator) override; protected: // Get the stashed 'KvDB_ pointer' without ref. template diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/kv/sync_able_kvdb_connection.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/kv/sync_able_kvdb_connection.cpp index 7468acda..94275ffe 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/kv/sync_able_kvdb_connection.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/kv/sync_able_kvdb_connection.cpp @@ -447,7 +447,7 @@ void SyncAbleKvDBConnection::SetGenCloudVersionCallback(const GenerateCloudVersi int SyncAbleKvDBConnection::SetReceiveDataInterceptor(const DataInterceptor &interceptor) { - auto kvDB = GetDB(); + auto *kvDB = GetDB(); if (kvDB == nullptr) { return -E_INVALID_CONNECTION; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.cpp index 89164443..39b70b6e 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.cpp @@ -150,12 +150,13 @@ int DatabaseOper::RekeyRecover(const KvDBProperties &property) return E_OK; } -int DatabaseOper::CheckSecurityOption(const std::string &filePath, const KvDBProperties &property) const +int DatabaseOper::CheckSecurityOption(const std::string &filePath, const KvDBProperties &property) { SecurityOption dbSecOpt; dbSecOpt.securityFlag = property.GetSecFlag(); dbSecOpt.securityLabel = property.GetSecLabel(); - RuntimeContext::GetInstance()->SetSecurityOption(filePath, dbSecOpt); + // set backup file security label first, avoid file loss label + (void)RuntimeContext::GetInstance()->SetSecurityOption(filePath, dbSecOpt); SecurityOption secOption; int errCode = RuntimeContext::GetInstance()->GetSecurityOption(filePath, secOption); if (errCode != E_OK && errCode != -E_NOT_SUPPORT) { diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.h b/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.h index c46ef085..eaaae546 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/operation/database_oper.h @@ -72,7 +72,6 @@ protected: // export begin int ExecuteExport(const std::string &filePath, const CipherPassword &passwd, const KvDBProperties &property) const; - private: int CreateStatusCtrlFile(const KvDBProperties &property, std::string &orgCtrlFile, std::string &newCtrlFile); @@ -98,10 +97,10 @@ private: int PackExportedDatabase(const std::string &fileDir, const std::string &packedFile, const KvDBProperties &property) const; - int CheckSecurityOption(const std::string &filePath, const KvDBProperties &property) const; - int CreateBackupDirForExport(const KvDBProperties &property, std::string ¤tDir, std::string &backupDir) const; + static int CheckSecurityOption(const std::string &filePath, const KvDBProperties &property); + std::string deviceId_; }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage.cpp index 6b24e765..b0bde0a8 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage.cpp @@ -547,16 +547,18 @@ int RelationalSyncAbleStorage::SaveSyncDataItems(const QueryObject &object, std: LOGE("Find remote schema failed. err=%d", errCode); return errCode; } + if (!IsSetDistributedSchema(query.GetTableName(), localSchema)) { + return -E_SCHEMA_MISMATCH; + } if (query.IsUseLocalSchema()) { // remote send always with its table col sort filterSchema.SetDistributedSchema(localSchema.GetDistributedSchema()); } StoreInfo info = GetStoreInfo(); - auto inserter = RelationalSyncDataInserter::CreateInserter(deviceName, query, storageEngine_->GetSchema(), - filterSchema.GetSyncFieldInfo(query.GetTableName()), info); - ChangedData data; - data.properties.isP2pSyncDataChange = !dataItems.empty(); + SchemaInfo schemaInfo = {storageEngine_->GetSchema(), storageEngine_->GetTrackerSchema()}; + auto inserter = RelationalSyncDataInserter::CreateInserter( + deviceName, query, schemaInfo, filterSchema.GetSyncFieldInfo(query.GetTableName()), info); inserter.SetEntries(dataItems); auto *handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); @@ -577,10 +579,15 @@ int RelationalSyncAbleStorage::SaveSyncDataItems(const QueryObject &object, std: } DBDfxAdapter::StartTracing(); + handle->SetTableMode(localSchema.GetTableMode()); errCode = handle->SaveSyncItems(inserter); + ChangedData data = inserter.GetChangedData(); + data.properties.isP2pSyncDataChange = !dataItems.empty(); DBDfxAdapter::FinishTracing(); - if (errCode == E_OK) { + bool isEmptyChangedData = data.field.empty() && data.primaryData[OP_INSERT].empty() && + data.primaryData[OP_UPDATE].empty() && data.primaryData[OP_DELETE].empty(); + if (errCode == E_OK && !isEmptyChangedData) { // dataItems size > 0 now because already check before // all dataItems will write into db now, so need to observer notify here // if some dataItems will not write into db in the future, observer notify here need change @@ -744,30 +751,24 @@ int RelationalSyncAbleStorage::GetCompressionAlgo(std::set &a int RelationalSyncAbleStorage::RegisterObserverAction(uint64_t connectionId, const StoreObserver *observer, const RelationalObserverAction &action) { - int errCode = E_OK; - TaskHandle handle = ConcurrentAdapter::ScheduleTaskH([this, connectionId, observer, action, &errCode] () mutable { - ConcurrentAdapter::AdapterAutoLock(dataChangeDeviceMutex_); - ResFinalizer finalizer([this]() { ConcurrentAdapter::AdapterAutoUnLock(dataChangeDeviceMutex_); }); - auto it = dataChangeCallbackMap_.find(connectionId); - if (it != dataChangeCallbackMap_.end()) { - if (it->second.find(observer) != it->second.end()) { - LOGE("obsever already registered"); - errCode = -E_ALREADY_SET; - return; - } - if (it->second.size() >= DBConstant::MAX_OBSERVER_COUNT) { - LOGE("The number of relational observers has been over limit"); - errCode = -E_MAX_LIMITS; - return; - } - it->second[observer] = action; - } else { - dataChangeCallbackMap_[connectionId][observer] = action; + ConcurrentAdapter::AdapterAutoLock(dataChangeDeviceMutex_); + ResFinalizer finalizer([this]() { ConcurrentAdapter::AdapterAutoUnLock(dataChangeDeviceMutex_); }); + auto it = dataChangeCallbackMap_.find(connectionId); + if (it != dataChangeCallbackMap_.end()) { + if (it->second.find(observer) != it->second.end()) { + LOGE("obsever already registered"); + return -E_ALREADY_SET; } - LOGI("register relational observer ok"); - }, nullptr, &dataChangeCallbackMap_); - ADAPTER_WAIT(handle); - return errCode; + if (it->second.size() >= DBConstant::MAX_OBSERVER_COUNT) { + LOGE("The number of relational observers has been over limit"); + return -E_MAX_LIMITS; + } + it->second[observer] = action; + } else { + dataChangeCallbackMap_[connectionId][observer] = action; + } + LOGI("register relational observer ok"); + return E_OK; } int RelationalSyncAbleStorage::UnRegisterObserverAction(uint64_t connectionId, const StoreObserver *observer) @@ -776,28 +777,22 @@ int RelationalSyncAbleStorage::UnRegisterObserverAction(uint64_t connectionId, c EraseDataChangeCallback(connectionId); return E_OK; } - int errCode = -E_NOT_FOUND; - TaskHandle handle = ConcurrentAdapter::ScheduleTaskH([this, connectionId, observer, &errCode] () mutable { - ConcurrentAdapter::AdapterAutoLock(dataChangeDeviceMutex_); - ResFinalizer finalizer([this]() { ConcurrentAdapter::AdapterAutoUnLock(dataChangeDeviceMutex_); }); - auto it = dataChangeCallbackMap_.find(connectionId); - if (it == dataChangeCallbackMap_.end()) { - return; - } + ConcurrentAdapter::AdapterAutoLock(dataChangeDeviceMutex_); + ResFinalizer finalizer([this]() { ConcurrentAdapter::AdapterAutoUnLock(dataChangeDeviceMutex_); }); + auto it = dataChangeCallbackMap_.find(connectionId); + if (it != dataChangeCallbackMap_.end()) { auto action = it->second.find(observer); - if (action == it->second.end()) { - return; - } - it->second.erase(action); - LOGI("unregister relational observer."); - if (it->second.empty()) { - dataChangeCallbackMap_.erase(it); - LOGI("observer for this delegate is zero now"); + if (action != it->second.end()) { + it->second.erase(action); + LOGI("unregister relational observer."); + if (it->second.empty()) { + dataChangeCallbackMap_.erase(it); + LOGI("observer for this delegate is zero now"); + } + return E_OK; } - errCode = E_OK; - }, nullptr, &dataChangeCallbackMap_); - ADAPTER_WAIT(handle); - return errCode; + } + return -E_NOT_FOUND; } void RelationalSyncAbleStorage::ExecuteDataChangeCallback( @@ -1356,6 +1351,15 @@ int RelationalSyncAbleStorage::CleanCloudData(ClearMode mode, const std::vector< return errCode; } +int RelationalSyncAbleStorage::ClearCloudLogVersion(const std::vector &tableNameList) +{ + if (transactionHandle_ == nullptr) { + LOGE("[RelationalSyncAbleStorage][ClearCloudLogVersion] the transaction has not been started"); + return -E_INVALID_DB; + } + return transactionHandle_->DoClearCloudLogVersion(tableNameList); +} + int RelationalSyncAbleStorage::GetCloudTableSchema(const TableName &tableName, TableSchema &tableSchema) { std::shared_lock readLock(schemaMgrMutex_); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage_extend.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage_extend.cpp index 2284aa79..4e98a047 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_able_storage_extend.cpp @@ -149,6 +149,7 @@ int RelationalSyncAbleStorage::UpdateAssetStatusForAssetOnly(const std::string & void RelationalSyncAbleStorage::PrintCursorChange(const std::string &tableName) { + std::lock_guard lock(cursorChangeMutex_); auto iter = cursorChangeMap_.find(tableName); if (iter == cursorChangeMap_.end()) { return; @@ -160,6 +161,7 @@ void RelationalSyncAbleStorage::PrintCursorChange(const std::string &tableName) void RelationalSyncAbleStorage::SaveCursorChange(const std::string &tableName, uint64_t currCursor) { + std::lock_guard lock(cursorChangeMutex_); auto iter = cursorChangeMap_.find(tableName); if (iter == cursorChangeMap_.end()) { std::pair initCursors = {currCursor, currCursor}; @@ -282,5 +284,51 @@ int RelationalSyncAbleStorage::GetLockStatusByGid(const std::string &tableName, ReleaseHandle(handle); return errCode; } + +bool RelationalSyncAbleStorage::IsExistTableContainAssets() +{ + std::shared_ptr cloudSchema = nullptr; + int errCode = GetCloudDbSchema(cloudSchema); + if (errCode != E_OK) { + LOGE("Cannot get cloud schema: %d when check contain assets table", errCode); + return false; + } + if (cloudSchema == nullptr) { + LOGE("Not set cloud schema when check contain assets table"); + return false; + } + auto schema = GetSchemaInfo(); + for (const auto &table : cloudSchema->tables) { + auto tableInfo = schema.GetTable(table.name); + if (tableInfo.GetTableName().empty()) { + continue; // ignore not distributed table + } + for (const auto &field : table.fields) { + if (field.type == TYPE_INDEX || field.type == TYPE_INDEX) { + return true; + } + } + } + return false; +} + +bool RelationalSyncAbleStorage::IsSetDistributedSchema(const std::string &tableName, RelationalSchemaObject &schemaObj) +{ + if (schemaObj.GetTableMode() == DistributedTableMode::COLLABORATION) { + const std::vector &tables = schemaObj.GetDistributedSchema().tables; + if (tables.empty()) { + LOGE("[RelationalSyncAbleStorage] Distributed schema not set in COLLABORATION mode"); + return false; + } + auto iter = std::find_if(tables.begin(), tables.end(), [tableName](const DistributedTable &table) { + return DBCommon::CaseInsensitiveCompare(table.tableName, tableName); + }); + if (iter == tables.end()) { + LOGE("[RelationalSyncAbleStorage] table name mismatch"); + return false; + } + } + return true; +} } -#endif \ No newline at end of file +#endif diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.cpp index 699701a3..3a72ce04 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.cpp @@ -46,17 +46,19 @@ int SaveSyncDataStmt::ResetStatements(bool isNeedFinalize) } RelationalSyncDataInserter RelationalSyncDataInserter::CreateInserter(const std::string &deviceName, - const QueryObject &query, const RelationalSchemaObject &localSchema, const std::vector &remoteFields, + const QueryObject &query, const SchemaInfo &schemaInfo, const std::vector &remoteFields, const StoreInfo &info) { RelationalSyncDataInserter inserter; inserter.SetHashDevId(DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceName))); inserter.SetRemoteFields(remoteFields); inserter.SetQuery(query); - TableInfo localTable = localSchema.GetTable(query.GetTableName()); + TableInfo localTable = schemaInfo.localSchema.GetTable(query.GetTableName()); + localTable.SetTrackerTable(schemaInfo.trackerSchema.GetTrackerTable(localTable.GetTableName())); + localTable.SetDistributedTable(schemaInfo.localSchema.GetDistributedTable(query.GetTableName())); inserter.SetLocalTable(localTable); - inserter.SetTableMode(localSchema.GetTableMode()); - if (localSchema.GetTableMode() == DistributedTableMode::COLLABORATION) { + inserter.SetTableMode(schemaInfo.localSchema.GetTableMode()); + if (schemaInfo.localSchema.GetTableMode() == DistributedTableMode::COLLABORATION) { inserter.SetInsertTableName(localTable.GetTableName()); } else { inserter.SetInsertTableName(DBCommon::GetDistributedTableName(deviceName, localTable.GetTableName(), info)); @@ -122,8 +124,11 @@ int RelationalSyncDataInserter::GetInsertStatement(sqlite3 *db, sqlite3_stmt *&s } colName.pop_back(); dataFormat.pop_back(); - - std::string sql = "INSERT OR REPLACE INTO '" + insertTableName_ + "'" + + std::string sql = "INSERT "; + if (mode_ == DistributedTableMode::SPLIT_BY_DEVICE) { + sql += "OR REPLACE "; + } + sql += "INTO '" + insertTableName_ + "'" + "(" + colName + ") VALUES(" + dataFormat + ");"; int errCode = SQLiteUtils::GetStatement(db, sql, stmt); if (errCode != E_OK) { @@ -132,8 +137,87 @@ int RelationalSyncDataInserter::GetInsertStatement(sqlite3 *db, sqlite3_stmt *&s return errCode; } +int RelationalSyncDataInserter::GetDbValueByRowId(sqlite3 *db, const std::vector &fieldList, + const int64_t rowid, std::vector &values) +{ + if (fieldList.empty()) { + LOGW("[RelationalSyncDataInserter][GetDbValueByRowId] fieldList is empty"); + return E_OK; + } + sqlite3_stmt *getValueStmt = nullptr; + std::string sql = "SELECT "; + for (const auto &col : fieldList) { + sql += "data.'" + col + "',"; + } + sql.pop_back(); + sql += " FROM '" + localTable_.GetTableName() + "' as data WHERE " + std::string(DBConstant::SQLITE_INNER_ROWID) + + " = ?;"; + int errCode = SQLiteUtils::GetStatement(db, sql, getValueStmt); + if (errCode != E_OK) { + LOGE("[RelationalSyncDataInserter][GetDbValueByRowId] failed to prepare statmement"); + return errCode; + } + + SQLiteUtils::BindInt64ToStatement(getValueStmt, 1, rowid); + errCode = SQLiteUtils::StepWithRetry(getValueStmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + VBucket bucket; + errCode = SQLiteRelationalUtils::GetSelectVBucket(getValueStmt, bucket); + if (errCode != E_OK) { + LOGE("[RelationalSyncDataInserter][GetDbValueByRowId] failed to convert sql result to values"); + int ret = E_OK; + SQLiteUtils::ResetStatement(getValueStmt, true, ret); + return errCode; + } + for (auto value : bucket) { + values.push_back(value.second); + } + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGW("[RelationalSyncDataInserter][GetDbValueByRowId] found no data in db"); + errCode = E_OK; + } + int ret = E_OK; + SQLiteUtils::ResetStatement(getValueStmt, true, ret); + return errCode; +} + +int RelationalSyncDataInserter::GetObserverDataByRowId(sqlite3 *db, int64_t rowid, ChangeType type) +{ + std::vector primaryValues; + std::vector primaryKeys; + + if (localTable_.IsMultiPkTable() || localTable_.IsNoPkTable()) { + primaryKeys.push_back(DBConstant::ROWID); + primaryValues.push_back(rowid); + } + std::vector pkList; + for (const auto &primaryKey : localTable_.GetPrimaryKey()) { + if (primaryKey.second != DBConstant::ROWID) { + pkList.push_back(primaryKey.second); + } + } + + data_.field = primaryKeys; + if (pkList.empty()) { + data_.primaryData[type].push_back(primaryValues); + return E_OK; + } + + std::vector dbValues; + int errCode = GetDbValueByRowId(db, pkList, rowid, dbValues); + if (errCode != E_OK) { + LOGE("[RelationalSyncDataInserter][GetObserverDataByRowId] failed to get db values"); + data_.primaryData[type].push_back(primaryValues); + return errCode; + } + data_.field.insert(data_.field.end(), pkList.begin(), pkList.end()); + primaryValues.insert(primaryValues.end(), dbValues.begin(), dbValues.end()); + data_.primaryData[type].push_back(primaryValues); + return errCode; +} + int RelationalSyncDataInserter::SaveData(bool isUpdate, const DataItem &dataItem, - SaveSyncDataStmt &saveSyncDataStmt) + SaveSyncDataStmt &saveSyncDataStmt, std::map &saveVals) { sqlite3_stmt *&stmt = isUpdate ? saveSyncDataStmt.updateDataStmt : saveSyncDataStmt.insertDataStmt; std::set filterSet; @@ -141,13 +225,17 @@ int RelationalSyncDataInserter::SaveData(bool isUpdate, const DataItem &dataItem for (const auto &primaryKey : localTable_.GetIdentifyKey()) { filterSet.insert(primaryKey); } + auto distributedPk = localTable_.GetSyncDistributedPk(); + if (!distributedPk.empty()) { + filterSet.insert(distributedPk.begin(), distributedPk.end()); + } } if (stmt == nullptr) { LOGW("skip save data %s", DBCommon::StringMiddleMasking(DBCommon::VectorToHexString(dataItem.hashKey)).c_str()); return E_OK; } - int errCode = BindSaveDataStatement(isUpdate, dataItem, filterSet, stmt); + int errCode = BindSaveDataStatement(isUpdate, dataItem, filterSet, stmt, saveVals); if (errCode != E_OK) { LOGE("Bind data failed, errCode=%d.", errCode); int ret = E_OK; @@ -157,12 +245,12 @@ int RelationalSyncDataInserter::SaveData(bool isUpdate, const DataItem &dataItem errCode = SQLiteUtils::StepWithRetry(stmt, false); int ret = E_OK; - SQLiteUtils::ResetStatement(stmt, false, ret); + SQLiteUtils::ResetStatement(stmt, false, true, ret); return errCode; } int RelationalSyncDataInserter::BindSaveDataStatement(bool isExist, const DataItem &dataItem, - const std::set &filterSet, sqlite3_stmt *stmt) + const std::set &filterSet, sqlite3_stmt *stmt, std::map &saveVals) { OptRowDataWithLog data; // deserialize by remote field info @@ -182,14 +270,17 @@ int RelationalSyncDataInserter::BindSaveDataStatement(bool isExist, const DataIt dataIdx++; continue; // skip fields which is orphaned in remote } - if (filterSet.find(it.GetFieldName()) != filterSet.end()) { - dataIdx++; - continue; // skip fields when update - } if (dataIdx >= data.optionalData.size()) { LOGD("field over size. cnt:%d, data size:%d", dataIdx, data.optionalData.size()); break; // cnt should less than optionalData size. } + Type saveVal; + CloudStorageUtils::SaveChangedDataByType(data.optionalData[dataIdx], saveVal); + saveVals[it.GetFieldName()] = saveVal; + if (filterSet.find(it.GetFieldName()) != filterSet.end()) { + dataIdx++; + continue; // skip fields when update + } errCode = SQLiteUtils::BindDataValueByType(stmt, data.optionalData[dataIdx], bindIdx++); if (errCode != E_OK) { LOGE("Bind data failed, errCode:%d, cid:%zu.", errCode, dataIdx); @@ -235,33 +326,39 @@ int RelationalSyncDataInserter::GetDeleteSyncDataStmt(sqlite3 *db, sqlite3_stmt int RelationalSyncDataInserter::GetSaveLogStatement(sqlite3 *db, sqlite3_stmt *&logStmt, sqlite3_stmt *&queryStmt) { - std::string conflictPk; - std::string selCondition; - if (mode_ == DistributedTableMode::COLLABORATION) { - conflictPk = "ON CONFLICT(hash_key)"; - selCondition = " WHERE hash_key = ?;"; - } else { - conflictPk = "ON CONFLICT(hash_key, device)"; - selCondition = " WHERE hash_key = ? AND device = ?;"; - } const std::string tableName = DBConstant::RELATIONAL_PREFIX + query_.GetTableName() + "_log"; std::string dataFormat = "?, '" + hashDevId_ + "', ?, ?, ?, ?, ?"; - std::string columnList = "data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key"; - std::string sql = "INSERT INTO " + tableName + + TrackerTable trackerTable = localTable_.GetTrackerTable(); + if (trackerTable.GetExtendNames().empty()) { + dataFormat += ", ?"; + } else { + dataFormat += ", (select json_object("; + for (const auto &extendColName : trackerTable.GetExtendNames()) { + dataFormat += "'" + extendColName + "'," + extendColName + ","; + } + dataFormat.pop_back(); + dataFormat += ") from "; + dataFormat += localTable_.GetTableName() + " where _rowid_ = ?)"; + } + std::string columnList = "data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key, extend_field"; + std::string sql = "INSERT OR REPLACE INTO " + tableName + " (" + columnList + ", cursor) VALUES (" + dataFormat + "," + - CloudStorageUtils::GetSelectIncCursorSql(query_.GetTableName()) +") " + conflictPk + - " DO UPDATE SET data_key = excluded.data_key, device = excluded.device," - " ori_device = excluded.ori_device, timestamp = excluded.timestamp, wtimestamp = excluded.wtimestamp," - " flag = excluded.flag, cursor = excluded.cursor;"; + CloudStorageUtils::GetSelectIncCursorSql(query_.GetTableName()) +");"; int errCode = SQLiteUtils::GetStatement(db, sql, logStmt); if (errCode != E_OK) { LOGE("[info statement] Get log statement fail! errCode:%d", errCode); return errCode; } - std::string selectSql = "SELECT " + columnList + " FROM " + tableName + selCondition; + std::string selectSql = "select " + columnList + " from " + tableName; + if (mode_ == DistributedTableMode::COLLABORATION) { + selectSql += " where hash_key = ?;"; + } else { + selectSql += " where hash_key = ? and device = ?;"; + } errCode = SQLiteUtils::GetStatement(db, selectSql, queryStmt); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(logStmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(logStmt, true, ret); LOGE("[info statement] Get query statement fail! errCode:%d", errCode); } return errCode; @@ -291,11 +388,8 @@ int RelationalSyncDataInserter::Iterate(const std::function &s int errCode = E_OK; for (auto &it : entries_) { it.dev = hashDevId_; - errCode = saveSyncDataItem(it); - if (errCode != E_OK) { - LOGE("Save sync data item failed. err=%d", errCode); - break; - } + int ret = saveSyncDataItem(it); + errCode = errCode == E_OK ? ret : errCode; } return errCode; } @@ -310,6 +404,10 @@ int RelationalSyncDataInserter::GetUpdateStatement(sqlite3 *db, sqlite3_stmt *&s for (const auto &primaryKey : localTable_.GetIdentifyKey()) { identifyKeySet.insert(primaryKey); } + auto distributedPk = localTable_.GetSyncDistributedPk(); + if (!distributedPk.empty()) { + identifyKeySet.insert(distributedPk.begin(), distributedPk.end()); + } std::string updateValue; const auto &localTableFields = localTable_.GetFields(); for (const auto &it : remoteFields_) { @@ -362,10 +460,20 @@ int RelationalSyncDataInserter::BindHashKeyAndDev(const DataItem &dataItem, sqli } int RelationalSyncDataInserter::SaveSyncLog(sqlite3 *db, sqlite3_stmt *statement, sqlite3_stmt *queryStmt, - const DataItem &dataItem, int64_t rowid) + const DataItem &dataItem, std::map &saveVals) { + if (std::get_if(&saveVals[DBConstant::ROWID]) == nullptr) { + LOGE("[RelationalSyncDataInserter] Invalid args because of no rowid!"); + return -E_INVALID_ARGS; + } + auto updateCursor = CloudStorageUtils::GetCursorIncSql(query_.GetTableName()); + int errCode = SQLiteUtils::ExecuteRawSQL(db, updateCursor); + if (errCode != E_OK) { + LOGE("[RelationalSyncDataInserter] update cursor failed %d", errCode); + return errCode; + } LogInfo logInfoGet; - int errCode = SQLiteRelationalUtils::GetLogInfoPre(queryStmt, mode_, dataItem, logInfoGet); + errCode = SQLiteRelationalUtils::GetLogInfoPre(queryStmt, mode_, dataItem, logInfoGet); LogInfo logInfoBind; logInfoBind.hashKey = dataItem.hashKey; logInfoBind.device = dataItem.dev; @@ -384,17 +492,36 @@ int RelationalSyncDataInserter::SaveSyncLog(sqlite3 *db, sqlite3_stmt *statement } // bind - SQLiteUtils::BindInt64ToStatement(statement, 1, rowid); // 1 means dataKey index + int bindIndex = 0; + // 1 means dataKey index + SQLiteUtils::BindInt64ToStatement(statement, ++bindIndex, std::get(saveVals[DBConstant::ROWID])); std::vector originDev(logInfoBind.originDev.begin(), logInfoBind.originDev.end()); - SQLiteUtils::BindBlobToStatement(statement, 2, originDev); // 2 means ori_dev index - SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); // 3 means timestamp index - SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimestamp); // 4 means w_timestamp index - SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); // 5 means flag index - SQLiteUtils::BindBlobToStatement(statement, 6, logInfoBind.hashKey); // 6 means hashKey index + SQLiteUtils::BindBlobToStatement(statement, ++bindIndex, originDev); // 2 means ori_dev index + SQLiteUtils::BindInt64ToStatement(statement, ++bindIndex, logInfoBind.timestamp); // 3 means timestamp index + SQLiteUtils::BindInt64ToStatement(statement, ++bindIndex, logInfoBind.wTimestamp); // 4 means w_timestamp index + SQLiteUtils::BindInt64ToStatement(statement, ++bindIndex, logInfoBind.flag); // 5 means flag index + SQLiteUtils::BindBlobToStatement(statement, ++bindIndex, logInfoBind.hashKey); // 6 means hashKey index + BindExtendFieldOrRowid(statement, saveVals, bindIndex); // bind extend_field or rowid errCode = SQLiteUtils::StepWithRetry(statement, false); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { return E_OK; } return errCode; } + +ChangedData &RelationalSyncDataInserter::GetChangedData() +{ + return data_; +} + +void RelationalSyncDataInserter::BindExtendFieldOrRowid(sqlite3_stmt *&stmt, std::map &saveVals, + int bindIndex) +{ + TrackerTable trackerTable = localTable_.GetTrackerTable(); + if (trackerTable.GetExtendNames().empty()) { + SQLiteUtils::BindTextToStatement(stmt, ++bindIndex, ""); + } else { + SQLiteUtils::BindInt64ToStatement(stmt, ++bindIndex, std::get(saveVals[DBConstant::ROWID])); + } +} } // namespace DistributedDB \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.h b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.h index 48173d93..d233a47a 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/relational/relational_sync_data_inserter.h @@ -39,14 +39,18 @@ struct SaveSyncDataStmt { int ResetStatements(bool isNeedFinalize); }; +struct SchemaInfo { + const RelationalSchemaObject &localSchema; + const RelationalSchemaObject &trackerSchema; +}; + class RelationalSyncDataInserter { public: RelationalSyncDataInserter() = default; ~RelationalSyncDataInserter() = default; static RelationalSyncDataInserter CreateInserter(const std::string &deviceName, const QueryObject &query, - const RelationalSchemaObject &localSchema, const std::vector &remoteFields, - const StoreInfo &info); + const SchemaInfo &schemaInfo, const std::vector &remoteFields, const StoreInfo &info); void SetHashDevId(const std::string &hashDevId); // Set remote fields in cid order @@ -63,9 +67,12 @@ public: int Iterate(const std::function &); - int SaveData(bool isUpdate, const DataItem &dataItem, SaveSyncDataStmt &saveSyncDataStmt); + int GetObserverDataByRowId(sqlite3 *db, int64_t rowid, ChangeType type); + + int SaveData(bool isUpdate, const DataItem &dataItem, SaveSyncDataStmt &saveSyncDataStmt, + std::map &saveVals); int BindSaveDataStatement(bool isExist, const DataItem &dataItem, const std::set &filterSet, - sqlite3_stmt *stmt); + sqlite3_stmt *stmt, std::map &saveVals); int PrepareStatement(sqlite3 *db, SaveSyncDataStmt &stmt); int GetDeleteLogStmt(sqlite3 *db, sqlite3_stmt *&stmt); @@ -74,7 +81,9 @@ public: int BindHashKeyAndDev(const DataItem &dataItem, sqlite3_stmt *stmt, int beginIndex); int SaveSyncLog(sqlite3 *db, sqlite3_stmt *statement, sqlite3_stmt *queryStmt, const DataItem &dataItem, - int64_t rowid); + std::map &saveVals); + + ChangedData &GetChangedData(); private: int GetInsertStatement(sqlite3 *db, sqlite3_stmt *&stmt); @@ -83,6 +92,11 @@ private: int GetUpdateStatement(sqlite3 *db, sqlite3_stmt *&stmt); + void GetPrimaryKeys(std::vector &primaryKeys); + int GetDbValueByRowId(sqlite3 *db, const std::vector &fieldList, + const int64_t rowid, std::vector &values); + void BindExtendFieldOrRowid(sqlite3_stmt *&stmt, std::map &saveVals, int bindIndex); + std::string hashDevId_; std::vector remoteFields_; std::vector entries_; @@ -90,6 +104,7 @@ private: QueryObject query_; std::string insertTableName_; // table name to save sync data DistributedTableMode mode_ = DistributedTableMode::SPLIT_BY_DEVICE; + ChangedData data_; }; } #endif // RELATIONAL_SYNC_DATA_INSERTER_H \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/single_ver_natural_store_connection.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/single_ver_natural_store_connection.cpp index 9d50169d..92bf09c3 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/single_ver_natural_store_connection.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/single_ver_natural_store_connection.cpp @@ -83,7 +83,7 @@ bool SingleVerNaturalStoreConnection::CheckAndGetKeyLen(const std::vector & return true; } -int SingleVerNaturalStoreConnection::CheckSyncEntriesValid([[gnu::unused]] const std::vector &entries) const +int SingleVerNaturalStoreConnection::CheckSyncEntriesValid(const std::vector &entries) const { return E_OK; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.cpp index b482eb60..deeb6e1f 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -91,7 +91,8 @@ QueryObject::QueryObject(const QueryExpression &queryExpression) isAssetsOnly_ = queryExpressions.IsAssetsOnly(); groupNum_ = queryExpressions.GetGroupNum(); assetsGroupMap_ = queryExpressions.GetAssetsOnlyGroupMap(); - isValidForAssetsOnly_ = queryExpressions.GetExpressionStatusForAssetsOnly() == E_OK; + assetsOnlyErrFlag_ = queryExpressions.GetExpressionStatusForAssetsOnly(); + isUseFromTables_ = queryExpressions.IsUseFromTables(); } QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey, @@ -579,9 +580,9 @@ AssetsGroupMap QueryObject::GetAssetsOnlyGroupMap() const return assetsGroupMap_; } -bool QueryObject::IsValidForAssetsOnly() const +int QueryObject::AssetsOnlyErrFlag() const { - return isValidForAssetsOnly_; + return assetsOnlyErrFlag_; } void QueryObject::SetUseLocalSchema(bool isUse) @@ -604,5 +605,9 @@ void QueryObject::SetRemoteDev(const std::string &dev) remoteDev_ = dev; } +bool QueryObject::IsUseFromTables() const +{ + return isUseFromTables_; +} } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.h index 33b0bfd2..2c8b2c37 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_object.h @@ -72,7 +72,7 @@ public: AssetsGroupMap GetAssetsOnlyGroupMap() const; - bool IsValidForAssetsOnly() const; + int AssetsOnlyErrFlag() const; #ifdef RELATIONAL_STORE int SetSchema(const RelationalSchemaObject &schemaObj); // The interface can only be used in relational query. @@ -86,6 +86,8 @@ public: void SetRemoteDev(const std::string &dev); std::string GetRemoteDev() const; + + bool IsUseFromTables() const; protected: explicit QueryObject(const QueryExpression &queryExpression); static std::vector GetQueryExpressions(const Query &query); @@ -104,7 +106,8 @@ protected: uint32_t groupNum_ = 0; bool isAssetsOnly_ = false; AssetsGroupMap assetsGroupMap_; - bool isValidForAssetsOnly_ = false; + int assetsOnlyErrFlag_ = E_OK; + bool isUseFromTables_ = false; private: int Parse(); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp index 314e9e07..62a9aaac 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp @@ -216,6 +216,7 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) (void)parcel.WriteUInt32(context.queryObjNodes.size()); parcel.EightByteAlign(); if (parcel.IsError()) { + LOGE("[QuerySyncObject] [SerializeData] parcel is error."); return -E_INVALID_ARGS; } for (const QueryObjNode &node : context.queryObjNodes) { @@ -238,6 +239,7 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) } // QUERY_SYNC_OBJECT_VERSION_1 end. parcel.EightByteAlign(); if (parcel.IsError()) { // parcel almost success + LOGE("[QuerySyncObject] [SerializeData] After version 1, parcel is error."); return -E_INVALID_ARGS; } return E_OK; @@ -245,6 +247,7 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) void QuerySyncObject::SetCloudGid(const std::vector &cloudGid) { + int emptyGidCount = 0; for (size_t i = 0; i < cloudGid.size(); i+= MAX_VALUE_SIZE) { size_t end = std::min(i + MAX_VALUE_SIZE, cloudGid.size()); if (!queryObjNodes_.empty()) { @@ -261,6 +264,7 @@ void QuerySyncObject::SetCloudGid(const std::vector &cloudGid) std::vector subCloudGid(cloudGid.begin() + i, cloudGid.begin() + end); for (const auto &gid : subCloudGid) { if (gid.empty()) { + emptyGidCount++; continue; } FieldValue fieldValue; @@ -269,6 +273,7 @@ void QuerySyncObject::SetCloudGid(const std::vector &cloudGid) } queryObjNodes_.emplace_back(objNode); } + LOGI("[QuerySyncObject] [SetCloudGid] gid empty count is %d", emptyGidCount); } namespace { @@ -283,6 +288,7 @@ int DeSerializeVersion1Data(uint32_t version, Parcel &parcel, std::string &table uint32_t keysSize = 0; (void)parcel.ReadUInt32(keysSize); if (keysSize > DBConstant::MAX_INKEYS_SIZE) { + LOGE("[DeSerializeVersion1Data] keys size %" PRIu32 " is too large", keysSize); return -E_PARSE_FAIL; } for (uint32_t i = 0; i < keysSize; ++i) { @@ -318,6 +324,7 @@ int QuerySyncObject::DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj) parcel.EightByteAlign(); // Due to historical reasons, the limit of query node size was incorrectly set to MAX_QUERY_NODE_SIZE + 1 if (parcel.IsError() || nodesSize > MAX_QUERY_NODE_SIZE + 1) { // almost success + LOGE("[QuerySyncObject] [DeSerializeData] parcel is error or nodes size is too large."); return -E_INVALID_ARGS; } for (size_t i = 0; i < nodesSize; i++) { @@ -338,6 +345,7 @@ int QuerySyncObject::DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj) } // QUERY_SYNC_OBJECT_VERSION_1 end. if (parcel.IsError()) { // almost success + LOGE("[QuerySyncObject] [DeSerializeData] After version 1, parcel is error."); return -E_INVALID_ARGS; } queryObj = QuerySyncObject(context.queryObjNodes, context.prefixKey, keys); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/cloud_sync_log_table_manager.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/cloud_sync_log_table_manager.cpp index 671c0daf..e9374270 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/cloud_sync_log_table_manager.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/cloud_sync_log_table_manager.cpp @@ -81,7 +81,7 @@ void CloudSyncLogTableManager::GetIndexSql(const TableInfo &table, std::vector 0) { sqliteStorageEngine_->SetConnectionFlag(true); @@ -443,8 +448,16 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName, } } + auto mode = sqliteStorageEngine_->GetProperties().GetDistributedTableMode(); + std::string localIdentity; // collaboration mode need local identify + if (mode == DistributedTableMode::COLLABORATION) { + int errCode = syncAbleEngine_->GetLocalIdentity(localIdentity); + if (errCode != E_OK || localIdentity.empty()) { + LOGW("Get local identity failed: %d", errCode); + } + } bool schemaChanged = false; - int errCode = sqliteStorageEngine_->CreateDistributedTable(tableName, DBCommon::TransferStringToHex(""), + int errCode = sqliteStorageEngine_->CreateDistributedTable(tableName, DBCommon::TransferStringToHex(localIdentity), schemaChanged, syncType, trackerSchemaChanged); if (errCode != E_OK) { LOGE("Create distributed table failed. %d", errCode); @@ -495,10 +508,54 @@ int SQLiteRelationalStore::CleanCloudData(ClearMode mode) return errCode; } + +int SQLiteRelationalStore::ClearCloudWatermark(const std::set &tableNames) +{ + RelationalSchemaObject localSchema = sqliteStorageEngine_->GetSchema(); + TableInfoMap tables = localSchema.GetTables(); + std::vector cloudTableNameList; + bool isClearAllTables = tableNames.empty(); + for (const auto &tableInfo : tables) { + if (tableInfo.second.GetSharedTableMark()) { + continue; + } + std::string tableName = tableInfo.first; + std::string maskedName = DBCommon::StringMiddleMasking(tableName); + bool isTargetTable = tableNames.count(tableName) > 0; + if (tableInfo.second.GetTableSyncType() == CLOUD_COOPERATION && (isClearAllTables || isTargetTable)) { + cloudTableNameList.push_back(tableName); + LOGI("[RelationalStore] cloud watermark of table %s will be cleared, original name length: %zu", + maskedName.c_str(), tableName.length()); + } else if (!isClearAllTables && isTargetTable) { + LOGW("[RelationalStore] table %s ignored when clear cloud watermark, original name length: %zu", + maskedName.c_str(), tableName.length()); + } + } + if (cloudTableNameList.empty()) { + LOGI("[RelationalStore] no target tables found for clear cloud watermark"); + return E_OK; + } + if (cloudSyncer_ == nullptr) { + LOGE("[RelationalStore] cloudSyncer was not initialized when clear cloud watermark"); + return -E_INVALID_DB; + } + int errCode = cloudSyncer_->ClearCloudWatermark(cloudTableNameList); + if (errCode != E_OK) { + LOGE("[RelationalStore] failed to clear cloud watermark, %d.", errCode); + } + return errCode; +} #endif int SQLiteRelationalStore::RemoveDeviceData() { + auto mode = static_cast(sqliteStorageEngine_->GetProperties().GetIntProp( + RelationalDBProperties::DISTRIBUTED_TABLE_MODE, static_cast(DistributedTableMode::SPLIT_BY_DEVICE))); + if (mode == DistributedTableMode::COLLABORATION) { + LOGE("Not support remove all device data in collaboration mode."); + return -E_NOT_SUPPORT; + } + std::vector tableNameList = GetAllDistributedTableName(); if (tableNameList.empty()) { return E_OK; @@ -543,6 +600,13 @@ int SQLiteRelationalStore::RemoveDeviceData() int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std::string &tableName) { + auto mode = static_cast(sqliteStorageEngine_->GetProperties().GetIntProp( + RelationalDBProperties::DISTRIBUTED_TABLE_MODE, static_cast(DistributedTableMode::SPLIT_BY_DEVICE))); + if (mode == DistributedTableMode::COLLABORATION) { + LOGE("Not support remove device data in collaboration mode."); + return -E_NOT_SUPPORT; + } + TableInfoMap tables = sqliteStorageEngine_->GetSchema().GetTables(); // TableInfoMap auto iter = tables.find(tableName); if (tables.empty() || (!tableName.empty() && iter == tables.end())) { @@ -1044,24 +1108,29 @@ int SQLiteRelationalStore::SetIAssetLoader(const std::shared_ptr & int SQLiteRelationalStore::ChkSchema(const TableName &tableName) { - // check schema is ok - int errCode = E_OK; + // check schema first then compare columns to avoid change the origin return error code + if (storageEngine_ == nullptr) { + LOGE("[RelationalStore][ChkSchema] storageEngine was not initialized"); + return -E_INVALID_DB; + } + int errCode = storageEngine_->ChkSchema(tableName); + if (errCode != E_OK) { + LOGE("[SQLiteRelationalStore][ChkSchema] ChkSchema failed %d.", errCode); + return errCode; + } auto *handle = GetHandle(false, errCode); if (handle == nullptr) { LOGE("[SQLiteRelationalStore][ChkSchema] handle is nullptr"); return errCode; } + RelationalSchemaObject localSchema = storageEngine_->GetSchemaInfo(); + handle->SetLocalSchema(localSchema); errCode = handle->CompareSchemaTableColumns(tableName); - ReleaseHandle(handle); if (errCode != E_OK) { LOGE("[SQLiteRelationalStore][ChkSchema] local schema info incompatible %d.", errCode); - return errCode; - } - if (storageEngine_ == nullptr) { - LOGE("[RelationalStore][ChkSchema] storageEngine was not initialized"); - return -E_INVALID_DB; } - return storageEngine_->ChkSchema(tableName); + ReleaseHandle(handle); + return errCode; } #ifdef USE_DISTRIBUTEDDB_CLOUD @@ -1129,6 +1198,27 @@ int SQLiteRelationalStore::CheckBeforeSync(const CloudSyncOption &option) return E_OK; } +int SQLiteRelationalStore::CheckAssetsOnlyValid(const QuerySyncObject &querySyncObject, const CloudSyncOption &option) +{ + if (!querySyncObject.IsAssetsOnly()) { + return E_OK; + } + if (option.mode != SyncMode::SYNC_MODE_CLOUD_FORCE_PULL) { + LOGE("[RelationalStore] not support mode %d when sync with assets only", option.mode); + return -E_NOT_SUPPORT; + } + if (option.priorityLevel != CloudDbConstant::PRIORITY_TASK_MAX_LEVEL) { + LOGE("[RelationalStore] priorityLevel must be 2 when sync with assets only, now is %d", + option.priorityLevel); + return -E_INVALID_ARGS; + } + if (querySyncObject.AssetsOnlyErrFlag() == -E_INVALID_ARGS) { + LOGE("[RelationalStore] the query statement of assets only is incorrect."); + return -E_INVALID_ARGS; + } + return E_OK; +} + int SQLiteRelationalStore::CheckQueryValid(const CloudSyncOption &option) { if (option.compensatedSyncOnly) { @@ -1161,16 +1251,9 @@ int SQLiteRelationalStore::CheckQueryValid(const CloudSyncOption &option) DBCommon::StringMiddleMasking(tableName).c_str(), tableName.length()); return -E_NOT_SUPPORT; } - if (item.IsAssetsOnly()) { - if (option.mode != SyncMode::SYNC_MODE_CLOUD_FORCE_PULL) { - LOGE("[RelationalStore] not support mode %d when sync with assets only", option.mode); - return -E_NOT_SUPPORT; - } - if (option.priorityLevel != CloudDbConstant::PRIORITY_TASK_MAX_LEVEL) { - LOGE("[RelationalStore] priorityLevel must be 2 when sync with assets only, now is %d", - option.priorityLevel); - return -E_INVALID_ARGS; - } + errCode = CheckAssetsOnlyValid(item, option); + if (errCode != E_OK) { + return errCode; } } errCode = CheckTableName(syncTableNames); @@ -1662,7 +1745,15 @@ int SQLiteRelationalStore::SetDistributedSchema(const DistributedSchema &schema) LOGE("[RelationalStore] engine was not initialized"); return -E_INVALID_DB; } - auto [errCode, isSchemaChange] = sqliteStorageEngine_->SetDistributedSchema(schema); + auto mode = sqliteStorageEngine_->GetProperties().GetDistributedTableMode(); + std::string localIdentity; // collaboration mode need local identify + if (mode == DistributedTableMode::COLLABORATION) { + int errCode = syncAbleEngine_->GetLocalIdentity(localIdentity); + if (errCode != E_OK || localIdentity.empty()) { + LOGW("Get local identity failed: %d", errCode); + } + } + auto [errCode, isSchemaChange] = sqliteStorageEngine_->SetDistributedSchema(schema, localIdentity); if (errCode != E_OK) { return errCode; } @@ -1703,5 +1794,86 @@ int SQLiteRelationalStore::GetDownloadingAssetsCount(int32_t &count) ReleaseHandle(handle); return errCode; } + +int SQLiteRelationalStore::SetTableMode(DistributedTableMode tableMode) +{ + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore][SetTableMode] sqliteStorageEngine was not initialized"); + return -E_INVALID_DB; + } + if (sqliteStorageEngine_->GetProperties().GetDistributedTableMode() != tableMode) { + auto schema = sqliteStorageEngine_->GetSchema(); + for (const auto &tableMap : schema.GetTables()) { + if (tableMap.second.GetTableSyncType() == TableSyncType::DEVICE_COOPERATION) { + LOGW("[RelationalStore][SetTableMode] Can not set table mode for table %s[%zu]", + DBCommon::StringMiddleMasking(tableMap.first).c_str(), tableMap.first.size()); + return -E_NOT_SUPPORT; + } + } + } + RelationalDBProperties properties = sqliteStorageEngine_->GetProperties(); + properties.SetIntProp(RelationalDBProperties::DISTRIBUTED_TABLE_MODE, static_cast(tableMode)); + sqliteStorageEngine_->SetProperties(properties); + LOGI("[RelationalStore][SetTableMode] Set table mode to %d successful", static_cast(tableMode)); + return E_OK; +} + +int SQLiteRelationalStore::OperateDataStatus(uint32_t dataOperator) +{ + LOGI("[RelationalStore] OperateDataStatus %" PRIu32, dataOperator); + if ((dataOperator & static_cast(DataOperator::UPDATE_TIME)) == 0) { + return E_OK; + } + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore] sqliteStorageEngine was not initialized when operate data status"); + return -E_INVALID_DB; + } + auto schema = sqliteStorageEngine_->GetSchema(); + std::vector operateTables; + for (const auto &tableMap : schema.GetTables()) { + if (tableMap.second.GetTableSyncType() == TableSyncType::DEVICE_COOPERATION) { + operateTables.push_back(tableMap.second.GetTableName()); + } + } + return OperateDataStatusInner(operateTables); +} + +int SQLiteRelationalStore::OperateDataStatusInner(const std::vector &tables) const +{ + int errCode = E_OK; + SQLiteSingleVerRelationalStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + LOGE("[RelationalStore][OperateDataStatus] Get handle failed %d when operate data status", errCode); + return errCode; + } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + LOGE("[RelationalStore][OperateDataStatus] Start transaction failed %d when operate data status", errCode); + ReleaseHandle(handle); + return errCode; + } + sqlite3 *db = nullptr; + errCode = handle->GetDbHandle(db); + if (errCode != E_OK) { + LOGE("[RelationalStore][OperateDataStatus] Get db failed %d when operate data status", errCode); + (void)handle->Rollback(); + ReleaseHandle(handle); + return errCode; + } + errCode = SQLiteRelationalUtils::OperateDataStatus(db, tables); + if (errCode == E_OK) { + errCode = handle->Commit(); + if (errCode != E_OK) { + LOGE("[RelationalStore][OperateDataStatus] Commit failed %d when operate data status", errCode); + } + } else { + int ret = handle->Rollback(); + if (ret != E_OK) { + LOGE("[RelationalStore][OperateDataStatus] Rollback failed %d when operate data status", ret); + } + } + ReleaseHandle(handle); + return errCode; +} } // namespace DistributedDB #endif diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 1af8b423..95fd3a85 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -99,6 +99,8 @@ public: int GetDownloadingAssetsCount(int32_t &count); + int SetTableMode(DistributedTableMode tableMode); + #ifdef USE_DISTRIBUTEDDB_CLOUD int PrepareAndSetCloudDbSchema(const DataBaseSchema &schema); @@ -106,6 +108,8 @@ public: int CleanCloudData(ClearMode mode); + int ClearCloudWatermark(const std::set &tableNames); + int SetCloudDB(const std::shared_ptr &cloudDb); int Sync(const CloudSyncOption &option, const SyncProcessCallback &onProcess, uint64_t taskId); @@ -114,6 +118,8 @@ public: SyncProcess GetCloudTaskStatus(uint64_t taskId); #endif + + int OperateDataStatus(uint32_t dataOperator); private: void ReleaseResources(); @@ -152,6 +158,8 @@ private: int CheckBeforeSync(const CloudSyncOption &option); + int CheckAssetsOnlyValid(const QuerySyncObject &querySyncObject, const CloudSyncOption &option); + int CheckQueryValid(const CloudSyncOption &option); int CheckObjectValid(bool priorityTask, const std::vector &object, bool isFromTable); @@ -190,6 +198,8 @@ private: int CheckCloudSchema(const DataBaseSchema &schema); #endif + + int OperateDataStatusInner(const std::vector &tables) const; // use for sync Interactive std::shared_ptr syncAbleEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index a9ddc5dd..774fe957 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -17,6 +17,7 @@ #include "db_errno.h" #include "log_print.h" #include "sqlite_relational_utils.h" +#include "res_finalizer.h" namespace DistributedDB { SQLiteRelationalStoreConnection::SQLiteRelationalStoreConnection(SQLiteRelationalStore *store) @@ -28,8 +29,8 @@ SQLiteRelationalStoreConnection::SQLiteRelationalStoreConnection(SQLiteRelationa return; } UnlockObj(); + ResFinalizer finalizer([this]() { LockObj(); }); store->StopSync(GetConnectionId()); - LockObj(); }); } // Close and release the connection. @@ -201,6 +202,21 @@ int SQLiteRelationalStoreConnection::DoClean(ClearMode mode) } return errCode; } + +int SQLiteRelationalStoreConnection::ClearCloudWatermark(const std::set &tableNames) +{ + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed when clear watermark!"); + return -E_INVALID_CONNECTION; + } + + int errCode = store->ClearCloudWatermark(tableNames); + if (errCode != E_OK) { + LOGE("[RelationalConnection] failed to clear cloud watermark, %d.", errCode); + } + return errCode; +} #endif int SQLiteRelationalStoreConnection::RemoveDeviceData(const std::string &device) @@ -238,15 +254,6 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) return -E_INVALID_CONNECTION; } - { - AutoLock lockGuard(this); - if (IsKilled()) { - // If this happens, users are using a closed connection. - LOGE("Sync on a closed connection."); - return -E_STALE; - } - IncObjRef(this); - } ISyncer::SyncParma syncParam; syncParam.devices = info.devices; syncParam.mode = info.mode; @@ -254,20 +261,15 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.isQuerySync = true; syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); - syncParam.onFinalize = [this]() { - DecObjRef(this); - }; if (syncParam.syncQuery.GetSortType() != SortType::NONE) { - LOGE("not support order by timestamp"); - DecObjRef(this); + LOGE("not support order by timestamp, type: %d", static_cast(syncParam.syncQuery.GetSortType())); return -E_NOT_SUPPORT; } - int errCode = store->Sync(syncParam, GetConnectionId()); - if (errCode != E_OK) { - DecObjRef(this); - return errCode; + if (syncParam.syncQuery.GetValidStatus() != E_OK) { + LOGE("invalid sync query origin valid status %d", syncParam.syncQuery.GetValidStatus()); + return syncParam.syncQuery.GetValidStatus(); } - return E_OK; + return store->Sync(syncParam, GetConnectionId()); } int SQLiteRelationalStoreConnection::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) @@ -503,5 +505,25 @@ int SQLiteRelationalStoreConnection::GetDownloadingAssetsCount(int32_t &count) } return store->GetDownloadingAssetsCount(count); } + +int SQLiteRelationalStoreConnection::SetTableMode(DistributedTableMode tableMode) +{ + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null when set distributed table mode."); + return -E_INVALID_CONNECTION; + } + return store->SetTableMode(tableMode); +} + +int SQLiteRelationalStoreConnection::OperateDataStatus(uint32_t dataOperator) +{ + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null when operate data status."); + return -E_INVALID_CONNECTION; + } + return store->OperateDataStatus(dataOperator); +} } #endif \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 5d4bd5ca..70dcf9d4 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -61,9 +61,13 @@ public: int SetDistributedDbSchema(const DistributedSchema &schema) override; int GetDownloadingAssetsCount(int32_t &count) override; + + int SetTableMode(DistributedTableMode tableMode) override; #ifdef USE_DISTRIBUTEDDB_CLOUD int DoClean(ClearMode mode) override; + int ClearCloudWatermark(const std::set &tableNames) override; + int32_t GetCloudSyncTaskCount() override; int SetCloudDB(const std::shared_ptr &cloudDb) override; @@ -78,6 +82,8 @@ public: SyncProcess GetCloudTaskStatus(uint64_t taskId) override; #endif + + int OperateDataStatus(uint32_t dataOperator) override; protected: int Pragma(int cmd, void *parameter) override; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp index e8a18f5f..74df0413 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.cpp @@ -14,14 +14,15 @@ */ #include "sqlite_relational_utils.h" -#include "db_common.h" -#include "db_errno.h" +#include "cloud/cloud_db_constant.h" #include "cloud/cloud_db_types.h" -#include "sqlite_utils.h" #include "cloud/cloud_storage_utils.h" +#include "db_common.h" +#include "db_errno.h" #include "res_finalizer.h" #include "runtime_context.h" -#include "cloud/cloud_db_constant.h" +#include "sqlite_utils.h" +#include "time_helper.h" namespace DistributedDB { int SQLiteRelationalUtils::GetDataValueByType(sqlite3_stmt *statement, int cid, DataValue &value) @@ -425,8 +426,9 @@ int SQLiteRelationalUtils::QueryCount(sqlite3 *db, const std::string &tableName, } else { LOGE("Failed to get the count. %d", errCode); } - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteRelationalUtils::GetCursor(sqlite3 *db, const std::string &tableName, uint64_t &cursor) @@ -467,12 +469,13 @@ int SQLiteRelationalUtils::GetCursor(sqlite3 *db, const std::string &tableName, } void GetFieldsNeedContain(const TableInfo &tableInfo, const std::vector &syncFields, - std::set &fieldsNeedContain, std::set &requiredNotNullFields) + std::set &fieldsNeedContain, std::set &fieldsNotDecrease, + std::set &requiredNotNullFields) { // should not decrease distributed field for (const auto &syncField : syncFields) { if (!tableInfo.IsPrimaryKey(syncField.GetFieldName())) { - fieldsNeedContain.insert(syncField.GetFieldName()); + fieldsNotDecrease.insert(syncField.GetFieldName()); } } const std::vector &uniqueDefines = tableInfo.GetUniqueDefine(); @@ -511,6 +514,20 @@ bool CheckRequireFieldsInMap(const std::set &fieldsNeedContain, return true; } +bool IsMarkUniqueColumnInvalid(const TableInfo &tableInfo, const std::vector &originFields) +{ + int count = 0; + for (const auto &field : originFields) { + if (field.isSpecified && tableInfo.IsUniqueField(field.colName)) { + count++; + if (count > 1) { + return true; + } + } + } + return false; +} + bool IsDistributedPkInvalid(const TableInfo &tableInfo, const std::set &distributedPk, const std::vector &originFields) @@ -524,22 +541,24 @@ bool IsDistributedPkInvalid(const TableInfo &tableInfo, if (tableInfo.IsNoPkTable()) { return false; } + if (!distributedPk.empty() && distributedPk.size() != tableInfo.GetPrimaryKey().size()) { + return true; + } if (tableInfo.GetAutoIncrement()) { if (distributedPk.empty()) { return false; } - auto uniqueDefine = tableInfo.GetUniqueDefine(); - bool isMissMatch = true; - for (const auto &item : uniqueDefine) { - // unique index field count should be same - if (item.size() != distributedPk.size()) { - continue; - } - if (distributedPk == DBCommon::TransformToCaseInsensitive(item)) { - isMissMatch = false; - break; - } + auto uniqueAndPkDefine = tableInfo.GetUniqueAndPkDefine(); + if (IsMarkUniqueColumnInvalid(tableInfo, originFields)) { + LOGE("Mark more than one unique column specified in auto increment table: %s, tableName len: %zu", + DBCommon::StringMiddleMasking(tableInfo.GetTableName()).c_str(), tableInfo.GetTableName().length()); + return true; } + auto find = std::any_of(uniqueAndPkDefine.begin(), uniqueAndPkDefine.end(), [&distributedPk](const auto &item) { + // unique index field count should be same + return item.size() == distributedPk.size() && distributedPk == DBCommon::TransformToCaseInsensitive(item); + }); + bool isMissMatch = !find; if (isMissMatch) { LOGE("Miss match distributed pk size %zu in auto increment table %s", distributedPk.size(), DBCommon::StringMiddleMasking(tableInfo.GetTableName()).c_str()); @@ -562,6 +581,27 @@ bool IsDistributedPkInvalid(const TableInfo &tableInfo, return false; } +bool IsDistributedSchemaSupport(const TableInfo &tableInfo, const std::vector &fields) +{ + if (!tableInfo.GetAutoIncrement()) { + return true; + } + bool isSyncPk = false; + bool isSyncOtherSpecified = false; + for (const auto &item : fields) { + if (tableInfo.IsPrimaryKey(item.colName) && item.isP2pSync) { + isSyncPk = true; + } else if (item.isSpecified && item.isP2pSync) { + isSyncOtherSpecified = true; + } + } + if (isSyncPk && isSyncOtherSpecified) { + LOGE("Not support sync with auto increment pk and other specified col"); + return false; + } + return true; +} + int CheckDistributedSchemaFields(const TableInfo &tableInfo, const std::vector &syncFields, const std::vector &fields) { @@ -569,13 +609,20 @@ int CheckDistributedSchemaFields(const TableInfo &tableInfo, const std::vector distributedPk; + bool isNoPrimaryKeyTable = tableInfo.IsNoPkTable(); for (const auto &field : fields) { if (!tableInfo.IsFieldExist(field.colName)) { LOGE("Column[%s [%zu]] not found in table", DBCommon::StringMiddleMasking(field.colName).c_str(), field.colName.size()); return -E_SCHEMA_MISMATCH; } + if (isNoPrimaryKeyTable && field.isSpecified) { + return -E_SCHEMA_MISMATCH; + } if (field.isSpecified && field.isP2pSync) { distributedPk.insert(field.colName); } @@ -584,14 +631,19 @@ int CheckDistributedSchemaFields(const TableInfo &tableInfo, const std::vector fieldsNeedContain; + std::set fieldsNotDecrease; std::set requiredNotNullFields; - GetFieldsNeedContain(tableInfo, syncFields, fieldsNeedContain, requiredNotNullFields); + GetFieldsNeedContain(tableInfo, syncFields, fieldsNeedContain, fieldsNotDecrease, requiredNotNullFields); std::map fieldsMap; for (auto &field : fields) { fieldsMap.insert({field.colName, field.isP2pSync}); } if (!CheckRequireFieldsInMap(fieldsNeedContain, fieldsMap)) { LOGE("The required fields are not found in fieldsMap"); + return -E_SCHEMA_MISMATCH; + } + if (!CheckRequireFieldsInMap(fieldsNotDecrease, fieldsMap)) { + LOGE("The fields should not decrease"); return -E_DISTRIBUTED_FIELD_DECREASE; } if (!CheckRequireFieldsInMap(requiredNotNullFields, fieldsMap)) { @@ -601,6 +653,27 @@ int CheckDistributedSchemaFields(const TableInfo &tableInfo, const std::vector, CaseInsensitiveComparator> tableSchemaMap; + for (const auto &table : schema.tables) { + tableSchemaMap[table.tableName] = table.fields; + } + for (const auto &table : localSchema.tables) { + if (tableSchemaMap.find(table.tableName) == tableSchemaMap.end()) { + LOGE("[RDBUtils][CheckDistributedSchemaTables] table [%s [%zu]] missing", + DBCommon::StringMiddleMasking(table.tableName).c_str(), table.tableName.size()); + return -E_DISTRIBUTED_FIELD_DECREASE; + } + } + return E_OK; +} + int SQLiteRelationalUtils::CheckDistributedSchemaValid(const RelationalSchemaObject &schemaObj, const DistributedSchema &schema, SQLiteSingleVerRelationalStorageExecutor *executor) { @@ -614,6 +687,11 @@ int SQLiteRelationalUtils::CheckDistributedSchemaValid(const RelationalSchemaObj LOGE("[RDBUtils][CheckDistributedSchemaValid] sqlite handle failed %d", errCode); return errCode; } + errCode = CheckDistributedSchemaTables(schemaObj, schema); + if (errCode != E_OK) { + LOGE("[RDBUtils][CheckDistributedSchemaValid] Check tables fail %d", errCode); + return errCode; + } for (const auto &table : schema.tables) { if (table.tableName.empty()) { LOGE("[RDBUtils][CheckDistributedSchemaValid] Table name cannot be empty"); @@ -734,4 +812,90 @@ int SQLiteRelationalUtils::GetLogInfoPre(sqlite3_stmt *queryStmt, DistributedTab } return errCode; } + +int SQLiteRelationalUtils::OperateDataStatus(sqlite3 *db, const std::vector &tables) +{ + auto [errCode, time] = GetCurrentVirtualTime(db); + if (errCode != E_OK) { + return errCode; + } + LOGI("[SQLiteRDBUtils] %zu tables wait for update time to %s", tables.size(), time.c_str()); + for (const auto &table : tables) { + errCode = UpdateLocalDataModifyTime(db, table, time); + if (errCode != E_OK) { + LOGE("[SQLiteRDBUtils] %s table len %zu operate failed %d", DBCommon::StringMiddleMasking(table).c_str(), + table.size(), errCode); + break; + } + } + if (errCode == E_OK) { + LOGI("[SQLiteRDBUtils] Operate data all success"); + } + return errCode; +} + +int SQLiteRelationalUtils::UpdateLocalDataModifyTime(sqlite3 *db, const std::string &table, + const std::string &modifyTime) +{ + auto logTable = DBCommon::GetLogTableName(table); + bool isCreate = false; + auto errCode = SQLiteUtils::CheckTableExists(db, logTable, isCreate); + if (errCode != E_OK) { + LOGE("[SQLiteRDBUtils] Check table exist failed %d when update time", errCode); + return errCode; + } + if (!isCreate) { + LOGW("[SQLiteRDBUtils] Skip non exist log table %s len %zu when update time", + DBCommon::StringMiddleMasking(table).c_str(), table.size()); + return E_OK; + } + std::string operateSql = "UPDATE " + logTable + + " SET timestamp= _rowid_ + " + modifyTime + " WHERE flag & 0x02 != 0"; + errCode = SQLiteUtils::ExecuteRawSQL(db, operateSql); + if (errCode != E_OK) { + LOGE("[SQLiteRDBUtils] Update %s len %zu modify time failed %d", DBCommon::StringMiddleMasking(table).c_str(), + table.size(), errCode); + } + return errCode; +} + +int SQLiteRelationalUtils::GetMetaLocalTimeOffset(sqlite3 *db, int64_t &timeOffset) +{ + std::string sql = "SELECT value FROM " + DBCommon::GetMetaTableName() + " WHERE key=x'" + + DBCommon::TransferStringToHex(std::string(DBConstant::LOCALTIME_OFFSET_KEY)) + "';"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + return errCode; + } + int ret = E_OK; + errCode = SQLiteUtils::StepWithRetry(stmt); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + timeOffset = static_cast(sqlite3_column_int64(stmt, 0)); + if (timeOffset < 0) { + LOGE("[SQLiteRDBUtils] TimeOffset %" PRId64 "is invalid.", timeOffset); + SQLiteUtils::ResetStatement(stmt, true, ret); + return -E_INTERNAL_ERROR; + } + errCode = E_OK; + } + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; +} + +std::pair SQLiteRelationalUtils::GetCurrentVirtualTime(sqlite3 *db) +{ + int64_t localTimeOffset = 0; + std::pair res; + auto &[errCode, time] = res; + errCode = GetMetaLocalTimeOffset(db, localTimeOffset); + if (errCode != E_OK) { + LOGE("[SQLiteRDBUtils] Failed to get local timeOffset.%d", errCode); + return res; + } + Timestamp currentSysTime = TimeHelper::GetSysCurrentTime(); + Timestamp currentLocalTime = currentSysTime + static_cast(localTimeOffset); + time = std::to_string(currentLocalTime); + return res; +} } // namespace DistributedDB \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h index e57f8d2c..80d37b46 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_utils.h @@ -62,11 +62,19 @@ public: static int GetLogInfoPre(sqlite3_stmt *queryStmt, DistributedTableMode mode, const DataItem &dataItem, LogInfo &logInfoGet); + + static int OperateDataStatus(sqlite3 *db, const std::vector &tables); + + static int GetMetaLocalTimeOffset(sqlite3 *db, int64_t &timeOffset); + + static std::pair GetCurrentVirtualTime(sqlite3 *db); private: static int BindExtendStatementByType(sqlite3_stmt *statement, int cid, Type &typeVal); static int GetTypeValByStatement(sqlite3_stmt *stmt, int cid, Type &typeVal); static int GetBlobByStatement(sqlite3_stmt *stmt, int cid, Type &typeVal); + + static int UpdateLocalDataModifyTime(sqlite3 *db, const std::string &table, const std::string &modifyTime); }; } // namespace DistributedDB #endif // SQLITE_RELATIONAL_UTILS_H diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index 18e0cc54..04415fc8 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -153,7 +153,12 @@ int SaveSchemaToMetaTable(SQLiteSingleVerRelationalStorageExecutor *handle, cons { const Key schemaKey(DBConstant::RELATIONAL_SCHEMA_KEY.begin(), DBConstant::RELATIONAL_SCHEMA_KEY.end()); Value schemaVal; - DBCommon::StringToVector(schema.ToSchemaString(), schemaVal); + auto schemaStr = schema.ToSchemaString(); + if (schemaStr.size() > SchemaConstant::SCHEMA_STRING_SIZE_LIMIT) { + LOGE("schema is too large %zu", schemaStr.size()); + return -E_MAX_LIMITS; + } + DBCommon::StringToVector(schemaStr, schemaVal); int errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data if (errCode != E_OK) { LOGE("Save schema to meta table failed. %d", errCode); @@ -187,7 +192,7 @@ int SaveSyncTableTypeAndDropFlagToMeta(SQLiteSingleVerRelationalStorageExecutor LOGE("Save sync table type to meta table failed. %d", errCode); return errCode; } - DBCommon::StringToVector(DBConstant::TABLE_WAS_DROPPED + tableName, key); + DBCommon::StringToVector(DBConstant::TABLE_IS_DROPPED + tableName, key); errCode = handle->DeleteMetaData({ key }); if (errCode != E_OK) { LOGE("Save table drop flag to meta table failed. %d", errCode); @@ -472,6 +477,17 @@ int SQLiteSingleRelationalStorageEngine::SetTrackerTable(const TrackerSchema &sc ReleaseExecutor(handle); return ret; } + Key key; + DBCommon::StringToVector(SYNC_TABLE_TYPE + schema.tableName, key); + Value value; + DBCommon::StringToVector(tableInfo.GetTableSyncType() == DEVICE_COOPERATION ? DEVICE_TYPE : CLOUD_TYPE, value); + errCode = handle->PutKvData(key, value); + if (errCode != E_OK) { + LOGE("[SetTrackerTable] Save sync type to meta table failed: %d", errCode); + (void)handle->Rollback(); + ReleaseExecutor(handle); + return errCode; + } if (schema.trackerColNames.empty() && !schema.isTrackAction) { tracker.RemoveTrackerSchema(schema); @@ -1080,7 +1096,8 @@ int SQLiteSingleRelationalStorageEngine::UpdateExtendField(const DistributedDB:: return handle->Commit(); } -std::pair SQLiteSingleRelationalStorageEngine::SetDistributedSchema(const DistributedSchema &schema) +std::pair SQLiteSingleRelationalStorageEngine::SetDistributedSchema(const DistributedSchema &schema, + const std::string &localIdentity) { std::lock_guard autoLock(createDistributedTableMutex_); auto schemaObj = GetSchema(); @@ -1099,7 +1116,7 @@ std::pair SQLiteSingleRelationalStorageEngine::SetDistributedSchema(c LOGE("new schema version no upgrade old:%" PRIu32 " new:%" PRIu32, localSchema.version, schema.version); errCode = -E_INVALID_ARGS; } else { - errCode = SetDistributedSchemaInner(schemaObj, schema); + errCode = SetDistributedSchemaInner(schemaObj, schema, localIdentity); } if (errCode == E_OK) { SetSchema(schemaObj); @@ -1108,7 +1125,7 @@ std::pair SQLiteSingleRelationalStorageEngine::SetDistributedSchema(c } int SQLiteSingleRelationalStorageEngine::SetDistributedSchemaInner(RelationalSchemaObject &schemaObj, - const DistributedSchema &schema) + const DistributedSchema &schema, const std::string &localIdentity) { int errCode = E_OK; auto *handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, @@ -1137,12 +1154,19 @@ int SQLiteSingleRelationalStorageEngine::SetDistributedSchemaInner(RelationalSch continue; } tableInfo.SetDistributedTable(schemaObj.GetDistributedTable(table.tableName)); - errCode = handle->RenewTableTrigger(schemaObj.GetTableMode(), tableInfo, tableInfo.GetTableSyncType()); + errCode = handle->RenewTableTrigger(schemaObj.GetTableMode(), tableInfo, tableInfo.GetTableSyncType(), + localIdentity); if (errCode != E_OK) { LOGE("Failed to refresh trigger while setting up distributed schema: %d", errCode); (void)handle->Rollback(); return errCode; } + errCode = handle->UpdateHashKey(schemaObj.GetTableMode(), tableInfo, tableInfo.GetTableSyncType()); + if (errCode != E_OK) { + LOGE("Failed to update hash_key while setting up distributed schema: %d", errCode); + (void)handle->Rollback(); + return errCode; + } } errCode = SaveSchemaToMetaTable(handle, schemaObj); if (errCode != E_OK) { diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h index c7e8f359..ce9b1709 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -62,7 +62,7 @@ public: const std::map &sharedTableOriginNames); int UpdateExtendField(const TrackerSchema &schema); - std::pair SetDistributedSchema(const DistributedSchema &schema); + std::pair SetDistributedSchema(const DistributedSchema &schema, const std::string &localIdentity); protected: StorageExecutor *NewSQLiteStorageExecutor(sqlite3 *dbHandle, bool isWrite, bool isMemDb) override; int Upgrade(sqlite3 *db) override; @@ -124,7 +124,8 @@ private: int CheckIfExistUserTable(SQLiteSingleVerRelationalStorageExecutor *&handle, const DataBaseSchema &cloudSchema, const std::map &alterTableNames, const RelationalSchemaObject &schema); - int SetDistributedSchemaInner(RelationalSchemaObject &schemaObj, const DistributedSchema &schema); + int SetDistributedSchemaInner(RelationalSchemaObject &schemaObj, const DistributedSchema &schema, + const std::string &localIdentity); RelationalSchemaObject schema_; RelationalSchemaObject trackerSchema_; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_continue_token.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_continue_token.cpp index 1a66c016..00b85401 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_continue_token.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_continue_token.cpp @@ -51,7 +51,8 @@ int SQLiteSingleVerRelationalContinueToken::GetStatement(sqlite3 *db, sqlite3_st errCode = GetMissQueryStatement(db, fullStmt); } if (errCode != E_OK) { - SQLiteUtils::ResetStatement(queryStmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(queryStmt, true, ret); } return errCode; } @@ -133,7 +134,8 @@ int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqli return errCode; ERROR: - SQLiteUtils::ResetStatement(stmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.cpp index f04ef6de..439a6305 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.cpp @@ -48,8 +48,10 @@ static constexpr const char *SET_FLAG_LOCAL_AND_CLEAN_WAIT_COMPENSATED_SYNC = "( static constexpr const char *SET_FLAG_CLEAN_WAIT_COMPENSATED_SYNC = "(CASE WHEN data_key = -1 and " "FLAG & 0x02 = 0x02 THEN FLAG & (~0x10) & (~0x20) ELSE (FLAG | 0x20) & (~0x10) END)"; static constexpr const char *FLAG_IS_LOGIC_DELETE = "FLAG & 0x08 != 0"; // see if 3th bit of a flag is logic delete +// set data logic delete, exist passport, delete, not compensated, cloud and consistency +static constexpr const char *SET_FLAG_WHEN_LOGOUT = "(FLAG | 0x08 | 0x800 | 0x01) & (~0x12) & (~0x20)"; // set data logic delete, exist passport, delete, not compensated and cloud -static constexpr const char *SET_FLAG_WHEN_LOGOUT = "(FLAG | 0x08 | 0x800 | 0x01) & (~0x12)"; +static constexpr const char *SET_FLAG_WHEN_LOGOUT_FOR_SHARE_TABLE = "(FLAG | 0x08 | 0x800 | 0x01) & (~0x12)"; static constexpr const char *DATA_IS_DELETE = "data_key = -1 AND FLAG & 0X08 = 0"; // see if data is delete static constexpr const char *UPDATE_CURSOR_SQL = "cursor=update_cursor()"; static constexpr const int SET_FLAG_ZERO_MASK = ~0x04; // clear 2th bit of flag @@ -155,26 +157,9 @@ int GetExistedDataTimeOffset(sqlite3 *db, const std::string &tableName, bool isM timeOffset = static_cast(sqlite3_column_int64(stmt, 0)); errCode = E_OK; } - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; -} - -int GetMetaLocalTimeOffset(sqlite3 *db, int64_t &timeOffset) -{ - std::string sql = "SELECT value FROM " + DBCommon::GetMetaTableName() + " WHERE key=x'" + - DBCommon::TransferStringToHex(std::string(DBConstant::LOCALTIME_OFFSET_KEY)) + "';"; - sqlite3_stmt *stmt = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, stmt); - if (errCode != E_OK) { - return errCode; - } - errCode = SQLiteUtils::StepWithRetry(stmt); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - timeOffset = static_cast(sqlite3_column_int64(stmt, 0)); - errCode = E_OK; - } - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } } @@ -195,13 +180,10 @@ std::string GetExtendValue(const TrackerTable &trackerTable) return extendValue; } -int SQLiteSingleVerRelationalStorageExecutor::GeneLogInfoForExistedData(sqlite3 *db, const std::string &identity, - const TableInfo &tableInfo, std::unique_ptr &logMgrPtr) +int SQLiteSingleVerRelationalStorageExecutor::GeneLogInfoForExistedDataInner(sqlite3 *db, const std::string &identity, + const TableInfo &tableInfo, std::unique_ptr &logMgrPtr, bool isTrackerTable) { std::string tableName = tableInfo.GetTableName(); - if (tableInfo.GetTableSyncType() == TableSyncType::DEVICE_COOPERATION) { - return UpdateTrackerTableTimeStamp(db, identity, tableInfo, logMgrPtr, false); - } int64_t timeOffset = 0; int errCode = GetExistedDataTimeOffset(db, tableName, isMemDb_, timeOffset); if (errCode != E_OK) { @@ -218,12 +200,18 @@ int SQLiteSingleVerRelationalStorageExecutor::GeneLogInfoForExistedData(sqlite3 static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY)); TrackerTable trackerTable = tableInfo.GetTrackerTable(); trackerTable.SetTableName(tableName); - std::string calPrimaryKeyHash = logMgrPtr->CalcPrimaryKeyHash("a.", tableInfo, identity); + const std::string prefix = "a."; + std::string calPrimaryKeyHash = logMgrPtr->CalcPrimaryKeyHash(prefix, tableInfo, identity); std::string sql = "INSERT OR REPLACE INTO " + logTable + " SELECT " + rowid + ", '', '', " + timeOffsetStr + " + " + rowid + ", " + timeOffsetStr + " + " + rowid + ", " + flag + ", " + calPrimaryKeyHash + ", '', "; sql += GetExtendValue(tableInfo.GetTrackerTable()); - sql += ", 0, '', '', 0 FROM '" + tableName + "' AS a WHERE 1=1;"; + sql += ", 0, '', '', 0 FROM '" + tableName + "' AS a "; + if (isTrackerTable) { + sql += " WHERE 1 = 1;"; + } else { + sql += "WHERE NOT EXISTS (SELECT 1 FROM " + logTable + " WHERE data_key = a._rowid_);"; + } errCode = trackerTable.ReBuildTempTrigger(db, TriggerMode::TriggerModeEnum::INSERT, [db, &sql]() { int ret = SQLiteUtils::ExecuteRawSQL(db, sql); if (ret != E_OK) { @@ -234,6 +222,15 @@ int SQLiteSingleVerRelationalStorageExecutor::GeneLogInfoForExistedData(sqlite3 return errCode; } +int SQLiteSingleVerRelationalStorageExecutor::GeneLogInfoForExistedData(sqlite3 *db, const std::string &identity, + const TableInfo &tableInfo, std::unique_ptr &logMgrPtr, bool isTrackerTable) +{ + if (tableInfo.GetTableSyncType() == TableSyncType::DEVICE_COOPERATION) { + return UpdateTrackerTable(db, identity, tableInfo, logMgrPtr, false); + } + return GeneLogInfoForExistedDataInner(db, identity, tableInfo, logMgrPtr, isTrackerTable); +} + int SQLiteSingleVerRelationalStorageExecutor::ResetLogStatus(std::string &tableName) { int errCode = SetLogTriggerStatus(false); @@ -250,38 +247,47 @@ int SQLiteSingleVerRelationalStorageExecutor::ResetLogStatus(std::string &tableN return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::UpdateTrackerTableTimeStamp(sqlite3 *db, const std::string &identity, - const TableInfo &tableInfo, std::unique_ptr &logMgrPtr, bool isRowReplace) +int SQLiteSingleVerRelationalStorageExecutor::UpdateTrackerTable(sqlite3 *db, const std::string &identity, + const TableInfo &tableInfo, std::unique_ptr &logMgrPtr, bool isTimestampOnly) { - int64_t errCode = SetLogTriggerStatus(false); + int errCode = SetLogTriggerStatus(false); if (errCode != E_OK) { return errCode; } - int64_t localTimeOffset = 0; - errCode = GetMetaLocalTimeOffset(db, localTimeOffset); + std::string currentLocalTimeStr; + std::tie(errCode, currentLocalTimeStr) = SQLiteRelationalUtils::GetCurrentVirtualTime(db); if (errCode != E_OK) { LOGE("Failed to get local timeOffset.%d", errCode); return errCode; } std::string tableName = tableInfo.GetTableName(); std::string logTable = DBCommon::GetLogTableName(tableName); + TrackerTable trackerTable = tableInfo.GetTrackerTable(); + trackerTable.SetTableName(tableName); + if (isTimestampOnly) { + std::string sql = "update " + logTable + " set timestamp = " + currentLocalTimeStr + + " + data_key, wtimestamp = " + currentLocalTimeStr + " + data_key where data_key != -1;"; + errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("Failed to initialize device type log data.%d", errCode); + return errCode; + } + return UpgradedLogForExistedData(tableInfo, false); + } std::string rowid = std::string(DBConstant::SQLITE_INNER_ROWID); std::string flag = std::to_string(static_cast(LogInfoFlag::FLAG_LOCAL) | - static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY)); - Timestamp currentSysTime = TimeHelper::GetSysCurrentTime(); - Timestamp currentLocalTime = currentSysTime + localTimeOffset; - std::string currentLocalTimeStr = std::to_string(currentLocalTime); - std::string insertPrefix = isRowReplace ? "INSERT OR REPLACE INTO " : "INSERT INTO "; - std::string insertSuffix = isRowReplace ? ";" : " " + logMgrPtr->GetConflictPkSql(tableInfo) + " DO UPDATE SET " - "data_key=excluded.data_key, device=excluded.device, ori_device=excluded.ori_device, " - "flag=excluded.flag, cloud_gid=excluded.cloud_gid, extend_field=excluded.extend_field, " - "cursor=excluded.cursor, version=excluded.version, sharing_resource=excluded.sharing_resource, " - "status=excluded.status;"; + static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY)); std::string calPrimaryKeyHash = logMgrPtr->CalcPrimaryKeyHash("a.", tableInfo, identity); - std::string sql = insertPrefix + logTable + " SELECT " + rowid + ", '', '', " + - currentLocalTimeStr + " + " + rowid + ", " + currentLocalTimeStr + " + " + rowid + ", " + flag + ", " + - calPrimaryKeyHash + ", '', " + "'', '', '', '', 0 FROM '" + tableName + "' AS a WHERE 1=1" + insertSuffix; - errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + std::string sql = "INSERT OR REPLACE INTO " + logTable + " SELECT " + rowid + ", '', '', " + currentLocalTimeStr + + " + " + rowid + ", " + currentLocalTimeStr + " + " + rowid + ", " + flag + ", " + calPrimaryKeyHash + + ", '', '', '', '', '', 0 FROM '" + tableName + "' AS a WHERE 1=1;"; + errCode = trackerTable.ReBuildTempTrigger(db, TriggerMode::TriggerModeEnum::INSERT, [db, &sql]() { + int ret = SQLiteUtils::ExecuteRawSQL(db, sql); + if (ret != E_OK) { + LOGE("Failed to initialize cloud type log data.%d", ret); + } + return ret; + }); if (errCode != E_OK) { LOGE("Failed to initialize device type log data.%d", errCode); } @@ -310,19 +316,23 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateRelationalLogTable(Distribut std::string tableName = table.GetTableName(); if (!isUpgraded) { if (table.GetTrackerTable().GetTableName().empty()) { - // user table -> distributed table - errCode = GeneLogInfoForExistedData(dbHandle_, identity, table, tableManager); + errCode = GeneLogInfoForExistedData(dbHandle_, identity, table, tableManager, false); } else if (table.GetTableSyncType() == TableSyncType::DEVICE_COOPERATION) { // tracker table -> distributed device table - errCode = UpdateTrackerTableTimeStamp(dbHandle_, tableName, table, tableManager, true); + errCode = UpdateTrackerTable(dbHandle_, tableName, table, tableManager, true); } else { // tracker table -> distributed cloud table errCode = ResetLogStatus(tableName); } - if (errCode != E_OK) { - return errCode; + } else { + if (table.GetTrackerTable().GetTableName().empty()) { + errCode = GeneLogInfoForExistedDataInner(dbHandle_, identity, table, tableManager, false); } } + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] generate log isUpgraded %d failed %d.", static_cast(isUpgraded), errCode); + return errCode; + } // add trigger errCode = tableManager->AddRelationalLogTableTrigger(dbHandle_, table, identity); @@ -478,8 +488,9 @@ int GetDeviceTableName(sqlite3 *handle, const std::string &tableName, const std: deviceTables.emplace_back(realTableName); } while (true); - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } std::vector GetUpgradeFields(const TableInfo &oldTableInfo, const TableInfo &newTableInfo) @@ -713,7 +724,7 @@ void GetCloudLog(sqlite3_stmt *logStatement, VBucket &logInfo, uint32_t &totalSi void GetCloudExtraLog(sqlite3_stmt *logStatement, VBucket &flags) { - flags.insert_or_assign(CloudDbConstant::ROWID, + flags.insert_or_assign(DBConstant::ROWID, static_cast(sqlite3_column_int64(logStatement, DATA_KEY_INDEX))); flags.insert_or_assign(CloudDbConstant::TIMESTAMP, static_cast(sqlite3_column_int64(logStatement, TIMESTAMP_INDEX))); @@ -776,9 +787,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v } errCode = SQLiteUtils::GetColumnBlobValue(statement, 0, value); // only one result. - END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; +END: + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const @@ -807,8 +819,9 @@ int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Va errCode = E_OK; } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const @@ -821,6 +834,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &k return errCode; } errCode = SqliteMetaExecutor::GetAllKeys(statement, isMemDb_, keys); - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem, @@ -893,15 +909,26 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { errCode = E_OK; } - SQLiteUtils::ResetStatement(stmt, false, errCode); // Finalize outside. - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, false, ret); // Finalize outside. + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(const DataItem &dataItem, bool isUpdate, - SaveSyncDataStmt &saveStmt, RelationalSyncDataInserter &inserter, int64_t &rowid) + SaveSyncDataStmt &saveStmt, RelationalSyncDataInserter &inserter, std::map &saveVals) { + if (std::get_if(&saveVals[DBConstant::ROWID]) == nullptr) { + LOGE("[SaveSyncDataItem] Invalid args because of no rowid!"); + return -E_INVALID_ARGS; + } if ((dataItem.flag & DataItem::DELETE_FLAG) != 0) { - rowid = -1; + int errCode = inserter.GetObserverDataByRowId(dbHandle_, std::get(saveVals[DBConstant::ROWID]), + ChangeType::OP_DELETE); + if (errCode != E_OK) { + LOGE("[SaveSyncDataItem] Failed to get primary data before deletion, errCode: %d", errCode); + return errCode; + } + saveVals[DBConstant::ROWID] = static_cast(-1); return DeleteSyncDataItem(dataItem, inserter, saveStmt.rmDataStmt); } // we don't know the rowid if user drop device table @@ -914,12 +941,17 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(const DataItem &d return errCode; } } - int errCode = inserter.SaveData(isUpdate, dataItem, saveStmt); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - if (!isUpdate) { - rowid = SQLiteUtils::GetLastRowId(dbHandle_); - } - errCode = E_OK; + int errCode = inserter.SaveData(isUpdate, dataItem, saveStmt, saveVals); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + return errCode; + } + if (!isUpdate) { + saveVals[DBConstant::ROWID] = SQLiteUtils::GetLastRowId(dbHandle_); + } + errCode = inserter.GetObserverDataByRowId(dbHandle_, std::get(saveVals[DBConstant::ROWID]), + isUpdate ? ChangeType::OP_UPDATE : ChangeType::OP_INSERT); + if (errCode != E_OK) { + LOGE("Get observer data by rowid failed, errCode=%d.", errCode); } return errCode; } @@ -943,8 +975,9 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncLog(const DataItem &data if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { errCode = E_OK; } - SQLiteUtils::ResetStatement(stmt, false, errCode); // Finalize outside. - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, false, ret); // Finalize outside. + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::ProcessMissQueryData(const DataItem &item, @@ -975,7 +1008,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDataConflictDefeated(const Da isDefeated = false; // no need to solve conflict except miss query data return E_OK; } - if (dataItem.dev != logInfoGet.device) { + if (!isExist || dataItem.dev != logInfoGet.device) { // defeated if item timestamp is earlier than raw data isDefeated = (dataItem.timestamp <= logInfoGet.timestamp); } @@ -1001,10 +1034,16 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(RelationalSyncDat if ((item.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) != 0) { return ProcessMissQueryData(item, inserter, saveStmt.rmDataStmt, saveStmt.rmLogStmt); } + if (!isExist && ((item.flag & DataItem::DELETE_FLAG) != 0)) { + LOGI("[RelationalStorageExecutor][SaveSyncDataItem] Delete non-exist data. Nothing to save."); + return E_OK; + } bool isUpdate = isExist && mode_ == DistributedTableMode::COLLABORATION; - errCode = SaveSyncDataItem(item, isUpdate, saveStmt, inserter, rowid); + std::map saveVals; + saveVals[DBConstant::ROWID] = rowid; + errCode = SaveSyncDataItem(item, isUpdate, saveStmt, inserter, saveVals); if (errCode == E_OK || errCode == -E_NOT_FOUND) { - errCode = inserter.SaveSyncLog(dbHandle_, saveStmt.saveLogStmt, saveStmt.queryStmt, item, rowid); + errCode = inserter.SaveSyncLog(dbHandle_, saveStmt.saveLogStmt, saveStmt.queryStmt, item, saveVals); } return errCode; } @@ -1044,11 +1083,16 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(RelationalSyncDataIn } } - int errCode = SaveSyncDataItems(inserter); + int errCode = SetLogTriggerStatus(false); + if (errCode != E_OK) { + goto END; + } + errCode = SaveSyncDataItems(inserter); if (errCode != E_OK) { LOGE("Save sync data items failed. errCode=%d", errCode); goto END; } + errCode = SetLogTriggerStatus(true); END: if (useTrans) { if (errCode == E_OK) { @@ -1214,9 +1258,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector(sqlite3_column_int64(stmt, 0))); // 0 is index errCode = E_OK; } - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); if (errCode != E_OK) { maxTimestamp = 0; return errCode; @@ -1717,6 +1765,12 @@ int SQLiteSingleVerRelationalStorageExecutor::CleanCloudDataOnLogTable(const std return SQLiteUtils::ExecuteRawSQL(dbHandle_, cleanLogSql); } +int SQLiteSingleVerRelationalStorageExecutor::ClearVersionOnLogTable(const std::string &logTableName) +{ + std::string cleanLogSql = "UPDATE " + logTableName + " SET " + VERSION + " = '';"; + return SQLiteUtils::ExecuteRawSQL(dbHandle_, cleanLogSql); +} + int SQLiteSingleVerRelationalStorageExecutor::CleanUploadFinishedFlag(const std::string &tableName) { // unset upload finished flag @@ -1773,7 +1827,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SetDataOnUserTableWithLogicDelete( { UpdateCursorContext context; int errCode = SQLiteRelationalUtils::GetCursor(dbHandle_, tableName, context.cursor); - LOGI("removeData on userTable:%s length:%d start and cursor is %llu.", + LOGI("removeData on userTable:%s length:%zu start and cursor is %llu.", DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), context.cursor); errCode = CreateFuncUpdateCursor(context, &UpdateCursor); if (errCode != E_OK) { @@ -1802,7 +1856,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SetDataOnUserTableWithLogicDelete( LOGE("Failed to deal logic delete data flag on usertable, %d.", errCode); return errCode; } - LOGI("removeData on userTable:%s length:%d finish and cursor is %llu.", + LOGI("removeData on userTable:%s length:%zu finish and cursor is %llu.", DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), context.cursor); errCode = SetCursor(tableName, context.cursor); if (errCode != E_OK) { @@ -1817,17 +1871,17 @@ int SQLiteSingleVerRelationalStorageExecutor::SetDataOnShareTableWithLogicDelete { UpdateCursorContext context; int errCode = SQLiteRelationalUtils::GetCursor(dbHandle_, tableName, context.cursor); - LOGI("removeData on shareTable:%s length:%d start and cursor is %llu.", + LOGI("removeData on shareTable:%s length:%zu start and cursor is %llu.", DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), context.cursor); errCode = CreateFuncUpdateCursor(context, &UpdateCursor); if (errCode != E_OK) { LOGE("Failed to create updateCursor func on shareTable errCode=%d.", errCode); return errCode; } - std::string sql = "UPDATE '" + logTableName + "' SET " + CloudDbConstant::FLAG + " = " + SET_FLAG_WHEN_LOGOUT + - ", " + VERSION + " = '', " + DEVICE_FIELD + " = '', " + CLOUD_GID_FIELD + " = '', " + - SHARING_RESOURCE + " = '', " + UPDATE_CURSOR_SQL + " WHERE (NOT (" + DATA_IS_DELETE + ") " + - " AND NOT (" + FLAG_IS_LOGIC_DELETE + "));"; + std::string sql = "UPDATE '" + logTableName + "' SET " + CloudDbConstant::FLAG + " = " + + SET_FLAG_WHEN_LOGOUT_FOR_SHARE_TABLE + ", " + VERSION + " = '', " + DEVICE_FIELD + " = '', " + + CLOUD_GID_FIELD + " = '', " + SHARING_RESOURCE + " = '', " + UPDATE_CURSOR_SQL + + " WHERE (NOT (" + DATA_IS_DELETE + ") " + " AND NOT (" + FLAG_IS_LOGIC_DELETE + "));"; errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, sql); // here just clear updateCursor func, fail will not influence other function (void)CreateFuncUpdateCursor(context, nullptr); @@ -1843,7 +1897,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SetDataOnShareTableWithLogicDelete LOGE("Failed to deal logic delete data flag on shareTable, %d.", errCode); return errCode; } - LOGI("removeData on shareTable:%s length:%d finish and cursor is %llu.", + LOGI("removeData on shareTable:%s length:%zu finish and cursor is %llu.", DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), context.cursor); return SetCursor(tableName, context.cursor); } @@ -1958,5 +2012,27 @@ int SQLiteSingleVerRelationalStorageExecutor::GetLockStatusByGid(const std::stri SQLiteUtils::ResetStatement(stmt, true, resetRet); return errCode; } + +int SQLiteSingleVerRelationalStorageExecutor::UpdateHashKey(DistributedTableMode mode, const TableInfo &tableInfo, + TableSyncType syncType) +{ + auto tableManager = LogTableManagerFactory::GetTableManager(mode, syncType); + auto logName = DBCommon::GetLogTableName(tableInfo.GetTableName()); + std::string sql = "UPDATE " + logName + " SET hash_key = hash_key || '_old' where data_key in " + + "(select _rowid_ from '" + tableInfo.GetTableName() + "');"; + int errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, sql); + if (errCode != E_OK) { + return errCode; + } + sql = "UPDATE " + logName + " SET hash_key = data.hash_value FROM (SELECT _rowid_, " + + tableManager->CalcPrimaryKeyHash("dataTable.", tableInfo, "") + " AS hash_value " + + "FROM '" + tableInfo.GetTableName() + "' AS dataTable) AS data WHERE data._rowid_ = " + logName + ".data_key;"; + return SQLiteUtils::ExecuteRawSQL(dbHandle_, sql); +} + +void SQLiteSingleVerRelationalStorageExecutor::SetTableMode(DistributedTableMode mode) +{ + mode_ = mode; +} } // namespace DistributedDB #endif diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h index 6f3093ca..8acac8e9 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor.h @@ -144,10 +144,11 @@ public: int FillCloudAssetForDownload(const TableSchema &tableSchema, VBucket &vBucket, bool isDownloadSuccess, uint64_t &currCursor); - int FillCloudAssetForAsyncDownload(const TableSchema &tableSchema, VBucket &vBucket, bool isDownloadSuccess); + int DoCleanInner(ClearMode mode, const std::vector &tableNameList, const RelationalSchemaObject &localSchema, std::vector &assets, std::vector ¬ifyTableList); + int DoClearCloudLogVersion(const std::vector &tableNameList); int FillCloudAssetForUpload(OpType opType, const TableSchema &tableSchema, const CloudSyncBatch &data); int FillCloudVersionForUpload(const OpType opType, const CloudSyncData &data); @@ -181,7 +182,8 @@ public: int CheckIfExistUserTable(const std::string &tableName); void SetLogicDelete(bool isLogicDelete); - int RenewTableTrigger(DistributedTableMode mode, const TableInfo &tableInfo, TableSyncType syncType); + int RenewTableTrigger(DistributedTableMode mode, const TableInfo &tableInfo, TableSyncType syncType, + const std::string &localIdentity = ""); std::pair GetAssetsByGidOrHashKey(const TableSchema &tableSchema, const std::string &gid, const Bytes &hashKey, VBucket &assets); @@ -197,7 +199,7 @@ public: void SetMarkFlagOption(MarkFlagOption option); - int UpgradedLogForExistedData(TableInfo &tableInfo, bool schemaChanged); + int UpgradedLogForExistedData(const TableInfo &tableInfo, bool schemaChanged); int UpdateRecordFlag(const std::string &tableName, const std::string &sql, const LogInfo &logInfo); @@ -231,13 +233,12 @@ public: int GetLocalDataCount(const std::string &tableName, int &dataCount, int &logicDeleteDataCount); + int GetCloudDataCount(const std::string &tableName, DownloadData &downloadData, int64_t &count); + int UpdateExtendField(const std::string &tableName, const std::set &extendColNames); int UpdateDeleteDataExtendField(const std::string &tableName, const std::string &lowVersionExtendColName, const std::set &oldExtendColNames, const std::set &extendColNames); - - void CheckAndCreateTrigger(const TrackerTable &trackerTable); - int GetDownloadingAssetsCount(const TableSchema &tableSchema, int32_t &totalCount); int GetDownloadingCount(const std::string &tableName, int32_t &count); @@ -254,11 +255,15 @@ public: int CleanDownloadingFlagByGid(const std::string &tableName, const std::string &gid, VBucket dbAssets); - int CompareSchemaTableColumns(const std::string &tableName); - void CheckAndCreateTrigger(const TableInfo &table); int GetLockStatusByGid(const std::string &tableName, const std::string &gid, LockStatus &status); + + int CompareSchemaTableColumns(const std::string &tableName); + + int UpdateHashKey(DistributedTableMode mode, const TableInfo &tableInfo, TableSyncType syncType); + + void SetTableMode(DistributedTableMode mode); private: int DoCleanLogs(const std::vector &tableNameList, const RelationalSchemaObject &localSchema); @@ -267,6 +272,8 @@ private: int CleanCloudDataOnLogTable(const std::string &logTableName, ClearMode mode); + int ClearVersionOnLogTable(const std::string &logTableName); + int CleanCloudDataAndLogOnUserTable(const std::string &tableName, const std::string &logTableName, const RelationalSchemaObject &localSchema); @@ -303,7 +310,7 @@ private: int SaveSyncDataItem(RelationalSyncDataInserter &inserter, SaveSyncDataStmt &saveStmt, DataItem &item); int SaveSyncDataItem(const DataItem &dataItem, bool isUpdate, SaveSyncDataStmt &saveStmt, - RelationalSyncDataInserter &inserter, int64_t &rowid); + RelationalSyncDataInserter &inserter, std::map &saveVals); int DeleteSyncDataItem(const DataItem &dataItem, RelationalSyncDataInserter &inserter, sqlite3_stmt *&rmDataStmt); @@ -320,7 +327,10 @@ private: void SetTableInfo(const TableInfo &tableInfo); // When put or get sync data, must call the func first. int GeneLogInfoForExistedData(sqlite3 *db, const std::string &identity, const TableInfo &tableInfo, - std::unique_ptr &logMgrPtr); + std::unique_ptr &logMgrPtr, bool isTrackerTable); + + int GeneLogInfoForExistedDataInner(sqlite3 *db, const std::string &identity, const TableInfo &tableInfo, + std::unique_ptr &logMgrPtr, bool isTrackerTable); int CleanExtendAndCursorForDeleteData(const std::string &tableName); @@ -402,8 +412,8 @@ private: int GetDeleteStatementForCloudSync(const TableSchema &tableSchema, const std::set &pkSet, const VBucket &vBucket, sqlite3_stmt *&deleteStmt); - int UpdateTrackerTableTimeStamp(sqlite3 *db, const std::string &identity, const TableInfo &tableInfo, - std::unique_ptr &logMgrPtr, bool isRowReplace); + int UpdateTrackerTable(sqlite3 *db, const std::string &identity, const TableInfo &tableInfo, + std::unique_ptr &logMgrPtr, bool isTimestampOnly); int DeleteCloudData(const std::string &tableName, const VBucket &vBucket, const TableSchema &tableSchema, const TrackerTable &trackerTable); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor_extend.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor_extend.cpp index 75b9e38d..007fe7e5 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_executor_extend.cpp @@ -208,7 +208,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetQueryLogStatement(const TableSc errCode = SQLiteUtils::BindTextToStatement(selectStmt, index, cloudGid); if (errCode != E_OK) { LOGE("Bind cloud gid to query log statement failed. %d", errCode); - SQLiteUtils::ResetStatement(selectStmt, true, errCode); + SQLiteUtils::ResetStatement(selectStmt, true, ret); return errCode; } } @@ -339,6 +339,23 @@ int SQLiteSingleVerRelationalStorageExecutor::DoCleanInner(ClearMode mode, return errCode; } +int SQLiteSingleVerRelationalStorageExecutor::DoClearCloudLogVersion(const std::vector &tableNameList) +{ + int errCode = E_OK; + for (const auto &tableName: tableNameList) { + std::string logTableName = DBCommon::GetLogTableName(tableName); + LOGD("[Storage Executor] Start clear version on log table."); + errCode = ClearVersionOnLogTable(logTableName); + if (errCode != E_OK) { + std::string maskedName = DBCommon::StringMiddleMasking(tableName); + LOGE("[Storage Executor] failed to clear version of cloud data on log table %s (name length is %zu), %d", + maskedName.c_str(), maskedName.length(), errCode); + return errCode; + } + } + return errCode; +} + int SQLiteSingleVerRelationalStorageExecutor::DoCleanLogs(const std::vector &tableNameList, const RelationalSchemaObject &localSchema) { @@ -551,6 +568,44 @@ END: return errCode != E_OK ? errCode : ret; } +int SQLiteSingleVerRelationalStorageExecutor::GetCloudDataCount(const std::string &tableName, + DownloadData &downloadData, int64_t &count) +{ + if (downloadData.data.empty()) { + return E_OK; + } + int errCode = E_OK; + sqlite3_stmt *queryStmt = nullptr; + std::string querySql = "SELECT COUNT(*) FROM '" + DBCommon::GetLogTableName(tableName) + + "' WHERE FLAG & 0x02 = 0 AND CLOUD_GID IN ("; + for (const VBucket &vBucket : downloadData.data) { + std::string cloudGid; + errCode = CloudStorageUtils::GetValueFromVBucket(CloudDbConstant::GID_FIELD, vBucket, cloudGid); + if (errCode != E_OK) { + LOGE("get gid for query log statement failed, %d", errCode); + return -E_CLOUD_ERROR; + } + querySql += "'" + cloudGid + "',"; + } + querySql.pop_back(); + querySql += ");"; + errCode = SQLiteUtils::GetStatement(dbHandle_, querySql, queryStmt); + if (errCode != E_OK) { // LCOV_EXCL_BR_LINE + LOGE("Get query count statement failed, %d", errCode); + return errCode; + } + errCode = SQLiteUtils::StepWithRetry(queryStmt); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { // LCOV_EXCL_BR_LINE + count = static_cast(sqlite3_column_int64(queryStmt, 0)); + errCode = E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + int ret = E_OK; + SQLiteUtils::ResetStatement(queryStmt, true, ret); + return errCode != E_OK ? errCode : ret; +} + int SQLiteSingleVerRelationalStorageExecutor::GetCloudAssets(const std::string &tableName, const std::vector &fieldInfos, const std::vector &dataKeys, std::vector &assets) { @@ -607,13 +662,22 @@ int SQLiteSingleVerRelationalStorageExecutor::PutCloudSyncData(const std::string std::map statisticMap = {}; errCode = ExecutePutCloudData(tableName, tableSchema, trackerTable, downloadData, statisticMap); + if (errCode != E_OK) { + LOGE("ExecutePutCloudData failed, %d", errCode); + } int ret = SetLogTriggerStatus(true); if (ret != E_OK) { LOGE("Fail to set log trigger on, %d", ret); } - LOGI("save cloud data:%d, ins:%d, upd:%d, del:%d, only gid:%d, flag zero:%d, flag one:%d, upd timestamp:%d," - "clear gid:%d, not handle:%d, lock:%d", - errCode, statisticMap[static_cast(OpType::INSERT)], statisticMap[static_cast(OpType::UPDATE)], + int64_t count = 0; + int errCodeCount = GetCloudDataCount(tableName, downloadData, count); + if (errCodeCount != E_OK) { + LOGW("get cloud data count failed, %d", errCodeCount); + } + LOGI("save cloud data of table %s [length %zu]:%d, cloud data count:%lld, ins:%d, upd:%d, del:%d, only gid:%d," + "flag zero:%d, flag one:%d, upd timestamp:%d, clear gid:%d, not handle:%d, lock:%d", + DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size(), errCode, count, + statisticMap[static_cast(OpType::INSERT)], statisticMap[static_cast(OpType::UPDATE)], statisticMap[static_cast(OpType::DELETE)], statisticMap[static_cast(OpType::ONLY_UPDATE_GID)], statisticMap[static_cast(OpType::SET_CLOUD_FORCE_PUSH_FLAG_ZERO)], statisticMap[static_cast(OpType::SET_CLOUD_FORCE_PUSH_FLAG_ONE)], @@ -668,14 +732,14 @@ int SQLiteSingleVerRelationalStorageExecutor::InsertCloudData(VBucket &vBucket, if (putDataMode_ == PutDataMode::SYNC) { CloudStorageUtils::PrepareToFillAssetFromVBucket(vBucket, CloudStorageUtils::FillAssetBeforeDownload); } + int ret = E_OK; errCode = BindValueToUpsertStatement(vBucket, tableSchema.fields, insertStmt); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(insertStmt, true, errCode); + SQLiteUtils::ResetStatement(insertStmt, true, ret); return errCode; } // insert data errCode = SQLiteUtils::StepWithRetry(insertStmt, false); - int ret = E_OK; SQLiteUtils::ResetStatement(insertStmt, true, ret); if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { LOGE("insert data failed when save cloud data:%d, reset stmt:%d", errCode, ret); @@ -733,14 +797,14 @@ int SQLiteSingleVerRelationalStorageExecutor::InsertLogRecord(const TableSchema return errCode; } + int ret = E_OK; errCode = BindValueToInsertLogStatement(vBucket, tableSchema, trackerTable, insertLogStmt); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(insertLogStmt, true, errCode); + SQLiteUtils::ResetStatement(insertLogStmt, true, ret); return errCode; } errCode = SQLiteUtils::StepWithRetry(insertLogStmt, false); - int ret = E_OK; SQLiteUtils::ResetStatement(insertLogStmt, true, ret); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { return ret; @@ -915,7 +979,7 @@ int SQLiteSingleVerRelationalStorageExecutor::BindValueToInsertLogStatement(VBuc return errCode; } - vBucket[CloudDbConstant::ROW_ID_FIELD_NAME] = rowid; // fill rowid to cloud data to notify user + vBucket[DBConstant::ROWID] = rowid; // fill rowid to cloud data to notify user return BindHashKeyAndGidToInsertLogStatement(vBucket, tableSchema, trackerTable, insertLogStmt, bindIndex); } @@ -1002,7 +1066,8 @@ int SQLiteSingleVerRelationalStorageExecutor::GetUpdateDataTableStatement(const errCode = BindValueToUpsertStatement(vBucket, updateFields, updateStmt); if (errCode != E_OK) { LOGE("bind value to update statement failed when update cloud data, %d", errCode); - SQLiteUtils::ResetStatement(updateStmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(updateStmt, true, ret); } return errCode; } @@ -1020,15 +1085,16 @@ int SQLiteSingleVerRelationalStorageExecutor::UpdateCloudData(VBucket &vBucket, } // update data + int ret = E_OK; errCode = SQLiteUtils::StepWithRetry(updateStmt, false); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { errCode = E_OK; } else { LOGE("update data failed when save cloud data:%d", errCode); - SQLiteUtils::ResetStatement(updateStmt, true, errCode); + SQLiteUtils::ResetStatement(updateStmt, true, ret); return errCode; } - SQLiteUtils::ResetStatement(updateStmt, true, errCode); + SQLiteUtils::ResetStatement(updateStmt, true, ret); // update log errCode = UpdateLogRecord(vBucket, tableSchema, OpType::UPDATE); @@ -1540,8 +1606,9 @@ int SQLiteSingleVerRelationalStorageExecutor::GetUploadCountInner(const Timestam } else { LOGE("Failed to get the count to be uploaded. %d", errCode); } - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerRelationalStorageExecutor::GetUploadCount(const Timestamp ×tamp, bool isCloudForcePush, @@ -1873,17 +1940,18 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDownloadAssetGid(const TableSch std::vector &gids, int64_t beginTime, bool abortWithLimit) { std::string sql = "SELECT cloud_gid FROM " + DBCommon::GetLogTableName(tableSchema.name) + - " WHERE flag&0x1000=0x1000 AND timestamp >= ?;"; + " WHERE flag&0x1000=0x1000 AND timestamp > ?;"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); if (errCode != E_OK) { LOGE("[RDBExecutor]Get gid statement failed, %d", errCode); return errCode; } + int ret = E_OK; errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, beginTime); if (errCode != E_OK) { LOGE("[RDBExecutor] bind time failed %d when get download asset gid", errCode); - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } uint32_t count = 0; @@ -1907,7 +1975,6 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDownloadAssetGid(const TableSch break; } } while (errCode == E_OK); - int ret = E_OK; SQLiteUtils::ResetStatement(stmt, true, ret); return errCode == E_OK ? ret : errCode; } @@ -1934,9 +2001,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDownloadAssetRecordsByGid(const LOGE("Get downloading asset records statement failed, %d", errCode); return errCode; } + int ret = E_OK; errCode = SQLiteUtils::BindTextToStatement(stmt, 1, gid); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } errCode = SQLiteUtils::StepWithRetry(stmt); @@ -1958,7 +2026,6 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDownloadAssetRecordsByGid(const } else { LOGE("step get downloading asset records statement failed %d.", errCode); } - int ret = E_OK; SQLiteUtils::ResetStatement(stmt, true, ret); return errCode == E_OK ? ret : errCode; } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp index f0cbd2bf..73cb0655 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_ver_relational_storage_extend_executor.cpp @@ -82,10 +82,11 @@ int SQLiteSingleVerRelationalStorageExecutor::GetFillDownloadAssetStatement(cons LOGE("Get fill asset statement failed, %d.", errCode); return errCode; } + int ret = E_OK; for (size_t i = 0; i < fields.size(); ++i) { errCode = BindOneField(i + 1, vBucket, fields[i], stmt); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } } @@ -108,11 +109,12 @@ int SQLiteSingleVerRelationalStorageExecutor::CleanDownloadingFlagByGid(const st errCode, DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); return errCode; } + int ret = E_OK; errCode = SQLiteUtils::BindTextToStatement(stmt, 1, gid); if (errCode != E_OK) { LOGE("[RDBExecutor]bind gid failed when clean downloading flag:%d, tableName:%s, length:%zu", errCode, DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } errCode = SQLiteUtils::StepWithRetry(stmt); @@ -122,7 +124,6 @@ int SQLiteSingleVerRelationalStorageExecutor::CleanDownloadingFlagByGid(const st LOGE("[RDBExecutor]clean downloading flag failed:%d, tableName:%s, length:%zu", errCode, DBCommon::StringMiddleMasking(tableName).c_str(), tableName.size()); } - int ret = E_OK; SQLiteUtils::ResetStatement(stmt, true, ret); if (ret != E_OK) { LOGE("[RDBExecutor]reset stmt when clean downloading flag:%d, tableName:%s, length:%zu", @@ -393,12 +394,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateTrackerTable(const TrackerTa if (errCode != E_OK) { return errCode; } - errCode = CleanExtendAndCursorForDeleteData(table.GetTableName()); - if (errCode != E_OK) { - LOGE("clean tracker log info for deleted data failed %d.", errCode); - return errCode; - } - errCode = GeneLogInfoForExistedData(dbHandle_, "", table, tableManager); + errCode = GeneLogInfoForExistedData(dbHandle_, "", table, tableManager, true); if (errCode != E_OK) { LOGE("general tracker log info for existed data failed %d.", errCode); return errCode; @@ -449,17 +445,20 @@ int SQLiteSingleVerRelationalStorageExecutor::ExecuteSql(const SqlCondition &con LOGE("Execute sql failed when prepare stmt."); return errCode; } + if (condition.readOnly && !SQLiteUtils::IsStmtReadOnly(statement)) { + LOGW("[ExecuteSql] The condition is read-only, but SQL is not read-only"); + } size_t bindCount = static_cast(sqlite3_bind_parameter_count(statement)); if (bindCount > condition.bindArgs.size() || bindCount < condition.bindArgs.size()) { LOGE("Sql bind args mismatch."); SQLiteUtils::ResetStatement(statement, true, errCode); return -E_INVALID_ARGS; } + int ret = E_OK; for (size_t i = 0; i < condition.bindArgs.size(); i++) { Type type = condition.bindArgs[i]; errCode = SQLiteRelationalUtils::BindStatementByType(statement, i + 1, type); if (errCode != E_OK) { - int ret = E_OK; SQLiteUtils::ResetStatement(statement, true, ret); return errCode; } @@ -468,13 +467,11 @@ int SQLiteSingleVerRelationalStorageExecutor::ExecuteSql(const SqlCondition &con VBucket bucket; errCode = SQLiteRelationalUtils::GetSelectVBucket(statement, bucket); if (errCode != E_OK) { - int ret = E_OK; SQLiteUtils::ResetStatement(statement, true, ret); return errCode; } records.push_back(std::move(bucket)); } - int ret = E_OK; SQLiteUtils::ResetStatement(statement, true, ret); return errCode == -E_FINISHED ? (ret == E_OK ? E_OK : ret) : errCode; } @@ -512,7 +509,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetClearWaterMarkTables( return E_OK; } -int SQLiteSingleVerRelationalStorageExecutor::UpgradedLogForExistedData(TableInfo &tableInfo, bool schemaChanged) +int SQLiteSingleVerRelationalStorageExecutor::UpgradedLogForExistedData(const TableInfo &tableInfo, bool schemaChanged) { std::string logTable = DBCommon::GetLogTableName(tableInfo.GetTableName()); if (schemaChanged) { @@ -1047,7 +1044,8 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckIfExistUserTable(const std::s errCode = SQLiteUtils::BindTextToStatement(statement, 1, tableName); if (errCode != E_OK) { LOGE("[RDBExecutor] Bind table name failed: %d.", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return errCode; } if (SQLiteUtils::StepWithRetry(statement) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { @@ -1072,12 +1070,15 @@ int SQLiteSingleVerRelationalStorageExecutor::GetCloudDeleteSql(const std::strin if (isLogicDelete_) { // cursor already increased by DeleteCloudData, can be assigned directly here // 1001 which is logicDelete|cloudForcePush|local|delete - sql += "flag = flag&" + std::string(CONSISTENT_FLAG) + "|" + - std::to_string(static_cast(LogInfoFlag::FLAG_DELETE) | - static_cast(LogInfoFlag::FLAG_LOGIC_DELETE)) + ", cursor = " + std::to_string(cursor) + " "; + sql += "flag = (flag&" + std::string(CONSISTENT_FLAG) + "|" + + std::to_string(static_cast(LogInfoFlag::FLAG_DELETE) | + static_cast(LogInfoFlag::FLAG_LOGIC_DELETE)) + + ")&" + std::to_string(~static_cast(LogInfoFlag::FLAG_CLOUD_UPDATE_LOCAL)) + + ", cursor = " + std::to_string(cursor) + " "; } else { - sql += "data_key = -1, flag = flag&" + std::string(CONSISTENT_FLAG) + "|" + - std::to_string(static_cast(LogInfoFlag::FLAG_DELETE)) + ", sharing_resource = ''"; + sql += "data_key = -1, flag = (flag&" + std::string(CONSISTENT_FLAG) + "|" + + std::to_string(static_cast(LogInfoFlag::FLAG_DELETE)) + ")&" + + std::to_string(~static_cast(LogInfoFlag::FLAG_CLOUD_UPDATE_LOCAL)) + ", sharing_resource = ''"; errCode = SetCursor(table, cursor + 1); if (errCode == E_OK) { sql += ", cursor = " + std::to_string(cursor + 1) + " "; @@ -1141,10 +1142,10 @@ int SQLiteSingleVerRelationalStorageExecutor::BindStmtWithCloudGidInner(const st } int SQLiteSingleVerRelationalStorageExecutor::RenewTableTrigger(DistributedTableMode mode, - const TableInfo &tableInfo, TableSyncType syncType) + const TableInfo &tableInfo, TableSyncType syncType, const std::string &localIdentity) { auto tableManager = LogTableManagerFactory::GetTableManager(mode, syncType); - return tableManager->AddRelationalLogTableTrigger(dbHandle_, tableInfo, ""); + return tableManager->AddRelationalLogTableTrigger(dbHandle_, tableInfo, localIdentity); } int SQLiteSingleVerRelationalStorageExecutor::DoCleanAssetId(const std::string &tableName, @@ -1293,23 +1294,24 @@ int SQLiteSingleVerRelationalStorageExecutor::CleanAssetsIdOnUserTable(const std std::string cleanAssetIdSql = "UPDATE " + tableName + " SET " + fieldName + " = ? WHERE " + std::string(DBConstant::SQLITE_INNER_ROWID) + " = " + std::to_string(rowId) + ";"; sqlite3_stmt *stmt = nullptr; + int ret = E_OK; int errCode = SQLiteUtils::GetStatement(dbHandle_, cleanAssetIdSql, stmt); if (errCode != E_OK) { // LCOV_EXCL_BR_LINE LOGE("Get statement failed, %d", errCode); - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } errCode = SQLiteUtils::BindBlobToStatement(stmt, 1, assetsValue, false); if (errCode != E_OK) { // LCOV_EXCL_BR_LINE - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } errCode = SQLiteUtils::StepWithRetry(stmt); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { // LCOV_EXCL_BR_LINE errCode = E_OK; } - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } std::pair SQLiteSingleVerRelationalStorageExecutor::GetAssetsByGidOrHashKey( @@ -1375,18 +1377,19 @@ int SQLiteSingleVerRelationalStorageExecutor::InitGetAssetStmt(const std::string return errCode; } int index = 1; + int ret = E_OK; if (!gid.empty()) { errCode = SQLiteUtils::BindTextToStatement(stmt, index++, gid); if (errCode != E_OK) { LOGE("bind gid failed %d.", errCode); - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } } errCode = SQLiteUtils::BindBlobToStatement(stmt, index, hashKey); if (errCode != E_OK) { LOGE("bind hash failed %d.", errCode); - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); } return errCode; } @@ -1745,9 +1748,10 @@ int64_t SQLiteSingleVerRelationalStorageExecutor::GetDataFlag() std::string SQLiteSingleVerRelationalStorageExecutor::GetUpdateDataFlagSql(const VBucket &data) { - std::string retentionFlag = "flag = flag & " + - std::to_string(static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY) | - static_cast(LogInfoFlag::FLAG_ASSET_DOWNLOADING_FOR_ASYNC)); + std::string retentionFlag = "flag = (flag & " + + std::to_string(static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY) | + static_cast(LogInfoFlag::FLAG_ASSET_DOWNLOADING_FOR_ASYNC)) + + ") | " + std::to_string(static_cast(LogInfoFlag::FLAG_CLOUD_UPDATE_LOCAL)); if (putDataMode_ == PutDataMode::SYNC) { if (CloudStorageUtils::IsAssetsContainDownloadRecord(data)) { return retentionFlag; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_executor_utils.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_executor_utils.cpp index d4d26020..5569bfa9 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_executor_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_executor_utils.cpp @@ -61,6 +61,10 @@ int SqliteCloudKvExecutorUtils::GetCloudData(const CloudSyncConfig &config, cons Timestamp SqliteCloudKvExecutorUtils::GetMaxTimeStamp(const std::vector &dataExtend) { Timestamp maxTimeStamp = 0; + if (dataExtend.empty()) { + LOGE("[SqliteCloudKvExecutorUtils] [GetMaxTimeStamp] data extend is empty."); + return maxTimeStamp; + } VBucket lastRecord = dataExtend.back(); auto it = lastRecord.find(CloudDbConstant::MODIFY_FIELD); if (it != lastRecord.end() && maxTimeStamp < static_cast(std::get(it->second))) { @@ -180,7 +184,7 @@ void SqliteCloudKvExecutorUtils::GetCloudLog(sqlite3_stmt *stmt, VBucket &logInf void SqliteCloudKvExecutorUtils::GetCloudExtraLog(sqlite3_stmt *stmt, VBucket &flags) { - flags.insert_or_assign(CloudDbConstant::ROWID, + flags.insert_or_assign(DBConstant::ROWID, static_cast(sqlite3_column_int64(stmt, CLOUD_QUERY_ROW_ID_INDEX))); flags.insert_or_assign(CloudDbConstant::TIMESTAMP, static_cast(sqlite3_column_int64(stmt, CLOUD_QUERY_MODIFY_TIME_INDEX))); @@ -197,19 +201,23 @@ int SqliteCloudKvExecutorUtils::GetCloudKvData(sqlite3_stmt *stmt, VBucket &data { int errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_KEY, CLOUD_QUERY_KEY_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudKvData] Get key failed:%d", errCode); return errCode; } errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_VALUE, CLOUD_QUERY_VALUE_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudKvData] Get value failed:%d", errCode); return errCode; } errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_DEVICE, CLOUD_QUERY_DEV_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudKvData] Get device failed:%d", errCode); return errCode; } errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_ORI_DEVICE, CLOUD_QUERY_ORI_DEV_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudKvData] Get ori device failed:%d", errCode); return errCode; } data.insert_or_assign(CloudDbConstant::CLOUD_KV_FIELD_DEVICE_CREATE_TIME, @@ -233,6 +241,7 @@ int SqliteCloudKvExecutorUtils::GetCloudKvBlobData(const std::string &keyStr, in if (tmp.empty()) { errCode = RuntimeContext::GetInstance()->GetLocalIdentity(tmp); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] Get local identity failed %d", errCode); return errCode; } tmp = DBCommon::TransferHashString(tmp); @@ -252,6 +261,7 @@ std::pair SqliteCloudKvExecutorUtils::GetLogInfo(sqlite3 * std::string keyStr; errCode = CloudStorageUtils::GetValueFromVBucket(CloudDbConstant::CLOUD_KV_FIELD_KEY, cloudData, keyStr); if (errCode == -E_NOT_FOUND) { + LOGW("[SqliteCloudKvExecutorUtils] Get key not found."); errCode = E_OK; } if (errCode != E_OK) { @@ -491,7 +501,7 @@ int SqliteCloudKvExecutorUtils::OperateOtherUserLog(sqlite3 *db, bool isMemory, return errCode; } sqlite3_stmt *logStmt = nullptr; - std::string sql = "UPDATE naturalbase_kv_aux_sync_data_log SET cloud_flag = cloud_flag & ~0x2000" + std::string sql = "UPDATE naturalbase_kv_aux_sync_data_log SET cloud_flag = cloud_flag & ~0x2000 " "WHERE userid != ? AND hash_key = ?"; errCode = SQLiteUtils::GetStatement(db, sql, logStmt); if (errCode != E_OK) { @@ -1342,10 +1352,12 @@ int SqliteCloudKvExecutorUtils::GetCloudVersionRecordData(sqlite3_stmt *stmt, VB { int errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_KEY, CLOUD_QUERY_KEY_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudVersionRecordData] Get key failed:%d", errCode); return errCode; } errCode = GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_VALUE, CLOUD_QUERY_VALUE_INDEX, stmt, data, totalSize); if (errCode != E_OK) { + LOGE("[SqliteCloudKvExecutorUtils] [GetCloudVersionRecordData] Get value failed:%d", errCode); return errCode; } return GetCloudKvBlobData(CloudDbConstant::CLOUD_KV_FIELD_DEVICE, CLOUD_QUERY_DEV_INDEX, stmt, data, totalSize); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.cpp index 524f469b..25e473f2 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.cpp @@ -578,7 +578,8 @@ bool SqliteCloudKvStore::IsTagCloudUpdateLocal(const LogInfo &localInfo, const L // if local not delete and cloud is different user, insert data to local by timestamp if (localInfo.dataKey != -1 && (localInfo.flag & static_cast(LogInfoFlag::FLAG_LOCAL)) == 0 && (localInfo.cloud_flag & static_cast(LogInfoFlag::FLAG_LOGIN_USER)) == 0 && - localInfo.wTimestamp > cloudInfo.wTimestamp) { + (localInfo.flag & static_cast(LogInfoFlag::FLAG_CLOUD_WRITE)) == + static_cast(LogInfoFlag::FLAG_CLOUD_WRITE) && localInfo.wTimestamp > cloudInfo.wTimestamp) { return false; } std::string cloudInfoDev; @@ -751,4 +752,70 @@ int SqliteCloudKvStore::GetLocalDataCount(const std::string &tableName, int &dat storageHandle_->RecycleStorageExecutor(handle); return errCode; } + +int SqliteCloudKvStore::OperateDataStatus(uint32_t dataOperator) +{ + LOGI("[SqliteCloudKvStore] OperateDataStatus %" PRIu32, dataOperator); + if ((dataOperator & static_cast(DataOperator::UPDATE_TIME)) == 0 && + (dataOperator & static_cast(DataOperator::RESET_UPLOAD_CLOUD)) == 0) { + return E_OK; + } + + Timestamp currentRawTime = storageHandle_->GetCurrentTimestamp(); + TimeOffset timeOffset = storageHandle_->GetLocalTimeOffsetForCloud(); + Timestamp currentSysTime = static_cast(static_cast(currentRawTime) - timeOffset); + auto currentVirtualTime = std::to_string(currentRawTime); + auto currentTime = std::to_string(currentSysTime); + + auto [errCode, handle] = storageHandle_->GetStorageExecutor(true); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Get handle failed: %d", errCode); + return errCode; + } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Start transaction failed %d when operate data status", errCode); + storageHandle_->RecycleStorageExecutor(handle); + return errCode; + } + errCode = OperateDataStatusInner(handle, currentVirtualTime, currentTime, dataOperator); + if (errCode == E_OK) { + errCode = handle->Commit(); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Commit failed %d when operate data status", errCode); + } + } else { + int ret = handle->Rollback(); + if (ret != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Rollback failed %d when operate data status", ret); + } + } + storageHandle_->RecycleStorageExecutor(handle); + return errCode; +} + +int SqliteCloudKvStore::OperateDataStatusInner(SQLiteSingleVerStorageExecutor *handle, + const std::string ¤tVirtualTime, const std::string ¤tTime, uint32_t dataOperator) +{ + sqlite3 *db = nullptr; + int errCode = handle->GetDbHandle(db); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Get db failed %d when operate data status", errCode); + return errCode; + } + if ((dataOperator & static_cast(DataOperator::UPDATE_TIME)) != 0) { + errCode = SQLiteUtils::UpdateLocalDataModifyTime(db, currentVirtualTime, currentTime); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Update local data modify time failed: %d", errCode); + return errCode; + } + } + if ((dataOperator & static_cast(DataOperator::RESET_UPLOAD_CLOUD)) != 0) { + errCode = SQLiteUtils::UpdateLocalDataCloudFlag(db); + if (errCode != E_OK) { + LOGE("[SqliteCloudKvStore][OperateDataStatus] Update local data cloud flag failed: %d", errCode); + } + } + return errCode; +} } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.h index ba351dad..2f3c98b1 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_cloud_kv_store.h @@ -121,6 +121,8 @@ public: std::pair> GetDownloadAssetRecords(const std::string &tableName, int64_t beginTime) override; + + int OperateDataStatus(uint32_t dataOperator); private: std::pair GetTransactionDbHandleAndMemoryStatus(); @@ -132,6 +134,9 @@ private: int ReviseOneLocalModTime(sqlite3_stmt *stmt, const ReviseModTimeInfo &data, bool isMemory); + int OperateDataStatusInner(SQLiteSingleVerStorageExecutor *handle, const std::string ¤tVirtualTime, + const std::string ¤tTime, uint32_t dataOperator); + KvStorageHandle *storageHandle_; std::mutex schemaMutex_; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_local_storage_executor.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_local_storage_executor.cpp index 56c10e75..a356cd66 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_local_storage_executor.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_local_storage_executor.cpp @@ -68,7 +68,8 @@ int SQLiteLocalStorageExecutor::Get(const Key &key, Value &value) const errCode = SQLiteUtils::GetColumnBlobValue(statement, 0, value); END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -123,7 +124,8 @@ int SQLiteLocalStorageExecutor::GetEntries(const Key &keyPrefix, } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -216,7 +218,8 @@ int SQLiteLocalStorageExecutor::Put(const Key &key, const Value &value) } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -246,7 +249,8 @@ int SQLiteLocalStorageExecutor::Delete(const Key &key) } } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp index 811bc30c..edbd2681 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.cpp @@ -17,6 +17,10 @@ #include "sqlite_log_table_manager.h" namespace DistributedDB { +namespace { + const int MAX_FIELD_NUM_IN_ONE_STATEMENT = 100; +} + int SqliteLogTableManager::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &table, const std::string &identity) { @@ -167,6 +171,29 @@ std::string SqliteLogTableManager::GetUpdateTimestamp(const TableInfo &table, bo defaultNewTime ? "get_sys_time(0)" : "timestamp"); } +std::string SqliteLogTableManager::GetUpdateSqlWhenFieldExceedsLimit(const std::vector &syncFields, + const std::string &matchValue, const std::string &missMatchValue) +{ + std::string sql = " CASE"; + int index = 0; + int maxNum = syncFields.size(); + while (index < maxNum) { + sql.append(" WHEN ("); + int beginIndex = index; + for (int i = beginIndex; i < (beginIndex + MAX_FIELD_NUM_IN_ONE_STATEMENT) && i < maxNum; i++) { + auto field = syncFields[i]; + sql.append("(").append("OLD.'").append(field).append("'!= NEW.'").append(field).append("') OR"); + index++; + } + // pop last "OR" + sql.pop_back(); + sql.pop_back(); + sql.append(") THEN ").append(matchValue); + } + sql.append(" ELSE ").append(missMatchValue).append(" END"); + return sql; +} + std::string SqliteLogTableManager::GetUpdateWithAssignSql(const TableInfo &table, const std::string &emptyValue, const std::string &matchValue, const std::string &missMatchValue) { @@ -174,6 +201,9 @@ std::string SqliteLogTableManager::GetUpdateWithAssignSql(const TableInfo &table if (syncFields.empty() || table.GetFields().size() <= syncFields.size()) { return emptyValue; } + if (syncFields.size() > MAX_FIELD_NUM_IN_ONE_STATEMENT) { + return GetUpdateSqlWhenFieldExceedsLimit(syncFields, matchValue, missMatchValue); + } std::string sql = " CASE WHEN ("; for (const auto &field : syncFields) { sql.append("(").append("OLD.'").append(field).append("'!= NEW.'").append(field).append("') OR"); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h index 41ac1ce7..67e5f54c 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_log_table_manager.h @@ -60,6 +60,9 @@ private: static int UpgradeKvSyncLogTable(const std::string &tableName, sqlite3 *db); static int CreateKvCloudFlagIndex(const std::string &tableName, sqlite3 *db); + + static std::string GetUpdateSqlWhenFieldExceedsLimit(const std::vector &syncFields, + const std::string &matchValue, const std::string &missMatchValue); }; } #endif // SQLITE_LOG_TABLE_MANAGER_H \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_meta_executor.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_meta_executor.cpp index fdc929b8..e9f2e26b 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_meta_executor.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_meta_executor.cpp @@ -44,16 +44,17 @@ int SqliteMetaExecutor::GetMetaKeysByKeyPrefix(const std::string &keyPre, sqlite Key keyPrefix; DBCommon::StringToVector(keyPre + '%', keyPrefix); + int ret = E_OK; errCode = SQLiteUtils::BindBlobToStatement(statement, 1, keyPrefix); // 1: bind index for prefix key if (errCode != E_OK) { LOGE("[SqliteMetaExecutor] Bind statement failed:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); + SQLiteUtils::ResetStatement(statement, true, ret); return errCode; } std::vector keys; errCode = GetAllKeys(statement, isMemDb, keys); - SQLiteUtils::ResetStatement(statement, true, errCode); + SQLiteUtils::ResetStatement(statement, true, ret); for (const auto &it : keys) { if (it.size() >= keyPre.size() + DBConstant::HASH_KEY_SIZE) { outKeys.insert({it.begin() + keyPre.size(), it.begin() + keyPre.size() + DBConstant::HASH_KEY_SIZE}); @@ -61,7 +62,7 @@ int SqliteMetaExecutor::GetMetaKeysByKeyPrefix(const std::string &keyPre, sqlite LOGW("[SqliteMetaExecutor] Get invalid key, size=%zu", it.size()); } } - return errCode; + return errCode != E_OK ? errCode : ret; } int SqliteMetaExecutor::GetAllKeys(sqlite3_stmt *statement, bool isMemDb, std::vector &keys) diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_multi_ver_transaction.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_multi_ver_transaction.cpp index 243639b6..c78126e7 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_multi_ver_transaction.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_multi_ver_transaction.cpp @@ -253,8 +253,9 @@ int SQLiteMultiVerTransaction::Get(const Key &key, Value &value) const } errCode = GetKeyAndValueByHashKey(statement, hashKey, readKey, value, false); END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteMultiVerTransaction::GetValueForTrimSlice(const Key &hashKey, const Version version, Value &value) const @@ -294,8 +295,9 @@ int SQLiteMultiVerTransaction::GetValueForTrimSlice(const Key &hashKey, const Ve } END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteMultiVerTransaction::GetEntries(const Key &keyPrefix, std::vector &entries) const @@ -318,7 +320,7 @@ int SQLiteMultiVerTransaction::GetEntries(const Key &keyPrefix, std::vector(); + if (naturalStore == nullptr) { + LOGE("[SingleVerConnection] DB is null when operate data status"); + return -E_INVALID_DB; + } + return naturalStore->OperateDataStatus(dataOperator); +} DEFINE_OBJECT_TAG_FACILITIES(SQLiteSingleVerNaturalStoreConnection) } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index e1755419..1b6b3f03 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -124,6 +124,7 @@ public: int SetCloudSyncConfig(const CloudSyncConfig &config) override; #endif + int OperateDataStatus(uint32_t dataOperator) override; private: int CheckMonoStatus(OperatePerm perm); diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_extend.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_extend.cpp index 3658730b..ba52b0e0 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_extend.cpp @@ -686,4 +686,14 @@ int SQLiteSingleVerNaturalStore::SetCloudSyncConfig(const CloudSyncConfig &confi sqliteCloudKvStore_->SetCloudSyncConfig(config); return E_OK; } + +int SQLiteSingleVerNaturalStore::OperateDataStatus(uint32_t dataOperator) +{ + std::lock_guard autoLock(cloudStoreMutex_); + if (sqliteCloudKvStore_ == nullptr) { + LOGE("[SingleVerNStore] DB is null when operate data status"); + return -E_INTERNAL_ERROR; + } + return sqliteCloudKvStore_->OperateDataStatus(dataOperator); +} } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index c34d3896..01f68d75 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -151,6 +151,10 @@ int SQLiteSingleVerStorageEngine::MigrateSyncDataByVersion(SQLiteSingleVerStorag std::vector dataItems; uint64_t minVerIncurCacheDb = 0; + if (handle == nullptr) { + LOGE("[MigrateSyncDataByVersion] handle is nullptr."); + return -E_INVALID_DB; + } int errCode = handle->GetMinVersionCacheData(dataItems, minVerIncurCacheDb); if (errCode != E_OK) { LOGE("[MigrateSyncDataByVersion]Fail to get cur data in cache! err[%d]", errCode); @@ -295,6 +299,10 @@ int SQLiteSingleVerStorageEngine::AttachMainDbAndCacheDb(SQLiteSingleVerStorageE // Judge the file corresponding to db by the engine status and attach it to another file int errCode = E_OK; std::string attachAbsPath; + if (handle == nullptr) { + LOGE("[AttachMainDbAndCacheDb] handle is nullptr."); + return -E_INVALID_DB; + } if (stateBeforeMigrate == EngineState::MAINDB) { attachAbsPath = GetDbDir(option_.subdir, DbType::CACHE) + "/" + DBConstant::SINGLE_VER_CACHE_STORE + DBConstant::DB_EXTENSION; diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 376167a8..a99a3c62 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -151,7 +151,8 @@ int SQLiteSingleVerStorageExecutor::GetKvData(SingleVerDataType type, const Key } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -241,7 +242,8 @@ int SQLiteSingleVerStorageExecutor::GetKvDataByHashKey(const Key &hashKey, Singl } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -272,7 +274,8 @@ int SQLiteSingleVerStorageExecutor::SaveKvData(SingleVerDataType type, const Key } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -336,7 +339,8 @@ int SQLiteSingleVerStorageExecutor::GetEntries(bool isGetValue, SingleVerDataTyp errCode = StepForResultEntries(isGetValue, statement, entries); END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -354,7 +358,8 @@ int SQLiteSingleVerStorageExecutor::GetEntries(QueryObject &queryObj, std::vecto errCode = StepForResultEntries(true, statement, entries); } - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -406,7 +411,8 @@ int SQLiteSingleVerStorageExecutor::GetCount(QueryObject &queryObj, int &count) } END: - SQLiteUtils::ResetStatement(countStatement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStatement, true, ret); return CheckCorruptedStatus(errCode); } @@ -454,7 +460,8 @@ int SQLiteSingleVerStorageExecutor::PrepareForSyncDataByTime(Timestamp begin, Ti ERROR: if (errCode != E_OK) { LOGE("Bind the timestamp for getting sync data error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); } return CheckCorruptedStatus(errCode); @@ -544,7 +551,8 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector } errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -558,7 +566,8 @@ int SQLiteSingleVerStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &tim } return E_OK; // do not release statement when success ERR: - SQLiteUtils::ResetStatement(stmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); return errCode; } @@ -646,8 +656,9 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataWithQuery(const QueryObject &quer LOGE("Get sync data with query failed. %d", errCode); } END: - SQLiteUtils::ResetStatement(fullStmt, true, errCode); - SQLiteUtils::ResetStatement(queryStmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(fullStmt, true, ret); + SQLiteUtils::ResetStatement(queryStmt, true, ret); return CheckCorruptedStatus(errCode); } @@ -729,7 +740,8 @@ int SQLiteSingleVerStorageExecutor::OpenResultSet(const Key &keyPrefix, int &cou } END: - SQLiteUtils::ResetStatement(countStatement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStatement, true, ret); if (errCode != E_OK) { CloseResultSet(); } @@ -781,7 +793,8 @@ int SQLiteSingleVerStorageExecutor::OpenResultSet(QueryObject &queryObj, int &co } END: - SQLiteUtils::ResetStatement(countStatement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStatement, true, ret); if (errCode != E_OK) { CloseResultSet(); } @@ -800,14 +813,15 @@ int SQLiteSingleVerStorageExecutor::OpenResultSetForCacheRowIdMode(const Key &ke return CheckCorruptedStatus(errCode); } errCode = SQLiteUtils::BindPrefixKey(getResultRowIdStatement_, 1, keyPrefix); // first argument index is 1 + int ret = E_OK; if (errCode != E_OK) { LOGE("[SqlSinExe][OpenResSetRowId][PrefixKey] Bind rowid stmt fail, errCode=%d", errCode); - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); return CheckCorruptedStatus(errCode); } errCode = OpenResultSetForCacheRowIdModeCommon(rowIdCache, cacheLimit, count); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); } return errCode; } @@ -831,15 +845,16 @@ int SQLiteSingleVerStorageExecutor::OpenResultSetForCacheRowIdMode(QueryObject & } errCode = helper.GetQuerySqlStatement(dbHandle_, true, getResultRowIdStatement_); + int ret = E_OK; if (errCode != E_OK) { LOGE("[SqlSinExe][OpenResSetRowId][Query] Get Stmt fail, errCode=%d", errCode); // The GetQuerySqlStatement does not self rollback(BAD...), so we have to reset the stmt here. - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); return errCode; } errCode = OpenResultSetForCacheRowIdModeCommon(rowIdCache, cacheLimit, count); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); } return errCode; } @@ -1108,7 +1123,8 @@ int SQLiteSingleVerStorageExecutor::GetDeviceIdentifier(PragmaEntryDeviceIdentif } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -1396,8 +1412,9 @@ int SQLiteSingleVerStorageExecutor::GetAllMetaKeys(std::vector &keys) const } errCode = SqliteMetaExecutor::GetAllKeys(statement, isMemDb_, keys); - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::GetAllSyncedEntries(const std::string &hashDev, @@ -1433,8 +1450,9 @@ int SQLiteSingleVerStorageExecutor::GetAllSyncedEntries(const std::string &hashD errCode = GetAllEntries(statement, entries); END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::GetAllEntries(sqlite3_stmt *statement, std::vector &entries) const @@ -1659,7 +1677,8 @@ int SQLiteSingleVerStorageExecutor::DeleteLocalDataInner(SingleVerNaturalStoreCo } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -1698,7 +1717,8 @@ int SQLiteSingleVerStorageExecutor::EraseSyncData(const Key &hashKey) LOGE("erase data failed:%d", errCode); } END: - SQLiteUtils::ResetStatement(stmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); return CheckCorruptedStatus(errCode); } @@ -1827,9 +1847,10 @@ int SQLiteSingleVerStorageExecutor::InitResultSet(const Key &keyPrefix, sqlite3_ return E_OK; ERROR: - SQLiteUtils::ResetStatement(countStmt, true, errCode); - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); - SQLiteUtils::ResetStatement(getResultEntryStatement_, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStmt, true, ret); + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); + SQLiteUtils::ResetStatement(getResultEntryStatement_, true, ret); return CheckCorruptedStatus(errCode); } @@ -1848,7 +1869,8 @@ int SQLiteSingleVerStorageExecutor::InitResultSetCount(QueryObject &queryObj, sq errCode = helper.GetCountSqlStatement(dbHandle_, countStmt); if (errCode != E_OK) { LOGE("Get count bind statement error:%d", errCode); - SQLiteUtils::ResetStatement(countStmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStmt, true, ret); } return errCode; } @@ -1865,7 +1887,8 @@ int SQLiteSingleVerStorageExecutor::InitResultSetContent(QueryObject &queryObj) errCode = helper.GetQuerySqlStatement(dbHandle_, true, getResultRowIdStatement_); if (errCode != E_OK) { LOGE("[SqlSinExe][InitResSetContent] Bind result set rowid statement of query error:%d", errCode); - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, ret); return errCode; } errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_SYNC_DATA_BY_ROWID_SQL, getResultEntryStatement_); @@ -2244,49 +2267,4 @@ bool SQLiteSingleVerStorageExecutor::IsPrintTimestamp() return false; } } - -int SQLiteSingleVerStorageExecutor::BindSyncDataTime(sqlite3_stmt *statement, const DataItem &dataItem, bool isUpdate) -{ - int errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_SYNC_STAMP_INDEX, dataItem.timestamp); - if (errCode != E_OK) { - LOGE("Bind saved sync data stamp failed:%d", errCode); - return errCode; - } - - const int writeTimeIndex = isUpdate ? BIND_SYNC_UPDATE_W_TIME_INDEX : BIND_SYNC_W_TIME_INDEX; - errCode = SQLiteUtils::BindInt64ToStatement(statement, writeTimeIndex, dataItem.writeTimestamp); - if (errCode != E_OK) { - LOGE("Bind saved sync data write stamp failed:%d", errCode); - return errCode; - } - - const int modifyTimeIndex = isUpdate ? BIND_SYNC_UPDATE_MODIFY_TIME_INDEX : BIND_SYNC_MODIFY_TIME_INDEX; - errCode = SQLiteUtils::BindInt64ToStatement(statement, modifyTimeIndex, dataItem.modifyTime); - if (errCode != E_OK) { - LOGE("Bind saved sync data modify time failed:%d", errCode); - return errCode; - } - - const int createTimeIndex = isUpdate ? BIND_SYNC_UPDATE_CREATE_TIME_INDEX : BIND_SYNC_CREATE_TIME_INDEX; - errCode = SQLiteUtils::BindInt64ToStatement(statement, createTimeIndex, dataItem.createTime); - if (errCode != E_OK) { - LOGE("Bind saved sync data create time failed:%d", errCode); - return errCode; - } - - if (IsPrintTimestamp()) { - LOGI("Write timestamp:%" PRIu64 " timestamp:%" PRIu64 ", flag:%" PRIu64 " modifyTime:%" PRIu64 " createTime:%" - PRIu64, dataItem.writeTimestamp, dataItem.timestamp, dataItem.flag, dataItem.modifyTime, - dataItem.createTime); - } - return errCode; -} - -int SQLiteSingleVerStorageExecutor::CreateCloudLogTable() -{ - if (dbHandle_ == nullptr) { - return -E_INVALID_DB; - } - return SqliteLogTableManager::CreateKvSyncLogTable(dbHandle_); -} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index 5c28b663..bd2c272f 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -119,7 +119,8 @@ int SQLiteSingleVerStorageExecutor::RemoveDeviceDataInCacheMode(const std::strin } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -147,7 +148,8 @@ int SQLiteSingleVerStorageExecutor::GetMinVersionCacheData( LOGE("Failed to get all the data items by the min version:[%d]", errCode); } - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -187,7 +189,8 @@ int SQLiteSingleVerStorageExecutor::MigrateRmDevData(const DataItem &dataItem) c errCode = E_OK; } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -250,7 +253,8 @@ int SQLiteSingleVerStorageExecutor::GetMaxVersionInCacheDb(uint64_t &maxVersion) maxVersion = 0; errCode = E_OK; } - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -303,6 +307,7 @@ int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(std::vector &da { int errCode = E_OK; sqlite3_stmt *stmt = nullptr; + int ret = E_OK; for (auto &item : dataItems) { if ((item.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == 0) { continue; @@ -328,9 +333,9 @@ int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(std::vector &da LOGE("Check miss query data item failed. %d", errCode); break; } - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); } - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); return CheckCorruptedStatus(errCode); } @@ -453,7 +458,8 @@ int SQLiteSingleVerStorageExecutor::DelCacheDbDataByVersion(uint64_t version) co } END: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -622,8 +628,9 @@ int SQLiteSingleVerStorageExecutor::GetExpandedCheckSql(QueryObject query, DataI } DBCommon::StringToVector(expandedSql, dataItem.value); END: - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::SaveSyncDataItemInCacheMode(DataItem &dataItem, const DeviceInfo &deviceInfo, @@ -701,7 +708,8 @@ int SQLiteSingleVerStorageExecutor::PutLocalDataToCacheDB(const LocalDataItem &d } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -801,8 +809,9 @@ int SQLiteSingleVerStorageExecutor::GetMinTimestampInCacheDB(Timestamp &minStamp } ERROR: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::InitMigrateTimestampOffset() @@ -946,6 +955,7 @@ int SQLiteSingleVerStorageExecutor::DeleteMetaData(const std::vector &keys) return errCode; } + int ret = E_OK; for (const auto &key : keys) { errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // first arg. if (errCode != E_OK) { @@ -957,10 +967,10 @@ int SQLiteSingleVerStorageExecutor::DeleteMetaData(const std::vector &keys) break; } errCode = E_OK; - SQLiteUtils::ResetStatement(statement, false, errCode); + SQLiteUtils::ResetStatement(statement, false, ret); } - SQLiteUtils::ResetStatement(statement, true, errCode); + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } @@ -983,7 +993,8 @@ int SQLiteSingleVerStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPref } } - SQLiteUtils::ResetStatement(statement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_extend.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_extend.cpp index ead11db6..3ee2eb31 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_extend.cpp @@ -36,6 +36,51 @@ namespace { constexpr const char *HWM_HEAD = "naturalbase_cloud_meta_sync_data_"; } +int SQLiteSingleVerStorageExecutor::BindSyncDataTime(sqlite3_stmt *statement, const DataItem &dataItem, bool isUpdate) +{ + int errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_SYNC_STAMP_INDEX, dataItem.timestamp); + if (errCode != E_OK) { + LOGE("Bind saved sync data stamp failed:%d", errCode); + return errCode; + } + + const int writeTimeIndex = isUpdate ? BIND_SYNC_UPDATE_W_TIME_INDEX : BIND_SYNC_W_TIME_INDEX; + errCode = SQLiteUtils::BindInt64ToStatement(statement, writeTimeIndex, dataItem.writeTimestamp); + if (errCode != E_OK) { + LOGE("Bind saved sync data write stamp failed:%d", errCode); + return errCode; + } + + const int modifyTimeIndex = isUpdate ? BIND_SYNC_UPDATE_MODIFY_TIME_INDEX : BIND_SYNC_MODIFY_TIME_INDEX; + errCode = SQLiteUtils::BindInt64ToStatement(statement, modifyTimeIndex, dataItem.modifyTime); + if (errCode != E_OK) { + LOGE("Bind saved sync data modify time failed:%d", errCode); + return errCode; + } + + const int createTimeIndex = isUpdate ? BIND_SYNC_UPDATE_CREATE_TIME_INDEX : BIND_SYNC_CREATE_TIME_INDEX; + errCode = SQLiteUtils::BindInt64ToStatement(statement, createTimeIndex, dataItem.createTime); + if (errCode != E_OK) { + LOGE("Bind saved sync data create time failed:%d", errCode); + return errCode; + } + + if (IsPrintTimestamp()) { + LOGI("Write timestamp:%" PRIu64 " timestamp:%" PRIu64 ", flag:%" PRIu64 " modifyTime:%" PRIu64 " createTime:%" + PRIu64 ", key size:%" PRIu32 ", value size:%" PRIu32, dataItem.writeTimestamp, dataItem.timestamp, + dataItem.flag, dataItem.modifyTime, dataItem.createTime, dataItem.key.size(), dataItem.value.size()); + } + return errCode; +} + +int SQLiteSingleVerStorageExecutor::CreateCloudLogTable() +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + return SqliteLogTableManager::CreateKvSyncLogTable(dbHandle_); +} + int SQLiteSingleVerStorageExecutor::CloudExcuteRemoveOrUpdate(const std::string &sql, const std::string &deviceName, const std::string &user, bool isUserBlobType) { @@ -292,11 +337,14 @@ int SQLiteSingleVerStorageExecutor::GetEntries(const std::string &device, std::v DBCommon::StringToVector(hashDev, blobDev); errCode = SQLiteUtils::BindBlobToStatement(stmt, BIND_GET_ENTRIES_DEVICE_INDEX, blobDev); if (errCode != E_OK) { - LOGE("[SQLiteSingleVerStorageExecutor] Bind hash device to statement failed:%d", errCode); + LOGE("[SQLiteSingleVerStorageExecutor] Bind hash device[%s] to statement failed:%d", + DBCommon::TransferStringToHex(hashDev).c_str(), errCode); return errCode; } errCode = StepForResultEntries(true, stmt, entries); if (errCode != E_OK) { + LOGE("[SQLiteSingleVerStorageExecutor] Get device[%s] entries failed:%d", + DBCommon::TransferStringToHex(hashDev).c_str(), errCode); return errCode; } LOGD("[SQLiteSingleVerStorageExecutor] Get %zu entries by device", entries.size()); @@ -319,12 +367,18 @@ int SQLiteSingleVerStorageExecutor::PrepareForUnSyncTotalByTime(Timestamp begin, errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); if (errCode != E_OK) { - LOGE("Bind the timestamp for getting sync num error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("Bind the begin timestamp for getting sync num error:%d", errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); return CheckCorruptedStatus(errCode); } errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); + if (errCode != E_OK) { + LOGE("Bind the end timestamp for getting sync num error:%d", errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + } return CheckCorruptedStatus(errCode); } @@ -345,7 +399,8 @@ int SQLiteSingleVerStorageExecutor::GetCountValue(sqlite3_stmt *&countStatement, } else { errCode = -E_UNEXPECTED_DATA; } - SQLiteUtils::ResetStatement(countStatement, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(countStatement, true, ret); return CheckCorruptedStatus(errCode); } diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp index 7f2d8356..af2d7635 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp @@ -75,6 +75,7 @@ int SQLiteSingleVerStorageExecutor::CheckMissQueryDataItems(sqlite3_stmt *&stmt, const DeviceInfo &deviceInfo, std::vector &dataItems) { int errCode = E_OK; + int ret = E_OK; for (auto &item : dataItems) { if ((item.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) != 0 && !item.key.empty()) { errCode = helper.BindSyncDataCheckStmt(stmt, item.key); @@ -87,10 +88,10 @@ int SQLiteSingleVerStorageExecutor::CheckMissQueryDataItems(sqlite3_stmt *&stmt, LOGE("Check miss query data item failed. %d", errCode); return errCode; } - SQLiteUtils::ResetStatement(stmt, false, errCode); + SQLiteUtils::ResetStatement(stmt, false, ret); } } - return errCode; + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(QueryObject query, std::vector &dataItems, @@ -122,7 +123,8 @@ int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(QueryObject query, std::v if (errCode != E_OK) { LOGE("check data with query failed. %d", errCode); } - SQLiteUtils::ResetStatement(stmt, true, errCode); + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); return CheckCorruptedStatus(errCode); } @@ -228,6 +230,7 @@ int SQLiteSingleVerStorageExecutor::RemoveSubscribeTriggerWaterMark(const std::v LOGE("Get remove trigger water mark statement failed. %d", errCode); return errCode; } + int ret = E_OK; for (const auto &id : subscribeIds) { errCode = SQLiteUtils::BindTextToStatement(statement, 1, DBConstant::SUBSCRIBE_QUERY_PREFIX + id); if (errCode != E_OK) { @@ -241,10 +244,10 @@ int SQLiteSingleVerStorageExecutor::RemoveSubscribeTriggerWaterMark(const std::v LOGE("Remove trigger water mark failed. %d", errCode); break; } - SQLiteUtils::ResetStatement(statement, false, errCode); + SQLiteUtils::ResetStatement(statement, false, ret); } - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteSingleVerStorageExecutor::GetTriggers(const std::string &namePreFix, std::vector &triggerNames) @@ -256,9 +259,10 @@ int SQLiteSingleVerStorageExecutor::GetTriggers(const std::string &namePreFix, s return errCode; } + int ret = E_OK; errCode = SQLiteUtils::BindTextToStatement(stmt, 1, namePreFix + "%"); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(stmt, true, errCode); + SQLiteUtils::ResetStatement(stmt, true, ret); LOGE("Bind trigger name prefix to statement failed. %d", errCode); return errCode; } @@ -278,7 +282,7 @@ int SQLiteSingleVerStorageExecutor::GetTriggers(const std::string &namePreFix, s } } while (true); - SQLiteUtils::ResetStatement(stmt, true, errCode); - return errCode; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode != E_OK ? errCode : ret; } } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 04cb1941..f07a061e 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "sqlite_import.h" #include "securec.h" @@ -128,8 +129,7 @@ void SQLiteUtils::SqliteLogCallback(void *data, int err, const char *msg) if (verboseLog) { LOGD("[SQLite] Error[%d] sys[%d] %s ", err, errno, sqlite3_errstr(err)); } - } else if (errType == SQLITE_WARNING || errType == SQLITE_IOERR || - errType == SQLITE_CORRUPT || errType == SQLITE_CANTOPEN) { + } else if (errType == SQLITE_WARNING || errType == SQLITE_IOERR || errType == SQLITE_CANTOPEN) { LOGI("[SQLite] Error[%d], sys[%d], %s, msg: %s ", err, errno, sqlite3_errstr(err), SQLiteUtils::Anonymous(logMsg).c_str()); } else { @@ -179,6 +179,9 @@ END: (void)sqlite3_close_v2(dbTemp); dbTemp = nullptr; } + struct stat curStat; + stat(fileUrl.c_str(), &curStat); + LOGI("[SQLite] open database result: %d, inode: %llu", errCode, curStat.st_ino); return errCode; } @@ -311,6 +314,11 @@ int SQLiteUtils::BindBlobToStatement(sqlite3_stmt *statement, int index, const s } void SQLiteUtils::ResetStatement(sqlite3_stmt *&statement, bool isNeedFinalize, int &errCode) +{ + ResetStatement(statement, isNeedFinalize, false, errCode); +} + +void SQLiteUtils::ResetStatement(sqlite3_stmt *&statement, bool isNeedFinalize, bool isIgnoreResetRet, int &errCode) { if (statement == nullptr) { return; @@ -321,7 +329,7 @@ void SQLiteUtils::ResetStatement(sqlite3_stmt *&statement, bool isNeedFinalize, if (!isNeedFinalize) { // reset the statement firstly. innerCode = sqlite3_reset(statement); - if (innerCode != SQLITE_OK) { + if (innerCode != SQLITE_OK && !isIgnoreResetRet) { LOGE("[SQLiteUtils] reset statement error:%d, sys:%d", innerCode, errno); isNeedFinalize = true; } else { @@ -629,8 +637,9 @@ int SQLiteUtils::AttachNewDatabaseInner(sqlite3 *db, CipherType type, const Ciph } END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int SQLiteUtils::CreateMetaDatabase(const std::string &metaDbPath) @@ -700,8 +709,9 @@ int SQLiteUtils::CheckIntegrity(sqlite3 *db, const std::string &sql) } else { errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; } - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } #ifdef RELATIONAL_STORE @@ -719,9 +729,10 @@ int AnalysisSchemaSqlAndTrigger(sqlite3 *db, const std::string &tableName, Table return errCode; } errCode = SQLiteUtils::BindTextToStatement(statement, 1, tableName); + int ret = E_OK; if (errCode != E_OK) { LOGE("[AnalysisSchema] Bind table name failed:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); + SQLiteUtils::ResetStatement(statement, true, ret); return errCode; } @@ -745,8 +756,8 @@ int AnalysisSchemaSqlAndTrigger(sqlite3 *db, const std::string &tableName, Table break; } } while (true); - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + SQLiteUtils::ResetStatement(statement, true, ret); + return errCode != E_OK ? errCode : ret; } int GetSchemaIndexList(sqlite3 *db, const std::string &tableName, std::vector &indexList, @@ -779,8 +790,9 @@ int GetSchemaIndexList(sqlite3 *db, const std::string &tableName, std::vector(isReadOnly); +} + +int SQLiteUtils::UpdateLocalDataModifyTime(sqlite3 *db, const std::string &virtualTime, const std::string &modifyTime) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + bool isCreate = false; + std::string tableName = DBConstant::KV_SYNC_TABLE_NAME; + auto errCode = SQLiteUtils::CheckTableExists(db, tableName, isCreate); + if (errCode != E_OK) { + LOGE("[SQLiteUtils] Check table exist failed %d when update modify time", errCode); + return errCode; + } + if (!isCreate) { + LOGW("[SQLiteUtils] non exist table when update time"); + return E_OK; + } + std::string updateTimeSql = "UPDATE " + tableName + " SET timestamp = _rowid_ + " + virtualTime + + ", w_timestamp = _rowid_ + " + virtualTime + ", modify_time = _rowid_ + " + modifyTime + + " WHERE flag & 0x02 != 0;"; + errCode = SQLiteUtils::ExecuteRawSQL(db, updateTimeSql); + if (errCode != E_OK) { + LOGE("[SQLiteUtils] Update modify time failed %d", errCode); + } + return errCode; +} + +int SQLiteUtils::UpdateLocalDataCloudFlag(sqlite3 *db) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + bool isCreate = false; + std::string logTableName = "naturalbase_kv_aux_sync_data_log"; + auto errCode = SQLiteUtils::CheckTableExists(db, logTableName, isCreate); + if (errCode != E_OK) { + LOGE("[SQLiteUtils] Check log table exist failed %d when update cloud flag", errCode); + return errCode; + } + if (!isCreate) { + LOGW("[SQLiteUtils] non exist log table when update cloud flag"); + return E_OK; + } + std::string tableName = DBConstant::KV_SYNC_TABLE_NAME; + errCode = SQLiteUtils::CheckTableExists(db, tableName, isCreate); + if (errCode != E_OK) { + LOGE("[SQLiteUtils] Check table exist failed %d when update cloud flag", errCode); + return errCode; + } + if (!isCreate) { + LOGW("[SQLiteUtils] non exist table when update cloud flag"); + return E_OK; + } + std::string updateCloudFlagSql = "UPDATE " + logTableName + " SET cloud_flag = cloud_flag & ~0x400 " + "WHERE hash_key IN (SELECT hash_key FROM " + tableName + + " WHERE flag & 0x02 != 0);"; + errCode = SQLiteUtils::ExecuteRawSQL(db, updateCloudFlagSql); + if (errCode != E_OK) { + LOGE("[SQLiteUtils] Update cloud flag failed %d", errCode); + } + return errCode; +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/storage/src/storage_proxy.cpp b/kv_store/frameworks/libs/distributeddb/storage/src/storage_proxy.cpp index b338aab4..0899eafe 100644 --- a/kv_store/frameworks/libs/distributeddb/storage/src/storage_proxy.cpp +++ b/kv_store/frameworks/libs/distributeddb/storage/src/storage_proxy.cpp @@ -381,6 +381,19 @@ int StorageProxy::CleanCloudData(ClearMode mode, const std::vector return store_->CleanCloudData(mode, tableNameList, localSchema, assets); } +int StorageProxy::ClearCloudLogVersion(const std::vector &tableNameList) +{ + std::shared_lock readLock(storeMutex_); + if (store_ == nullptr) { + return -E_INVALID_DB; + } + if (!transactionExeFlag_.load()) { + LOGE("[StorageProxy][ClearCloudLogVersion] the transaction has not been started"); + return -E_TRANSACT_STATE; + } + return store_->ClearCloudLogVersion(tableNameList); +} + int StorageProxy::ReleaseContinueToken(ContinueToken &continueStmtToken) { return store_->ReleaseCloudDataToken(continueStmtToken); @@ -441,7 +454,7 @@ int StorageProxy::GetPrimaryColNamesWithAssetsFields(const TableName &tableName, } } if (colNames.empty() || colNames.size() > 1) { - (void)colNames.insert(colNames.begin(), CloudDbConstant::ROW_ID_FIELD_NAME); + (void)colNames.insert(colNames.begin(), DBConstant::ROWID); } return E_OK; } @@ -850,30 +863,13 @@ int StorageProxy::GetLockStatusByGid(const std::string &tableName, const std::st return store_->GetLockStatusByGid(tableName, gid, status); } -bool StorageProxy::IsContainAssetsTable() +bool StorageProxy::IsExistTableContainAssets() { std::shared_lock readLock(storeMutex_); if (store_ == nullptr) { LOGE("store is nullptr when check contain assets table"); return false; } - std::shared_ptr cloudSchema = nullptr; - int errCode = store_->GetCloudDbSchema(cloudSchema); - if (errCode != E_OK) { - LOGE("Cannot get cloud schema: %d when check contain assets table", errCode); - return false; - } - if (cloudSchema == nullptr) { - LOGE("Not set cloud schema when check contain assets table"); - return false; - } - for (const auto &table : cloudSchema->tables) { - for (const auto &field : table.fields) { - if (field.type == TYPE_INDEX || field.type == TYPE_INDEX) { - return true; - } - } - } - return false; + return store_->IsExistTableContainAssets(); } } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_merge_strategy.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_merge_strategy.cpp index 526eebaa..b487c62b 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_merge_strategy.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_merge_strategy.cpp @@ -83,8 +83,9 @@ OpType CloudMergeStrategy::TagLocallyNewer(const LogInfo &localInfo, const LogIn OpType CloudMergeStrategy::TagLoginUserAndUpdate(const LogInfo &localInfo, const LogInfo &cloudInfo) { - if (JudgeKvScene() && (localInfo.cloud_flag & static_cast(LogInfoFlag::FLAG_LOGIN_USER)) == 0 - && localInfo.wTimestamp > cloudInfo.wTimestamp) { + if (JudgeKvScene() && (localInfo.flag & static_cast(LogInfoFlag::FLAG_LOCAL)) == 0 && + (localInfo.cloud_flag & static_cast(LogInfoFlag::FLAG_LOGIN_USER)) == 0 + && localInfo.timestamp > cloudInfo.timestamp) { return OpType::NOT_HANDLE; } return TagUpdateLocal(cloudInfo, localInfo); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.cpp index 46878a2a..f6bcc943 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.cpp @@ -33,7 +33,7 @@ int CloudSyncUtils::GetCloudPkVals(const VBucket &datum, const std::vector &pkColNames) { - return pkColNames.size() == 1 && pkColNames[0] != CloudDbConstant::ROW_ID_FIELD_NAME; + return pkColNames.size() == 1 && pkColNames[0] != DBConstant::ROWID; } void CloudSyncUtils::RemoveDataExceptExtendInfo(VBucket &datum, const std::vector &pkColNames) @@ -153,7 +153,8 @@ bool CloudSyncUtils::NeedSaveData(const LogInfo &localLogInfo, const LogInfo &cl localLogInfo.cloudGid == cloudLogInfo.cloudGid && localLogInfo.sharingResource == cloudLogInfo.sharingResource && localLogInfo.version == cloudLogInfo.version && - (localLogInfo.flag & static_cast(LogInfoFlag::FLAG_WAIT_COMPENSATED_SYNC)) == 0; + (localLogInfo.flag & static_cast(LogInfoFlag::FLAG_WAIT_COMPENSATED_SYNC)) == 0 && + !localLogInfo.isNeedUpdateAsset; return !isSame; } @@ -729,7 +730,16 @@ std::tuple CloudSyncUtils::GetDownloadListByGid( } Type prefix; std::vector pkVal; - OpType strategy = OpType::UPDATE; + OpType strategy; + if ((dataInfo.logInfo.flag & static_cast(LogInfoFlag::FLAG_CLOUD_UPDATE_LOCAL)) == + static_cast(LogInfoFlag::FLAG_CLOUD_UPDATE_LOCAL)) { + strategy = OpType::UPDATE; + } else if ((dataInfo.logInfo.flag & static_cast(LogInfoFlag::FLAG_DELETE)) == + static_cast(LogInfoFlag::FLAG_DELETE)) { + strategy = OpType::DELETE; + } else { + strategy = OpType::INSERT; + } errCode = CloudSyncUtils::GetCloudPkVals(dataInfo.primaryKeys, pkColNames, dataInfo.logInfo.dataKey, pkVal); if (errCode != E_OK) { LOGE("[CloudSyncUtils] HandleTagAssets cannot get primary key value list. %d", errCode); @@ -966,4 +976,33 @@ bool CloudSyncUtils::IsAssetOnlyData(VBucket &queryData, AssetsMap &assetsMap, b } return true; } + +int CloudSyncUtils::ClearCloudWatermark(const std::vector &tableNameList, + std::shared_ptr &storageProxy) +{ + for (const auto &tableName: tableNameList) { + LOGD("[CloudSyncUtils] Start clear cloud watermark."); + int ret = storageProxy->CleanWaterMark(tableName); + if (ret != E_OK) { + std::string maskedName = DBCommon::StringMiddleMasking(tableName); + LOGE("[CloudSyncUtils] failed to clear watermark. err: %d. table: %s, name length: %zu", + ret, maskedName.c_str(), maskedName.length()); + return ret; + } + } + int errCode = storageProxy->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + LOGE("[CloudSyncUtils] failed to start Transaction before clear cloud log version, %d", errCode); + return errCode; + } + + errCode = storageProxy->ClearCloudLogVersion(tableNameList); + if (errCode != E_OK) { + LOGE("[CloudSyncUtils] failed to clear log version, %d.", errCode); + storageProxy->Rollback(); + return errCode; + } + + return storageProxy->Commit(); +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.h b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.h index dd5cd76e..b8960b60 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_sync_utils.h @@ -137,6 +137,9 @@ public: const std::map &gidAssetsMap, const AssetsMap &assetsMap); static bool IsAssetOnlyData(VBucket &queryData, AssetsMap &assetsMap, bool isDownloading); + + static int ClearCloudWatermark(const std::vector &tableNameList, + std::shared_ptr &storageProxy); private: static void InsertOrReplaceChangedDataByType(ChangeType type, std::vector &pkVal, ChangedData &changedData); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.cpp index 6344da0d..d0c91c94 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.cpp @@ -158,8 +158,12 @@ int CloudSyncer::TriggerSync() void CloudSyncer::SetProxyUser(const std::string &user) { std::lock_guard autoLock(dataLock_); - storageProxy_->SetUser(user); - currentContext_.notifier->SetUser(user); + if (storageProxy_ != nullptr) { + storageProxy_->SetUser(user); + } + if (currentContext_.notifier != nullptr) { + currentContext_.notifier->SetUser(user); + } currentContext_.currentUserIndex = currentContext_.currentUserIndex + 1; cloudDB_.SwitchCloudDB(user); } @@ -225,7 +229,9 @@ int CloudSyncer::DoSync(TaskId taskId) bool isNeedFirstDownload = false; { std::lock_guard autoLock(dataLock_); - needUpload = currentContext_.strategy->JudgeUpload(); + if (currentContext_.strategy != nullptr) { + needUpload = currentContext_.strategy->JudgeUpload(); + } // 1. if the locker is already exist, directly reuse the lock, no need do the first download // 2. if the task(resume task) is already be tagged need upload data, no need do the first download isNeedFirstDownload = (currentContext_.locker == nullptr) && (!currentContext_.isNeedUpload); @@ -491,7 +497,7 @@ int CloudSyncer::UpdateChangedData(SyncParam ¶m, DownloadList &assetsDownloa VBucket &datum = param.downloadData.data[j]; std::vector primaryValues; ret = CloudSyncUtils::GetCloudPkVals(datum, param.changedData.field, - std::get(datum[CloudDbConstant::ROW_ID_FIELD_NAME]), primaryValues); + std::get(datum[DBConstant::ROWID]), primaryValues); if (ret != E_OK) { LOGE("[CloudSyncer] updateChangedData cannot get primaryValues"); return ret; @@ -503,7 +509,7 @@ int CloudSyncer::UpdateChangedData(SyncParam ¶m, DownloadList &assetsDownloa VBucket &datum = param.downloadData.data[downloadIndex]; size_t insertIdx = std::get<1>(tuple); std::vector &pkVal = std::get<5>(assetsDownloadList[insertIdx]); // 5 means primary key list - pkVal[0] = datum[CloudDbConstant::ROW_ID_FIELD_NAME]; + pkVal[0] = datum[DBConstant::ROWID]; } for (const auto &tuple : param.withoutRowIdData.updateData) { size_t downloadIndex = std::get<0>(tuple); @@ -520,7 +526,7 @@ int CloudSyncer::UpdateChangedData(SyncParam ¶m, DownloadList &assetsDownloa } // no primary key or composite primary key, the first element is rowid param.changedData.primaryData[ChangeType::OP_UPDATE][updateIndex][0] = - datum[CloudDbConstant::ROW_ID_FIELD_NAME]; + datum[DBConstant::ROWID]; } return ret; } @@ -842,8 +848,6 @@ int CloudSyncer::SaveData(CloudSyncer::TaskId taskId, SyncParam ¶m) param.info.downLoadInfo.total += param.downloadData.data.size(); param.info.retryInfo.downloadBatchOpCount = 0; int ret = E_OK; - DownloadList assetsDownloadList; - param.assetsDownloadList = assetsDownloadList; param.deletePrimaryKeySet.clear(); param.dupHashKeySet.clear(); CloudSyncUtils::ClearWithoutData(param); @@ -1035,29 +1039,33 @@ int CloudSyncer::DoDownloadAssets(SyncParam ¶m) } if (ret != E_OK) { LOGE("[CloudSyncer] Cannot notify downloadAssets due to error %d", ret); + } else { + param.assetsDownloadList.clear(); } return ret; } -int CloudSyncer::SaveDataNotifyProcess(CloudSyncer::TaskId taskId, SyncParam ¶m) +int CloudSyncer::SaveDataNotifyProcess(CloudSyncer::TaskId taskId, SyncParam ¶m, bool downloadAssetOnly) { - ChangedData changedData; InnerProcessInfo tmpInfo = param.info; - param.changedData = changedData; - param.downloadData.opType.resize(param.downloadData.data.size()); - param.downloadData.existDataKey.resize(param.downloadData.data.size()); - param.downloadData.existDataHashKey.resize(param.downloadData.data.size()); - int ret = SaveDataInTransaction(taskId, param); - if (ret != E_OK) { - return ret; - } - // call OnChange to notify changedData object first time (without Assets) - ret = NotifyChangedDataInCurrentTask(std::move(param.changedData)); - if (ret != E_OK) { - LOGE("[CloudSyncer] Cannot notify changed data due to error %d", ret); - return ret; + int ret = E_OK; + if (!downloadAssetOnly) { + ChangedData changedData; + param.changedData = changedData; + param.downloadData.opType.resize(param.downloadData.data.size()); + param.downloadData.existDataKey.resize(param.downloadData.data.size()); + param.downloadData.existDataHashKey.resize(param.downloadData.data.size()); + ret = SaveDataInTransaction(taskId, param); + if (ret != E_OK) { + return ret; + } + // call OnChange to notify changedData object first time (without Assets) + ret = NotifyChangedDataInCurrentTask(std::move(param.changedData)); + if (ret != E_OK) { + LOGE("[CloudSyncer] Cannot notify changed data due to error %d", ret); + return ret; + } } - ret = DoDownloadAssets(param); if (ret == -E_TASK_PAUSED) { param.info = tmpInfo; @@ -1097,6 +1105,13 @@ int CloudSyncer::DoDownload(CloudSyncer::TaskId taskId, bool isFirstDownload) LOGE("[CloudSyncer] get sync param for download failed %d", errCode); return errCode; } + for (auto it = param.assetsDownloadList.begin(); it != param.assetsDownloadList.end();) { + if (std::get(*it) != OpType::DELETE) { + it = param.assetsDownloadList.erase(it); + } else { + it++; + } + } (void)storageProxy_->CreateTempSyncTrigger(param.tableName); errCode = DoDownloadInner(taskId, param, isFirstDownload); (void)storageProxy_->ClearAllTempSyncTrigger(); @@ -1699,6 +1714,12 @@ int CloudSyncer::CleanCloudData(ClearMode mode, const std::vector & return errCode; } +int CloudSyncer::ClearCloudWatermark(const std::vector &tableNameList) +{ + std::lock_guard lock(syncMutex_); + return CloudSyncUtils::ClearCloudWatermark(tableNameList, storageProxy_); +} + int CloudSyncer::CleanWaterMarkInMemory(const std::set &tableNameList) { std::lock_guard lock(syncMutex_); @@ -1969,13 +1990,20 @@ int CloudSyncer::DownloadOneBatch(TaskId taskId, SyncParam ¶m, bool isFirstD if (ret != E_OK) { return ret; } - bool abort = false; - ret = DownloadDataFromCloud(taskId, param, abort, isFirstDownload); - if (abort) { + ret = DownloadDataFromCloud(taskId, param, isFirstDownload); + if (ret != E_OK) { return ret; } + bool downloadAssetOnly = false; + if (param.downloadData.data.empty()) { + if (param.assetsDownloadList.empty()) { + return E_OK; + } + LOGI("[CloudSyncer] Resume task need download assets"); + downloadAssetOnly = true; + } // Save data in transaction, update cloud water mark, notify process and changed data - ret = SaveDataNotifyProcess(taskId, param); + ret = SaveDataNotifyProcess(taskId, param, downloadAssetOnly); if (ret == -E_TASK_PAUSED) { return ret; } @@ -2079,7 +2107,7 @@ int CloudSyncer::GetSyncParamForDownload(TaskId taskId, SyncParam ¶m) param.isAssetsOnly = queryObject.IsAssetsOnly(); param.groupNum = queryObject.GetGroupNum(); param.assetsGroupMap = queryObject.GetAssetsOnlyGroupMap(); - param.isVaildForAssetsOnly = queryObject.IsValidForAssetsOnly(); + param.isVaildForAssetsOnly = queryObject.AssetsOnlyErrFlag() == E_OK; param.isForcePullAseets = IsModeForcePull(taskId) && queryObject.IsContainQueryNodes(); return ret; } @@ -2102,8 +2130,7 @@ bool CloudSyncer::IsCurrentTableResume(TaskId taskId, bool upload) return upload == resumeTaskInfos_[taskId].upload; } -int CloudSyncer::DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool &abort, - bool isFirstDownload) +int CloudSyncer::DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool isFirstDownload) { // Get cloud data after cloud water mark param.info.tableStatus = ProcessStatus::PROCESSING; @@ -2120,7 +2147,6 @@ int CloudSyncer::DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool &ab std::lock_guard autoLock(dataLock_); param.info.tableStatus = ProcessStatus::FINISHED; currentContext_.notifier->UpdateProcess(param.info); - abort = true; return ret; } @@ -2130,7 +2156,6 @@ int CloudSyncer::DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool &ab std::lock_guard autoLock(dataLock_); param.info.tableStatus = ProcessStatus::FINISHED; currentContext_.notifier->UpdateProcess(param.info); - abort = true; return ret; } @@ -2144,7 +2169,6 @@ int CloudSyncer::DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool &ab if (isFirstDownload) { NotifyInEmptyDownload(taskId, param.info); } - abort = true; } return E_OK; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.h b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.h index 59dcae01..0d773580 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer.h @@ -60,6 +60,8 @@ public: int CleanCloudData(ClearMode mode, const std::vector &tableNameList, const RelationalSchemaObject &localSchema); + int ClearCloudWatermark(const std::vector &tableNameList); + int CleanKvCloudData(std::function &removeFunc); int CleanWaterMarkInMemory(const std::set &tableNameList); @@ -166,6 +168,8 @@ protected: virtual int DoDownloadInNeed(const CloudTaskInfo &taskInfo, const bool needUpload, bool isFirstDownload); + void SetNeedUpload(bool isNeedUpload); + void DoFinished(TaskId taskId, int errCode); virtual int DoDownload(CloudSyncer::TaskId taskId, bool isFirstDownload); @@ -246,7 +250,7 @@ protected: int DoDownloadAssets(SyncParam ¶m); - int SaveDataNotifyProcess(CloudSyncer::TaskId taskId, SyncParam ¶m); + int SaveDataNotifyProcess(CloudSyncer::TaskId taskId, SyncParam ¶m, bool downloadAssetOnly); void NotifyInBatchUpload(const UploadParam &uploadParam, const InnerProcessInfo &innerProcessInfo, bool lastBatch); @@ -334,7 +338,7 @@ protected: bool IsCurrentTableResume(TaskId taskId, bool upload); - int DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool &abort, bool isFirstDownload); + int DownloadDataFromCloud(TaskId taskId, SyncParam ¶m, bool isFirstDownload); size_t GetDownloadAssetIndex(TaskId taskId); @@ -412,7 +416,7 @@ protected: void SetProxyUser(const std::string &user); - bool MergeTaskInfo(const std::shared_ptr &cloudSchema, TaskId taskId); + void MergeTaskInfo(const std::shared_ptr &cloudSchema, TaskId taskId); void RemoveTaskFromQueue(int32_t priorityLevel, TaskId taskId); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer_extend.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer_extend.cpp index 99c32f27..1e9c5350 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/cloud/cloud_syncer_extend.cpp @@ -466,7 +466,7 @@ void CloudSyncer::SeparateNormalAndFailAssets(const std::map 0) { failedAssets[key] = std::move(tempFailedAssets); @@ -831,6 +831,12 @@ void CloudSyncer::UpdateProcessInfoWithoutUpload(CloudSyncer::TaskId taskId, con } } +void CloudSyncer::SetNeedUpload(bool isNeedUpload) +{ + std::lock_guard autoLock(dataLock_); + currentContext_.isNeedUpload = isNeedUpload; +} + int CloudSyncer::DoDownloadInNeed(const CloudTaskInfo &taskInfo, const bool needUpload, bool isFirstDownload) { std::vector needNotifyTables; @@ -838,11 +844,15 @@ int CloudSyncer::DoDownloadInNeed(const CloudTaskInfo &taskInfo, const bool need std::string table; { std::lock_guard autoLock(dataLock_); + if (currentContext_.processRecorder == nullptr) { + LOGE("[CloudSyncer] process recorder of current context is nullptr."); + return -E_INTERNAL_ERROR; + } if (currentContext_.processRecorder->IsDownloadFinish(currentContext_.currentUserIndex, taskInfo.table[i])) { continue; } - LOGD("[CloudSyncer] try download table, index: %zu, table name: %s, length: %u", + LOGD("[CloudSyncer] try download table, index: %zu, table name: %s, length: %zu", i, DBCommon::StringMiddleMasking(taskInfo.table[i]).c_str(), taskInfo.table[i].length()); currentContext_.tableName = taskInfo.table[i]; table = currentContext_.tableName; @@ -863,10 +873,7 @@ int CloudSyncer::DoDownloadInNeed(const CloudTaskInfo &taskInfo, const bool need } // count > 0 means current table need upload actually if (count > 0) { - { - std::lock_guard autoLock(dataLock_); - currentContext_.isNeedUpload = true; - } + SetNeedUpload(true); continue; } needNotifyTables.emplace_back(table); @@ -914,41 +921,28 @@ int CloudSyncer::TryToAddSyncTask(CloudTaskInfo &&taskInfo) } auto taskId = taskInfo.taskId; cloudTaskInfos_[taskId] = std::move(taskInfo); - if (cloudTaskInfos_[taskId].priorityTask) { - taskQueue_.insert({cloudTaskInfos_[taskId].priorityLevel, taskId}); - LOGI("[CloudSyncer] Add priority task ok, storeId %.3s, priorityLevel %" PRId32 ", taskId %" PRIu64 " async %d", - cloudTaskInfos_[taskId].storeId.c_str(), - cloudTaskInfos_[taskId].priorityLevel, - cloudTaskInfos_[taskId].taskId, - static_cast(cloudTaskInfos_[taskId].asyncDownloadAssets)); - MarkCurrentTaskPausedIfNeed(taskInfo); - return E_OK; - } - if (!MergeTaskInfo(cloudSchema, taskId)) { - taskQueue_.insert({cloudTaskInfos_[taskId].priorityLevel, taskId}); - LOGI("[CloudSyncer] Add task ok, storeId %.3s, priorityLevel %" PRId32 ", taskId %" PRIu64 " async %d", - cloudTaskInfos_[taskId].storeId.c_str(), - cloudTaskInfos_[taskId].priorityLevel, - cloudTaskInfos_[taskId].taskId, - static_cast(cloudTaskInfos_[taskId].asyncDownloadAssets)); - MarkCurrentTaskPausedIfNeed(taskInfo); + taskQueue_.insert({cloudTaskInfos_[taskId].priorityLevel, taskId}); + LOGI("[CloudSyncer]Add task ok, storeId %.3s, priority %d, priorityLevel %" PRId32 ", taskId %" PRIu64 " async %d", + cloudTaskInfos_[taskId].storeId.c_str(), cloudTaskInfos_[taskId].priorityTask, + cloudTaskInfos_[taskId].priorityLevel, cloudTaskInfos_[taskId].taskId, + static_cast(cloudTaskInfos_[taskId].asyncDownloadAssets)); + MarkCurrentTaskPausedIfNeed(taskInfo); + if (!cloudTaskInfos_[taskId].priorityTask) { + MergeTaskInfo(cloudSchema, taskId); } return E_OK; } -bool CloudSyncer::MergeTaskInfo(const std::shared_ptr &cloudSchema, TaskId taskId) +void CloudSyncer::MergeTaskInfo(const std::shared_ptr &cloudSchema, TaskId taskId) { if (!cloudTaskInfos_[taskId].merge) { // LCOV_EXCL_BR_LINE - return false; + return; } bool isMerge = false; - bool mergeHappen = false; TaskId checkTaskId = taskId; do { std::tie(isMerge, checkTaskId) = TryMergeTask(cloudSchema, checkTaskId); - mergeHappen = mergeHappen || isMerge; } while (isMerge); - return mergeHappen; } void CloudSyncer::RemoveTaskFromQueue(int32_t priorityLevel, TaskId taskId) @@ -992,7 +986,7 @@ std::pair CloudSyncer::TryMergeTask(const std::shared_ptr nextTryTask) { // LCOV_EXCL_BR_LINE std::tie(beMergeTask, nextTryTask) = SwapTwoTaskAndCopyTable(beMergeTask, nextTryTask); } AdjustTableBasedOnSchema(cloudSchema, cloudTaskInfos_[nextTryTask]); @@ -1118,7 +1112,7 @@ int CloudSyncer::HandleBatchUpload(UploadParam &uploadParam, InnerProcessInfo &i break; } SetUploadDataFlag(uploadParam.taskId, uploadData); - LOGI("[CloudSyncer] Write local water after upload one batch, table[%s length[%u]], water[%llu]", + LOGI("[CloudSyncer] Write local water after upload one batch, table[%s length[%zu]], water[%llu]", DBCommon::StringMiddleMasking(uploadData.tableName).c_str(), uploadData.tableName.length(), uploadParam.localMark); RecordWaterMark(uploadParam.taskId, uploadParam.localMark); @@ -1290,7 +1284,7 @@ int CloudSyncer::DoUploadByMode(const std::string &tableName, UploadParam &uploa CloudSyncData uploadData(tableName, uploadParam.mode); SetUploadDataFlag(uploadParam.taskId, uploadData); auto [err, localWater] = GetLocalWater(tableName, uploadParam); - LOGI("[CloudSyncer] Get local water before upload result: %d, table[%s length[%u]], water[%llu]", err, + LOGI("[CloudSyncer] Get local water before upload result: %d, table[%s length[%zu]], water[%llu]", err, DBCommon::StringMiddleMasking(tableName).c_str(), tableName.length(), localWater); if (err != E_OK) { return err; @@ -1372,13 +1366,12 @@ int CloudSyncer::GenerateTaskIdIfNeed(CloudTaskInfo &taskInfo) taskInfo.storeId.c_str()); return -E_INVALID_ARGS; } - lastTaskId_ = std::max(lastTaskId_, taskInfo.taskId); LOGI("[CloudSyncer] Sync with taskId %" PRIu64 " storeId %.3s", taskInfo.taskId, taskInfo.storeId.c_str()); return E_OK; } - lastTaskId_++; - if (lastTaskId_ == UINT64_MAX) { - lastTaskId_ = 1u; + lastTaskId_--; + if (lastTaskId_ == INVALID_TASK_ID) { + lastTaskId_ = UINT64_MAX; } taskInfo.taskId = lastTaskId_; return E_OK; @@ -1636,7 +1629,7 @@ void CloudSyncer::CheckDataAfterDownload(const std::string &tableName) int logicDeleteDataCount = 0; int errCode = storageProxy_->GetLocalDataCount(tableName, dataCount, logicDeleteDataCount); if (errCode == E_OK) { - LOGI("[CloudSyncer] Check local data after download[%s[%u]], data count: %d, logic delete data count: %d", + LOGI("[CloudSyncer] Check local data after download[%s[%zu]], data count: %d, logic delete data count: %d", DBCommon::StringMiddleMasking(tableName).c_str(), tableName.length(), dataCount, logicDeleteDataCount); } else { LOGW("[CloudSyncer] Get local data after download fail: %d", errCode); @@ -1649,7 +1642,7 @@ void CloudSyncer::WaitCurTaskFinished() std::unique_lock uniqueLock(dataLock_); if (currentContext_.currentTaskId != INVALID_TASK_ID) { LOGI("[CloudSyncer] begin wait current task %" PRIu64 " finished", currentContext_.currentTaskId); - contextCv_.wait(uniqueLock, [this]() { + contextCv_.wait_for(uniqueLock, std::chrono::milliseconds(DBConstant::MAX_TIMEOUT), [this]() { return currentContext_.currentTaskId == INVALID_TASK_ID; }); LOGI("[CloudSyncer] current task has been finished"); @@ -1681,7 +1674,7 @@ void CloudSyncer::TriggerAsyncDownloadAssetsInTaskIfNeed(bool isFirstDownload) void CloudSyncer::TriggerAsyncDownloadAssetsIfNeed() { - if (!storageProxy_->IsContainAssetsTable()) { + if (!storageProxy_->IsExistTableContainAssets()) { LOGD("[CloudSyncer] No exist table contain assets, skip async download asset check"); return; } @@ -1693,7 +1686,10 @@ void CloudSyncer::TriggerAsyncDownloadAssetsIfNeed() static_cast(asyncTaskId_), static_cast(closed_)); return; } - lastTaskId_++; + lastTaskId_--; + if (lastTaskId_ == INVALID_TASK_ID) { + lastTaskId_ = UINT64_MAX; + } asyncTaskId_ = lastTaskId_; taskId = asyncTaskId_; IncObjRef(this); @@ -1731,10 +1727,10 @@ void CloudSyncer::BackgroundDownloadAssetsTask() }); if (errCode == E_OK) { // increase download count success - DecObjRef(this); CancelDownloadListener(); DoBackgroundDownloadAssets(); RuntimeContext::GetInstance()->GetAssetsDownloadManager()->FinishDownload(); + DecObjRef(this); return; } if (listener != nullptr) { @@ -1789,7 +1785,7 @@ void CloudSyncer::DoBackgroundDownloadAssets() } else { LOGW("[CloudSyncer] BackgroundDownloadAssetsByTable table %s failed %d", DBCommon::StringMiddleMasking(tableQueue.front()).c_str(), errCode); - tableQueue.pop_front(); + allDownloadFinish = false; } } } while (!allDownloadFinish); @@ -2107,6 +2103,9 @@ uint32_t CloudSyncer::GetCurrentTableUploadBatchIndex() void CloudSyncer::ResetCurrentTableUploadBatchIndex() { std::lock_guard autoLock(dataLock_); + if (currentContext_.notifier == nullptr) { + return; + } currentContext_.notifier->ResetUploadBatchIndex(currentContext_.tableName); } @@ -2164,14 +2163,15 @@ bool CloudSyncer::TryToInitQueryAndUserListForCompensatedSync(TaskId triggerTask } std::vector userList; CloudSyncUtils::GetUserListForCompensatedSync(cloudDB_, users, userList); - { - std::lock_guard autoLock(dataLock_); - cloudTaskInfos_[triggerTaskId].users = userList; - cloudTaskInfos_[triggerTaskId].table.clear(); - cloudTaskInfos_[triggerTaskId].queryList.clear(); - cloudTaskInfos_[triggerTaskId].table.push_back(syncQuery[0].GetRelationTableName()); - cloudTaskInfos_[triggerTaskId].queryList.push_back(syncQuery[0]); + std::lock_guard autoLock(dataLock_); + if (cloudTaskInfos_.find(triggerTaskId) == cloudTaskInfos_.end()) { + return false; } + cloudTaskInfos_[triggerTaskId].users = userList; + cloudTaskInfos_[triggerTaskId].table.clear(); + cloudTaskInfos_[triggerTaskId].queryList.clear(); + cloudTaskInfos_[triggerTaskId].table.push_back(syncQuery[0].GetRelationTableName()); + cloudTaskInfos_[triggerTaskId].queryList.push_back(syncQuery[0]); return true; } } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/ability_sync.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/ability_sync.cpp index efcbb30d..7a4af6fa 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/ability_sync.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/ability_sync.cpp @@ -1252,7 +1252,8 @@ RelationalSyncOpinion AbilitySync::MakeRelationSyncOpinion(const AbilitySyncRequ { uint8_t remoteSchemaType = packet->GetSchemaType(); RelationalSchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); - return SchemaNegotiate::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); + return SchemaNegotiate::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType, + packet->GetSoftwareVersion()); } int AbilitySync::HandleKvAckSchemaParam(const AbilitySyncAckPacket *recvPacket, @@ -1288,7 +1289,8 @@ int AbilitySync::HandleRelationAckSchemaParam(const AbilitySyncAckPacket *recvPa std::string remoteSchema = recvPacket->GetSchema(); uint8_t remoteSchemaType = recvPacket->GetSchemaType(); auto localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); - auto localOpinion = SchemaNegotiate::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); + auto localOpinion = SchemaNegotiate::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType, + recvPacket->GetSoftwareVersion()); auto localStrategy = SchemaNegotiate::ConcludeSyncStrategy(localOpinion, recvPacket->GetRelationalSyncOpinion()); (static_cast(context))->SetRelationalSyncStrategy(localStrategy, true); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/generic_syncer.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/generic_syncer.cpp index 90b771be..772dee8f 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/generic_syncer.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/generic_syncer.cpp @@ -388,11 +388,11 @@ void GenericSyncer::AddSyncOperation(ISyncEngine *engine, SyncOperation *operati return; } - LOGD("[Syncer] AddSyncOperation, sync id: %d.", operation->GetSyncId()); + LOGD("[Syncer] AddSyncOperation, sync id: %" PRIu32 ".", operation->GetSyncId()); engine->AddSyncOperation(operation); if (operation->CheckIsAllFinished()) { - LOGD("[Syncer] AddSyncOperation, sync id: %d, but all finished.", operation->GetSyncId()); + LOGD("[Syncer] AddSyncOperation, sync id: %" PRIu32 ", but all finished.", operation->GetSyncId()); return; } @@ -905,6 +905,10 @@ int GenericSyncer::SyncPreCheck(const SyncParma ¶m) const void GenericSyncer::InitSyncOperation(SyncOperation *operation, const SyncParma ¶m) { + if (syncInterface_ == nullptr) { + LOGE("[GenericSyncer] [InitSyncOperation] syncInterface_ is nullptr."); + return; + } operation->SetIdentifier(syncInterface_->GetIdentifier()); operation->SetSyncProcessCallFun(param.onSyncProcess); operation->Initialize(); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.cpp index a812917e..80fc35ac 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.cpp @@ -217,13 +217,14 @@ int Metadata::SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &i return errCode; } -void Metadata::GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash) +int Metadata::GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash) { int errCode = GetMetaDataValueFromDB(deviceId, isNeedHash, outValue); if (errCode == -E_NOT_FOUND && RuntimeContext::GetInstance()->IsTimeChanged()) { outValue = {}; outValue.syncMark = static_cast(SyncMark::SYNC_MARK_TIME_CHANGE); } + return errCode; } int Metadata::SerializeMetaData(const MetaDataValue &inValue, std::vector &outValue) @@ -449,7 +450,7 @@ void Metadata::GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue) { std::lock_guard lockGuard(metadataLock_); MetaDataValue metadata; - int errCode = GetMetaDataValueFromDB(deviceId, true, metadata); + int errCode = GetMetaDataValue(deviceId, metadata, true); if (errCode == E_OK) { outValue = metadata.dbCreateTime; return; @@ -462,7 +463,7 @@ int Metadata::SetDbCreateTime(const DeviceID &deviceId, uint64_t inValue, bool i { std::lock_guard lockGuard(metadataLock_); MetaDataValue metadata; - int errCode = GetMetaDataValueFromDB(deviceId, isNeedHash, metadata); + int errCode = GetMetaDataValue(deviceId, metadata, isNeedHash); if (errCode == E_OK) { if (metadata.dbCreateTime != 0 && metadata.dbCreateTime != inValue) { metadata.clearDeviceDataMark = REMOVE_DEVICE_DATA_MARK; @@ -484,7 +485,7 @@ int Metadata::ResetMetaDataAfterRemoveData(const DeviceID &deviceId) { std::lock_guard lockGuard(metadataLock_); MetaDataValue metadata; - int errCode = GetMetaDataValueFromDB(deviceId, true, metadata); + int errCode = GetMetaDataValue(deviceId, metadata, true); if (errCode == E_OK) { metadata.clearDeviceDataMark = 0; return SaveMetaDataValue(deviceId, metadata, true); @@ -496,7 +497,7 @@ void Metadata::GetRemoveDataMark(const DeviceID &deviceId, uint64_t &outValue) { std::lock_guard lockGuard(metadataLock_); MetaDataValue metadata; - int errCode = GetMetaDataValueFromDB(deviceId, true, metadata); + int errCode = GetMetaDataValue(deviceId, metadata, true); if (errCode == E_OK) { outValue = metadata.clearDeviceDataMark; return; @@ -531,7 +532,7 @@ uint64_t Metadata::GetQueryLastTimestamp(const DeviceID &deviceId, const std::st if (errCode == E_OK && iter != queryIdMap_.end()) { iter->second.erase(hashqueryId); } - return StringToLong(value); + return static_cast(StringToLong(value)); } void Metadata::RemoveQueryFromRecordSet(const DeviceID &deviceId, const std::string &queryId) @@ -605,7 +606,7 @@ int Metadata::GetWaterMarkInfoFromDB(const std::string &dev, bool isNeedHash, Wa // read from db avoid watermark update in diff process std::lock_guard lockGuard(metadataLock_); MetaDataValue metadata; - int errCode = GetMetaDataValueFromDB(dev, isNeedHash, metadata); + int errCode = GetMetaDataValue(dev, metadata, isNeedHash); if (errCode == -E_NOT_FOUND) { LOGD("[Metadata] not found meta value"); return E_OK; diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.h b/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.h index c483b86a..4df789a0 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/meta_data.h @@ -183,7 +183,7 @@ private: int SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &inValue, bool isNeedHash = true); // sync module need hash devices id - void GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash); + int GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash); static int SerializeMetaData(const MetaDataValue &inValue, std::vector &outValue); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.cpp index 167d671d..84f35ce3 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.cpp @@ -1611,12 +1611,23 @@ void SingleVerDataSync::SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTask WaterMark localMark = 0; GetLocalWaterMark(curType, packet->GetQueryId(), context, localMark); ackPacket.SetRecvCode(recvCode); + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = SingleVerDataSyncUtils::GetRecvDataTimeRange( + SyncOperation::GetSyncType(packet->GetMode()), packet->GetData(), isUpdateWaterMark); + bool isPacketWaterLower = false; // send ack packet if ((recvCode == E_OK) && (maxSendDataTime != 0)) { ackPacket.SetData(maxSendDataTime + 1); // + 1 to next start } else if (recvCode != WATER_MARK_INVALID) { WaterMark mark = 0; GetPeerWaterMark(curType, packet->GetQueryId(), context->GetDeviceId(), mark); + if (recvCode == -E_NEED_ABILITY_SYNC && packet->GetLocalWaterMark() < mark) { + LOGI("[DataSync][SetAckPacket] packetLocalMark=%" PRIu64 ",lockMark=%" PRIu64, packet->GetLocalWaterMark(), + mark); + isPacketWaterLower = true; + dataTime.endTime = packet->GetLocalWaterMark(); + mark = packet->GetLocalWaterMark(); + } ackPacket.SetData(mark); } std::vector reserved {localMark}; @@ -1632,10 +1643,20 @@ void SingleVerDataSync::SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTask if (curType == SyncType::QUERY_SYNC_TYPE && recvCode != WATER_MARK_INVALID) { WaterMark deletedPeerMark; GetPeerDeleteSyncWaterMark(context->GetDeleteSyncId(), deletedPeerMark); + if (recvCode == -E_NEED_ABILITY_SYNC && packet->GetDeletedWaterMark() < deletedPeerMark) { + LOGI("[DataSync][SetAckPacket] packetDeletedMark=%" PRIu64 ",deletedMark=%" PRIu64, + packet->GetDeletedWaterMark(), deletedPeerMark); + isPacketWaterLower = true; + dataTime.deleteEndTime = packet->GetDeletedWaterMark(); + deletedPeerMark = packet->GetDeletedWaterMark(); + } reserved.push_back(deletedPeerMark); // query sync mode, reserve[2] store deletedPeerMark value } ackPacket.SetReserved(reserved); ackPacket.SetVersion(version); + if (isPacketWaterLower) { + UpdatePeerWaterMarkInner(*packet, dataTime, isUpdateWaterMark, context); + } } int SingleVerDataSync::GetReSendData(SyncEntry &syncData, SingleVerSyncTaskContext *context, @@ -1827,7 +1848,7 @@ int SingleVerDataSync::ControlCmdStart(SingleVerSyncTaskContext *context) if (errCode != E_OK) { return errCode; } - ControlRequestPacket* packet = new (std::nothrow) SubscribeRequest(); + auto packet = new (std::nothrow) SubscribeRequest(); if (packet == nullptr) { LOGE("[DataSync][ControlCmdStart] new SubscribeRequest error"); return -E_OUT_OF_MEMORY; @@ -1851,7 +1872,7 @@ int SingleVerDataSync::ControlCmdStart(SingleVerSyncTaskContext *context) int SingleVerDataSync::ControlCmdRequestRecv(SingleVerSyncTaskContext *context, const Message *message) { - const ControlRequestPacket *packet = message->GetObject(); + const SubscribeRequest *packet = message->GetObject(); if (packet == nullptr) { return -E_INVALID_ARGS; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.h b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.h index 5c1a758c..1e07eb1a 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync.h @@ -240,7 +240,7 @@ protected: int ControlCmdStartCheck(SingleVerSyncTaskContext *context); - int SendControlPacket(ControlRequestPacket *packet, SingleVerSyncTaskContext *context); + int SendControlPacket(SubscribeRequest *packet, SingleVerSyncTaskContext *context); int ControlCmdRequestRecvPre(SingleVerSyncTaskContext *context, const Message *message); int SubscribeRequestRecvPre(SingleVerSyncTaskContext *context, const SubscribeRequest *packet, diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_extend.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_extend.cpp index cbfeb722..a7b75567 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_extend.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_extend.cpp @@ -90,7 +90,7 @@ int SingleVerDataSync::ControlCmdStartCheck(SingleVerSyncTaskContext *context) return E_OK; } -int SingleVerDataSync::SendControlPacket(ControlRequestPacket *packet, SingleVerSyncTaskContext *context) +int SingleVerDataSync::SendControlPacket(SubscribeRequest *packet, SingleVerSyncTaskContext *context) { Message *message = new (std::nothrow) Message(CONTROL_SYNC_MESSAGE); if (message == nullptr) { @@ -102,10 +102,10 @@ int SingleVerDataSync::SendControlPacket(ControlRequestPacket *packet, SingleVer uint32_t packetLen = packet->CalculateLen(); int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { - delete packet; - packet = nullptr; delete message; message = nullptr; + delete packet; + packet = nullptr; LOGE("[DataSync][SendControlPacket] set external object failed errCode=%d", errCode); return errCode; } @@ -155,7 +155,7 @@ int SingleVerDataSync::ControlCmdRequestRecvPre(SingleVerSyncTaskContext *contex if (context == nullptr || message == nullptr) { return -E_INVALID_ARGS; } - const ControlRequestPacket *packet = message->GetObject(); + const SubscribeRequest *packet = message->GetObject(); if (packet == nullptr) { return -E_INVALID_ARGS; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_utils.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_utils.cpp index f9eb3b76..986d841b 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_utils.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_data_sync_utils.cpp @@ -345,20 +345,16 @@ bool SingleVerDataSyncUtils::IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyn if (inMsg == nullptr) { return false; } - if (inMsg->GetMessageId() != CONTROL_SYNC_MESSAGE) { + if (inMsg->GetMessageId() != CONTROL_SYNC_MESSAGE || inMsg->GetMessageType() != TYPE_REQUEST) { return false; } - const ControlRequestPacket *packet = inMsg->GetObject(); + auto packet = inMsg->GetObject(); if (packet == nullptr) { return false; } uint32_t controlCmdType = packet->GetcontrolCmdType(); - if (controlCmdType == ControlCmdType::SUBSCRIBE_QUERY_CMD && inMsg->GetMessageType() == TYPE_REQUEST) { - const SubscribeRequest *subPacket = inMsg->GetObject(); - if (subPacket == nullptr) { - return false; - } - query = subPacket->GetQuery(); + if (controlCmdType == ControlCmdType::SUBSCRIBE_QUERY_CMD) { + query = packet->GetQuery(); LOGI("[SingleVerDataSync] receive sub scribe query cmd,begin to trigger query auto sync"); return true; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.cpp index 106797be..e392bb45 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.cpp @@ -32,18 +32,23 @@ int SingleVerRelationalSyncer::Initialize(ISyncInterface *syncInterface, bool is int SingleVerRelationalSyncer::Sync(const SyncParma ¶m, uint64_t connectionId) { - if (QuerySyncPreCheck(param) != E_OK) { - return -E_NOT_SUPPORT; + int errCode = QuerySyncPreCheck(param); + if (errCode != E_OK) { + return errCode; } return GenericSyncer::Sync(param, connectionId); } int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId) { + if (syncInterface_ == nullptr) { + LOGE("[SingleVerRelationalSyncer] [PrepareSync] syncInterface_ is nullptr."); + return -E_INTERNAL_ERROR; + } const auto &syncInterface = static_cast(syncInterface_); std::vector tablesQuery; if (param.isQuerySync) { - tablesQuery.push_back(param.syncQuery); + tablesQuery = GetQuerySyncObject(param); } else { tablesQuery = syncInterface->GetTablesQuery(); } @@ -78,8 +83,8 @@ int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint LOGI("[SingleVerRelationalSyncer] SubSyncId %" PRIu32 " create by SyncId %" PRIu32 ", hashTableName = %s", subSyncId, syncId, STR_MASK(DBCommon::TransferStringToHex(hashTableName))); subParam.syncQuery = table; - subParam.onComplete = [this, subSyncId, syncId, param](const std::map &devicesMap) { - DoOnSubSyncComplete(subSyncId, syncId, param, devicesMap); + subParam.onComplete = [this, subSyncId, syncId, subParam](const std::map &devicesMap) { + DoOnSubSyncComplete(subSyncId, syncId, subParam, devicesMap); }; { std::lock_guard lockGuard(syncMapLock_); @@ -105,8 +110,11 @@ void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, co std::lock_guard lockGuard(syncMapLock_); fullSyncIdMap_[syncId].erase(subSyncId); allFinish = fullSyncIdMap_[syncId].empty(); + TableStatus tableStatus; + tableStatus.tableName = param.syncQuery.GetRelationTableName(); for (const auto &item : devicesMap) { - resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = static_cast(item.second); + tableStatus.status = static_cast(item.second); + resMap_[syncId][item.first].push_back(tableStatus); } } // block sync do callback in sync function @@ -131,7 +139,7 @@ void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t sy return; } std::map> syncRes; - std::map> tmpMap; + std::map> tmpMap; { std::lock_guard lockGuard(syncMapLock_); tmpMap = resMap_[syncId]; @@ -139,7 +147,7 @@ void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t sy for (const auto &devicesRes : tmpMap) { for (const auto &tableRes : devicesRes.second) { syncRes[devicesRes.first].push_back( - {tableRes.first, static_cast(tableRes.second)}); + {tableRes.tableName, static_cast(tableRes.status)}); } } param.relationOnComplete(syncRes); @@ -178,25 +186,29 @@ int SingleVerRelationalSyncer::SyncConditionCheck(const SyncParma ¶m, const if (!param.isQuerySync) { return E_OK; } - QuerySyncObject query = param.syncQuery; - int errCode = static_cast(storage)->CheckAndInitQueryCondition(query); - if (errCode != E_OK) { - LOGE("[SingleVerRelationalSyncer] QuerySyncObject check failed"); - return errCode; + auto queryList = GetQuerySyncObject(param); + const RelationalSchemaObject schemaObj = static_cast(storage)->GetSchemaInfo(); + const std::vector &sTable = schemaObj.GetDistributedSchema().tables; + if (schemaObj.GetTableMode() == DistributedTableMode::COLLABORATION && sTable.empty()) { + LOGE("[SingleVerRelationalSyncer] Distributed schema not set in COLLABORATION mode"); + return -E_SCHEMA_MISMATCH; } - const RelationalSchemaObject &schemaObj = static_cast(storage)->GetSchemaInfo(); - if (schemaObj.GetTableMode() == DistributedTableMode::COLLABORATION) { - const std::vector &sTable = schemaObj.GetDistributedSchema().tables; - if (sTable.empty()) { - LOGE("[SingleVerRelationalSyncer] Distributed schema not set in COLLABORATION mode"); - return -E_SCHEMA_MISMATCH; + for (auto &item : queryList) { + int errCode = static_cast(storage)->CheckAndInitQueryCondition(item); + if (errCode != E_OK) { + LOGE("[SingleVerRelationalSyncer] table %s[length: %zu] query check failed %d", + DBCommon::StringMiddleMasking(item.GetTableName()).c_str(), item.GetTableName().size(), errCode); + return errCode; } - auto iter = std::find_if(sTable.begin(), sTable.end(), [¶m](const DistributedTable &table) { - return table.tableName == param.syncQuery.GetTableName(); - }); - if (iter == sTable.end()) { - LOGE("[SingleVerRelationalSyncer] table name mismatch"); - return -E_SCHEMA_MISMATCH; + if (schemaObj.GetTableMode() == DistributedTableMode::COLLABORATION) { + auto iter = std::find_if(sTable.begin(), sTable.end(), [&item](const DistributedTable &table) { + return DBCommon::ToLowerCase(table.tableName) == DBCommon::ToLowerCase(item.GetTableName()); + }); + if (iter == sTable.end()) { + LOGE("[SingleVerRelationalSyncer] table %s[length: %zu] mismatch distributed schema", + DBCommon::StringMiddleMasking(item.GetTableName()).c_str(), item.GetTableName().size()); + return -E_SCHEMA_MISMATCH; + } } } if (param.mode == SUBSCRIBE_QUERY) { @@ -207,10 +219,6 @@ int SingleVerRelationalSyncer::SyncConditionCheck(const SyncParma ¶m, const int SingleVerRelationalSyncer::QuerySyncPreCheck(const SyncParma ¶m) const { - if (!param.syncQuery.GetRelationTableNames().empty()) { - LOGE("[SingleVerRelationalSyncer] sync with not support query"); - return -E_NOT_SUPPORT; - } if (!param.isQuerySync) { return E_OK; } @@ -218,11 +226,29 @@ int SingleVerRelationalSyncer::QuerySyncPreCheck(const SyncParma ¶m) const LOGE("[SingleVerRelationalSyncer] sync with not support push_pull mode"); return -E_NOT_SUPPORT; } - if (param.syncQuery.GetRelationTableName().empty()) { + if (param.syncQuery.IsUseFromTables() && param.syncQuery.GetRelationTableNames().empty()) { + LOGE("[SingleVerRelationalSyncer] sync with from table but no table found"); + return -E_INVALID_ARGS; + } + if (!param.syncQuery.IsUseFromTables() && param.syncQuery.GetRelationTableName().empty()) { LOGE("[SingleVerRelationalSyncer] sync with empty table"); return -E_NOT_SUPPORT; } return E_OK; } + +std::vector SingleVerRelationalSyncer::GetQuerySyncObject(const SyncParma ¶m) +{ + std::vector res; + auto tables = param.syncQuery.GetRelationTableNames(); + if (!tables.empty()) { + for (const auto &it : tables) { + res.emplace_back(Query::Select(it)); + } + } else { + res.push_back(param.syncQuery); + } + return res; +} } #endif \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.h b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.h index 04102c8d..df637af8 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_relational_syncer.h @@ -52,9 +52,11 @@ private: int QuerySyncPreCheck(const SyncParma ¶m) const; + static std::vector GetQuerySyncObject(const SyncParma ¶m); + mutable std::mutex syncMapLock_; std::map> fullSyncIdMap_; - std::map>> resMap_; + std::map>> resMap_; }; } #endif diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_serialize_manager.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_serialize_manager.cpp index 9be5cdaa..569cac55 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_serialize_manager.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_serialize_manager.cpp @@ -538,7 +538,7 @@ int SingleVerSerializeManager::AckPacketSyncerPartDeSerializationV1(Parcel &parc int SingleVerSerializeManager::ControlPacketCalculateLen(const Message *inMsg, uint32_t &len) { - auto packet = inMsg->GetObject(); + auto packet = inMsg->GetObject(); if (packet == nullptr || packet->GetcontrolCmdType() >= INVALID_CONTROL_CMD) { LOGE("[ControlPacketSerialization] invalid control cmd"); return -E_INVALID_ARGS; @@ -551,7 +551,7 @@ int SingleVerSerializeManager::ControlPacketCalculateLen(const Message *inMsg, u int SingleVerSerializeManager::ControlPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) { - auto packet = inMsg->GetObject(); + auto packet = inMsg->GetObject(); if (packet == nullptr || packet->GetcontrolCmdType() >= INVALID_CONTROL_CMD) { LOGE("[ControlPacketSerialization] invalid control cmd"); return -E_INVALID_ARGS; @@ -641,7 +641,7 @@ ERROR: int SingleVerSerializeManager::ControlRequestSerialization(Parcel &parcel, const Message *inMsg) { - auto packet = inMsg->GetObject(); + auto packet = inMsg->GetObject(); if (packet == nullptr) { return -E_INVALID_ARGS; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_state_machine.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_state_machine.cpp index 986f9abe..205ca3da 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_state_machine.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_state_machine.cpp @@ -537,6 +537,10 @@ Event SingleVerSyncStateMachine::GetEventAfterTimeSync(int mode) const Event SingleVerSyncStateMachine::DoSyncTaskFinished() { StopWatchDog(); + if (dataSync_ == nullptr || communicator_ == nullptr || syncContext_ == nullptr) { + LOGE("[SingleVerSyncStateMachine] [DoSyncTaskFinished] dataSync_ or communicator_ or syncContext_ is nullptr."); + return TransformErrCodeToEvent(-E_OUT_OF_MEMORY); + } dataSync_->ClearSyncStatus(); auto timeout = communicator_->GetTimeout(syncContext_->GetDeviceId()); RefObject::AutoLock lock(syncContext_); @@ -877,7 +881,8 @@ int SingleVerSyncStateMachine::GetSyncOperationStatus(int errCode) const { -E_NOT_REGISTER, SyncOperation::OP_NOT_SUPPORT }, { -E_DENIED_SQL, SyncOperation::OP_DENIED_SQL }, { -E_REMOTE_OVER_SIZE, SyncOperation::OP_MAX_LIMITS }, - { -E_INVALID_PASSWD_OR_CORRUPTED_DB, SyncOperation::OP_NOTADB_OR_CORRUPTED } + { -E_INVALID_PASSWD_OR_CORRUPTED_DB, SyncOperation::OP_NOTADB_OR_CORRUPTED }, + { -E_DISTRIBUTED_SCHEMA_NOT_FOUND, SyncOperation::OP_SCHEMA_INCOMPATIBLE } }; const auto &result = std::find_if(std::begin(stateNodes), std::end(stateNodes), [errCode](const auto &node) { return node.errCode == errCode; diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_task_context.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_task_context.cpp index 3183a486..9ef63989 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_task_context.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/singlever/single_ver_sync_task_context.cpp @@ -44,10 +44,12 @@ int SingleVerSyncTaskContext::Initialize(const std::string &deviceId, ISyncInter { if (deviceId.empty() || syncInterface == nullptr || metadata == nullptr || communicator == nullptr) { + LOGE("[SingleVerSyncTaskContext] [Initialize] parameter is invalid."); return -E_INVALID_ARGS; } stateMachine_ = new (std::nothrow) SingleVerSyncStateMachine; if (stateMachine_ == nullptr) { + LOGE("[SingleVerSyncTaskContext] [Initialize] stateMachine_ is nullptr."); return -E_OUT_OF_MEMORY; } deviceId_ = deviceId; @@ -92,6 +94,7 @@ ERROR_OUT: int SingleVerSyncTaskContext::AddSyncOperation(SyncOperation *operation) { if (operation == nullptr) { + LOGE("[SingleVerSyncTaskContext] [AddSyncOperation] operation is nullptr."); return -E_INVALID_ARGS; } @@ -121,6 +124,7 @@ int SingleVerSyncTaskContext::AddSyncOperation(SyncOperation *operation) auto *newTarget = new (std::nothrow) SingleVerSyncTarget; if (newTarget == nullptr) { + LOGE("[SingleVerSyncTaskContext] [AddSyncOperation] newTarget is nullptr."); return -E_OUT_OF_MEMORY; } newTarget->SetSyncOperation(operation); diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/device/sync_engine.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/device/sync_engine.cpp index b64cd1e6..5d4a60e5 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/device/sync_engine.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/device/sync_engine.cpp @@ -70,6 +70,7 @@ int SyncEngine::Initialize(ISyncInterface *syncInterface, const std::shared_ptr< const InitCallbackParam &callbackParam) { if ((syncInterface == nullptr) || (metadata == nullptr)) { + LOGE("[SyncEngine] [Initialize] syncInterface or metadata is nullptr."); return -E_INVALID_ARGS; } int errCode = StartAutoSubscribeTimer(*syncInterface); @@ -702,6 +703,7 @@ int SyncEngine::ExecSyncTask(ISyncTaskContext *context) int SyncEngine::GetQueueCacheSize() const { + std::lock_guard lock(queueLock_); return queueCacheSize_; } @@ -713,6 +715,7 @@ void SyncEngine::SetQueueCacheSize(int size) unsigned int SyncEngine::GetDiscardMsgNum() const { + std::lock_guard lock(queueLock_); return discardMsgNum_; } diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.cpp b/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.cpp index c0369a08..f82841dc 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.cpp +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.cpp @@ -149,11 +149,4 @@ Timestamp TimeHelper::GetMonotonicTime() } return time; } - -Timestamp TimeHelper::GetCurrentLocalTime(int64_t &curTimeOffset, int64_t &localTimeOffset) -{ - Timestamp currentSysTime = GetSysCurrentTime(); - Timestamp currentLocalTime = currentSysTime + curTimeOffset + localTimeOffset; - return currentLocalTime; -} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.h b/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.h index f06ac32e..d3a79e43 100644 --- a/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.h +++ b/kv_store/frameworks/libs/distributeddb/syncer/src/time_helper.h @@ -60,8 +60,6 @@ public: static Timestamp GetMonotonicTime(); - static Timestamp GetCurrentLocalTime(int64_t &curTimeOffset, int64_t &localTimeOffset); - private: static std::mutex systemTimeLock_; static Timestamp lastSystemTimeUs_; diff --git a/kv_store/frameworks/libs/distributeddb/test/BUILD.gn b/kv_store/frameworks/libs/distributeddb/test/BUILD.gn index 8c83c045..d27aba5b 100644 --- a/kv_store/frameworks/libs/distributeddb/test/BUILD.gn +++ b/kv_store/frameworks/libs/distributeddb/test/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("../distributeddb.gni") -module_output_path = "kv_store/distributeddb" +module_output_path = "kv_store/kv_store/distributeddb" ############################################################################### config("gaussdb_rd_config") { @@ -127,10 +127,13 @@ ohos_source_set("src_file") { sources = distributeddb_src sources += distributeddb_cloud_src sources += [ + "unittest/common/common/basic_unit_test.cpp", "unittest/common/common/distributeddb_data_generate_unit_test.cpp", "unittest/common/common/distributeddb_tools_unit_test.cpp", + "unittest/common/common/kv_general_ut.cpp", "unittest/common/common/native_sqlite.cpp", "unittest/common/common/rdb_data_generator.cpp", + "unittest/common/common/rdb_general_ut.cpp", "unittest/common/common/system_time.cpp", "unittest/common/common/thread_pool_test_stub.cpp", "unittest/common/interfaces/process_system_api_adapter_impl.cpp", @@ -853,18 +856,38 @@ distributeddb_unittest("DistributedDBCloudKvStoreTest") { [ "unittest/common/syncer/cloud/distributeddb_cloud_kvstore_test.cpp" ] } +distributeddb_unittest("DistributedDBCloudAsyncDownloadAssetsTest") { + sources = [ "unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp" ] +} + +distributeddb_unittest("DistributedDBCloudSimpleAssetTest") { + sources = [ + "unittest/common/syncer/cloud/distributeddb_cloud_simple_asset_test.cpp", + ] +} + distributeddb_unittest("DistributedDBRDBCollaborationTest") { sources = [ "unittest/common/storage/distributeddb_rdb_collaboration_test.cpp" ] } -distributeddb_unittest("DistributedDBCloudAsyncDownloadAssetsTest") { - sources = [ "unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp" ] +distributeddb_unittest("DistributedDBBasicKVTest") { + sources = [ "unittest/common/store_test/kv/distributeddb_basic_kv_test.cpp" ] } -distributeddb_unittest("DistributedDBCloudSimpleAssetTest") { +distributeddb_unittest("DistributedDBKVDataStatusTest") { + sources = + [ "unittest/common/store_test/kv/distributeddb_kv_data_status_test.cpp" ] +} + +distributeddb_unittest("DistributedDBBasicRDBTest") { + sources = + [ "unittest/common/store_test/rdb/distributeddb_basic_rdb_test.cpp" ] +} + +distributeddb_unittest("DistributedDBRDBDataStatusTest") { sources = [ - "unittest/common/syncer/cloud/distributeddb_cloud_simple_asset_test.cpp", + "unittest/common/store_test/rdb/distributeddb_rdb_data_status_test.cpp", ] } @@ -875,6 +898,8 @@ group("unittest") { deps += [ ":DistributedDBAbilitySyncTest", ":DistributedDBAutoLaunchUnitTest", + ":DistributedDBBasicKVTest", + ":DistributedDBBasicRDBTest", ":DistributedDBCloudAssetCompareTest", ":DistributedDBCloudAssetsOperationSyncTest", ":DistributedDBCloudAsyncDownloadAssetsTest", @@ -942,6 +967,7 @@ group("unittest") { ":DistributedDBInterfacesTransactionSyncDBTest", ":DistributedDBInterfacesTransactionTest", ":DistributedDBJsonPrecheckUnitTest", + ":DistributedDBKVDataStatusTest", ":DistributedDBMetaDataTest", ":DistributedDBMockSyncModuleTest", ":DistributedDBMultiVerP2PSyncTest", @@ -949,6 +975,7 @@ group("unittest") { ":DistributedDBNotificationChainTest", ":DistributedDBParcelTest", ":DistributedDBRDBCollaborationTest", + ":DistributedDBRDBDataStatusTest", ":DistributedDBRelationalCloudSyncableStorageTest", ":DistributedDBRelationalEncryptedDbTest", ":DistributedDBRelationalGetDataTest", diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/json_fuzzer/json_fuzzer.cpp b/kv_store/frameworks/libs/distributeddb/test/fuzztest/json_fuzzer/json_fuzzer.cpp index a1d345b8..7671d045 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/json_fuzzer/json_fuzzer.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/json_fuzzer/json_fuzzer.cpp @@ -1602,7 +1602,8 @@ void TestGrdDbApGrdGetItem002Fuzz() } GRD_Flush(g_db, 1); - uint32_t begin = 0, end = MAX_SIZE_NUM; + uint32_t begin = 0; + uint32_t end = MAX_SIZE_NUM; GRD_FilterOptionT option = {}; option.mode = KV_SCAN_RANGE; option.begin = { &begin, sizeof(uint32_t) }; @@ -1610,7 +1611,8 @@ void TestGrdDbApGrdGetItem002Fuzz() GRD_ResultSet *resultSet = nullptr; GRD_KVFilter(g_db, COLLECTION_NAME, &option, &resultSet); - uint32_t keySize, valueSize; + uint32_t keySize; + uint32_t valueSize; GRD_KVGetSize(resultSet, &keySize, &valueSize); resultSet = nullptr; diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/jsoninner_fuzzer/jsoninner_fuzzer.cpp b/kv_store/frameworks/libs/distributeddb/test/fuzztest/jsoninner_fuzzer/jsoninner_fuzzer.cpp index 3f6262e9..ce3d7570 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/jsoninner_fuzzer/jsoninner_fuzzer.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/jsoninner_fuzzer/jsoninner_fuzzer.cpp @@ -1611,7 +1611,8 @@ void TestGrdDbApGrdGetItem002Fuzz() } GRD_FlushInner(g_db, 1); - uint32_t begin = 0, end = MAX_SIZE_NUM; + uint32_t begin = 0; + uint32_t end = MAX_SIZE_NUM; GRD_FilterOptionT option = {}; option.mode = KV_SCAN_RANGE; option.begin = { &begin, sizeof(uint32_t) }; @@ -1619,7 +1620,8 @@ void TestGrdDbApGrdGetItem002Fuzz() GRD_ResultSet *resultSet = nullptr; GRD_KVFilterInner(g_db, COLLECTION_NAME, &option, &resultSet); - uint32_t keySize, valueSize; + uint32_t keySize; + uint32_t valueSize; GRD_KVGetSizeInner(resultSet, &keySize, &valueSize); resultSet = nullptr; diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/parseckeck_fuzzer/project.xml b/kv_store/frameworks/libs/distributeddb/test/fuzztest/parseckeck_fuzzer/project.xml index be92381e..3034a779 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/parseckeck_fuzzer/project.xml +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/parseckeck_fuzzer/project.xml @@ -16,7 +16,7 @@ - 1000 + 800 30 diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/BUILD.gn b/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/BUILD.gn index 0cbf9a21..a0423aac 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/BUILD.gn +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/BUILD.gn @@ -83,6 +83,7 @@ ohos_fuzztest("QueryFuzzTest") { sources = distributeddb_src sources += distributeddb_cloud_src sources += [ + "../common/fuzzer_data.cpp", "//foundation/distributeddatamgr/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_data_generate_unit_test.cpp", "query_fuzzer.cpp", ] diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/query_fuzzer.cpp b/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/query_fuzzer.cpp index e8c8e4ed..11085e24 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/query_fuzzer.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/query_fuzzer/query_fuzzer.cpp @@ -14,6 +14,7 @@ */ #include "query_fuzzer.h" +#include "fuzzer_data.h" #include "get_query_info.h" using namespace DistributedDB; @@ -111,6 +112,23 @@ void FuzzIsNull(const uint8_t* data, size_t size) std::string rawString(reinterpret_cast(data), size); Query query = Query::Select().IsNull(rawString); } + +void FuzzAssetsOnly(const uint8_t *data, size_t size) +{ + DistributedDBTest::FuzzerData fuzzerData(data, size); + uint32_t len = fuzzerData.GetUInt32(); + const int lenMod = 30; // 30 is mod for string vector size + std::string tableName = fuzzerData.GetString(len % lenMod); + std::string fieldName = fuzzerData.GetString(len % lenMod); + std::map> assets; + assets[fuzzerData.GetString(len % lenMod)] = fuzzerData.GetStringSet(len % lenMod); + Query query = Query::Select().From(tableName) + .BeginGroup() + .EqualTo(fieldName, fuzzerData.GetInt()) + .And() + .AssetsOnly(assets) + .EndGroup(); +} } /* Fuzzer entry point */ @@ -134,6 +152,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) OHOS::FuzzIn(data, size); OHOS::FuzzNotIn(data, size); OHOS::FuzzIsNull(data, size); + OHOS::FuzzAssetsOnly(data, size); return 0; } diff --git a/kv_store/frameworks/libs/distributeddb/test/fuzztest/storage_fuzzer/storage_fuzzer.cpp b/kv_store/frameworks/libs/distributeddb/test/fuzztest/storage_fuzzer/storage_fuzzer.cpp index f28a29cf..6c34394c 100644 --- a/kv_store/frameworks/libs/distributeddb/test/fuzztest/storage_fuzzer/storage_fuzzer.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/fuzztest/storage_fuzzer/storage_fuzzer.cpp @@ -62,6 +62,7 @@ static std::condition_variable g_cv; static bool g_alreadyNotify = false; DistributedDB::StoreObserver::StoreChangedInfo g_changedData; static constexpr const int MOD = 1000; // 1000 is mod +static constexpr const int MODNUM = 10; class StorageFuzzer { public: @@ -281,7 +282,7 @@ void InitLogicDeleteData(sqlite3 *&db, const std::string &tableName, uint64_t nu void InitDataStatus(const std::string &tableName, int count, sqlite3 *db) { int type = 4; // the num of different status - for (int i = 1; i <= type * count; i++) { + for (int i = 1; i <= (type * count) % MODNUM; i++) { std::string sql = "INSERT INTO " + tableName + " VALUES(" + std::to_string(i) + ", 'zhangsan" + std::to_string(i) + "');"; RdbTestUtils::ExecSql(db, sql); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.cpp new file mode 100644 index 00000000..c775dd24 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "basic_unit_test.h" + +namespace DistributedDB { +using namespace testing::ext; +using namespace DistributedDBUnitTest; +std::string g_testDir; +void BasicUnitTest::SetUpTestCase() +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("[BasicUnitTest] Rm test db files error!"); + } +} + +void BasicUnitTest::TearDownTestCase() +{ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("[BasicUnitTest] Rm test db files error!"); + } +} + +void BasicUnitTest::SetUp() +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + ASSERT_EQ(communicatorAggregator_, nullptr); + communicatorAggregator_ = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(communicatorAggregator_ != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(communicatorAggregator_); +} + +void BasicUnitTest::TearDown() +{ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("[BasicUnitTest] Rm test db files error."); + } + RuntimeContext::GetInstance()->SetCommunicatorAggregator(nullptr); + communicatorAggregator_ = nullptr; +} + +int BasicUnitTest::InitDelegate(const StoreInfo &info, const std::string &deviceId) +{ + if (communicatorAggregator_ == nullptr) { + LOGE("[BasicUnitTest] Init delegate with null aggregator"); + return -E_INTERNAL_ERROR; + } + communicatorAggregator_->SetRemoteDeviceId(deviceId); + int errCode = InitDelegate(info); + if (errCode != E_OK) { + LOGE("[BasicUnitTest] Init delegate failed %d", errCode); + } else { + SetDevice(info, deviceId); + } + return errCode; +} + +std::string BasicUnitTest::GetTestDir() +{ + return g_testDir; +} + +std::string BasicUnitTest::GetDevice(const StoreInfo &info) const +{ + std::lock_guard autoLock(deviceMutex_); + auto iter = deviceMap_.find(info); + if (iter == deviceMap_.end()) { + LOGW("[BasicUnitTest] Not exist device app %s store %s user %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return ""; + } + return iter->second; +} + +void BasicUnitTest::SetDevice(const StoreInfo &info, const std::string &device) +{ + std::lock_guard autoLock(deviceMutex_); + deviceMap_[info] = device; + LOGW("[BasicUnitTest] Set device app %s store %s user %s device %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str(), device.c_str()); +} +} \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.h new file mode 100644 index 00000000..0dfa21ac --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/basic_unit_test.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BASIC_UNIT_TEST_H +#define BASIC_UNIT_TEST_H +#include + +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "virtual_communicator_aggregator.h" + +namespace DistributedDB { + +class BasicUnitTest : public testing::Test { +public: + static void SetUpTestCase(); + static void TearDownTestCase(); + void SetUp() override; + void TearDown() override; +protected: + int InitDelegate(const StoreInfo &info, const std::string &deviceId); + virtual int InitDelegate(const StoreInfo &info) = 0; + virtual int CloseDelegate(const StoreInfo &info) = 0; + virtual void CloseAllDelegate() = 0; + std::string GetDevice(const StoreInfo &info) const; + void SetDevice(const StoreInfo &info, const std::string &device); + static std::string GetTestDir(); + struct StoreComparator { + bool operator() (const StoreInfo &source, const StoreInfo &target) const + { + return source.userId < target.userId || source.appId < target.appId || source.storeId < target.storeId; + } + }; + VirtualCommunicatorAggregator *communicatorAggregator_ = nullptr; + mutable std::mutex deviceMutex_; + std::map deviceMap_; +}; +} +#endif // BASIC_UNIT_TEST_H diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp index edb68a09..e1ea4bc8 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -179,6 +179,10 @@ namespace { const std::string SCHEMA_TABLE_MODE_COLLABORATION = R"("TABLE_MODE": "COLLABORATION",)"; const std::string SCHEMA_TABLE_MODE_SPLIT_BY_DEVICE = R"("TABLE_MODE": "SPLIT_BY_DEVICE",)"; const std::string SCHEMA_TABLE_MODE_INVALID = R"("TABLE_MODE": "SPLIT_BY_USER",)"; + const std::string DISTRIBUTED_VALID_VERSION = R"("DISTRIBUTED_SCHEMA": { "VERSION": 1 })"; + const std::string DISTRIBUTED_INVALID_SMALL_VERSION = R"("DISTRIBUTED_SCHEMA": { "VERSION": -1 })"; + const std::string DISTRIBUTED_INVALID_LARGE_VERSION = R"("DISTRIBUTED_SCHEMA": { "VERSION": 5000000000 })"; + const std::string DISTRIBUTED_INVALID_NAN_VERSION = R"("DISTRIBUTED_SCHEMA": { "VERSION": "not a number" })"; const std::string SCHEMA_TABLE_STR = R""("TABLES": [{ "NAME": "FIRST", @@ -304,7 +308,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest001, EXPECT_EQ(schemaObj2.GetTable("FIRST").GetUniqueDefine().size(), 2u); RelationalSyncOpinion op = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, schemaObj2.ToSchemaString(), - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_EQ(op.size(), 2u); EXPECT_EQ(op.at("FIRST").permitSync, true); @@ -484,6 +488,47 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest006, EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); } +/** + * @tc.name: RelationalSchemaParseTest007 + * @tc.desc: test parse for distributed version in schema + * @tc.type: FUNC + * @tc.require: + * @tc.author: liuhongyang + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest007, TestSize.Level1) +{ + RelationalSchemaObject schemaObj; + std::string schemaOtherField = SCHEMA_VERSION_STR_2_1 + SCHEMA_TABLE_MODE_COLLABORATION + SCHEMA_TYPE_STR_RELATIVE + + SCHEMA_TABLE_STR + ","; + /** + * @tc.steps: step1. call ParseFromSchemaString with a version less than the min of uint32_t + * @tc.expected: step1. return -E_SCHEMA_PARSE_FAIL. + */ + std::string schema = "{" + schemaOtherField + DISTRIBUTED_INVALID_SMALL_VERSION + "}"; + int errCode = schemaObj.ParseFromSchemaString(schema); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + /** + * @tc.steps: step2. call ParseFromSchemaString with a version greater than the max of uint32_t + * @tc.expected: step2. return -E_SCHEMA_PARSE_FAIL. + */ + schema = "{" + schemaOtherField + DISTRIBUTED_INVALID_LARGE_VERSION + "}"; + errCode = schemaObj.ParseFromSchemaString(schema); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + /** + * @tc.steps: step3. call ParseFromSchemaString with a version that is not a number + * @tc.expected: step3. return -E_SCHEMA_PARSE_FAIL. + */ + schema = "{" + schemaOtherField + DISTRIBUTED_INVALID_NAN_VERSION + "}"; + errCode = schemaObj.ParseFromSchemaString(schema); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + /** + * @tc.steps: step4. call ParseFromSchemaString with a normal version + * @tc.expected: step4. return E_OK. + */ + schema = "{" + schemaOtherField + DISTRIBUTED_VALID_VERSION + "}"; + errCode = schemaObj.ParseFromSchemaString(schema); + EXPECT_EQ(errCode, E_OK); +} /** * @tc.name: RelationalSchemaCompareTest001 @@ -499,7 +544,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest001 EXPECT_EQ(errCode, E_OK); RelationalSyncOpinion opinion = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA, - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_EQ(opinion.at("FIRST").permitSync, true); EXPECT_EQ(opinion.at("FIRST").checkOnReceive, false); EXPECT_EQ(opinion.at("FIRST").requirePeerConvert, false); @@ -519,7 +564,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest002 EXPECT_EQ(errCode, E_OK); RelationalSyncOpinion opinion = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA_V2_1, - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_EQ(opinion.at("FIRST").permitSync, true); EXPECT_EQ(opinion.at("FIRST").checkOnReceive, false); EXPECT_EQ(opinion.at("FIRST").requirePeerConvert, false); @@ -539,7 +584,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest003 EXPECT_EQ(errCode, E_OK); RelationalSyncOpinion opinion = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA, - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(opinion.empty()); } @@ -557,7 +602,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest004 EXPECT_EQ(errCode, E_OK); RelationalSyncOpinion opinion = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA_V2_1, - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(opinion.empty()); } @@ -576,7 +621,7 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest005 schemaObj.SetTableMode(DistributedTableMode::COLLABORATION); RelationalSyncOpinion opinion = SchemaNegotiate::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA_V2_1, - static_cast(SchemaType::RELATIVE)); + static_cast(SchemaType::RELATIVE), SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(opinion.empty()); } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp index fc4fbb98..9469dd06 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp @@ -34,6 +34,7 @@ #include "db_common.h" #include "db_constant.h" #include "generic_single_ver_kv_entry.h" +#include "get_query_info.h" #include "platform_specific.h" #include "res_finalizer.h" #include "runtime_config.h" @@ -714,9 +715,18 @@ void DistributedDBToolsUnitTest::BlockSync(RelationalStoreDelegate &delegate, co }; DBStatus callStatus = delegate.Sync(devices, syncMode, query, callBack, true); EXPECT_EQ(callStatus, OK); + QueryExpression queryExpression = GetQueryInfo::GetQueryExpression(query); + std::vector syncTables; + if (queryExpression.IsUseFromTables()) { + syncTables = queryExpression.GetTables(); + } else { + syncTables = {queryExpression.GetTableName()}; + } for (const auto &tablesRes : statusMap) { - for (const auto &tableStatus : tablesRes.second) { - EXPECT_EQ(tableStatus.status, exceptStatus); + ASSERT_EQ(tablesRes.second.size(), syncTables.size()); + for (uint32_t i = 0; i < syncTables.size(); i++) { + EXPECT_EQ(tablesRes.second[i].status, exceptStatus); + EXPECT_EQ(tablesRes.second[i].tableName, syncTables[i]); } } } @@ -750,9 +760,9 @@ void KvStoreObserverUnitTest::OnChange(DistributedDB::StoreObserver::StoreChange void KvStoreObserverUnitTest::OnChange(DistributedDB::Origin origin, const std::string &originalId, DistributedDB::ChangedData &&data) { - (void)origin; - (void)originalId; - (void)data; + callCount_++; + changedData_[data.tableName] = data; + LOGD("data change when cloud sync, origin = %d, tableName = %s", origin, data.tableName.c_str()); KvStoreObserver::OnChange(origin, originalId, std::move(data)); } @@ -763,6 +773,7 @@ void KvStoreObserverUnitTest::ResetToZero() inserted_.clear(); updated_.clear(); deleted_.clear(); + changedData_.clear(); } unsigned long KvStoreObserverUnitTest::GetCallCount() const @@ -790,6 +801,11 @@ bool KvStoreObserverUnitTest::IsCleared() const return isCleared_; } +std::unordered_map KvStoreObserverUnitTest::GetChangedData() const +{ + return changedData_; +} + RelationalStoreObserverUnitTest::RelationalStoreObserverUnitTest() : callCount_(0) { } @@ -1641,6 +1657,7 @@ DistributedDB::ICloudSyncStorageHook *RelationalTestUtils::GetRDBStorageHook(con return nullptr; } auto engine = static_cast(store)->GetStorageEngine(); + RefObject::DecObjRef(store); return static_cast(engine); } @@ -1657,4 +1674,45 @@ DistributedDB::Origin RelationalStoreObserverUnitTest::GetLastOrigin() const { return lastOrigin_; } + +void DistributedDBToolsUnitTest::BlockSync(KvStoreNbDelegate *delegate, DistributedDB::DBStatus expectDBStatus, + DistributedDB::CloudSyncOption option, DistributedDB::DBStatus expectSyncResult) +{ + if (delegate == nullptr) { + return; + } + std::mutex dataMutex; + std::condition_variable cv; + bool finish = false; + SyncProcess last; + auto callback = [expectDBStatus, &last, &cv, &dataMutex, &finish, &option](const std::map &process) { + size_t notifyCnt = 0; + for (const auto &item: process) { + LOGD("user = %s, status = %d, errCode = %d", item.first.c_str(), item.second.process, item.second.errCode); + if (item.second.process != DistributedDB::FINISHED) { + continue; + } + EXPECT_EQ(item.second.errCode, expectDBStatus); + { + std::lock_guard autoLock(dataMutex); + notifyCnt++; + std::set userSet(option.users.begin(), option.users.end()); + if (notifyCnt == userSet.size()) { + finish = true; + last = item.second; + cv.notify_one(); + } + } + } + }; + auto actualRet = delegate->Sync(option, callback); + EXPECT_EQ(actualRet, expectSyncResult); + if (actualRet == OK) { + std::unique_lock uniqueLock(dataMutex); + cv.wait(uniqueLock, [&finish]() { + return finish; + }); + } +} } // namespace DistributedDBUnitTest diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h index a076b2c2..704df20d 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h @@ -235,6 +235,9 @@ public: static void BlockSync(DistributedDB::RelationalStoreDelegate &delegate, const DistributedDB::Query &query, DistributedDB::SyncMode syncMode, DistributedDB::DBStatus exceptStatus, const std::vector &devices); + + static void BlockSync(DistributedDB::KvStoreNbDelegate *delegate, DistributedDB::DBStatus expectDBStatus, + DistributedDB::CloudSyncOption option, DistributedDB::DBStatus expectSyncResult = DistributedDB::DBStatus::OK); private: static int OpenMockMultiDb(DatabaseInfo &dbInfo, DistributedDB::OpenDbProperties &properties); @@ -271,12 +274,14 @@ public: const std::list &GetEntriesUpdated() const; const std::list &GetEntriesDeleted() const; bool IsCleared() const; + std::unordered_map GetChangedData() const; private: unsigned long callCount_; bool isCleared_; std::list inserted_; std::list updated_; std::list deleted_; + std::unordered_map changedData_; }; class RelationalStoreObserverUnitTest : public DistributedDB::StoreObserver { diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp index ea7ab4c6..a020e68e 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp @@ -405,12 +405,13 @@ HWTEST_F(DistributedDBEventLoopTimerTest, EventLoopTimerTest007, TestSize.Level2 ASSERT_EQ(timer != nullptr, true); int counter = 1; // Interval: 1 * TIME_PIECE_100 EventTime lastTime = TimerTester::GetCurrentTime(); + std::atomic total = 0; errCode = timer->SetAction( - [timer, &counter, &lastTime](EventsMask revents) -> int { + [timer, &counter, &lastTime, &total](EventsMask revents) -> int { EventTime now = TimerTester::GetCurrentTime(); EventTime delta = now - lastTime; delta -= counter * TIME_PIECE_1000; - EXPECT_EQ(delta >= -TIME_INACCURACY && delta <= TIME_INACCURACY, true); + total += delta; if (++counter > RETRY_TIMES_5) { return -E_STALE; } @@ -421,6 +422,7 @@ HWTEST_F(DistributedDBEventLoopTimerTest, EventLoopTimerTest007, TestSize.Level2 } return E_OK; }, nullptr); + EXPECT_LE(std::abs(total), TIME_INACCURACY * RETRY_TIMES_5); EXPECT_EQ(errCode, E_OK); errCode = g_loop->Add(timer); EXPECT_EQ(errCode, E_OK); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.cpp new file mode 100644 index 00000000..cf413c03 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.cpp @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_general_ut.h" +#include "virtual_cloud_db.h" + +namespace DistributedDB { +void KVGeneralUt::SetUp() +{ + virtualCloudDb_ = std::make_shared(); + BasicUnitTest::SetUp(); +} + +void KVGeneralUt::TearDown() +{ + CloseAllDelegate(); + virtualCloudDb_ = nullptr; + BasicUnitTest::TearDown(); +} + +int KVGeneralUt::InitDelegate(const StoreInfo &info) +{ + KvStoreDelegateManager manager(info.appId, info.userId); + manager.SetKvStoreConfig(GetKvStoreConfig()); + KvStoreNbDelegate::Option option; + std::lock_guard autoLock(storeMutex_); + if (option_.has_value()) { + option = option_.value(); + } + KvStoreNbDelegate *store = nullptr; + DBStatus status = DBStatus::OK; + manager.GetKvStore(info.storeId, option, [&status, &store](DBStatus ret, KvStoreNbDelegate *delegate) { + status = ret; + store = delegate; + }); + if (status != DBStatus::OK) { + LOGE("[KVGeneralUt] Init delegate failed %d", static_cast(status)); + return -E_INTERNAL_ERROR; + } + stores_[info] = store; + LOGI("[KVGeneralUt] Init delegate app %s store %s user %s success", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return E_OK; +} + +int KVGeneralUt::CloseDelegate(const StoreInfo &info) +{ + std::lock_guard autoLock(storeMutex_); + auto iter = stores_.find(info); + if (iter == stores_.end()) { + LOGW("[KVGeneralUt] Close not exist delegate app %s store %s user %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return E_OK; + } + KvStoreDelegateManager manager(info.appId, info.userId); + manager.SetKvStoreConfig(GetKvStoreConfig()); + auto ret = manager.CloseKvStore(iter->second); + if (ret != DBStatus::OK) { + LOGI("[KVGeneralUt] Close delegate app %s store %s user %s failed %d", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str(), static_cast(ret)); + return -E_INTERNAL_ERROR; + } + LOGI("[KVGeneralUt] Close delegate app %s store %s user %s success", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + stores_.erase(iter); + return E_OK; +} + +void KVGeneralUt::CloseAllDelegate() +{ + std::vector infoList; + { + std::lock_guard autoLock(storeMutex_); + for (const auto &item : stores_) { + infoList.push_back(item.first); + } + } + for (const auto &info : infoList) { + (void)CloseDelegate(info); + } +} + +void KVGeneralUt::SetOption(const KvStoreNbDelegate::Option &option) +{ + std::lock_guard autoLock(storeMutex_); + option_ = option; +} + +KvStoreConfig KVGeneralUt::GetKvStoreConfig() +{ + KvStoreConfig config; + config.dataDir = GetTestDir(); + return config; +} + +StoreInfo KVGeneralUt::GetStoreInfo1() +{ + StoreInfo info; + info.userId = DistributedDBUnitTest::USER_ID; + info.storeId = DistributedDBUnitTest::STORE_ID_1; + info.appId = DistributedDBUnitTest::APP_ID; + return info; +} + +StoreInfo KVGeneralUt::GetStoreInfo2() +{ + StoreInfo info; + info.userId = DistributedDBUnitTest::USER_ID; + info.storeId = DistributedDBUnitTest::STORE_ID_2; + info.appId = DistributedDBUnitTest::APP_ID; + return info; +} + +KvStoreNbDelegate *KVGeneralUt::GetDelegate(const DistributedDB::StoreInfo &info) const +{ + std::lock_guard autoLock(storeMutex_); + auto iter = stores_.find(info); + if (iter == stores_.end()) { + LOGW("[KVGeneralUt] Not exist delegate app %s store %s user %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return nullptr; + } + return iter->second; +} + +void KVGeneralUt::BlockPush(const StoreInfo &from, const StoreInfo &to, DBStatus expectRet) +{ + auto fromStore = GetDelegate(from); + ASSERT_NE(fromStore, nullptr); + auto toDevice = GetDevice(to); + ASSERT_FALSE(toDevice.empty()); + std::map syncRet; + tool_.SyncTest(fromStore, {toDevice}, SyncMode::SYNC_MODE_PUSH_ONLY, syncRet); + for (const auto &item : syncRet) { + EXPECT_EQ(item.second, expectRet); + } +} + +DataBaseSchema KVGeneralUt::GetDataBaseSchema(bool invalidSchema) +{ + DataBaseSchema schema; + TableSchema tableSchema; + tableSchema.name = invalidSchema ? "invalid_schema_name" : CloudDbConstant::CLOUD_KV_TABLE_NAME; + Field field; + field.colName = CloudDbConstant::CLOUD_KV_FIELD_KEY; + field.type = TYPE_INDEX; + field.primary = true; + tableSchema.fields.push_back(field); + field.colName = CloudDbConstant::CLOUD_KV_FIELD_DEVICE; + field.primary = false; + tableSchema.fields.push_back(field); + field.colName = CloudDbConstant::CLOUD_KV_FIELD_ORI_DEVICE; + tableSchema.fields.push_back(field); + field.colName = CloudDbConstant::CLOUD_KV_FIELD_VALUE; + tableSchema.fields.push_back(field); + field.colName = CloudDbConstant::CLOUD_KV_FIELD_DEVICE_CREATE_TIME; + field.type = TYPE_INDEX; + tableSchema.fields.push_back(field); + schema.tables.push_back(tableSchema); + return schema; +} + +DBStatus KVGeneralUt::SetCloud(KvStoreNbDelegate *&delegate, bool invalidSchema) +{ + std::lock_guard autoLock(storeMutex_); + std::map> cloudDbs; + cloudDbs[DistributedDBUnitTest::USER_ID] = virtualCloudDb_; + delegate->SetCloudDB(cloudDbs); + std::map schemas; + schemas[DistributedDBUnitTest::USER_ID] = GetDataBaseSchema(invalidSchema); + return delegate->SetCloudDbSchema(schemas); +} + +DBStatus KVGeneralUt::GetDeviceEntries(KvStoreNbDelegate *delegate, const std::string &deviceId, bool isSelfDevice, + std::vector &entries) +{ + if (isSelfDevice) { + communicatorAggregator_->SetLocalDeviceId(deviceId); + } else { + communicatorAggregator_->SetLocalDeviceId(deviceId + "_"); + } + return delegate->GetDeviceEntries(deviceId, entries); +} + +void KVGeneralUt::BlockCloudSync(const StoreInfo &from, const std::string &deviceId, DBStatus expectRet) +{ + auto fromStore = GetDelegate(from); + ASSERT_NE(fromStore, nullptr); + + communicatorAggregator_->SetLocalDeviceId(deviceId); + CloudSyncOption syncOption; + syncOption.mode = SyncMode::SYNC_MODE_CLOUD_MERGE; + syncOption.users.push_back(DistributedDBUnitTest::USER_ID); + syncOption.devices.push_back("cloud"); + tool_.BlockSync(fromStore, DBStatus::OK, syncOption, expectRet); +} +} \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.h new file mode 100644 index 00000000..7f228eb4 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/kv_general_ut.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KV_GENERAL_UT_H +#define KV_GENERAL_UT_H + +#include "basic_unit_test.h" +#include "distributeddb_tools_unit_test.h" + +namespace DistributedDB { +class KVGeneralUt : public BasicUnitTest { +public: + void SetUp() override; + void TearDown() override; +protected: + int InitDelegate(const StoreInfo &info) override; + int CloseDelegate(const StoreInfo &info) override; + void CloseAllDelegate() override; + void SetOption(const KvStoreNbDelegate::Option &option); + KvStoreNbDelegate *GetDelegate(const StoreInfo &info) const; + void BlockPush(const StoreInfo &from, const StoreInfo &to, DBStatus expectRet = DBStatus::OK); + DBStatus SetCloud(KvStoreNbDelegate *&delegate, bool invalidSchema = false); + static DataBaseSchema GetDataBaseSchema(bool invalidSchema); + DBStatus GetDeviceEntries(KvStoreNbDelegate *delegate, const std::string &deviceId, bool isSelfDevice, + std::vector &entries); + void BlockCloudSync(const StoreInfo &from, const std::string &deviceId, DBStatus expectRet = DBStatus::OK); + static KvStoreConfig GetKvStoreConfig(); + static StoreInfo GetStoreInfo1(); + static StoreInfo GetStoreInfo2(); + mutable std::mutex storeMutex_; + std::optional option_; + std::map stores_; + DistributedDBUnitTest::DistributedDBToolsUnitTest tool_; + std::shared_ptr virtualCloudDb_; +}; +} +#endif // KV_GENERAL_UT_H + diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/process_communicator_test_stub.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/process_communicator_test_stub.h index 1a31ba62..4d420a1f 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/process_communicator_test_stub.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/process_communicator_test_stub.h @@ -85,8 +85,21 @@ public: { isCommErr = commErr; } + + DBStatus GetDataUserInfo(const uint8_t *data, uint32_t totalLen, const std::string &label, + std::vector &userInfos) override + { + userInfos = userInfos_; + return OK; + } + + void SetDataUserInfo(const std::vector &userInfos) + { + userInfos_ = userInfos; + } private: bool isCommErr = false; + std::vector userInfos_; }; } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.cpp index b770a478..3bbe98f7 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.cpp @@ -381,7 +381,7 @@ std::string RDBDataGenerator::GetUpdateSQL(const TableSchema &schema) } int RDBDataGenerator::InsertVirtualLocalDBData(int64_t begin, int64_t count, - DistributedDB::RelationalVirtualDevice *device, const DistributedDB::TableSchema &schema) + DistributedDB::RelationalVirtualDevice *device, const DistributedDB::TableSchema &schema, const bool isDelete) { if (device == nullptr) { return -E_INVALID_ARGS; @@ -394,6 +394,9 @@ int RDBDataGenerator::InsertVirtualLocalDBData(int64_t begin, int64_t count, FillTypeIntoDataValue(field, type, virtualRowData); } virtualRowData.logInfo.timestamp = TimeHelper::GetSysCurrentTime() + TimeHelper::BASE_OFFSET; + if (isDelete) { + virtualRowData.logInfo.flag = virtualRowData.logInfo.flag | DataItem::DELETE_FLAG; + } rows.push_back(virtualRowData); } return device->PutData(schema.name, rows); @@ -476,4 +479,46 @@ DistributedDB::TableSchema RDBDataGenerator::FlipTableSchema(const DistributedDB } return res; } + +int RDBDataGenerator::InitDatabaseWithSchemaInfo(const UtDateBaseSchemaInfo &schemaInfo, sqlite3 &db) +{ + int errCode = RelationalTestUtils::ExecSql(&db, "PRAGMA journal_mode=WAL;"); + if (errCode != SQLITE_OK) { + LOGE("[RDBDataGenerator] Execute sql failed %d", errCode); + return errCode; + } + for (const auto &tableInfo : schemaInfo.tablesInfo) { + errCode = InitTableWithSchemaInfo(tableInfo, db); + if (errCode != SQLITE_OK) { + LOGE("[RDBDataGenerator] Init table failed %d, %s", errCode, tableInfo.name.c_str()); + break; + } + } + return errCode; +} + +int RDBDataGenerator::InitTableWithSchemaInfo(const UtTableSchemaInfo &tableInfo, sqlite3 &db) +{ + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableInfo.name + "("; + for (const auto &fieldInfo : tableInfo.fieldInfo) { + sql += "'" + fieldInfo.field.colName + "' " + GetTypeText(fieldInfo.field.type); + if (fieldInfo.field.primary) { + sql += " PRIMARY KEY"; + if (fieldInfo.isAutoIncrement) { + sql += " AUTOINCREMENT"; + } + } + if (!fieldInfo.field.nullable && fieldInfo.field.type == TYPE_INDEX) { + sql += " NOT NULL ON CONFLICT IGNORE"; + } + sql += ","; + } + sql.pop_back(); + sql += ");"; + int errCode = RelationalTestUtils::ExecSql(&db, sql); + if (errCode != SQLITE_OK) { + LOGE("[RDBDataGenerator] Execute sql failed %d, sql is %s", errCode, sql.c_str()); + } + return errCode; +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.h index 6cd34c8a..3adad88b 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_data_generator.h @@ -20,6 +20,20 @@ #include "sqlite_utils.h" #include "virtual_cloud_db.h" namespace DistributedDBUnitTest { +struct UtFieldInfo { + DistributedDB::Field field; + bool isAutoIncrement = false; +}; + +struct UtTableSchemaInfo { + std::string name; + std::string sharedTableName; + std::vector fieldInfo; +}; + +struct UtDateBaseSchemaInfo { + std::vector tablesInfo; +}; class RDBDataGenerator { public: static int InitDatabase(const DistributedDB::DataBaseSchema &schema, sqlite3 &db); @@ -49,7 +63,7 @@ public: const DistributedDB::TableSchema &schema); static int InsertVirtualLocalDBData(int64_t begin, int64_t count, DistributedDB::RelationalVirtualDevice *device, - const DistributedDB::TableSchema &schema); + const DistributedDB::TableSchema &schema, const bool isDelete = false); static DistributedDB::DistributedSchema ParseSchema(const DistributedDB::DataBaseSchema &schema, bool syncOnlyPk = false); @@ -61,6 +75,10 @@ public: DistributedDB::RelationalVirtualDevice *device); static DistributedDB::TableSchema FlipTableSchema(const DistributedDB::TableSchema &origin); + + static int InitDatabaseWithSchemaInfo(const UtDateBaseSchemaInfo &schemaInfo, sqlite3 &db); + + static int InitTableWithSchemaInfo(const UtTableSchemaInfo &tableInfo, sqlite3 &db); private: static std::string GetTypeText(int type); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.cpp new file mode 100644 index 00000000..142941a8 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.cpp @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "rdb_general_ut.h" +#include "rdb_data_generator.h" + +using namespace DistributedDBUnitTest; + +namespace DistributedDB { +const Field intField = {"id", TYPE_INDEX, true, false}; +const Field stringField = {"name", TYPE_INDEX, false, false}; +const Field boolField = {"gender", TYPE_INDEX, false, true}; +const Field doubleField = {"height", TYPE_INDEX, false, true}; +const Field bytesField = {"photo", TYPE_INDEX, false, true}; +const Field assetsField = {"assert", TYPE_INDEX, false, true}; + +const std::vector g_defaultFiledInfo = { + {intField, true}, {stringField, false}, {boolField, false}, + {doubleField, false}, {bytesField, false}, {assetsField, false}, +}; + +UtDateBaseSchemaInfo g_defaultSchemaInfo = { + .tablesInfo = { + {.name = g_defaultTable1, .sharedTableName = g_defaultTable1 + "_shared", .fieldInfo = g_defaultFiledInfo}, + {.name = g_defaultTable2, .fieldInfo = g_defaultFiledInfo} + } +}; + +int RDBGeneralUt::InitDelegate(const StoreInfo &info) +{ + std::string storePath = GetTestDir() + "/" + info.storeId + ".db"; + sqlite3 *db = RelationalTestUtils::CreateDataBase(storePath); + if (db == nullptr) { + LOGE("[RDBGeneralUt] Create database failed %s", storePath.c_str()); + return -E_INVALID_DB; + } + UtDateBaseSchemaInfo schemaInfo = GetTableSchemaInfo(info); + int errCode = RDBDataGenerator::InitDatabaseWithSchemaInfo(schemaInfo, *db); + if (errCode != SQLITE_OK) { + LOGE("[RDBGeneralUt] Init database failed %d", errCode); + return errCode; + } + RelationalStoreManager mgr(info.appId, info.userId); + RelationalStoreDelegate *delegate = nullptr; + errCode = mgr.OpenStore(storePath, info.storeId, option_, delegate); + if ((errCode != E_OK )|| (delegate == nullptr)) { + LOGE("[RDBGeneralUt] Open store failed %d", errCode); + return errCode; + } + { + std::lock_guard lock(storeMutex_); + stores_[info] = delegate; + sqliteDb_[info] = db; + } + return E_OK; +} + +void RDBGeneralUt::SetOption(const RelationalStoreDelegate::Option& option) +{ + option_ = option; +} + +void RDBGeneralUt::AddSchemaInfo(const StoreInfo &info, const DistributedDBUnitTest::UtDateBaseSchemaInfo &schemaInfo) +{ + std::lock_guard lock(storeMutex_); + schemaInfoMap_[info] = schemaInfo; +} + +UtDateBaseSchemaInfo RDBGeneralUt::GetTableSchemaInfo(const StoreInfo &info) +{ + std::lock_guard lock(storeMutex_); + auto iter = schemaInfoMap_.find(info); + if (iter != schemaInfoMap_.end()) { + return iter->second; + } + return g_defaultSchemaInfo; +} + +DataBaseSchema RDBGeneralUt::GetSchema(const StoreInfo &info) +{ + UtDateBaseSchemaInfo schemaInfo = GetTableSchemaInfo(info); + DataBaseSchema schema; + for (auto &item : schemaInfo.tablesInfo) { + TableSchema tableSchema; + tableSchema.name = item.name; + tableSchema.sharedTableName = item.sharedTableName; + for (auto &fieldInfo : item.fieldInfo) { + tableSchema.fields.push_back(fieldInfo.field); + } + schema.tables.push_back(tableSchema); + } + return schema; +} + +TableSchema RDBGeneralUt::GetTableSchema(const StoreInfo &info, const std::string &tableName) +{ + UtDateBaseSchemaInfo schemaInfo = GetTableSchemaInfo(info); + for (auto &item : schemaInfo.tablesInfo) { + if (item.name != tableName) { + continue; + } + TableSchema tableSchema; + tableSchema.name = item.name; + tableSchema.sharedTableName = item.sharedTableName; + for (auto &fieldInfo : item.fieldInfo) { + tableSchema.fields.push_back(fieldInfo.field); + } + return tableSchema; + } + LOGD("[RDBGeneralUt] Table %s not found", tableName.c_str()); + return {"", "", {}}; +} + +std::vector RDBGeneralUt::GetAllTrackerSchema(const StoreInfo &info, + const std::vector &tables) +{ + UtDateBaseSchemaInfo schemaInfo = GetTableSchemaInfo(info); + std::vector res; + std::set trackerTables(tables.begin(), tables.end()); + for (const auto &item : schemaInfo.tablesInfo) { + if (trackerTables.find(item.name) == trackerTables.end()) { + continue; + } + TrackerSchema trackerSchema; + trackerSchema.tableName = item.name; + for (const auto &field : item.fieldInfo) { + trackerSchema.trackerColNames.insert(field.field.colName); + trackerSchema.extendColNames.insert(field.field.colName); + } + res.push_back(std::move(trackerSchema)); + } + return res; +} + +int RDBGeneralUt::CloseDelegate(const StoreInfo &info) +{ + std::lock_guard lock(storeMutex_); + auto storeIter = stores_.find(info); + if (storeIter != stores_.end()) { + RelationalStoreManager mgr(info.appId, info.userId); + EXPECT_EQ(mgr.CloseStore(storeIter->second), OK); + storeIter->second = nullptr; + stores_.erase(storeIter); + } + auto dbIter = sqliteDb_.find(info); + if (dbIter != sqliteDb_.end()) { + sqlite3_close_v2(dbIter->second); + dbIter->second = nullptr; + sqliteDb_.erase(dbIter); + } + auto schemaIter = schemaInfoMap_.find(info); + if (schemaIter != schemaInfoMap_.end()) { + schemaInfoMap_.erase(schemaIter); + } + return E_OK; +} + +void RDBGeneralUt::CloseAllDelegate() +{ + std::lock_guard lock(storeMutex_); + for (auto &iter : sqliteDb_) { + sqlite3_close_v2(iter.second); + iter.second = nullptr; + } + sqliteDb_.clear(); + for (auto &iter : stores_) { + RelationalStoreManager mgr(iter.first.appId, iter.first.userId); + EXPECT_EQ(mgr.CloseStore(iter.second), OK); + iter.second = nullptr; + } + stores_.clear(); + schemaInfoMap_.clear(); +} + +void RDBGeneralUt::SetUp() +{ + BasicUnitTest::SetUp(); +} + +void RDBGeneralUt::TearDown() +{ + CloseAllDelegate(); + BasicUnitTest::TearDown(); +} + +int RDBGeneralUt::InitDatabase(const StoreInfo &info) +{ + auto schema = GetSchema(info); + auto db = GetSqliteHandle(info); + if (db == nullptr) { + LOGE("[RDBGeneralUt] Get null sqlite when init database"); + return -E_INVALID_DB; + } + auto errCode = RDBDataGenerator::InitDatabase(schema, *db); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Init db failed %d app %s store %s user %s", errCode, info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + } + return errCode; +} + +sqlite3 *RDBGeneralUt::GetSqliteHandle(const StoreInfo &info) +{ + std::lock_guard autoLock(storeMutex_); + auto db = sqliteDb_.find(info); + if (db == sqliteDb_.end()) { + LOGE("[RDBGeneralUt] Not exist sqlite db app %s store %s user %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return nullptr; + } + return db->second; +} + +int RDBGeneralUt::InsertLocalDBData(int64_t begin, int64_t count, const StoreInfo &info) +{ + auto schema = GetSchema(info); + auto db = GetSqliteHandle(info); + if (db == nullptr) { + LOGE("[RDBGeneralUt] Get null sqlite when insert data"); + return -E_INVALID_DB; + } + auto errCode = RDBDataGenerator::InsertLocalDBData(begin, begin + count, db, schema); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Insert data failed %d app %s store %s user %s", errCode, info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + } else { + LOGI("[RDBGeneralUt] Insert data success begin %" PRId64 " count %" PRId64 " app %s store %s user %s", + begin, count, info.appId.c_str(), info.storeId.c_str(), info.userId.c_str()); + } + return errCode; +} + +int RDBGeneralUt::CreateDistributedTable(const StoreInfo &info, const std::string &table, TableSyncType type) +{ + auto store = GetDelegate(info); + if (store == nullptr) { + LOGE("[RDBGeneralUt] Get null delegate when create distributed table %s", table.c_str()); + return -E_INVALID_DB; + } + auto errCode = store->CreateDistributedTable(table, type); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Create distributed table failed %d app %s store %s user %s", errCode, info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + } + return errCode; +} + +int RDBGeneralUt::SetDistributedTables(const StoreInfo &info, const std::vector &tables, + TableSyncType type) +{ + int errCode = E_OK; + for (const auto &table : tables) { + errCode = CreateDistributedTable(info, table, type); + if (errCode != E_OK) { + return errCode; + } + } + if (option_.tableMode != DistributedTableMode::COLLABORATION) { + return E_OK; + } + auto store = GetDelegate(info); + if (store == nullptr) { + LOGE("[RDBGeneralUt] Get null delegate when set distributed tables"); + return -E_INVALID_DB; + } + auto schema = GetSchema(info); + errCode = store->SetDistributedSchema(RDBDataGenerator::ParseSchema(schema)); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Set distributed schema failed %d app %s store %s user %s", errCode, info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + } + return errCode; +} + +RelationalStoreDelegate *RDBGeneralUt::GetDelegate(const StoreInfo &info) +{ + std::lock_guard autoLock(storeMutex_); + auto db = stores_.find(info); + if (db == stores_.end()) { + LOGE("[RDBGeneralUt] Not exist delegate app %s store %s user %s", info.appId.c_str(), + info.storeId.c_str(), info.userId.c_str()); + return nullptr; + } + return db->second; +} + +void RDBGeneralUt::BlockPush(const StoreInfo &from, const StoreInfo &to, const std::string &table, DBStatus expectRet) +{ + auto store = GetDelegate(from); + ASSERT_NE(store, nullptr); + auto toDevice = GetDevice(to); + ASSERT_FALSE(toDevice.empty()); + Query query = Query::Select(table); + DistributedDBToolsUnitTest::BlockSync(*store, query, SYNC_MODE_PUSH_ONLY, expectRet, {toDevice}); +} + +int RDBGeneralUt::CountTableData(const StoreInfo &info, const std::string &table) +{ + return CountTableDataByDev(info, table, ""); +} + +int RDBGeneralUt::CountTableDataByDev(const StoreInfo &info, const std::string &table, const std::string &dev) +{ + auto db = GetSqliteHandle(info); + if (db == nullptr) { + LOGE("[RDBGeneralUt] Get null sqlite when count table %s", table.c_str()); + return 0; + } + bool isCreated = false; + int errCode = SQLiteUtils::CheckTableExists(db, table, isCreated); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Check table exist failed %d when count table %s", errCode, table.c_str()); + return 0; + } + if (!isCreated) { + LOGW("[RDBGeneralUt] Check table %s not exist", table.c_str()); + return 0; + } + + std::string cntSql = "SELECT count(*) FROM '" + table + "'"; + if (!dev.empty()) { + auto hex = DBCommon::TransferStringToHex(DBCommon::TransferHashString(dev)); + cntSql.append(" WHERE device='").append(hex).append("'"); + } + sqlite3_stmt *stmt = nullptr; + errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Check table %s failed by get stmt %d", table.c_str(), errCode); + return 0; + } + int count = 0; + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = sqlite3_column_int(stmt, 0); + LOGI("[RDBGeneralUt] Count table %s success exist %d data dev %s", table.c_str(), count, dev.c_str()); + } else { + LOGE("[RDBGeneralUt] Count table %s failed by %d dev %s", table.c_str(), errCode, dev.c_str()); + } + + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return count; +} + +int RDBGeneralUt::SetTrackerTables(const StoreInfo &info, const std::vector &tables) +{ + auto store = GetDelegate(info); + if (store == nullptr) { + LOGE("[RDBGeneralUt] Get null delegate when set tracker tables"); + return -E_INVALID_DB; + } + auto trackerSchema = GetAllTrackerSchema(info, tables); + for (const auto &trackerTable : trackerSchema) { + auto errCode = store->SetTrackerTable(trackerTable); + if (errCode != E_OK) { + LOGE("[RDBGeneralUt] Set tracker table %s failed %d app %s store %s user %s", + trackerTable.tableName.c_str(), errCode, + info.appId.c_str(), info.storeId.c_str(), info.userId.c_str()); + return errCode; + } + } + LOGI("[RDBGeneralUt] Set tracker %zu table success app %s store %s user %s", + trackerSchema.size(), info.appId.c_str(), info.storeId.c_str(), info.userId.c_str()); + return E_OK; +} +} \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.h new file mode 100644 index 00000000..1397fcd7 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/common/rdb_general_ut.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RDB_GENERAL_UT_H +#define RDB_GENERAL_UT_H + +#include "basic_unit_test.h" +#include "rdb_data_generator.h" +#include "relational_store_manager.h" + +namespace DistributedDB { +const std::string g_defaultTable1 = "defaultTable1"; +const std::string g_defaultTable2 = "defaultTable2"; + +class RDBGeneralUt : public BasicUnitTest { +public: + void SetUp() override; + void TearDown() override; +protected: + int InitDelegate(const StoreInfo &info) override; + int CloseDelegate(const StoreInfo &info) override; + void CloseAllDelegate() override; + + // If SetOption is not invoked before InitDelegate is invoked, the default data of Option is used to open store. + void SetOption(const RelationalStoreDelegate::Option& option); + // If AddSchemaInfo is not invoked before InitDelegate is invoked, g_defaultSchemaInfo is used to create table. + void AddSchemaInfo(const StoreInfo &info, const DistributedDBUnitTest::UtDateBaseSchemaInfo& schemaInfo); + DataBaseSchema GetSchema(const StoreInfo &info); + TableSchema GetTableSchema(const StoreInfo &info, const std::string &tableName = g_defaultTable1); + std::vector GetAllTrackerSchema(const StoreInfo &info, const std::vector &tables); + + int InitDatabase(const StoreInfo &info); + + int InsertLocalDBData(int64_t begin, int64_t count, const StoreInfo &info); + + int CreateDistributedTable(const StoreInfo &info, const std::string &table, + TableSyncType type = TableSyncType::DEVICE_COOPERATION); + + int SetDistributedTables(const StoreInfo &info, const std::vector &tables, + TableSyncType type = TableSyncType::DEVICE_COOPERATION); + + void BlockPush(const StoreInfo &from, const StoreInfo &to, const std::string &table, + DBStatus expectRet = DBStatus::OK); + + DistributedDBUnitTest::UtDateBaseSchemaInfo GetTableSchemaInfo(const StoreInfo &info); + sqlite3 *GetSqliteHandle(const StoreInfo &info); + RelationalStoreDelegate *GetDelegate(const StoreInfo &info); + + int CountTableData(const StoreInfo &info, const std::string &table); + + int CountTableDataByDev(const StoreInfo &info, const std::string &table, const std::string &dev); + + int SetTrackerTables(const StoreInfo &info, const std::vector &tables); + + RelationalStoreDelegate::Option option_; + mutable std::mutex storeMutex_; + std::map stores_; + std::map sqliteDb_; + std::map schemaInfoMap_; +}; +} +#endif // RDB_GENERAL_UT_H diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp index 6c874be3..0d2f5c9d 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp @@ -21,6 +21,7 @@ #include "iprocess_communicator.h" #include "log_print.h" #include "distributeddb_communicator_common.h" +#include "process_communicator_test_stub.h" using namespace DistributedDB; @@ -206,14 +207,18 @@ void AdapterStub::DeliverBytes(const std::string &srcTarget, const uint8_t *byte std::lock_guard onReceiveLockGuard(onReceiveMutex_); if (onReceiveHandle_) { uint32_t headLength = 0; - std::string userId; - CheckAndGetDataHeadInfo(bytes, length, headLength, userId); - onReceiveHandle_(srcTarget, bytes + headLength, length - headLength, userId); + GetDataHeadInfo(bytes, headLength); + std::vector userInfos; + GetDataUserInfo(bytes, userInfos); + std::shared_ptr processCommunicator = + std::make_shared(); + processCommunicator->SetDataUserInfo(userInfos); + DataUserInfoProc userInfoProc = {bytes, length, processCommunicator}; + onReceiveHandle_(srcTarget, bytes + headLength, length - headLength, userInfoProc); } } -void AdapterStub::CheckAndGetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength, - std::string &userId) +void AdapterStub::GetDataHeadInfo(const uint8_t *data, uint32_t &headLength) { auto info = reinterpret_cast(data); NetToHost(info->magic); @@ -221,15 +226,24 @@ void AdapterStub::CheckAndGetDataHeadInfo(const uint8_t *data, uint32_t totalLen NetToHost(info->length); NetToHost(info->version); headLength = info->length; - userId = ""; + } else { + headLength = 0; + } +} + +void AdapterStub::GetDataUserInfo(const uint8_t *data, std::vector &userInfos) +{ + auto info = reinterpret_cast(data); + NetToHost(info->magic); + if (info->magic == ExtendHeaderHandleTest::MAGIC_NUM) { + UserInfo userInfo; for (uint8_t i = 0; i < BUFF_LEN; i++) { if (info->userId[i] == 0) { - return; + break; } - userId.push_back(info->userId[i]); + userInfo.receiveUser.push_back(info->userId[i]); } - } else { - headLength = 0; + userInfos.push_back(userInfo); } } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h index 88b773a9..13c8396f 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h @@ -51,8 +51,9 @@ public: std::shared_ptr GetExtendHeaderHandle(const ExtendInfo ¶mInfo) override; - void CheckAndGetDataHeadInfo(const uint8_t *data, uint32_t totalLen, uint32_t &headLength, - std::string &userId); + void GetDataHeadInfo(const uint8_t *data, uint32_t &headLength); + + void GetDataUserInfo(const uint8_t *data, std::vector &userInfos); /* * Extended Part diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp index 03522eb7..9b0b725a 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp @@ -857,21 +857,24 @@ HWTEST_F(DistributedDBCommunicatorDeepTest, NetworkAdapter005, TestSize.Level1) /** * @tc.steps: step2. adapter recv data with no permission */ - EXPECT_CALL(*processCommunicator, CheckAndGetDataHeadInfo).WillRepeatedly( - [](const uint8_t *, uint32_t, uint32_t &, std::vector &) { + EXPECT_CALL(*processCommunicator, GetDataHeadInfo).WillRepeatedly([](const uint8_t *, uint32_t, uint32_t &) { return NO_PERMISSION; }); onDataReceive(deviceInfos, data.get(), 1); - EXPECT_CALL(*processCommunicator, CheckAndGetDataHeadInfo).WillRepeatedly( - [](const uint8_t *, uint32_t, uint32_t &, std::vector &userIds) { - userIds.emplace_back("1"); + EXPECT_CALL(*processCommunicator, GetDataHeadInfo).WillRepeatedly([](const uint8_t *, uint32_t, uint32_t &) { + return OK; + }); + EXPECT_CALL(*processCommunicator, GetDataUserInfo).WillRepeatedly( + [](const uint8_t *, uint32_t, const std::string &, std::vector &userInfos) { + UserInfo userId = {"1"}; + userInfos.emplace_back(userId); return OK; }); /** * @tc.steps: step3. adapter recv data with no callback */ onDataReceive(deviceInfos, data.get(), 1); - adapter->RegBytesReceiveCallback([](const std::string &, const uint8_t *, uint32_t, const std::string &) { + adapter->RegBytesReceiveCallback([](const std::string &, const uint8_t *, uint32_t, const DataUserInfoProc &) { }, nullptr); onDataReceive(deviceInfos, data.get(), 1); RuntimeContext::GetInstance()->StopTaskPool(); @@ -942,11 +945,10 @@ HWTEST_F(DistributedDBCommunicatorDeepTest, NetworkAdapter007, TestSize.Level1) InitAdapter(adapter, processCommunicator, onDataReceive, onDeviceChange); ASSERT_NE(onDeviceChange, nullptr); /** - * @tc.steps: step1. CheckAndGetDataHeadInfo return invalid headLen + * @tc.steps: step1. GetDataHeadInfo return invalid headLen * @tc.expected: step1. adapter check this len */ - EXPECT_CALL(*processCommunicator, CheckAndGetDataHeadInfo).WillOnce([](const uint8_t *, uint32_t, uint32_t &headLen, - std::vector &) { + EXPECT_CALL(*processCommunicator, GetDataHeadInfo).WillOnce([](const uint8_t *, uint32_t, uint32_t &headLen) { headLen = UINT32_MAX; return OK; }); @@ -956,7 +958,7 @@ HWTEST_F(DistributedDBCommunicatorDeepTest, NetworkAdapter007, TestSize.Level1) */ int callByteReceiveCount = 0; int res = adapter->RegBytesReceiveCallback([&callByteReceiveCount](const std::string &, const uint8_t *, uint32_t, - const std::string &) { + const DataUserInfoProc &) { callByteReceiveCount++; }, nullptr); EXPECT_EQ(res, E_OK); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/mock_process_communicator.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/mock_process_communicator.h index 3f0e2771..0c60f3ea 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/mock_process_communicator.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/communicator/mock_process_communicator.h @@ -30,8 +30,8 @@ public: MOCK_METHOD0(GetLocalDeviceInfos, DeviceInfos()); MOCK_METHOD0(GetRemoteOnlineDeviceInfosList, std::vector()); MOCK_METHOD1(IsSameProcessLabelStartedOnPeerDevice, bool(const DeviceInfos &)); - MOCK_METHOD4(CheckAndGetDataHeadInfo, DBStatus(const uint8_t *, uint32_t, uint32_t &, - std::vector &)); + MOCK_METHOD3(GetDataHeadInfo, DBStatus(const uint8_t *, uint32_t, uint32_t &)); + MOCK_METHOD4(GetDataUserInfo, DBStatus(const uint8_t *, uint32_t, const std::string &, std::vector &)); }; } #endif // MOCK_PROCESS_COMMUNICATOR_H diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_ext_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_ext_test.cpp index 8c3aa975..363c76a2 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_ext_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_ext_test.cpp @@ -600,6 +600,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest002, ClientObserver clientObserver = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); /** * @tc.steps:step3. insert data into sync_data, check observer. @@ -637,6 +638,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest002, ClientObserver clientObserver2 = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc2, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver2), OK); + RegisterDbHook(db); sql = "update " + tableName + " set name = 'lisi2' where id = 2;"; EXPECT_EQ(RelationalTestUtils::ExecSql(db, sql), E_OK); WaitAndResetNotify(); @@ -715,6 +717,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest004, ClientObserver clientObserver = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); /** * @tc.steps:step3. insert data into sync_data, check observer. @@ -778,6 +781,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest005, ClientObserver clientObserver = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); /** * @tc.steps:step3. begin transaction and commit. @@ -853,6 +857,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest006, ClientObserver clientObserver = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); /** * @tc.steps:step3. begin transaction and commit. @@ -882,6 +887,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest006, * @tc.expected: step5. check observer ok. */ EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); sql = "insert into " + tableName1 + " VALUES(7, 'zhangjiu');"; EXPECT_EQ(RelationalTestUtils::ExecSql(db, sql), E_OK); WaitAndResetNotify(); @@ -918,6 +924,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, TriggerObserverTest007, ClientObserver clientObserver = std::bind(&DistributedDBCloudInterfacesRelationalExtTest::ClientObserverFunc, this, std::placeholders::_1); EXPECT_EQ(RegisterClientObserver(db, clientObserver), OK); + RegisterDbHook(db); /** * @tc.steps:step3. insert data into sync_data, check observer. @@ -1576,6 +1583,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, RegisterStoreObserverTes */ auto storeObserver = std::make_shared(); EXPECT_EQ(RegisterStoreObserver(db, storeObserver), OK); + RegisterDbHook(db); EXPECT_TRUE(g_changedData.empty()); int dataCounts = 10; // 10 is count of insert options. int begin = 0; @@ -1623,6 +1631,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, RegisterStoreObserverTes EXPECT_NE(db, nullptr); auto storeObserver = std::make_shared(); EXPECT_EQ(RegisterStoreObserver(db, storeObserver), OK); + RegisterDbHook(db); EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); std::string tableName = "primary_test"; CreateTableForStoreObserver(db, tableName); @@ -1674,6 +1683,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, RegisterStoreObserverTes auto storeObserver2 = std::make_shared(); EXPECT_EQ(RegisterStoreObserver(db, storeObserver1), OK); EXPECT_EQ(RegisterStoreObserver(db, storeObserver2), OK); + RegisterDbHook(db); EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); std::string tableName = "primary_test"; CreateTableForStoreObserver(db, tableName); @@ -1734,7 +1744,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalExtTest, RegisterStoreObserverTes EXPECT_NE(db, nullptr); auto storeObserver = std::make_shared(); EXPECT_EQ(RegisterStoreObserver(db, storeObserver), OK); - EXPECT_EQ(RegisterStoreObserver(db, storeObserver), INVALID_ARGS); + EXPECT_EQ(RegisterStoreObserver(db, storeObserver), OK); EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_remove_device_data_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_remove_device_data_test.cpp index 03c7fda7..61168fdf 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_remove_device_data_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_cloud_interfaces_relational_remove_device_data_test.cpp @@ -1932,7 +1932,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalRemoveDeviceDataTest, CleanCloudD */ DeleteCloudTableRecordByGid(0, 2); CloudDBSyncUtilsTest::callSync(g_tables, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, g_delegate); - + /** * @tc.steps: step6. call Sync with cloud merge strategy. * @tc.expected: OK. @@ -1983,7 +1983,7 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalRemoveDeviceDataTest, CleanCloudD std::string device; ASSERT_EQ(g_delegate->RemoveDeviceData(device, DistributedDB::FLAG_AND_DATA), DBStatus::OK); std::string sql = "select count(*) from " + DBCommon::GetLogTableName(g_tables[0]) + - " where flag & 0x82B == 0x82B;"; + " where flag & 0x809 == 0x809;"; EXPECT_EQ(sqlite3_exec(db, sql.c_str(), CloudDBSyncUtilsTest::QueryCountCallback, reinterpret_cast(20), nullptr), SQLITE_OK); DropLogicDeletedData(db, g_tables[0], 0); @@ -2004,5 +2004,47 @@ HWTEST_F(DistributedDBCloudInterfacesRelationalRemoveDeviceDataTest, CleanCloudD CheckCloudTotalCount(g_tables, {20, 20}); CloseDb(); } + +/* + * @tc.name: CleanCloudDataTest030 + * @tc.desc: Test flag_and_data and logic delete to remove cloud and inconsistency data. + * @tc.type: FUNC + * @tc.require: + * @tc.author: tankaisheng + */ +HWTEST_F(DistributedDBCloudInterfacesRelationalRemoveDeviceDataTest, CleanCloudDataTest030, TestSize.Level0) +{ + /** + * @tc.steps: step1. Set data is logicDelete + * @tc.expected: OK. + */ + bool logicDelete = true; + auto data = static_cast(&logicDelete); + g_delegate->Pragma(LOGIC_DELETE_SYNC_DATA, data); + /** + * @tc.steps: step2. make data: 20 records on cloud + * @tc.expected: OK. + */ + int64_t paddingSize = 50; + int cloudCount = 50; + InsertCloudTableRecord(0, cloudCount, paddingSize, false); + /** + * @tc.steps: step3. call Sync with cloud merge strategy. + * @tc.expected: OK. + */ + g_virtualAssetLoader->SetDownloadStatus(CLOUD_ASSET_SPACE_INSUFFICIENT); + CloudDBSyncUtilsTest::callSync(g_tables, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, g_delegate); + /** + * @tc.steps: step4. remove device data and check log num. + * @tc.expected: OK. + */ + ASSERT_EQ(g_delegate->RemoveDeviceData("", DistributedDB::FLAG_AND_DATA), DBStatus::OK); + std::string sql = "select count(*) from " + DBCommon::GetLogTableName(g_tables[0]) + + " where flag & 0x22 = 0;"; + EXPECT_EQ(sqlite3_exec(db, sql.c_str(), CloudDBSyncUtilsTest::QueryCountCallback, + reinterpret_cast(50), nullptr), SQLITE_OK); + DropLogicDeletedData(db, g_tables[0], 0); + CloseDb(); +} } #endif // RELATIONAL_STORE \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp index 49a7fc40..cacbdb4a 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp @@ -111,6 +111,10 @@ namespace { { return E_OK; } + + void ClearOnlineLabel() override + { + } private: CommunicatorLackCallback lackCallback_; }; diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_rd_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_rd_test.cpp index 65d6201f..2c6befc7 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_rd_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_rd_test.cpp @@ -1084,6 +1084,7 @@ HWTEST_F(DistributedDBInterfacesImportAndExportRdTest, SeparaDbExportAndImport, g_mgr.GetKvStore(singleStoreId, option, g_kvNbDelegateCallback); ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); option.passwd = g_passwd1; option.isEncryptedDb = true; g_mgr.GetKvStore(singleStoreId, option, g_kvNbDelegateCallback); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp index 4bc4836b..47a57962 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp @@ -1365,4 +1365,124 @@ HWTEST_F(DistributedDBInterfacesImportAndExportTest, ImportTest001, TestSize.Lev EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); EXPECT_EQ(g_mgr.DeleteKvStore(singleStoreId), OK); } + + +/** + * @tc.name: CheckSecurityLabel001 + * @tc.desc: Test check label with set label. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBInterfacesImportAndExportTest, CheckSecurityLabel001, TestSize.Level0) +{ + std::shared_ptr adapter = std::make_shared(); + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter); + /** + * @tc.steps: step1. Pre-create folder dir + */ + std::string singleFileName = g_exportFileDir + "/CheckSecurityLabel001.$$"; + std::string singleStoreId = "distributed_CheckSecurityLabel001"; + KvStoreNbDelegate::Option option = {true, false, false}; + option.secOption = {SecurityLabel::S1, SecurityFlag::ECE}; + g_mgr.GetKvStore(singleStoreId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + /** + * @tc.steps: step2. Specify the path to export the non-encrypted board database. + * @tc.expected: step2. Returns OK + */ + CipherPassword passwd; + EXPECT_EQ(g_kvNbDelegatePtr->Export(singleFileName, passwd), OK); + /** + * @tc.steps: step3. Clear label before import and import again. + * @tc.expected: step3. Import will set label + */ + SecurityOption before; + adapter->GetSecurityOption(singleFileName, before); + adapter->ResetSecOptDic(); + EXPECT_EQ(g_kvNbDelegatePtr->Import(singleFileName, passwd), OK); + SecurityOption after; + adapter->GetSecurityOption(singleFileName, before); + EXPECT_NE(before, after); + /** + * @tc.steps: step4. Release resource. + * @tc.expected: step4. OK + */ + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore(singleStoreId), OK); + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); +} + +/** + * @tc.name: CheckSecurityLabel002 + * @tc.desc: Test import after changing the security level + * @tc.type: FUNC + * @tc.require: + * @tc.author: liuhongyang + */ +HWTEST_F(DistributedDBInterfacesImportAndExportTest, CheckSecurityLabel002, TestSize.Level0) +{ + std::shared_ptr adapter = std::make_shared(); + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter); + /** + * @tc.steps: step1. Pre-create folder dir + */ + std::string singleFileName = g_exportFileDir + "/CheckSecurityLabel002.$$"; + std::string singleStoreId = "distributed_CheckSecurityLabel002"; + KvStoreNbDelegate::Option option = {true, false, false}; + option.secOption = {SecurityLabel::S2, SecurityFlag::ECE}; + g_mgr.GetKvStore(singleStoreId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + /** + * @tc.steps: step2. Specify the path to export the encrypted database. + * @tc.expected: step2. Returns OK. + */ + CipherPassword passwd; + EXPECT_EQ(g_kvNbDelegatePtr->Export(singleFileName, passwd), OK); + /** + * @tc.steps: step3. Close and reopen the store with a lower secOpt + * @tc.expected: step3. Returns OK. + */ + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore(singleStoreId), OK); + KvStoreNbDelegate::Option option2 = {true, false, false}; + option2.secOption = {SecurityLabel::S1, SecurityFlag::ECE}; + g_mgr.GetKvStore(singleStoreId, option2, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + /** + * @tc.steps: step4. Make systemApi mock file system and try to import + * @tc.expected: step4. Import will fail with SECURITY_OPTION_CHECK_ERROR + */ + adapter->SetNeedValidateBeforeSet(true); + EXPECT_EQ(g_kvNbDelegatePtr->Import(singleFileName, passwd), SECURITY_OPTION_CHECK_ERROR); + /** + * @tc.steps: step5. Close and reopen the store with a higher secOpt + * @tc.expected: step5. Returns OK. + */ + adapter->SetNeedValidateBeforeSet(false); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore(singleStoreId), OK); + KvStoreNbDelegate::Option option3 = {true, false, false}; + option3.secOption = {SecurityLabel::S3, SecurityFlag::SECE}; + g_mgr.GetKvStore(singleStoreId, option3, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + /** + * @tc.steps: step6. Make systemApi mock file system and try to import + * @tc.expected: step6. Import will success with OK + */ + adapter->SetNeedValidateBeforeSet(true); + EXPECT_EQ(g_kvNbDelegatePtr->Import(singleFileName, passwd), OK); + /** + * @tc.steps: step7. Release resource. + * @tc.expected: step7. OK + */ + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore(singleStoreId), OK); + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); + g_junkFilesList.push_back(singleFileName); +} #endif // OMIT_ENCRYPT diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_extend_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_extend_test.cpp index 998ccaca..4a75ff22 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_extend_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_extend_test.cpp @@ -895,7 +895,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateExtendTest, InvalidQueryTest002, TestS * @tc.name: SyncRangeQuery001 * @tc.desc: test sync query with range * @tc.type: FUNC - * @tc.require: DTS2023112110763 + * @tc.require: * @tc.author: mazhao */ HWTEST_F(DistributedDBInterfacesNBDelegateExtendTest, SyncRangeQuery001, TestSize.Level3) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_rd_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_rd_test.cpp index 75071421..95d6d11d 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_rd_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_rd_test.cpp @@ -257,7 +257,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, CombineTest001, TestSize.Level * @tc.name: SingleVerGetLocalEntries001 * @tc.desc: Test GetEntries interface for the single ver database. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wangbingquan */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerGetLocalEntries001, TestSize.Level1) @@ -326,7 +326,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerGetLocalEntries001, T * @tc.name: ResultSetTest001 * @tc.desc: Test the NbDelegate for result set function. * @tc.type: FUNC - * @tc.require: AR000D08KT + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, ResultSetTest001, TestSize.Level1) @@ -415,7 +415,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, ResultSetTest001, TestSize.Lev * @tc.name: PutBatchVerify001 * @tc.desc: This test case use to verify the putBatch interface function * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, PutBatchVerify001, TestSize.Level1) @@ -458,7 +458,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, PutBatchVerify001, TestSize.Le * @tc.name: SingleVerPutBatch001 * @tc.desc: Check for illegal parameters * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch001, TestSize.Level1) @@ -498,7 +498,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch001, TestSize * @tc.name: SingleVerPutBatch002 * @tc.desc: PutBatch normal insert function test. * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch002, TestSize.Level1) @@ -563,7 +563,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch002, TestSize * @tc.name: SingleVerPutBatch003 * @tc.desc: Check interface atomicity * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch003, TestSize.Level1) @@ -633,7 +633,7 @@ static void PreparePutBatch004(vector &entrys1, vector &entrys2, v * @tc.name: SingleVerPutBatch004 * @tc.desc: Check interface data insertion and update functions. * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatch004, TestSize.Level1) @@ -745,7 +745,7 @@ static void CreatEntrys(int recordSize, vector &keys, vector &values * @tc.name: SingleVerDeleteBatch001 * @tc.desc: Check for illegal parameters. * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatch001, TestSize.Level1) @@ -848,7 +848,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatch001, TestS * @tc.name: SingleVerDeleteBatch002 * @tc.desc: Check normal delete batch ability. * @tc.type: FUNC - * @tc.require: AR000DPTQ8 + * @tc.require: * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatch002, TestSize.Level1) @@ -918,7 +918,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatch002, TestS * @tc.name: SingleVerPutBatchObserver001 * @tc.desc: Test the observer function of PutBatch() interface. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver001, TestSize.Level1) @@ -985,7 +985,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver001, * @tc.name: SingleVerPutBatchObserver002 * @tc.desc: Test the observer function of PutBatch() for invalid input. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver002, TestSize.Level4) @@ -1071,7 +1071,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver002, * @tc.name: SingleVerPutBatchObserver003 * @tc.desc: Test the observer function of PutBatch() update function. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver003, TestSize.Level1) @@ -1137,7 +1137,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver003, * @tc.name: SingleVerPutBatchObserver004 * @tc.desc: Test the observer function of PutBatch(), same keys handle. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver004, TestSize.Level1) @@ -1211,7 +1211,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerPutBatchObserver004, * @tc.name: SingleVerDeleteBatchObserver001 * @tc.desc: Test the observer function of DeleteBatch() interface. * @tc.type: FUNC - * @tc.require: AR000DPTTA + * @tc.require: * @tc.author: wumin */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatchObserver001, TestSize.Level1) @@ -1271,7 +1271,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerDeleteBatchObserver00 * @tc.name: SingleVerGetSecurityOption001 * @tc.desc: Test GetSecurityOption interface for the single ver database. * @tc.type: FUNC - * @tc.require: AR000EV1G2 + * @tc.require: * @tc.author: liuwenkai */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerGetSecurityOption001, TestSize.Level1) @@ -1318,7 +1318,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerGetSecurityOption001, * @tc.name: SingleVerGetSecurityOption002 * @tc.desc: Test GetSecurityOption interface for the single ver database. * @tc.type: FUNC - * @tc.require: AR000EV1G2 + * @tc.require: * @tc.author: liuwenkai */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, SingleVerGetSecurityOption002, TestSize.Level1) @@ -1514,7 +1514,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, MaxLogCheckPoint001, TestSize. * @tc.name: OpenStorePathCheckTest001 * @tc.desc: Test open store with same label but different path. * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, OpenStorePathCheckTest001, TestSize.Level1) @@ -2016,8 +2016,17 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, RdRangeQuery002, TestSize.Leve EXPECT_EQ(resultSet, nullptr); /** - * @tc.steps:step5. Close and delete KV store - * @tc.expected: step5. Returns OK. + * @tc.steps: step6. Use range query conditions to obtain the resultset and check the resultset + * @tc.expected: step6. The expected data are 0, 1, 2, 3, 4, 5. + */ + DistributedDB::Query query = Query::Select().Range({}, {}); + vector returnEntries; + EXPECT_EQ(g_kvNbDelegatePtr->GetEntries(query, returnEntries), OK); + EXPECT_EQ(returnEntries.size(), 6); // 6 means data: 0, 1, 2, 3, 4, 5. + + /** + * @tc.steps:step7. Close and delete KV store + * @tc.expected: step7. Returns OK. */ g_mgr.CloseKvStore(g_kvNbDelegatePtr); EXPECT_EQ(g_mgr.DeleteKvStore("RdRangeQuery002"), OK); @@ -2236,7 +2245,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, RdRangeQuery005, TestSize.Leve * @tc.name: RdRangeQuery006 * @tc.desc: Test GetEntries with rd kernel and the query filter is not Range. * @tc.type: FUNC - * @tc.require: DTS2024022106199 + * @tc.require: * @tc.author: mazhao */ HWTEST_F(DistributedDBInterfacesNBDelegateRdTest, RdRangeQuery006, TestSize.Level1) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_sync_test.cpp index 95ea6b0a..cb07e529 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_sync_test.cpp @@ -217,7 +217,7 @@ void DistributedDBInterfacesRelationalSyncTest::TearDown() * @tc.name: RelationalSyncTest001 * @tc.desc: Test with sync interface, table is not a distributed table * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest001, TestSize.Level1) @@ -236,7 +236,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest001, TestS * @tc.name: RelationalSyncTest002 * @tc.desc: Test with sync interface, query is not support * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest002, TestSize.Level1) @@ -255,7 +255,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest002, TestS * @tc.name: RelationalSyncTest003 * @tc.desc: Test with sync interface, query is invalid format * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest003, TestSize.Level1) @@ -274,7 +274,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest003, TestS * @tc.name: RelationalSyncTest004 * @tc.desc: Test with sync interface, query use invalid field * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest004, TestSize.Level1) @@ -293,7 +293,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest004, TestS * @tc.name: RelationalSyncTest005 * @tc.desc: Test with sync interface, query table has been modified * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest005, TestSize.Level1) @@ -315,7 +315,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest005, TestS * @tc.name: RelationalSyncTest006 * @tc.desc: Test with sync interface, query is not set table name * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest006, TestSize.Level1) @@ -334,7 +334,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest006, TestS * @tc.name: RelationalSyncTest007 * @tc.desc: Test with sync interface, distributed table has empty column type * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest007, TestSize.Level1) @@ -359,7 +359,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest007, TestS * @tc.name: RelationalSyncTest008 * @tc.desc: Test sync with rebuilt table * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest008, TestSize.Level1) @@ -416,7 +416,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest008, TestS * @tc.name: RelationalSyncTest009 * @tc.desc: Test sync with invalid query * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest009, TestSize.Level1) @@ -455,7 +455,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest009, TestS * @tc.name: RelationalSyncTest010 * @tc.desc: Test sync with shcema changed * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest010, TestSize.Level1) @@ -488,7 +488,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncTest010, TestS * @tc.name: UpdatePrimaryKeyTest001 * @tc.desc: Test update data's primary key * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, UpdatePrimaryKeyTest001, TestSize.Level1) @@ -514,7 +514,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, UpdatePrimaryKeyTest001, Tes * @tc.name: UpgradeTriggerTest001 * @tc.desc: Test upgrade from old version * @tc.type: FUNC - * @tc.require: AR000GK58F + * @tc.require: * @tc.author: lianhuix */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, UpgradeTriggerTest001, TestSize.Level1) @@ -806,6 +806,51 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, TableNameCaseInsensitiveTest }); } +/** + * @tc.name: TableNameCaseInsensitiveTest003 + * @tc.desc: Test distributed tables sync with table names in different case + * @tc.type: FUNC + * @tc.require: + * @tc.author: liuhongyang + */ +HWTEST_F(DistributedDBInterfacesRelationalSyncTest, TableNameCaseInsensitiveTest003, TestSize.Level1) +{ + /** + * @tc.steps:step1. Close the delegate, and open in Collaboration mode + * @tc.expected: step1. ok + */ + EXPECT_EQ(g_mgr.CloseStore(delegate), OK); + delegate = nullptr; + RelationalStoreDelegate::Option option; + option.tableMode = DistributedTableMode::COLLABORATION; + EXPECT_EQ(g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, option, delegate), OK); + ASSERT_NE(delegate, nullptr); + /** + * @tc.steps:step2. Create student_1 table in distributed mode, and insert one fake data + * @tc.expected: step2. ok + */ + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL_STUDENT), SQLITE_OK); + AddDeviceSchema(g_deviceB, db, "student_1"); + EXPECT_EQ(delegate->CreateDistributedTable("StUDent_1"), OK); + DistributedDB::DistributedSchema distributedSchema; + distributedSchema.tables.push_back({"sTudeNt_1", {{"id", true, true}, {"name"}, {"level"}, {"score"}}}); + EXPECT_EQ(delegate->SetDistributedSchema(distributedSchema), OK); + g_deviceB->SetDistributedSchema(distributedSchema); + g_deviceB->PutDeviceData("student_1", std::vector {{1001, "xue", 4, 91}}); // 4, 91 fake data + /** + * @tc.steps:step3. Sync with table name in different case + * @tc.expected: step3. ok + */ + std::vector devices = {DEVICE_B}; + Query query = Query::Select("sTudENT_1"); + DBStatus status = delegate->Sync(devices, SyncMode::SYNC_MODE_PULL_ONLY, query, + [&devices](const std::map> &devicesMap) { + EXPECT_EQ(devicesMap.size(), devices.size()); + EXPECT_EQ(devicesMap.at(DEVICE_B)[0].status, OK); + }, true); + EXPECT_EQ(status, OK); +} + HWTEST_F(DistributedDBInterfacesRelationalSyncTest, TableFieldsOrderTest001, TestSize.Level1) { EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL_STUDENT_IN_ORDER), SQLITE_OK); @@ -1078,7 +1123,7 @@ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RuntimeConfig001, TestSize.L * @tc.name: RelationalSyncRangeTest001 * @tc.desc: Test with sync interface, range query is not support * @tc.type: FUNC - * @tc.require: DTS2023112110763 + * @tc.require: * @tc.author: mazhao */ HWTEST_F(DistributedDBInterfacesRelationalSyncTest, RelationalSyncRangeTest001, TestSize.Level1) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp index b208abee..c72e1205 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -1628,7 +1628,7 @@ namespace { * @tc.require: * @tc.author: lianhuix */ -HWTEST_F(DistributedDBInterfacesRelationalTest, SqliteKeyWordTest001, TestSize.Level1) +HWTEST_F(DistributedDBInterfacesRelationalTest, SqliteKeyWordTest001, TestSize.Level0) { sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); ASSERT_NE(db, nullptr); @@ -1938,7 +1938,7 @@ HWTEST_F(DistributedDBInterfacesRelationalTest, CreateDistributedTableTest004, T } /** - * @tc.name: CloudRelationalStoreTest006 + * @tc.name: CreateDistributedTableTest006 * @tc.desc: Test create distributed table and execute transaction concurrently * @tc.type: FUNC * @tc.require: @@ -1978,4 +1978,84 @@ HWTEST_F(DistributedDBInterfacesRelationalTest, CreateDistributedTableTest006, T status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); } + +/** + * @tc.name: CreateDistributedTableTest007 + * @tc.desc: Test create distributed table after insert data + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBInterfacesRelationalTest, CreateDistributedTableTest007, TestSize.Level0) +{ + /** + * @tc.steps:step1. Prepare db and table + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + std::string t1 = "t1"; + std::string sql = "create table " + t1 + "(id integer);"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, sql), SQLITE_OK); + int64_t dataCount = 10; + for (int64_t i = 0; i < dataCount; i++) { + sql = "insert into " + t1 + " values(" + std::to_string(i) + ")"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, sql), SQLITE_OK); + } + /** + * @tc.steps:step2. open relational store, create distributed table + * @tc.expected: step2. Return OK. + */ + RelationalStoreDelegate *delegate = nullptr; + EXPECT_EQ(g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate), OK); + ASSERT_NE(delegate, nullptr); + EXPECT_EQ(delegate->CreateDistributedTable(t1), OK); + /** + * @tc.steps:step3. check log table + * @tc.expected: step3. Return OK. + */ + sql = "select flag, extend_field, cursor, status from " + DBCommon::GetLogTableName(t1) + " order by data_key;"; + sqlite3_stmt *stmt = nullptr; + EXPECT_EQ(SQLiteUtils::GetStatement(db, sql, stmt), E_OK); + int64_t actualCount = 0; + while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + actualCount++; + int64_t actualFlag = sqlite3_column_int64(stmt, 0); // index 0 is flag + std::string actualExtendVal; + EXPECT_EQ(SQLiteUtils::GetColumnTextValue(stmt, 1, actualExtendVal), E_OK); // index 1 is extend_field + int64_t actualCursor = sqlite3_column_int64(stmt, 2); // index 2 is cursor + int64_t actualStatus = sqlite3_column_int64(stmt, 3); // index 3 is status + EXPECT_EQ(actualFlag, static_cast(LogInfoFlag::FLAG_LOCAL) | + static_cast(LogInfoFlag::FLAG_DEVICE_CLOUD_INCONSISTENCY)); + EXPECT_EQ(actualExtendVal, ""); + EXPECT_EQ(actualCursor, actualCount); + EXPECT_EQ(actualStatus, 0); + } + EXPECT_EQ(actualCount, dataCount); + int errCode = E_OK; + SQLiteUtils::ResetStatement(stmt, true, errCode); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + DBStatus status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); +} + +/** + * @tc.name: StoreId001 + * @tc.desc: Test storeId support with dot + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBInterfacesRelationalTest, StoreId001, TestSize.Level0) +{ + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, APP_ID, STORE_ID_1, false), ""); + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, APP_ID, STORE_ID_1, true), ""); + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, APP_ID, STORE_ID_1, true), ""); + const std::string storeIdWithDot = STORE_ID_1 + "."; + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, APP_ID, storeIdWithDot, true), ""); + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, USER_ID, APP_ID, STORE_ID_1, false), ""); + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, USER_ID, APP_ID, STORE_ID_1, true), ""); + EXPECT_NE(RelationalStoreManager::GetRelationalStoreIdentifier(USER_ID, USER_ID, APP_ID, storeIdWithDot, true), ""); +} } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_tracker_table_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_tracker_table_test.cpp index ec16703f..14765c0c 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_tracker_table_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_tracker_table_test.cpp @@ -142,9 +142,8 @@ namespace { void CheckExtendAndCursor(uint64_t num, int start, const std::string &tableName, bool addNum = true) { int index = 0; - string querySql = "select json_extract(extend_field, '$.name'), cursor from " + - std::string(DBConstant::RELATIONAL_PREFIX) + tableName + "_log" + " where data_key <= " + - std::to_string(num); + string querySql = "select json_extract(extend_field, '$.name'), cursor from " + std::string(DBConstant::RELATIONAL_PREFIX) + tableName + + "_log" + " where data_key <= " + std::to_string(num); sqlite3_stmt *stmt = nullptr; EXPECT_EQ(SQLiteUtils::GetStatement(g_db, querySql, stmt), E_OK); while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { @@ -634,6 +633,20 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest009, */ BatchDeleteTableName2Data(num); CheckExtendAndCursor(num, 0); + + /** + * @tc.steps:step3. Set tracker, where extendColNames changed, then check the extend_field of deleted data + * @tc.expected: step3. Return OK. + */ + EXPECT_EQ(g_delegate->SetTrackerTable(g_normalSchema2), OK); + CheckExtendAndCursor(num, 0); + + /** + * @tc.steps:step4. Set tracker, where extendColNames no changed, then check the extend_field of deleted data + * @tc.expected: step4. Return OK. + */ + EXPECT_EQ(g_delegate->SetTrackerTable(g_normalSchema3), OK); + CheckExtendAndCursor(num, 0); CloseStore(); } @@ -821,6 +834,13 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest013, BatchDeleteTableName2Data(num); EXPECT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME2, CLOUD_COOPERATION), OK); BatchInsertTableName2Data(num); + + /** + * @tc.steps:step4. Set tracker, where extendColNames changed, then check the extend_field of deleted data + * @tc.expected: step4. Return OK. + */ + EXPECT_EQ(g_delegate->SetTrackerTable(g_normalSchema2), OK); + CheckExtendAndCursor(num, 0); CloseStore(); } @@ -1824,7 +1844,7 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, ExecuteSql011, TestS * @tc.steps:step2. ExecuteSql concurrently * @tc.expected: step2. Return OK. */ - std::thread readThread([&]() { + std::thread readThread([&](){ SqlCondition condition; std::vector records; condition.readOnly = true; @@ -1833,7 +1853,7 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, ExecuteSql011, TestS EXPECT_EQ(g_delegate->ExecuteSql(condition, records), OK); } }); - std::thread transactionThread([&]() { + std::thread transactionThread([&](){ SqlCondition condition; condition.readOnly = true; std::vector records; @@ -2550,9 +2570,10 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest042, BatchInsertTableName2Data(num); sqlite3_stmt *stmt = nullptr; EXPECT_EQ(SQLiteUtils::GetStatement( - g_db, "select timestamp from naturalbase_rdb_aux_worKer2_log where data_key = 1", stmt), E_OK); + g_db, "select timestamp,wtimestamp from naturalbase_rdb_aux_worKer2_log where data_key = 1", stmt), E_OK); ASSERT_EQ(SQLiteUtils::StepWithRetry(stmt, false), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); int64_t beforTime = static_cast(sqlite3_column_int64(stmt, 0)); + int64_t beforWTime = static_cast(sqlite3_column_int64(stmt, 1)); int errCode; SQLiteUtils::ResetStatement(stmt, true, errCode); @@ -2563,11 +2584,151 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest042, EXPECT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME2, DEVICE_COOPERATION), OK); stmt = nullptr; EXPECT_EQ(SQLiteUtils::GetStatement( - g_db, "select timestamp from naturalbase_rdb_aux_worKer2_log where data_key = 1", stmt), E_OK); + g_db, "select timestamp,wtimestamp from naturalbase_rdb_aux_worKer2_log where data_key = 1", stmt), E_OK); ASSERT_EQ(SQLiteUtils::StepWithRetry(stmt, false), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); int64_t afterTime = static_cast(sqlite3_column_int64(stmt, 0)); + int64_t afterWTime = static_cast(sqlite3_column_int64(stmt, 1)); SQLiteUtils::ResetStatement(stmt, true, errCode); EXPECT_NE(beforTime, afterTime); + EXPECT_NE(beforWTime, afterWTime); + CloseStore(); +} + +/** + * @tc.name: TrackerTableTest043 + * @tc.desc: Test whether to save syncType after setting up the tracking table + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest043, TestSize.Level0) +{ + /** + * @tc.steps:step1. SetTrackerTable + * @tc.expected: step1. Return OK. + */ + TrackerSchema schema = g_normalSchema1; + CreateMultiTable(); + OpenStore(); + EXPECT_EQ(g_delegate->SetTrackerTable(schema), OK); + /** + * @tc.steps:step2. Check sync type in metatable + * @tc.expected: step2. Return E_OK. + */ + std::string metaTableName = "naturalbase_rdb_aux_metadata"; + std::string sql = "select count(*) from " + metaTableName + " where key = ?"; + sqlite3_stmt* stmt = nullptr; + SQLiteUtils::GetStatement(g_db, sql, stmt); + std::string keyStr = "sync_table_type_" + schema.tableName; + Key key(keyStr.begin(), keyStr.end()); + SQLiteUtils::BindBlobToStatement(stmt, 1, key); + SQLiteUtils::StepWithRetry(stmt); + + int count = static_cast(sqlite3_column_int(stmt, 0)); + EXPECT_EQ(count, 1); + int errCode = E_OK; + SQLiteUtils::ResetStatement(stmt, true, errCode); + CloseStore(); +} + +void CheckCursor(int begin, int end, sqlite3 *db) +{ + std::string querySql = "select cursor from " + DBCommon::GetLogTableName(TABLE_NAME2) + " order by data_key;"; + sqlite3_stmt *stmt = nullptr; + EXPECT_EQ(SQLiteUtils::GetStatement(db, querySql, stmt), E_OK); + int64_t cursor = begin; + while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + std::string extendVal; + int64_t actualCursor = sqlite3_column_int64(stmt, 0); + EXPECT_EQ(actualCursor, cursor); + cursor++; + } + EXPECT_EQ(cursor, end + 1); + int errCode = E_OK; + SQLiteUtils::ResetStatement(stmt, true, errCode); +} + +/** + * @tc.name: TrackerTableTest044 + * @tc.desc: Test SetTrackerTable and CreateDistributedTable when there is data in the table + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest044, TestSize.Level0) +{ + /** + * @tc.steps:step1. SetTrackerTable on table2 + * @tc.expected: step1. Return OK. + */ + CreateMultiTable(); + OpenStore(); + TrackerSchema schema = g_normalSchema1; + EXPECT_EQ(g_delegate->SetTrackerTable(schema), OK); + /** + * @tc.steps:step2. CreateDistributedTable on table2 and insert data + * @tc.expected: step2. Return OK. + */ + uint64_t num = 10; + BatchInsertTableName2Data(num); + CheckCursor(1, 10, g_db); + EXPECT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME2, DEVICE_COOPERATION), DBStatus::OK); + /** + * @tc.steps:step3. Check log table + * @tc.expected: step3. Return OK. + */ + string querySql = "select json_extract(extend_field, '$.name') from " + DBCommon::GetLogTableName(TABLE_NAME2) + + " order by data_key;"; + sqlite3_stmt *stmt = nullptr; + EXPECT_EQ(SQLiteUtils::GetStatement(g_db, querySql, stmt), E_OK); + int count = 0; + while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + std::string extendVal; + EXPECT_EQ(SQLiteUtils::GetColumnTextValue(stmt, 0, extendVal), E_OK); + EXPECT_EQ(extendVal, "Local" + std::to_string(count)); + count++; + } + EXPECT_EQ(count, 10); + int errCode = E_OK; + SQLiteUtils::ResetStatement(stmt, true, errCode); + CheckCursor(11, 20, g_db); + CloseStore(); +} + +/** + * @tc.name: TrackerTableTest045 + * @tc.desc: Test SetTrackerTable and CreateDistributedTable when there is data in the table + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, TrackerTableTest045, TestSize.Level0) +{ + /** + * @tc.steps:step1. CreateDistributedTable on table2 and insert data + * @tc.expected: step1. Return OK. + */ + CreateMultiTable(); + OpenStore(); + uint64_t num = 10; + BatchInsertTableName2Data(num); + EXPECT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME2, DEVICE_COOPERATION), DBStatus::OK); + /** + * @tc.steps:step2. Check cursor + * @tc.expected: step2. Return OK. + */ + CheckCursor(1, 10, g_db); + /** + * @tc.steps:step3. SetTrackerTable on table2 + * @tc.expected: step3. Return WITH_INVENTORY_DATA. + */ + TrackerSchema schema = g_normalSchema1; + EXPECT_EQ(g_delegate->SetTrackerTable(schema), WITH_INVENTORY_DATA); + /** + * @tc.steps:step4. Check cursor + * @tc.expected: step4. Return OK. + */ + CheckCursor(11, 20, g_db); CloseStore(); } @@ -2642,7 +2803,7 @@ HWTEST_F(DistributedDBInterfacesRelationalTrackerTableTest, SchemaStrTest001, Te /** * @tc.name: TrackerTableTest041 - * @tc.desc: Test cursor increases when set tracker table after create distributed table by DEVICE_COOPERATION type + * @tc.desc: Test cursor increases when set tracker table after create distributed table by DEVICE_COOPERATION type * @tc.type: FUNC * @tc.require: * @tc.author: suyue diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp index 6b7ca36a..d5d11b97 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp @@ -38,7 +38,8 @@ namespace { ProcessSystemApiAdapterImpl::ProcessSystemApiAdapterImpl() : callback_(nullptr), isLocked_(false), - createDb_(false) + createDb_(false), + needValidateBeforeSet_(false) { } @@ -58,6 +59,19 @@ bool ProcessSystemApiAdapterImpl::IsAccessControlled() const return isLocked_; } +bool ProcessSystemApiAdapterImpl::SetSecurityOptionInner(const std::string &filePath, const SecurityOption &option) +{ + // file manager will return fail when trying to lower a secOpt + if (needValidateBeforeSet_ && + pathSecOptDic_.find(filePath) != pathSecOptDic_.end() && + pathSecOptDic_[filePath].securityLabel >= option.securityLabel) { + LOGW("[AdapterImpl] found path secOpt!"); + return pathSecOptDic_[filePath] == option; + } + pathSecOptDic_[filePath] = option; + return true; +} + DBStatus ProcessSystemApiAdapterImpl::SetSecurityOption(const std::string &filePath, const SecurityOption &option) { bool isExisted = OS::CheckPathExistence(filePath); @@ -71,10 +85,13 @@ DBStatus ProcessSystemApiAdapterImpl::SetSecurityOption(const std::string &fileP DIR *dirPtr = opendir(filePath.c_str()); if (dirPtr == nullptr) { LOGD("set path secOpt![%s] [%d] [%d]", filePath.c_str(), option.securityFlag, option.securityLabel); - pathSecOptDic_[filePath] = option; + if (!SetSecurityOptionInner(filePath, option)) { + return DB_ERROR; + } return OK; } + bool success = true; while (true) { direntPtr = readdir(dirPtr); // condition to exit the loop @@ -91,17 +108,20 @@ DBStatus ProcessSystemApiAdapterImpl::SetSecurityOption(const std::string &fileP if (direntPtr->d_type == DT_DIR) { SetSecurityOption(dirName, option); std::lock_guard lock(adapterlock_); - pathSecOptDic_[dirName] = option; + success = SetSecurityOptionInner(dirName, option) && success; LOGD("set path secOpt![%s] [%d] [%d]", dirName.c_str(), option.securityFlag, option.securityLabel); } else { std::lock_guard lock(adapterlock_); - pathSecOptDic_[dirName] = option; + success = SetSecurityOptionInner(dirName, option) && success; LOGD("set path secOpt![%s] [%d] [%d]", dirName.c_str(), option.securityFlag, option.securityLabel); continue; } } closedir(dirPtr); - pathSecOptDic_[filePath] = option; + success = SetSecurityOptionInner(filePath, option) && success; + if (!success) { + return DB_ERROR; + } return OK; } @@ -157,6 +177,13 @@ void ProcessSystemApiAdapterImpl::SetNeedCreateDb(bool isCreate) createDb_ = isCreate; } +void ProcessSystemApiAdapterImpl::SetNeedValidateBeforeSet(bool needValid) +{ + std::lock_guard lock(adapterlock_); + LOGW("[AdapterImpl] need valid before set is set to [%d]", needValid); + needValidateBeforeSet_ = needValid; +} + void ProcessSystemApiAdapterImpl::ResetSecOptDic() { pathSecOptDic_.clear(); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h index 0847fddf..7f8b718a 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h @@ -35,6 +35,7 @@ public: bool CheckDeviceSecurityAbility(const std::string &devId, const SecurityOption &option) const override; void SetLockStatus(bool isLock); void SetNeedCreateDb(bool isCreate); + void SetNeedValidateBeforeSet(bool needValid); void ResetSecOptDic(); void ResetAdapter(); @@ -43,11 +44,13 @@ public: std::map GetExistSecOpt() const; private: + bool SetSecurityOptionInner(const std::string &filePath, const SecurityOption &option); mutable std::mutex adapterlock_; OnAccessControlledEvent callback_; std::map pathSecOptDic_; bool isLocked_; bool createDb_; + bool needValidateBeforeSet_; std::function getSecurityOptionCallBack_; std::function checkDeviceCallBack_; }; diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_assets_operation_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_assets_operation_sync_test.cpp index 99562a41..0c84743a 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_assets_operation_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_assets_operation_sync_test.cpp @@ -85,6 +85,16 @@ void BlockSync(const Query &query, RelationalStoreDelegate *delegate, SyncMode s LOGW("end call sync"); } +int QueryCountCallback(void *data, int count, char **colValue, char **colName) +{ + if (count != 1) { + return 0; + } + auto expectCount = reinterpret_cast(data); + EXPECT_EQ(strtol(colValue[0], nullptr, 10), expectCount); // 10: decimal + return 0; +} + class DistributedDBCloudAssetsOperationSyncTest : public testing::Test { public: static void SetUpTestCase(); @@ -106,6 +116,7 @@ protected: void ForkDownloadAndRemoveAsset(DBStatus removeStatus, int &downLoadCount, int &removeCount); void InsertLocalAssetData(const std::string &assetHash); void InsertCloudAssetData(const std::string &assetHash); + void DeleteCloudDBData(int64_t beginGid, int64_t count, const std::string &tableName); void PrepareForAssetOperation010(); void UpdateAssetWhenSyncUpload(); DBStatus InsertRecordToCloud(const std::vector &record); @@ -874,6 +885,22 @@ void DistributedDBCloudAssetsOperationSyncTest::InsertCloudAssetData(const std:: virtualCloudDb_->BatchInsert(tableName_, std::move(record), extend); } +void DistributedDBCloudAssetsOperationSyncTest::DeleteCloudDBData(int64_t beginGid, int64_t count, + const std::string &tableName) +{ + Timestamp now = TimeHelper::GetSysCurrentTime(); + std::vector extend; + for (int64_t i = 0; i < count; ++i) { + VBucket log; + log.insert_or_assign(CloudDbConstant::CREATE_FIELD, (int64_t)now / CloudDbConstant::TEN_THOUSAND + i); + log.insert_or_assign(CloudDbConstant::MODIFY_FIELD, (int64_t)now / CloudDbConstant::TEN_THOUSAND + i); + log.insert_or_assign(CloudDbConstant::GID_FIELD, std::to_string(beginGid + i)); + extend.push_back(log); + } + ASSERT_EQ(virtualCloudDb_->BatchDelete(tableName, extend), DBStatus::OK); + std::this_thread::sleep_for(std::chrono::milliseconds(count)); +} + void DistributedDBCloudAssetsOperationSyncTest::PrepareForAssetOperation010() { InsertCloudAssetData("cloudAsset"); @@ -1753,5 +1780,51 @@ HWTEST_F(DistributedDBCloudAssetsOperationSyncTest, BatchAbnormalDownloadAsset00 EXPECT_EQ(virtualAssetLoader_->GetBatchRemoveCount(), 0u); // remove 0 times RuntimeContext::GetInstance()->SetBatchDownloadAssets(false); } + +/** + * @tc.name: SyncWithLogFlag001 + * @tc.desc: Test cloud update to local, flag will change. + * @tc.type: FUNC + * @tc.require: + * @tc.author: lg + */ +HWTEST_F(DistributedDBCloudAssetsOperationSyncTest, SyncWithLogFlag001, TestSize.Level0) +{ + /** + * @tc.steps:step1. Insert 5 records and sync. + * @tc.expected: step1. ok. + */ + const int actualCount = 5; + RelationalTestUtils::InsertCloudRecord(0, actualCount, tableName_, virtualCloudDb_); + InsertUserTableRecord(tableName_, 0, actualCount); + /** + * @tc.steps:step2. modify data and sync, check flag count. + * @tc.expected: step2. ok. + */ + UpdateCloudTableRecord(0, 3, true); + Query query = Query::Select().FromTable({ tableName_ }); + BlockSync(query, delegate_); + string checkSql = "select count(*) from naturalbase_rdb_aux_" + tableName_ + "_log where flag&0x4000=0x4000;"; + EXPECT_EQ(sqlite3_exec(db_, checkSql.c_str(), QueryCountCallback, + reinterpret_cast(3u), nullptr), SQLITE_OK); + /** + * @tc.steps:step3. delete local data and sync, check flag count. + * @tc.expected: step3. ok. + */ + std::string sql = "delete from " + tableName_ + " where id = 0;" ; + EXPECT_EQ(RelationalTestUtils::ExecSql(db_, sql), SQLITE_OK); + BlockSync(query, delegate_); + EXPECT_EQ(sqlite3_exec(db_, checkSql.c_str(), QueryCountCallback, + reinterpret_cast(2u), nullptr), SQLITE_OK); + + /** + * @tc.steps:step4. delete cloud data and sync, check flag count. + * @tc.expected: step4. ok. + */ + DeleteCloudDBData(1, 1, tableName_); + BlockSync(query, delegate_); + EXPECT_EQ(sqlite3_exec(db_, checkSql.c_str(), QueryCountCallback, + reinterpret_cast(1u), nullptr), SQLITE_OK); +} } #endif diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_check_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_check_sync_test.cpp index 0c0583a5..efe57233 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_check_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_check_sync_test.cpp @@ -1043,7 +1043,7 @@ HWTEST_F(DistributedDBCloudCheckSyncTest, CloudSyncTest008, TestSize.Level0) /** * @tc.name: CloudSyncTest009 - * @tc.desc: reopen database and sync + * @tc.desc: reopen database and sync * @tc.type: FUNC * @tc.require: * @tc.author: wangxiangdong @@ -1095,6 +1095,172 @@ HWTEST_F(DistributedDBCloudCheckSyncTest, CloudSyncTest009, TestSize.Level0) CheckCloudTableCount(tableName_, 0); } +/** + * @tc.name: CloudSyncTest010 + * @tc.desc: reopen database, recreate table with less columns and sync + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangxiangdong + */ +HWTEST_F(DistributedDBCloudCheckSyncTest, CloudSyncTest010, TestSize.Level0) +{ + /** + * @tc.steps: step1. insert 1 record to user table + * @tc.expected: step1. OK. + */ + const int actualCount = 1; + InsertUserTableRecord(tableName_, actualCount); + /** + * @tc.steps: step2. sync data to cloud + * @tc.expected: step2. OK. + */ + Query query = Query::Select().FromTable({ tableName_ }); + BlockSync(query, delegate_, g_actualDBStatus); + CheckCloudTableCount(tableName_, 1); + /** + * @tc.steps: step3. drop data table then close db + * @tc.expected: step3. OK. + */ + std::string deleteSql = "DROP TABLE IF EXISTS " + tableName_ + ";"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, deleteSql), DBStatus::OK); + EXPECT_EQ(mgr_->CloseStore(delegate_), DBStatus::OK); + delegate_ = nullptr; + /** + * @tc.steps: step4. recreate data table and reopen database + * @tc.expected: step4. OK. + */ + std::string createSql = "CREATE TABLE IF NOT EXISTS DistributedDBCloudCheckSyncTest(id INT PRIMARY KEY);"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createSql), DBStatus::OK); + RelationalStoreDelegate::Option option; + ASSERT_EQ(mgr_->OpenStore(storePath_, STORE_ID_1, option, delegate_), DBStatus::OK); + ASSERT_NE(delegate_, nullptr); + ASSERT_EQ(delegate_->CreateDistributedTable(tableName_, CLOUD_COOPERATION), DBStatus::SCHEMA_MISMATCH); + ASSERT_EQ(delegate_->SetCloudDB(virtualCloudDb_), DBStatus::OK); + ASSERT_EQ(delegate_->SetIAssetLoader(virtualAssetLoader_), DBStatus::OK); + DataBaseSchema dataBaseSchema = GetSchema(); + ASSERT_EQ(delegate_->SetCloudDbSchema(dataBaseSchema), DBStatus::OK); + communicatorAggregator_ = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(communicatorAggregator_ != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(communicatorAggregator_); + /** + * @tc.steps: step5. sync failed with SCHEMA_MISMATCH + * @tc.expected: step5. OK. + */ + BlockPrioritySync(query, delegate_, false, DBStatus::SCHEMA_MISMATCH); + CheckCloudTableCount(tableName_, 1); +} + +/** + * @tc.name: CloudSyncTest011 + * @tc.desc: reopen database, do not recreate table and sync + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangxiangdong + */ +HWTEST_F(DistributedDBCloudCheckSyncTest, CloudSyncTest011, TestSize.Level0) +{ + /** + * @tc.steps: step1. insert 1 record to user table + * @tc.expected: step1. OK. + */ + const int actualCount = 1; + InsertUserTableRecord(tableName_, actualCount); + /** + * @tc.steps: step2. sync data to cloud + * @tc.expected: step2. OK. + */ + Query query = Query::Select().FromTable({ tableName_ }); + BlockSync(query, delegate_, g_actualDBStatus); + CheckCloudTableCount(tableName_, 1); + /** + * @tc.steps: step3. drop data table then close db + * @tc.expected: step3. OK. + */ + std::string deleteSql = "DROP TABLE IF EXISTS " + tableName_ + ";"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, deleteSql), DBStatus::OK); + EXPECT_EQ(mgr_->CloseStore(delegate_), DBStatus::OK); + delegate_ = nullptr; + /** + * @tc.steps: step4. reopen database + * @tc.expected: step4. OK. + */ + RelationalStoreDelegate::Option option; + ASSERT_EQ(mgr_->OpenStore(storePath_, STORE_ID_1, option, delegate_), DBStatus::OK); + ASSERT_NE(delegate_, nullptr); + ASSERT_EQ(delegate_->SetCloudDB(virtualCloudDb_), DBStatus::OK); + ASSERT_EQ(delegate_->SetIAssetLoader(virtualAssetLoader_), DBStatus::OK); + DataBaseSchema dataBaseSchema = GetSchema(); + ASSERT_EQ(delegate_->SetCloudDbSchema(dataBaseSchema), DBStatus::OK); + communicatorAggregator_ = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(communicatorAggregator_ != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(communicatorAggregator_); + /** + * @tc.steps: step5. sync failed with SCHEMA_MISMATCH + * @tc.expected: step5. OK. + */ + BlockPrioritySync(query, delegate_, false, DBStatus::SCHEMA_MISMATCH); +} + +/** + * @tc.name: CloudSyncTest012 + * @tc.desc: insert data before re-SetDistributedTable and sync is ok + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangxiangdong + */ +HWTEST_F(DistributedDBCloudCheckSyncTest, CloudSyncTest012, TestSize.Level0) +{ + /** + * @tc.steps: step1. insert 1 record to user table + * @tc.expected: step1. OK. + */ + const int actualCount = 1; + InsertUserTableRecord(tableName_, actualCount); + /** + * @tc.steps: step2. sync data to cloud + * @tc.expected: step2. OK. + */ + Query query = Query::Select().FromTable({ tableName_ }); + BlockSync(query, delegate_, g_actualDBStatus); + CheckCloudTableCount(tableName_, 1); + /** + * @tc.steps: step3. drop data table then close db + * @tc.expected: step3. OK. + */ + std::string deleteSql = "DROP TABLE IF EXISTS " + tableName_ + ";"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, deleteSql), DBStatus::OK); + EXPECT_EQ(mgr_->CloseStore(delegate_), DBStatus::OK); + delegate_ = nullptr; + /** + * @tc.steps: step4. recreate data table and reopen database + * @tc.expected: step4. OK. + */ + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, g_createSQL), DBStatus::OK); + RelationalStoreDelegate::Option option; + ASSERT_EQ(mgr_->OpenStore(storePath_, STORE_ID_1, option, delegate_), DBStatus::OK); + ASSERT_NE(delegate_, nullptr); + ASSERT_EQ(delegate_->SetCloudDB(virtualCloudDb_), DBStatus::OK); + ASSERT_EQ(delegate_->SetIAssetLoader(virtualAssetLoader_), DBStatus::OK); + DataBaseSchema dataBaseSchema = GetSchema(); + ASSERT_EQ(delegate_->SetCloudDbSchema(dataBaseSchema), DBStatus::OK); + communicatorAggregator_ = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(communicatorAggregator_ != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(communicatorAggregator_); + /** + * @tc.steps: step5. insert data to new table + * @tc.expected: step5. OK. + */ + int begin = 1; + InsertUserTableRecord(tableName_, actualCount, begin); + /** + * @tc.steps: step6. sync and cloud data should be deleted + * @tc.expected: step6. OK. + */ + ASSERT_EQ(delegate_->CreateDistributedTable(tableName_, CLOUD_COOPERATION), DBStatus::OK); + BlockSync(query, delegate_, g_actualDBStatus); + CheckCloudTableCount(tableName_, 1); +} + /** * @tc.name: CloudSyncObserverTest001 * @tc.desc: test cloud sync multi observer diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_test.cpp index e55f1b7e..04f5cfd8 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_interfaces_relational_sync_test.cpp @@ -1050,12 +1050,15 @@ namespace { void CloseDb() { - delete g_observer; - g_virtualCloudDb = nullptr; if (g_delegate != nullptr) { EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); g_delegate = nullptr; } + if (g_observer != nullptr) { + delete g_observer; + g_observer = nullptr; + } + g_virtualCloudDb = nullptr; } void InitMockAssetLoader(DBStatus &status, int &index) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_meta_data_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_meta_data_test.cpp index df5c8f29..4df121dd 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_meta_data_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_meta_data_test.cpp @@ -59,6 +59,26 @@ namespace { sqlite3_close(db); } + string GetCreateTableSql(const string tableName) + { + return "CREATE TABLE IF NOT EXISTS " + tableName + "(name Text PRIMARY KEY);"; + } + + void CreateTables() + { + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + sqlite3_close(db); + return; + } + + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db, GetCreateTableSql(TABLE_NAME_1).c_str()), E_OK); + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db, GetCreateTableSql(TABLE_NAME_2).c_str()), E_OK); + sqlite3_close(db); + } + void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, RelationalDBProperties &properties) { @@ -387,4 +407,80 @@ namespace { EXPECT_EQ(obj.ClearAllTempSyncTrigger(), -E_INVALID_DB); EXPECT_EQ(obj.SetIAssetLoader(nullptr), -E_INVALID_DB); } + + /** + * @tc.name: ClearMetaDataTest001 + * @tc.desc: Check option mode + * @tc.type: FUNC + * @tc.require: + * @tc.author: lhy + */ + HWTEST_F(DistributedDBCloudMetaDataTest, ClearMetaDataTest001, TestSize.Level0) + { + /** + * @tc.steps: step1. Call ClearMetaData with invalid mode + * @tc.expected: step1. return INVALID_ARGS. + */ + ClearMetaDataOption option; + option.mode = ClearMetaDataMode::BUTT; + EXPECT_EQ(option.tableNameList.size(), 0u); + EXPECT_EQ(g_delegate->ClearMetaData(option), INVALID_ARGS); + /** + * @tc.steps: step2. Call ClearMetaData with valid mode + * @tc.expected: step2. return OK. + */ + option.mode = ClearMetaDataMode::CLOUD_WATERMARK; + EXPECT_EQ(g_delegate->ClearMetaData(option), OK); + } + + /** + * @tc.name: ClearMetaDataTest002 + * @tc.desc: Check the meta data are cleared based on the tableNameList + * @tc.type: FUNC + * @tc.require: + * @tc.author: lhy + */ + HWTEST_F(DistributedDBCloudMetaDataTest, ClearMetaDataTest002, TestSize.Level0) + { + /** + * @tc.steps: step1. Create the distributed table and set water mark + * @tc.expected: step1. OK + */ + CreateTables(); + ASSERT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME_1, CLOUD_COOPERATION), OK); + ASSERT_EQ(g_delegate->CreateDistributedTable(TABLE_NAME_2, CLOUD_COOPERATION), OK); + SetAndGetWaterMark(TABLE_NAME_1, "cloudMark1"); + SetAndGetWaterMark(TABLE_NAME_2, "cloudMark2"); + /** + * @tc.steps: step2. Call ClearMetaData with empty list + * @tc.expected: step2. Meta data of all tables are cleared + */ + ClearMetaDataOption option; + EXPECT_EQ(option.tableNameList.size(), 0u); + EXPECT_EQ(g_delegate->ClearMetaData(option), OK); + std::string retMark1; + std::string retMark2; + g_storageProxy = GetStorageProxy((ICloudSyncStorageInterface *) GetRelationalStore()); + EXPECT_EQ(g_storageProxy->GetCloudWaterMark(TABLE_NAME_1, retMark1), E_OK); + EXPECT_EQ(g_storageProxy->GetCloudWaterMark(TABLE_NAME_2, retMark2), E_OK); + EXPECT_EQ(retMark1, ""); + EXPECT_EQ(retMark2, ""); + /** + * @tc.steps: step3. Set water mark for 2 tables + * @tc.expected: step3. OK + */ + SetAndGetWaterMark(TABLE_NAME_1, "cloudMark1"); + SetAndGetWaterMark(TABLE_NAME_2, "cloudMark2"); + /** + * @tc.steps: step4. Call ClearMetaData with one table + * @tc.expected: step4. Return NOT_SUPPORT and meta is not cleared + */ + option.tableNameList.insert(TABLE_NAME_1); + EXPECT_EQ(g_delegate->ClearMetaData(option), NOT_SUPPORT); + g_storageProxy = GetStorageProxy((ICloudSyncStorageInterface *) GetRelationalStore()); + EXPECT_EQ(g_storageProxy->GetCloudWaterMark(TABLE_NAME_1, retMark1), E_OK); + EXPECT_EQ(g_storageProxy->GetCloudWaterMark(TABLE_NAME_2, retMark2), E_OK); + EXPECT_EQ(retMark1, "cloudMark1"); + EXPECT_EQ(retMark2, "cloudMark2"); + } } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_save_cloud_data_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_save_cloud_data_test.cpp index 87776736..8f076089 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_save_cloud_data_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/cloud/distributeddb_cloud_save_cloud_data_test.cpp @@ -283,8 +283,7 @@ namespace { vBucket[CloudDbConstant::GID_FIELD] = gidStr; DataInfoWithLog dataInfoWithLog; VBucket assetInfo; - EXPECT_EQ(storageProxy->GetInfoByPrimaryKeyOrGid(g_tableName, vBucket, true, dataInfoWithLog, assetInfo), - expectCode); + EXPECT_EQ(storageProxy->GetInfoByPrimaryKeyOrGid(g_tableName, vBucket, true, dataInfoWithLog, assetInfo), expectCode); if (expectCode == E_OK) { if (pkType == PrimaryKeyType::SINGLE_PRIMARY_KEY) { int64_t val = -1; @@ -849,7 +848,7 @@ namespace { if (expectCode == E_OK) { for (size_t i = 0; i < downloadData.opType.size(); i++) { if (downloadData.opType[i] == OpType::INSERT) { - EXPECT_TRUE(downloadData.data[i].find(CloudDbConstant::ROW_ID_FIELD_NAME) != + EXPECT_TRUE(downloadData.data[i].find(DBConstant::ROWID) != downloadData.data[i].end()); } } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_rdb_collaboration_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_rdb_collaboration_test.cpp index 7cb7f1e7..ee47387e 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_rdb_collaboration_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_rdb_collaboration_test.cpp @@ -13,6 +13,7 @@ * limitations under the License. */ #include +#include #include "cloud_db_sync_utils_test.h" #include "distributeddb_data_generate_unit_test.h" @@ -54,6 +55,8 @@ protected: static constexpr const char *DEVICE_SYNC_TABLE_UPGRADE = "DEVICE_SYNC_TABLE_UPGRADE"; static constexpr const char *DEVICE_SYNC_TABLE_AUTOINCREMENT = "DEVICE_SYNC_TABLE_AUTOINCREMENT"; static constexpr const char *CLOUD_SYNC_TABLE = "CLOUD_SYNC_TABLE"; + static constexpr const char *BIG_COLUMNS_TABLE = "BIG_COLUMNS_TABLE"; + static constexpr const char *INVALID_TABLE = "INVALID_TABLE"; }; void DistributedDBRDBCollaborationTest::SetUpTestCase() @@ -423,8 +426,7 @@ HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema006, TestSize.Level0) UnRegisterClientObserver(db_); } -std::string GetTriggerSql(const std::string &tableName, const std::string &triggerTypeName, sqlite3 *db) -{ +std::string GetTriggerSql(const std::string &tableName, const std::string &triggerTypeName, sqlite3 *db) { if (db == nullptr) { return ""; } @@ -637,464 +639,1945 @@ HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema012, TestSize.Level0) } /** - * @tc.name: NormalSync001 - * @tc.desc: Test set distributed schema and sync. + * @tc.name: SetSchema013 + * @tc.desc: Test set tracker table for device table and check if timestamp has changed * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: bty */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync001, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema013, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Create device table * @tc.expected: step1.ok */ ASSERT_NO_FATAL_FAILURE(InitDelegate()); - auto schema = GetSchema(); - auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); - deviceB_->SetDistributedSchema(distributedSchema); + DistributedSchema distributedSchema = GetDistributedSchema(DEVICE_SYNC_TABLE, {"pk", "int_field1"}); EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + TrackerSchema trackerSchema = { + .tableName = DEVICE_SYNC_TABLE, .extendColNames = {"int_field1"}, .trackerColNames = {"int_field1"} + }; /** - * @tc.steps: step2. Insert one data + * @tc.steps: step2. Insert one data and query timestamp * @tc.expected: step2.ok */ - auto tableSchema = GetTableSchema(); - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); + sqlite3_stmt *stmt = nullptr; + std::string sql = "select timestamp from " + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + + " where data_key=0"; + EXPECT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); + EXPECT_EQ(SQLiteUtils::StepWithRetry(stmt), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); + int64_t timestamp1 = static_cast(sqlite3_column_int64(stmt, 0)); + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); /** - * @tc.steps: step3. Sync to real device - * @tc.expected: step3.ok + * @tc.steps: step3. Set tracker table and query timestamp + * @tc.expected: step3.Equal */ - Query query = Query::Select(tableSchema.name); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + EXPECT_EQ(delegate_->SetTrackerTable(trackerSchema), WITH_INVENTORY_DATA); + EXPECT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); + EXPECT_EQ(SQLiteUtils::StepWithRetry(stmt), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); + int64_t timestamp2 = static_cast(sqlite3_column_int64(stmt, 0)); + SQLiteUtils::ResetStatement(stmt, true, ret); + EXPECT_EQ(timestamp1, timestamp2); } /** - * @tc.name: NormalSync002 - * @tc.desc: Test sync with diff distributed schema [high version -> low version]. + * @tc.name: SetSchema014 + * @tc.desc: Test set tracker table for device table and check if timestamp has changed * @tc.type: FUNC * @tc.require: * @tc.author: zqq */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync002, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema014, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION - * @tc.expected: step1.ok + * @tc.steps: step1. Create auto increment table and specified pk, distributed schema without not null field + * @tc.expected: step1. create distributed table ok but set schema failed */ + auto tableSchema = GetTableSchema(); + tableSchema.name = DEVICE_SYNC_TABLE_AUTOINCREMENT; + ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, true, *db_), E_OK); ASSERT_NO_FATAL_FAILURE(InitDelegate()); + DistributedSchema distributedSchema; + DistributedTable distributedTable; + distributedTable.tableName = tableSchema.name; + DistributedField distributedField; + distributedField.colName = "pk"; + distributedField.isSpecified = true; + distributedField.isP2pSync = true; + distributedTable.fields.push_back(distributedField); + distributedSchema.tables.push_back(distributedTable); + EXPECT_EQ(delegate_->CreateDistributedTable(tableSchema.name, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); /** - * @tc.steps: step2. DeviceB set schema [pk, int_field1, int_field2, int_field_upgrade] - * @tc.expected: step2.ok - */ - DataBaseSchema virtualSchema; - auto tableSchema = GetTableSchema(true); - tableSchema.name = DEVICE_SYNC_TABLE; - virtualSchema.tables.push_back(tableSchema); - auto distributedSchema = RDBDataGenerator::ParseSchema(virtualSchema); - deviceB_->SetDistributedSchema(distributedSchema); - /** - * @tc.steps: step3. Real device set schema [pk, int_field1, int_field2] - * @tc.expected: step3.ok - */ - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); - EXPECT_EQ(delegate_->SetDistributedSchema(RDBDataGenerator::ParseSchema(GetSchema())), OK); - /** - * @tc.steps: step4. Insert table info and virtual data into deviceB - * @tc.expected: step4.ok + * @tc.steps: step2. Distributed schema with not null field + * @tc.expected: step2. ok */ - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(DEVICE_SYNC_TABLE_UPGRADE, DEVICE_SYNC_TABLE, db_, deviceB_), - E_OK); + distributedSchema.tables.clear(); + distributedField.colName = "123"; + distributedField.isSpecified = false; + distributedTable.fields.push_back(distributedField); + distributedSchema.tables.push_back(distributedTable); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); /** - * @tc.steps: step5. Sync to real device - * @tc.expected: step5.ok - */ - Query query = Query::Select(tableSchema.name); - EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true), E_OK); + * @tc.steps: step3. Distributed schema with error specified + * @tc.expected: step3. SCHEMA_MISMATCH + */ + distributedSchema.tables.clear(); + distributedField.colName = "123"; + distributedField.isSpecified = true; + distributedTable.fields.push_back(distributedField); + distributedField.colName = "pk"; + distributedField.isP2pSync = false; + distributedTable.fields.push_back(distributedField); + distributedSchema.tables.push_back(distributedTable); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); } /** - * @tc.name: NormalSync003 - * @tc.desc: Test sync with diff distributed schema [low version -> high version]. + * @tc.name: SetSchema015 + * @tc.desc: Test call SetDistributedSchema with mark more than one unique col isSpecified true * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: tankaisheng */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync003, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema015, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Prepare db, tableMode is COLLABORATION * @tc.expected: step1.ok */ - ASSERT_NO_FATAL_FAILURE(InitDelegate()); - /** - * @tc.steps: step2. DeviceB set schema [pk, int_field1, int_field2] - * @tc.expected: step2.ok - */ - DataBaseSchema virtualSchema; - auto tableSchema = GetTableSchema(); - tableSchema.name = DEVICE_SYNC_TABLE_UPGRADE; - virtualSchema.tables.push_back(tableSchema); - auto distributedSchema = RDBDataGenerator::ParseSchema(virtualSchema); - deviceB_->SetDistributedSchema(distributedSchema); - /** - * @tc.steps: step3. Real device set schema [pk, int_field1, int_field2, int_field_upgrade] - * @tc.expected: step3.ok - */ - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE_UPGRADE, TableSyncType::DEVICE_COOPERATION), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); - EXPECT_EQ(delegate_->SetDistributedSchema(RDBDataGenerator::ParseSchema(GetSchema())), OK); - /** - * @tc.steps: step4. Insert table info and virtual data into deviceB - * @tc.expected: step4.ok - */ - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(DEVICE_SYNC_TABLE, DEVICE_SYNC_TABLE_UPGRADE, db_, deviceB_), - E_OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string createSql = "CREATE TABLE IF NOT EXISTS table_pk_int(integer_field INTEGER PRIMARY KEY AUTOINCREMENT," + "int_field INT, char_field CHARACTER(20) UNIQUE, clob_field CLOB UNIQUE, UNIQUE(char_field, clob_field));"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable("table_pk_int", TableSyncType::DEVICE_COOPERATION), OK); + /** - * @tc.steps: step5. Sync to real device - * @tc.expected: step5.ok + * @tc.steps: step2. Test mark more than one unique col isSpecified true + * @tc.expected: step2. return SCHEMA_MISMATCH */ - Query query = Query::Select(tableSchema.name); - EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true), E_OK); + DistributedSchema distributedSchema = {0, {{"table_pk_int", { + {"integer_field", false, false}, + {"int_field", true, false}, + {"char_field", true, true}, + {"clob_field", true, true}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); } /** - * @tc.name: NormalSync004 - * @tc.desc: Test sync when distributed schema was not set. + * @tc.name: SetSchema016 + * @tc.desc: Test set isSpecified to false after isSpecified was set to true * @tc.type: FUNC * @tc.require: * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync004, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema016, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Prepare db * @tc.expected: step1.ok */ - ASSERT_NO_FATAL_FAILURE(InitDelegate()); - auto tableSchema = GetTableSchema(); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string tableName = "multiPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + + "(pk1 INTEGER, pk2 INT, PRIMARY KEY (pk1, pk2));"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); /** - * @tc.steps: step2. Sync to real device - * @tc.expected: step2. return SCHEMA_MISMATCH. + * @tc.steps: step2. Test set distributed schema + * @tc.expected: step2. return OK */ - Query query = Query::Select(tableSchema.name); - DBStatus status = delegate_->Sync({UnitTestCommonConstant::DEVICE_B}, SYNC_MODE_PUSH_ONLY, query, nullptr, true); - EXPECT_EQ(status, SCHEMA_MISMATCH); + DistributedSchema schema1 = GetDistributedSchema(tableName, {"pk1", "pk2"}); + EXPECT_EQ(delegate_->SetDistributedSchema(schema1), OK); + /** + * @tc.steps: step3. Test set distributed schema + * @tc.expected: step3. return SCHEMA_MISMATCH + */ + DistributedSchema schema2 = GetDistributedSchema(tableName, {"pk1", "pk2"}); + DistributedField &field2 = schema2.tables.front().fields.front(); + field2.isSpecified = true; + EXPECT_EQ(delegate_->SetDistributedSchema(schema2), SCHEMA_MISMATCH); +} + +int GetHashKey(sqlite3 *db, const std::string &tableName, std::vector &hashKeys) { + if (db == nullptr) { + return -E_INVALID_DB; + } + + std::string sql = "select cast(hash_key as text) from " + DBCommon::GetLogTableName(tableName) + + " order by timestamp;"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_OK)) { + LOGE("prepare statement failed(%d), sys(%d), msg(%s)", errCode, errno, sqlite3_errmsg(db)); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(stmt); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + LOGE("[SQLiteUtils][ExecuteSQL] execute statement failed(%d), sys(%d), msg(%s)", + errCode, errno, sqlite3_errmsg(db)); + } else { + const unsigned char *result = sqlite3_column_text(stmt, 0); + hashKeys.push_back(reinterpret_cast(result)); + } + } while (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); + + int ret = E_OK; + SQLiteUtils::ResetStatement(stmt, true, ret); + return errCode; } /** - * @tc.name: NormalSync005 - * @tc.desc: Test sync with specified columns + * @tc.name: SetSchema017 + * @tc.desc: Test whether to update hash_key after setting up distributed schema * @tc.type: FUNC * @tc.require: * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync005, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema017, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Prepare db * @tc.expected: step1.ok */ ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); - DistributedSchema schema = GetDistributedSchema(DEVICE_SYNC_TABLE, {"pk", "int_field1"}); - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - deviceB_->SetDistributedSchema(schema); - EXPECT_EQ(delegate_->SetDistributedSchema(schema), OK); + std::string tableName = "multiPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + + "(pk1 INTEGER PRIMARY KEY AUTOINCREMENT, pk2 INT UNIQUE);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); /** - * @tc.steps: step2. Init some data + * @tc.steps: step2. Insert a record and get hash_key * @tc.expected: step2.ok */ - int64_t dataNum = 10; - EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, dataNum / 2, db_, GetTableSchema()), E_OK); - std::string sql = "update " + std::string(DEVICE_SYNC_TABLE) + " set int_field1 = 1, int_field2 = 2 where pk >= 0"; - ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); - auto tableSchema = GetTableSchema(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataNum, deviceB_, tableSchema), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + int dataCount = 10; + for (int i = 0; i < dataCount; i++) { + sql = "insert into " + tableName + " values (" + std::to_string(i) + ", " + std::to_string(i + 1) + ");"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + } + std::vector oldHashKeys; + EXPECT_EQ(GetHashKey(db_, tableName, oldHashKeys), E_OK); + ASSERT_EQ(oldHashKeys.size(), static_cast(dataCount)); /** - * @tc.steps: step3. Sync to real device and check data + * @tc.steps: step3. Set distributed schema and get old hash_key * @tc.expected: step3.ok */ - Query query = Query::Select(tableSchema.name); - EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_PULL, query, true), E_OK); - sql = "select int_field1, int_field2 from " + std::string(DEVICE_SYNC_TABLE) + " order by pk;"; - sqlite3_stmt *stmt = nullptr; - ASSERT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); - int dataIndex = 0; - while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - int64_t intField1Value = sqlite3_column_int64(stmt, 0); - int64_t intField2Value = sqlite3_column_int64(stmt, 1); - EXPECT_EQ(intField1Value, dataIndex); - dataIndex++; - if (dataIndex <= dataNum / 2) { - EXPECT_EQ(intField2Value, 2); - } else { - EXPECT_EQ(intField2Value, 0); - } + DistributedSchema schema1 = GetDistributedSchema(tableName, {"pk1", "pk2"}); + EXPECT_EQ(delegate_->SetDistributedSchema(schema1), OK); + std::vector newHashKeys1; + EXPECT_EQ(GetHashKey(db_, tableName, newHashKeys1), E_OK); + ASSERT_EQ(newHashKeys1.size(), static_cast(dataCount)); + for (int i = 0; i < dataCount; i++) { + EXPECT_EQ(oldHashKeys[i], newHashKeys1[i]); } - EXPECT_EQ(dataIndex, dataNum); - int errCode; - SQLiteUtils::ResetStatement(stmt, true, errCode); + /** + * @tc.steps: step4. Set another distributed schema and get old hash_key + * @tc.expected: step4.ok + */ + DistributedSchema schema2 = {0, {{"multiPriKeyTable", { + {"pk1", false, false}, + {"pk2", true, true}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(schema2), OK); + std::vector newHashKeys2; + EXPECT_EQ(GetHashKey(db_, tableName, newHashKeys2), E_OK); + ASSERT_EQ(newHashKeys2.size(), static_cast(dataCount)); + for (int i = 0; i < dataCount; i++) { + EXPECT_NE(newHashKeys1[i], newHashKeys2[i]); + } + EXPECT_NE(newHashKeys2, oldHashKeys); } /** - * @tc.name: NormalSync006 - * @tc.desc: Test set distributed schema and sync. + * @tc.name: SetSchema018 + * @tc.desc: Test no primary key table setting isSpecified * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync006, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema018, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Prepare db * @tc.expected: step1.ok */ - ASSERT_NO_FATAL_FAILURE(InitDelegate()); - auto schema = GetSchema(); - auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); - deviceB_->SetDistributedSchema(distributedSchema); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); - /** - * @tc.steps: step2. Insert one data - * @tc.expected: step2.ok - */ - auto tableSchema = GetTableSchema(); - EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); - /** - * @tc.steps: step3. Sync to real device - * @tc.expected: step3.ok - */ - Query query = Query::Select(tableSchema.name); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, OK, {deviceB_->GetDeviceId()}); + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string tableName = "noPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + + "(field_int1 INTEGER, field_int2 INT);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); /** - * @tc.steps: step4. Change schema to non-existent table name, then sync - * @tc.expected: step4.SCHEMA_MISMATCH + * @tc.steps: step2. Test set distributed schema + * @tc.expected: step2. return SCHEMA_MISMATCH */ - tableSchema.name = "not_config_table"; - ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, *db_), SQLITE_OK); - ASSERT_EQ(delegate_->CreateDistributedTable(tableSchema.name), OK); - Query query2 = Query::Select(tableSchema.name); - std::map> statusMap; - SyncStatusCallback callBack2; - DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, query2, callBack2, true); - EXPECT_EQ(callStatus, SCHEMA_MISMATCH); + DistributedSchema schema = GetDistributedSchema(tableName, {"field_int1"}); + DistributedField &field = schema.tables.front().fields.front(); + field.isSpecified = true; + EXPECT_EQ(delegate_->SetDistributedSchema(schema), SCHEMA_MISMATCH); } /** - * @tc.name: NormalSync007 - * @tc.desc: Test change distributed schema and sync. + * @tc.name: SetSchema019 + * @tc.desc: Test call SetDistributedSchema when unique col not set isP2pSync * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: tankaisheng */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync007, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema019, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Prepare db, tableMode is COLLABORATION * @tc.expected: step1.ok */ - ASSERT_NO_FATAL_FAILURE(InitDelegate()); - auto schema = GetSchema(); + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string createSql = "CREATE TABLE IF NOT EXISTS table_pk_integer(integer_field INTEGER UNIQUE," + "int_field INT, char_field CHARACTER(20), clob_field CLOB);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable("table_pk_integer", TableSyncType::DEVICE_COOPERATION), OK); + + /** + * @tc.steps: step2. Test mark unique col isP2pSync true + * @tc.expected: step2. return SCHEMA_MISMATCH + */ + DistributedSchema distributedSchema = {1, {{"table_pk_integer", { + {"int_field", true}, + {"char_field", true}, + {"clob_field", true}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); +} + +/** + * @tc.name: SetSchema020 + * @tc.desc: Test call SetDistributedSchema when unique col and pk isP2pSync + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema020, TestSize.Level0) +{ + + /** + * @tc.steps: step1. Prepare db, tableMode is COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string createSql = "CREATE TABLE IF NOT EXISTS table_pk_int(integer_field INTEGER PRIMARY KEY AUTOINCREMENT," + "int_field INT UNIQUE, char_field CHARACTER(20), clob_field CLOB);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable("table_pk_int", TableSyncType::DEVICE_COOPERATION), OK); + + /** + * @tc.steps: step2. Test mark unique col and pk isP2pSync true, specified unique col + * @tc.expected: step2. return NOT_SUPPORT + */ + DistributedSchema distributedSchema = {1, { + {"table_pk_int", { + {"integer_field", true}, + {"int_field", true, true}, + {"char_field", true}, + {"clob_field", true} + }}} + }; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), NOT_SUPPORT); +} + +/** + * @tc.name: SetSchema021 + * @tc.desc: Test call SetDistributedSchema when table contains 1000 columns + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema021, TestSize.Level1) +{ + /** + * @tc.steps: step1. create table which contains 1000 columns + * @tc.expected: step1. create table ok + */ + TableSchema tableSchema; + tableSchema.name = BIG_COLUMNS_TABLE; + Field field; + field.primary = true; + field.type = TYPE_INDEX; + field.colName = "pk"; + tableSchema.fields.push_back(field); + field.primary = false; + field.colName = "123"; + field.type = TYPE_INDEX; + tableSchema.fields.push_back(field); + const uint16_t fieldNum = 1000; + for (int i = 0; i < fieldNum; i++) { + field.colName = "field" + to_string(i); + tableSchema.fields.push_back(field); + } + ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, true, *db_), E_OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + EXPECT_EQ(delegate_->CreateDistributedTable(tableSchema.name, TableSyncType::DEVICE_COOPERATION), OK); + + /** + * @tc.steps: step2. set schema + * @tc.expected: step2. ok + */ + DistributedSchema distributedSchema; + DistributedTable distributedTable; + distributedTable.tableName = tableSchema.name; + DistributedField distributedField; + distributedField.isP2pSync = true; + distributedField.colName = "123"; + distributedField.isSpecified = false; + distributedTable.fields.push_back(distributedField); + for (int i = 0; i < fieldNum; i++) { + distributedField.colName = "field" + to_string(i); + distributedTable.fields.push_back(distributedField); + } + distributedSchema.tables.push_back(distributedTable); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); +} + +/** + * @tc.name: SetSchema022 + * @tc.desc: Test setting the distributed schema with table names named keywords. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema022, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table named keywords. + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + std::string tableName = "except"; + std::string sql = "CREATE TABLE IF NOT EXISTS '" + tableName + "'(field1 INTEGER, field2 INT);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + /** + * @tc.steps: step2. Create distributed table and set distributed schema + * @tc.expected: step2.ok + */ + auto distributedSchema = GetDistributedSchema(tableName, {"field1", "field2"}); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); +} + +/** + * @tc.name: SetSchema023 + * @tc.desc: Test SetDistributedSchema interface when tables decrease. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema023, TestSize.Level0) +{ + /** + * @tc.steps: step1. set distributed schema for the first time + * @tc.expected: step1. return OK + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + DistributedSchema distributedSchema = {1, {{DEVICE_SYNC_TABLE, {{"pk", true}, {"int_field1", true}}}, + {CLOUD_SYNC_TABLE, {{"pk", true}, {"int_field2", false}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + + /** + * @tc.steps: step2. decrease tables of input parameter to set distributed schema + * @tc.expected: step2. return DISTRIBUTED_FIELD_DECREASE + */ + distributedSchema = {2, {{CLOUD_SYNC_TABLE, {{"pk", true}, {"int_field2", false}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), DISTRIBUTED_FIELD_DECREASE); + distributedSchema = {2, {{DEVICE_SYNC_TABLE, {{"pk", true}, {"int_field1", true}}}, + {DEVICE_SYNC_TABLE_UPGRADE, {{"pk", true}, {"int_field2", false}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), DISTRIBUTED_FIELD_DECREASE); +} + +/** + * @tc.name: SetSchema024 + * @tc.desc: Test SetDistributedSchema out of max schema size limit. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema024, TestSize.Level0) +{ + /** + * @tc.steps: step1. set distributed schema for the first time + * @tc.expected: step1. return OK + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + /** + * @tc.steps: step2. generate 32 tables with 1000 field + * @tc.expected: step2. return OK + */ + auto basicSchema = GetTableSchema(); + const int fieldCount = 1000; + Field basicField; + basicField.type = TYPE_INDEX; + basicField.colName = "generate_field_"; + for (int i = 0; i < fieldCount; ++i) { + Field field = basicField; + field.colName += std::to_string(i); + basicSchema.fields.push_back(field); + } + /** + * @tc.steps: step3. create distributed table + * @tc.expected: step3. return OVER_MAX_LIMITS at last + */ + const int tableCount = 32; + DBStatus res = OK; + for (int i = 0; i < tableCount; ++i) { + TableSchema table = basicSchema; + table.name += std::to_string(i); + RDBDataGenerator::InitTable(table, false, *db_); + res = delegate_->CreateDistributedTable(table.name); + if (res != OK) { + break; + } + } + EXPECT_EQ(res, OVER_MAX_LIMITS); +} + +/** + * @tc.name: NormalSync001 + * @tc.desc: Test set distributed schema and sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync001, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); +} + +/** + * @tc.name: NormalSync002 + * @tc.desc: Test sync with diff distributed schema [high version -> low version]. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync002, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + /** + * @tc.steps: step2. DeviceB set schema [pk, int_field1, int_field2, int_field_upgrade] + * @tc.expected: step2.ok + */ + DataBaseSchema virtualSchema; + auto tableSchema = GetTableSchema(true); + tableSchema.name = DEVICE_SYNC_TABLE; + virtualSchema.tables.push_back(tableSchema); + auto distributedSchema = RDBDataGenerator::ParseSchema(virtualSchema); + deviceB_->SetDistributedSchema(distributedSchema); + /** + * @tc.steps: step3. Real device set schema [pk, int_field1, int_field2] + * @tc.expected: step3.ok + */ + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + EXPECT_EQ(delegate_->SetDistributedSchema(RDBDataGenerator::ParseSchema(GetSchema())), OK); + /** + * @tc.steps: step4. Insert table info and virtual data into deviceB + * @tc.expected: step4.ok + */ + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(DEVICE_SYNC_TABLE_UPGRADE, DEVICE_SYNC_TABLE, db_, deviceB_), + E_OK); + /** + * @tc.steps: step5. Sync to real device + * @tc.expected: step5.ok + */ + Query query = Query::Select(tableSchema.name); + EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true), E_OK); +} + +/** + * @tc.name: NormalSync003 + * @tc.desc: Test sync with diff distributed schema [low version -> high version]. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync003, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + /** + * @tc.steps: step2. DeviceB set schema [pk, int_field1, int_field2] + * @tc.expected: step2.ok + */ + DataBaseSchema virtualSchema; + auto tableSchema = GetTableSchema(); + tableSchema.name = DEVICE_SYNC_TABLE_UPGRADE; + virtualSchema.tables.push_back(tableSchema); + auto distributedSchema = RDBDataGenerator::ParseSchema(virtualSchema); + deviceB_->SetDistributedSchema(distributedSchema); + /** + * @tc.steps: step3. Real device set schema [pk, int_field1, int_field2, int_field_upgrade] + * @tc.expected: step3.ok + */ + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE_UPGRADE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + EXPECT_EQ(delegate_->SetDistributedSchema(RDBDataGenerator::ParseSchema(GetSchema())), OK); + /** + * @tc.steps: step4. Insert table info and virtual data into deviceB + * @tc.expected: step4.ok + */ + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(DEVICE_SYNC_TABLE, DEVICE_SYNC_TABLE_UPGRADE, db_, deviceB_), + E_OK); + /** + * @tc.steps: step5. Sync to real device + * @tc.expected: step5.ok + */ + Query query = Query::Select(tableSchema.name); + EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true), E_OK); +} + +/** + * @tc.name: NormalSync004 + * @tc.desc: Test sync when distributed schema was not set. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync004, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step2. Sync to real device + * @tc.expected: step2. return SCHEMA_MISMATCH. + */ + Query query = Query::Select(tableSchema.name); + DBStatus status = delegate_->Sync({UnitTestCommonConstant::DEVICE_B}, SYNC_MODE_PUSH_ONLY, query, nullptr, true); + EXPECT_EQ(status, SCHEMA_MISMATCH); +} + +/** + * @tc.name: NormalSync005 + * @tc.desc: Test sync with specified columns + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync005, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + DistributedSchema schema = GetDistributedSchema(DEVICE_SYNC_TABLE, {"pk", "int_field1"}); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + deviceB_->SetDistributedSchema(schema); + EXPECT_EQ(delegate_->SetDistributedSchema(schema), OK); + /** + * @tc.steps: step2. Init some data + * @tc.expected: step2.ok + */ + int64_t dataNum = 10; + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, dataNum / 2, db_, GetTableSchema()), E_OK); + std::string sql = "update " + std::string(DEVICE_SYNC_TABLE) + " set int_field1 = 1, int_field2 = 2 where pk >= 0"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + auto tableSchema = GetTableSchema(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataNum, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device and check data + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + EXPECT_EQ(deviceB_->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_PULL, query, true), E_OK); + sql = "select int_field1, int_field2 from " + std::string(DEVICE_SYNC_TABLE) + " order by pk;"; + sqlite3_stmt *stmt = nullptr; + ASSERT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); + int dataIndex = 0; + while (SQLiteUtils::StepWithRetry(stmt) == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + int64_t intField1Value = sqlite3_column_int64(stmt, 0); + int64_t intField2Value = sqlite3_column_int64(stmt, 1); + EXPECT_EQ(intField1Value, dataIndex); + dataIndex++; + if (dataIndex <= dataNum / 2) { + EXPECT_EQ(intField2Value, 2); + } else { + EXPECT_EQ(intField2Value, 0); + } + } + EXPECT_EQ(dataIndex, dataNum); + int errCode; + SQLiteUtils::ResetStatement(stmt, true, errCode); +} + +/** + * @tc.name: NormalSync006 + * @tc.desc: Test set distributed schema and sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync006, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. Change schema to non-existent table name, then sync + * @tc.expected: step4.SCHEMA_MISMATCH + */ + tableSchema.name = "not_config_table"; + ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, *db_), SQLITE_OK); + ASSERT_EQ(delegate_->CreateDistributedTable(tableSchema.name), OK); + Query query2 = Query::Select(tableSchema.name); + std::map> statusMap; + SyncStatusCallback callBack2; + DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, query2, callBack2, true); + EXPECT_EQ(callStatus, SCHEMA_MISMATCH); +} + +/** + * @tc.name: NormalSync007 + * @tc.desc: Test change distributed schema and sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync007, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE) + .append(" where pk=0 and int_field1 is null and int_field2 is null"); + int count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 1); + /** + * @tc.steps: step4. Change schema and sync again + * @tc.expected: step4.ok + */ + distributedSchema = RDBDataGenerator::ParseSchema(schema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE) + .append(" where pk=0 and int_field1=0 and int_field2=0"); + count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 1); + auto changeData = delegateObserver_->GetSavedChangedData()[std::string(DEVICE_SYNC_TABLE)]; + EXPECT_TRUE(changeData.properties.isP2pSyncDataChange); +} + +/** + * @tc.name: NormalSync008 + * @tc.desc: Test set distributed schema and sync with diff sort. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync008, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one data with diff sort col + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, + RDBDataGenerator::FlipTableSchema(tableSchema)), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE).append(" where pk=0;"); + int count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 1); + sql = std::string("select count(*) from ") + .append(RelationalStoreManager::GetDistributedLogTableName(DEVICE_SYNC_TABLE)) + .append(" where data_key=0 and cursor=1;"); + count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 1); +} + +/** + * @tc.name: NormalSync009 + * @tc.desc: Test if distributed table will be created when sync with COLLABORATION mode. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync009, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + std::string checkSql = "select count(*) from sqlite_master where type='table' and name != '" + + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + "' and name like 'naturalbase_rdb_aux_" + + std::string(DEVICE_SYNC_TABLE) + "_%'"; + int count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkSql, count), E_OK); + EXPECT_EQ(count, 0); + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. Check if the distributed table exists + * @tc.expected: step4.ok + */ + count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkSql, count), E_OK); + EXPECT_EQ(count, 0); +} + +/** + * @tc.name: NormalSync010 + * @tc.desc: Test sync without not null col. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync010, TestSize.Level0) +{ + /** + * @tc.steps: step1. Recreate not null table + * @tc.expected: step1.ok + */ + std::string sql = std::string("DROP TABLE ").append(DEVICE_SYNC_TABLE); + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, *db_), E_OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync without str col + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PULL_ONLY, query, nullptr, true); + EXPECT_EQ(callStatus, SCHEMA_MISMATCH); + /** + * @tc.steps: step4. Check if the distributed table exists + * @tc.expected: step4.ok + */ + int count = 0; + sql = std::string("select count(*) from ") + .append(RelationalStoreManager::GetDistributedLogTableName(DEVICE_SYNC_TABLE)); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 0); +} + +/** + * @tc.name: NormalSync011 + * @tc.desc: Test set the distributed schema first then sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: bty + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync011, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select().FromTable({tableSchema.name}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, OK, {deviceB_->GetDeviceId()}); +} + +/** + * @tc.name: NormalSync012 + * @tc.desc: Test sync with autoincrement table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync012, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create auto increment table and unique index + * @tc.expected: step1.ok + */ + auto tableSchema = GetTableSchema(); + tableSchema.name = DEVICE_SYNC_TABLE_AUTOINCREMENT; + ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, true, *db_), E_OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + tableSchema = GetTableSchema(false, true); + tableSchema.name = DEVICE_SYNC_TABLE_AUTOINCREMENT; + auto schema = GetSchema(); + schema.tables.push_back(tableSchema); + DistributedSchema distributedSchema = {0, {{tableSchema.name, { + {"pk", false, false}, + {"int_field1", true, false}, + {"int_field2", true, false}, + {"123", true, true}}}}}; + deviceB_->SetDistributedSchema(distributedSchema); + int errCode = SQLiteUtils::ExecuteRawSQL(db_, std::string("CREATE UNIQUE INDEX U_INDEX ON ") + .append(tableSchema.name).append("('123')")); + ASSERT_EQ(errCode, E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableSchema.name, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + ASSERT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, tableSchema), E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // sleep 100 ms + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 2, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string sql = std::string("select count(*) from ").append(tableSchema.name); + int count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 2); + /** + * @tc.steps: step4. Update date and sync again + * @tc.expected: step4.ok + */ + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + sql = std::string("select count(*) from ").append(tableSchema.name); + count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 2); +} + +/** + * @tc.name: NormalSync013 + * @tc.desc: Test chanage data after sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: lg + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync013, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. check changData + * @tc.expected: step4.ok + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + EXPECT_EQ(changeData[tableSchema.name].primaryData[0].size(), 10u); + EXPECT_EQ(changeData[tableSchema.name].field.size(), 1u); + /** + * @tc.steps: step5. Remove the last item and a cloud-only item, then check changeData + * @tc.expected: step5. Only the last item is in changeData and field is primary key + */ + delegateObserver_->ClearChangedData(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(9, 11, deviceB_, tableSchema, true), E_OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableSchema.name].primaryData[OP_DELETE].size(), 1u); + EXPECT_EQ(changeData[tableSchema.name].primaryData[OP_DELETE][0].size(), 1u); + ASSERT_EQ(changeData[tableSchema.name].field.size(), 1u); + EXPECT_EQ(changeData[tableSchema.name].field[0], "pk"); +} + +/** + * @tc.name: NormalSync014 + * @tc.desc: Test chanage data after sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: tankaisheng + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync014, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + DistributedSchema schema1 = GetDistributedSchema(DEVICE_SYNC_TABLE, {"int_field1"}); + EXPECT_EQ(delegate_->SetDistributedSchema(schema1), OK); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_COOPERATION); + /** + * @tc.steps: step2. Insert data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. check changData + * @tc.expected: step4.ok + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + EXPECT_EQ(changeData[tableSchema.name].primaryData[0].size(), 10u); + EXPECT_EQ(changeData[tableSchema.name].field.size(), 1u); + /** + * @tc.steps: step5. SetDistributedSchema again + * @tc.expected: step5.ok + */ + DistributedSchema schema2 = GetDistributedSchema(DEVICE_SYNC_TABLE, {"int_field1", "int_field2"}); + EXPECT_EQ(delegate_->SetDistributedSchema(schema2), OK); + /** + * @tc.steps: step6. Sync to real device + * @tc.expected: step6.ok + */ + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step7. check changData + * @tc.expected: step7.ok + */ + changeData = delegateObserver_->GetSavedChangedData(); + EXPECT_EQ(changeData[tableSchema.name].primaryData[1].size(), 10u); + EXPECT_EQ(changeData[tableSchema.name].field.size(), 1u); +} + +/** + * @tc.name: NormalSync015 + * @tc.desc: Test sync with multi primary key table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync015, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + std::string tableName = "multiPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + + "(pk1 INTEGER, pk2 INT, PRIMARY KEY (pk1, pk2));"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + /** + * @tc.steps: step2. Create distributed table and set distributed schema + * @tc.expected: step2.ok + */ + auto distributedSchema = GetDistributedSchema(tableName, {"pk1", "pk2"}); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step3. Init data and sync to real device + * @tc.expected: step3.ok + */ + TableSchema tableSchema; + tableSchema.name = tableName; + Field field; + field.primary = true; + field.type = TYPE_INDEX; + field.colName = "pk1"; + tableSchema.fields.push_back(field); + field.colName = "pk2"; + tableSchema.fields.push_back(field); + uint32_t dataCount = 10; + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableName, db_, deviceB_), E_OK); + Query query = Query::Select(tableName); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. check changData + * @tc.expected: step4.ok + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableName].primaryData[OP_INSERT].size(), dataCount); + for (uint32_t i = 0; i < dataCount; i++) { + EXPECT_EQ(changeData[tableName].primaryData[OP_INSERT][i].size(), 3u); // primary key (pk1, pk2) and rowid + } + EXPECT_EQ(changeData[tableName].field.size(), 3u); // primary key (pk1, pk2) and rowid + /** + * @tc.steps: step5. Remove the last item and a cloud-only item, then check changeData + * @tc.expected: step5. Only the last item is in changeData and field is key (pk1, pk2) and rowid + */ + delegateObserver_->ClearChangedData(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(dataCount - 1, dataCount + 1, deviceB_, tableSchema, true), + E_OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableName].primaryData[OP_DELETE].size(), 1u); + EXPECT_EQ(changeData[tableName].primaryData[OP_DELETE][0].size(), 3u); // primary key (pk1, pk2) and rowid + EXPECT_EQ(changeData[tableName].field.size(), 3u); // primary key (pk1, pk2) and rowid +} + +/** + * @tc.name: NormalSync016 + * @tc.desc: Test sync with diff specified field. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync016, TestSize.Level0) +{ + /** + * @tc.steps: step1. Prepare db, tableMode is COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + std::string createSql = "CREATE TABLE IF NOT EXISTS table_int(integer_field INTEGER PRIMARY KEY AUTOINCREMENT," + "int_field1 INT UNIQUE, int_field2 INT UNIQUE);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable("table_int", TableSyncType::DEVICE_COOPERATION), OK); + + /** + * @tc.steps: step2. Test mark one specified one is field1 another is field2 + * @tc.expected: step2. sync return SCHEMA_MISMATCH + */ + DistributedSchema distributedSchema = {0, {{"table_int", { + {"integer_field", false, false}, + {"int_field1", true, false}, + {"int_field2", true, true}}}}}; + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + distributedSchema = {0, {{"table_int", { + {"integer_field", false, false}, + {"int_field1", true, true}, + {"int_field2", true, false}}}}}; + deviceB_->SetDistributedSchema(distributedSchema); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv("table_int", db_, deviceB_), E_OK); + Query query = Query::Select("table_int"); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, SCHEMA_MISMATCH, + {deviceB_->GetDeviceId()}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, SCHEMA_MISMATCH, + {deviceB_->GetDeviceId()}); +} + +/** + * @tc.name: NormalSync017 + * @tc.desc: Test delete other device's data and sync + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync017, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE); + int count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 1); + /** + * @tc.steps: step4. Delete data and sync again + * @tc.expected: step4.ok + */ + EXPECT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // sleep 100 ms + sql = std::string("delete from ").append(DEVICE_SYNC_TABLE).append(" where 0 = 0"); + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE); + count = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); + EXPECT_EQ(count, 0); + auto changeData = delegateObserver_->GetSavedChangedData()[std::string(DEVICE_SYNC_TABLE)]; + EXPECT_TRUE(changeData.properties.isP2pSyncDataChange); +} + +/** + * @tc.name: NormalSync018 + * @tc.desc: Test sync with no primary key table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync018, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + std::string tableName = "noPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + "(pk1 INTEGER, pk2 INT);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + /** + * @tc.steps: step2. Create distributed table and set distributed schema + * @tc.expected: step2.ok + */ + auto distributedSchema = GetDistributedSchema(tableName, {"pk1", "pk2"}); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step3. Init data and sync to real device + * @tc.expected: step3.ok + */ + TableSchema tableSchema; + tableSchema.name = tableName; + Field field; + field.primary = true; + field.type = TYPE_INDEX; + field.colName = "pk1"; + tableSchema.fields.push_back(field); + field.colName = "pk2"; + tableSchema.fields.push_back(field); + uint32_t dataCount = 10; + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableName, db_, deviceB_), E_OK); + Query query = Query::Select(tableName); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. check changData + * @tc.expected: step4.ok + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableName].primaryData[OP_INSERT].size(), dataCount); + for (uint32_t i = 0; i < dataCount; i++) { + EXPECT_EQ(changeData[tableName].primaryData[OP_INSERT][i].size(), 1u); // rowid + } + EXPECT_EQ(changeData[tableName].field.size(), 1u); // rowid +} + +/** + * @tc.name: NormalSync019 + * @tc.desc: Test whether there is an observer notification when local win. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync019, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); deviceB_->SetDistributedSchema(distributedSchema); EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); /** - * @tc.steps: step2. Insert one data - * @tc.expected: step2.ok + * @tc.steps: step2. Insert a piece of data from the other end and then locally insert the same data. + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + std::string sql = std::string("insert into ").append(DEVICE_SYNC_TABLE).append("(pk) values (0)"); + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + /** + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. Check observer + * @tc.expected: step4.No data change notification + */ + auto onChangeCallCount = delegateObserver_->GetCloudCallCount(); + EXPECT_EQ(onChangeCallCount, 0u); +} + +/** + * @tc.name: NormalSync020 + * @tc.desc: Test set distributed schema after recreating the table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync020, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Sync a piece of data + * @tc.expected: step2.ok + */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step3. Recreate table and set distributed schema. + * @tc.expected: step3.ok + */ + std::string sql = std::string("drop table ").append(DEVICE_SYNC_TABLE); + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + RDBDataGenerator::InitTable(schema.tables.front(), false, *db_); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); +} + +/** + * @tc.name: NormalSync021 + * @tc.desc: Test sync multi table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync021, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE_UPGRADE, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step2. Insert 10 data + * @tc.expected: step2.ok + */ + int dataCount = 10; + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + tableSchema = GetTableSchema(true); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device and check data + * @tc.expected: step3.ok + */ + Query query = Query::Select().FromTable({DEVICE_SYNC_TABLE, DEVICE_SYNC_TABLE_UPGRADE}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE); + int actualCount = 0; + SQLiteUtils::GetCountBySql(db_, sql, actualCount); + EXPECT_EQ(actualCount, dataCount); + sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE_UPGRADE); + actualCount = 0; + SQLiteUtils::GetCountBySql(db_, sql, actualCount); + EXPECT_EQ(actualCount, dataCount); + /** + * @tc.steps: step4. Sync with invalid args + * @tc.expected: step4.invalid args + */ + query.And(); + DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, query, nullptr, true); + EXPECT_EQ(callStatus, NOT_SUPPORT); +} + +/** + * @tc.name: NormalSync022 + * @tc.desc: Test drop table after sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync022, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + std::string tableName = "noPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + "(field1 INTEGER, field2 INT);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + auto distributedSchema = GetDistributedSchema(tableName, {"field1", "field2"}); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert one piece of data at each end + * @tc.expected: step2.ok + */ + TableSchema tableSchema = {tableName, "", {{"field1", TYPE_INDEX}, {"field2", TYPE_INDEX}}}; + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + sql = "insert into " + tableName + " values(1, 1)"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + std::string checkLogCountSql = "select count(*) from " + DBCommon::GetLogTableName(tableName); + std::string checkDataCountSql = "select count(*) from " + tableName; + int logCount = 0; + int dataCount = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDataCountSql, dataCount), E_OK); + EXPECT_EQ(logCount, 1); + EXPECT_EQ(dataCount, 1); + /** + * @tc.steps: step3. Sync a piece of data + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableSchema.name); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDataCountSql, dataCount), E_OK); + EXPECT_EQ(dataCount, 2); + EXPECT_EQ(logCount, 2); + /** + * @tc.steps: step4. Drop table and check log count + * @tc.expected: step4.ok + */ + checkLogCountSql.append(" where data_key = -1 and flag&0x01 = 0x1;"); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(logCount, 0); + std::string dropTableSql = "drop table " + tableName; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, dropTableSql), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(logCount, 2); +} + +/** + * @tc.name: NormalSync023 + * @tc.desc: Test synchronization of autoIncremental primary key table with observer notices. + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync023, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + std::string tableName = "autoIncPriKeyTable"; + std::string sql = "CREATE TABLE IF NOT EXISTS " + tableName + + "(pk INTEGER PRIMARY KEY AUTOINCREMENT, field1 INT UNIQUE);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + DistributedSchema distributedSchema = {0u, {{tableName, {{"pk", false, false}, {"field1", true, true}}}}}; + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + /** + * @tc.steps: step2. Insert data + * @tc.expected: step2.ok + */ + uint32_t dataCount = 10; + TableSchema tableSchema = {tableName, "", {{"pk", TYPE_INDEX, true}, {"field1", TYPE_INDEX}}}; + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step3. Sync to real device and check data + * @tc.expected: step3.ok + */ + Query query = Query::Select(tableName); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. check changData + * @tc.expected: step4.ok + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableName].primaryData[OP_INSERT].size(), dataCount); + for (uint32_t i = 0; i < changeData[tableName].primaryData[OP_INSERT].size(); i++) { + auto *data = std::get_if(&changeData[tableName].primaryData[OP_INSERT][i].front()); + ASSERT_NE(data, nullptr); + EXPECT_EQ(static_cast(*data), i + 1); + } + ASSERT_EQ(changeData[tableName].field.size(), 1u); + EXPECT_EQ(changeData[tableName].field.front(), "pk"); +} + +/** + * @tc.name: NormalSync024 + * @tc.desc: Test set tracker table and sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: lg + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync024, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE_UPGRADE, TableSyncType::DEVICE_COOPERATION), OK); + + TrackerSchema trackerSchema = { + .tableName = DEVICE_SYNC_TABLE, .extendColNames = {"int_field1"}, .trackerColNames = {"int_field1"}}; + delegate_->SetTrackerTable(trackerSchema); + + /** + * @tc.steps: step2. close deviceA and reopen it + * @tc.expected: step2.ok + */ + RelationalStoreManager mgr(APP_ID, USER_ID); + EXPECT_EQ(mgr.CloseStore(delegate_), OK); + delegate_ = nullptr; + RelationalStoreDelegate::Option option; + option.tableMode = DistributedTableMode::COLLABORATION; + option.observer = delegateObserver_; + ASSERT_EQ(mgr.OpenStore(storePath_, STORE_ID_1, option, delegate_), OK); + ASSERT_NE(delegate_, nullptr); + + /** + * @tc.steps: step3. Insert 10 data + * @tc.expected: step3.ok + */ + int dataCount = 10; + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + tableSchema = GetTableSchema(true); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + /** + * @tc.steps: step4. Sync to real device and check data + * @tc.expected: step4.ok + */ + Query query = Query::Select().FromTable({DEVICE_SYNC_TABLE, DEVICE_SYNC_TABLE_UPGRADE}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + std::string checkLogCountSql = std::string("SELECT COUNT(*) FROM ") + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + + " AS a LEFT JOIN " + DEVICE_SYNC_TABLE + + " AS b ON (a.data_key = b._rowid_) WHERE a.data_key = b._rowid_;"; + int logCount = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(logCount, 10); +} + +/** + * @tc.name: NormalSync025 + * @tc.desc: Test drop table in split_by_device mode will only mark local data as delete in log table + * @tc.type: FUNC + * @tc.require: + * @tc.author: liuhongyang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync025, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create SPLIT_BY_DEVICE tables + * @tc.expected: step1. Ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::SPLIT_BY_DEVICE)); + std::string tableName = "noPKTable025"; + std::string createTableSql = "CREATE TABLE IF NOT EXISTS " + tableName + "(field1 INTEGER, field2 INT);"; + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createTableSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step2. Insert one piece of data at each end + * @tc.expected: step2. Ok */ - auto tableSchema = GetTableSchema(); + TableSchema tableSchema = {tableName, "", {{"field1", TYPE_INDEX}, {"field2", TYPE_INDEX}}}; ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); - /** - * @tc.steps: step3. Sync to real device - * @tc.expected: step3.ok + std::string sql = "insert into " + tableName + " values(1, 1)"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + std::string checkLogCountSql = "select count(*) from " + DBCommon::GetLogTableName(tableName); + std::string checkDataCountSql = "select count(*) from " + tableName; + int logCount = 0; + int dataCount = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDataCountSql, dataCount), E_OK); + EXPECT_EQ(logCount, 1); + EXPECT_EQ(dataCount, 1); + /** + * @tc.steps: step3. Sync to pull deviceB_ data + * @tc.expected: step3. One data in device table and one data in local table */ Query query = Query::Select(tableSchema.name); DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); - std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE) - .append(" where pk=0 and int_field1 is null and int_field2 is null"); - int count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); - /** - * @tc.steps: step4. Change schema and sync again - * @tc.expected: step4.ok - */ - distributedSchema = RDBDataGenerator::ParseSchema(schema); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); - sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE) - .append(" where pk=0 and int_field1=0 and int_field2=0"); - count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); - auto changeData = delegateObserver_->GetSavedChangedData()[std::string(DEVICE_SYNC_TABLE)]; - EXPECT_TRUE(changeData.properties.isP2pSyncDataChange); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDataCountSql, dataCount), E_OK); + EXPECT_EQ(dataCount, 1); + EXPECT_EQ(logCount, 2); + std::string checkDeviceTableCountSql = "select count(*) from " + + DBCommon::GetDistributedTableName(deviceB_->GetDeviceId(), tableName); + int deviceDataCount = 0; + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDeviceTableCountSql, deviceDataCount), E_OK); + EXPECT_EQ(deviceDataCount, 1); + /** + * @tc.steps: step4. Check delete count in log table before drop + * @tc.expected: step4. No deleted records + */ + checkLogCountSql.append(" where data_key = -1 and flag&0x01 = 0x1;"); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(logCount, 0); + /** + * @tc.steps: step5. Drop table and recreate + * @tc.expected: step5. Ok + */ + std::string dropTableSql = "drop table " + tableName; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, dropTableSql), E_OK); + ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, createTableSql), E_OK); + EXPECT_EQ(delegate_->CreateDistributedTable(tableName, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step6. Check count + * @tc.expected: step6. only the local record is marked as deleted + */ + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkLogCountSql, logCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDataCountSql, dataCount), E_OK); + EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkDeviceTableCountSql, deviceDataCount), E_OK); + EXPECT_EQ(logCount, 1); + EXPECT_EQ(dataCount, 0); + EXPECT_EQ(deviceDataCount, 1); } /** - * @tc.name: NormalSync008 - * @tc.desc: Test set distributed schema and sync with diff sort. + * @tc.name: SetStoreConfig001 + * @tc.desc: Test set store config. * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync008, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetStoreConfig001, TestSize.Level0) { /** - * @tc.steps: step1. Create device table and cloud table in COLLABORATION + * @tc.steps: step1. Create device table and cloud table in SPLIT_BY_DEVICE * @tc.expected: step1.ok */ - ASSERT_NO_FATAL_FAILURE(InitDelegate()); - auto schema = GetSchema(); - auto distributedSchema = RDBDataGenerator::ParseSchema(schema); - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); - deviceB_->SetDistributedSchema(distributedSchema); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::SPLIT_BY_DEVICE)); /** - * @tc.steps: step2. Insert one data with diff sort col + * @tc.steps: step2. Set store config. * @tc.expected: step2.ok */ - auto tableSchema = GetTableSchema(); - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, - RDBDataGenerator::FlipTableSchema(tableSchema)), E_OK); - ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); + EXPECT_EQ(delegate_->SetStoreConfig({DistributedTableMode::COLLABORATION}), OK); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); /** * @tc.steps: step3. Sync to real device * @tc.expected: step3.ok */ + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); Query query = Query::Select(tableSchema.name); DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); - std::string sql = std::string("select count(*) from ").append(DEVICE_SYNC_TABLE).append(" where pk=0;"); - int count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); - sql = std::string("select count(*) from ") - .append(RelationalStoreManager::GetDistributedLogTableName(DEVICE_SYNC_TABLE)) - .append(" where data_key=0 and cursor=1;"); - count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); } /** - * @tc.name: NormalSync009 - * @tc.desc: Test if distributed table will be created when sync with COLLABORATION mode. + * @tc.name: SetStoreConfig002 + * @tc.desc: Test set store config after create distributed table. * @tc.type: FUNC * @tc.require: * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync009, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, SetStoreConfig002, TestSize.Level0) +{ + /** + * @tc.steps: step1. Create device table and cloud table in SPLIT_BY_DEVICE + * @tc.expected: step1.ok + */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::SPLIT_BY_DEVICE)); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step2. Set store config. + * @tc.expected: step2. return not support + */ + EXPECT_EQ(delegate_->SetStoreConfig({DistributedTableMode::COLLABORATION}), NOT_SUPPORT); +} + +/** + * @tc.name: SetStoreConfig003 + * @tc.desc: Test set store config after create distributed table. + * @tc.type: FUNC + * @tc.require: + * @tc.author: lg + */ +HWTEST_F(DistributedDBRDBCollaborationTest, SetStoreConfig003, TestSize.Level0) { /** * @tc.steps: step1. Create device table and cloud table in COLLABORATION * @tc.expected: step1.ok */ + ASSERT_NO_FATAL_FAILURE(InitDelegate(DistributedTableMode::COLLABORATION)); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + /** + * @tc.steps: step2. Set store config. + * @tc.expected: step2. return not support + */ + EXPECT_EQ(delegate_->SetStoreConfig({DistributedTableMode::SPLIT_BY_DEVICE}), NOT_SUPPORT); +} + +/** + * @tc.name: InvalidSync001 + * @tc.desc: Test remote set empty distributed schema and sync. + * @tc.type: FUNC + * @tc.require: + * @tc.author: bty + */ +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync001, TestSize.Level0) +{ + /** + * @tc.steps: step1. Remote device set empty distributed schema + * @tc.expected: step1.ok + */ ASSERT_NO_FATAL_FAILURE(InitDelegate()); auto schema = GetSchema(); auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); - deviceB_->SetDistributedSchema(distributedSchema); EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + DistributedSchema emptySchema; + deviceB_->SetDistributedSchema(emptySchema); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); /** * @tc.steps: step2. Insert one data * @tc.expected: step2.ok */ auto tableSchema = GetTableSchema(); - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); /** * @tc.steps: step3. Sync to real device - * @tc.expected: step3.ok + * @tc.expected: step3.SCHEMA_MISMATCH */ - std::string checkSql = "select count(*) from sqlite_master where type='table' and name != '" + - DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + "' and name like 'naturalbase_rdb_aux_" + - std::string(DEVICE_SYNC_TABLE) + "_%'"; - int count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkSql, count), E_OK); - EXPECT_EQ(count, 0); Query query = Query::Select(tableSchema.name); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, SCHEMA_MISMATCH, + {deviceB_->GetDeviceId()}); /** - * @tc.steps: step4. Check if the distributed table exists - * @tc.expected: step4.ok + * @tc.steps: step4. Remove device data + * @tc.expected: step4. NOT_SUPPORT */ - count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, checkSql, count), E_OK); - EXPECT_EQ(count, 0); + EXPECT_EQ(delegate_->RemoveDeviceData("dev", DEVICE_SYNC_TABLE), NOT_SUPPORT); + EXPECT_EQ(delegate_->RemoveDeviceData(), NOT_SUPPORT); + EXPECT_EQ(delegate_->RemoveDeviceData("dev", ClearMode::DEFAULT), NOT_SUPPORT); } /** - * @tc.name: NormalSync010 - * @tc.desc: Test sync without not null col. + * @tc.name: InvalidSync002 + * @tc.desc: Test remote set distributed schema but table is not exist then sync. * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: tankaisheng */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync010, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync002, TestSize.Level0) { /** - * @tc.steps: step1. Recreate not null table + * @tc.steps: step1. Remote device set distributed schema but table is not exist * @tc.expected: step1.ok */ - std::string sql = std::string("DROP TABLE ").append(DEVICE_SYNC_TABLE); - ASSERT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); - auto tableSchema = GetTableSchema(); - ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, *db_), E_OK); ASSERT_NO_FATAL_FAILURE(InitDelegate()); auto schema = GetSchema(); auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); - deviceB_->SetDistributedSchema(distributedSchema); EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), SCHEMA_MISMATCH); + LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + DistributedSchema distributedSchema1; + deviceB_->SetDistributedSchema(distributedSchema1); /** * @tc.steps: step2. Insert one data * @tc.expected: step2.ok */ - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + auto tableSchema = GetTableSchema(); + EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); /** - * @tc.steps: step3. Sync without str col - * @tc.expected: step3.ok - */ - Query query = Query::Select(tableSchema.name); - DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PULL_ONLY, query, nullptr, true); - EXPECT_EQ(callStatus, SCHEMA_MISMATCH); - /** - * @tc.steps: step4. Check if the distributed table exists - * @tc.expected: step4.ok + * @tc.steps: step3. Sync to real device + * @tc.expected: step3.SCHEMA_MISMATCH */ - int count = 0; - sql = std::string("select count(*) from ") - .append(RelationalStoreManager::GetDistributedLogTableName(DEVICE_SYNC_TABLE)); - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 0); + Query query = Query::Select(DEVICE_SYNC_TABLE); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, SCHEMA_MISMATCH, + {deviceB_->GetDeviceId()}); } /** - * @tc.name: NormalSync011 - * @tc.desc: Test set the distributed schema first then sync. + * @tc.name: InvalidSync003 + * @tc.desc: Test sync of deletion when the deleted data has log locally but not found in the actual data table * @tc.type: FUNC * @tc.require: - * @tc.author: bty + * @tc.author: liuhongyang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync011, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync003, TestSize.Level0) { /** * @tc.steps: step1. Create device table and cloud table in COLLABORATION @@ -1104,122 +2587,165 @@ HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync011, TestSize.Level0) auto schema = GetSchema(); auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); deviceB_->SetDistributedSchema(distributedSchema); - EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", DEVICE_SYNC_TABLE); /** - * @tc.steps: step2. Insert one data + * @tc.steps: step2. Insert 10 data * @tc.expected: step2.ok */ auto tableSchema = GetTableSchema(); - EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 10, deviceB_, tableSchema), E_OK); ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); /** - * @tc.steps: step3. Sync to real device + * @tc.steps: step3. Sync to real device to make data consistent * @tc.expected: step3.ok */ Query query = Query::Select(tableSchema.name); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PUSH_ONLY, OK, {deviceB_->GetDeviceId()}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step4. Stop writing into log table and remove the first record locally + * @tc.expected: step4. ok and the target scenario is created + */ + std::string sql = "INSERT OR REPLACE INTO " + std::string(DBConstant::RELATIONAL_PREFIX) + "metadata" + + " VALUES ('log_trigger_switch', 'false');"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + sql = "DELETE FROM " + std::string(DEVICE_SYNC_TABLE) + " WHERE pk = 0;"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + sql = "INSERT OR REPLACE INTO " + std::string(DBConstant::RELATIONAL_PREFIX) + "metadata" + + " VALUES ('log_trigger_switch', 'true');"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db_, sql), E_OK); + /** + * @tc.steps: step5. Delete the first data in virtual device and sync to deviceB_ + * @tc.expected: step5. ok + */ + delegateObserver_->ClearChangedData(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema, true), E_OK); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); + /** + * @tc.steps: step6. check changeData + * @tc.expected: step6. OP_DELETE has one record but the record is empty + */ + auto changeData = delegateObserver_->GetSavedChangedData(); + ASSERT_EQ(changeData[tableSchema.name].primaryData[OP_DELETE].size(), 1u); + EXPECT_EQ(changeData[tableSchema.name].primaryData[OP_DELETE][0].size(), 0u); + ASSERT_EQ(changeData[tableSchema.name].field.size(), 1u); + EXPECT_EQ(changeData[tableSchema.name].field[0], "pk"); } /** - * @tc.name: NormalSync012 - * @tc.desc: Test sync with autoincrement table. + * @tc.name: InvalidSync004 + * @tc.desc: Test sync with empty tables * @tc.type: FUNC * @tc.require: - * @tc.author: zqq + * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, NormalSync012, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync004, TestSize.Level0) +{ + ASSERT_NO_FATAL_FAILURE(InitDelegate()); + Query query = Query::Select().FromTable({}); + DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, query, nullptr, true); + EXPECT_EQ(callStatus, INVALID_ARGS); +} + +/** + * @tc.name: InvalidSync005 + * @tc.desc: Test error returned by the other end during sync + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync005, TestSize.Level0) { /** - * @tc.steps: step1. Create auto increment table and unique index + * @tc.steps: step1. Create device table and cloud table in COLLABORATION * @tc.expected: step1.ok */ - auto tableSchema = GetTableSchema(); - tableSchema.name = DEVICE_SYNC_TABLE_AUTOINCREMENT; - ASSERT_EQ(RDBDataGenerator::InitTable(tableSchema, true, true, *db_), E_OK); ASSERT_NO_FATAL_FAILURE(InitDelegate()); - tableSchema = GetTableSchema(false, true); - tableSchema.name = DEVICE_SYNC_TABLE_AUTOINCREMENT; auto schema = GetSchema(); - schema.tables.push_back(tableSchema); auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); deviceB_->SetDistributedSchema(distributedSchema); - int errCode = SQLiteUtils::ExecuteRawSQL(db_, std::string("CREATE UNIQUE INDEX U_INDEX ON ") - .append(tableSchema.name).append("('123')")); - ASSERT_EQ(errCode, E_OK); - EXPECT_EQ(delegate_->CreateDistributedTable(tableSchema.name, TableSyncType::DEVICE_COOPERATION), OK); EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); /** - * @tc.steps: step2. Insert one data + * @tc.steps: step2. Prepare device B * @tc.expected: step2.ok */ - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); + int dataCount = 10; + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); /** - * @tc.steps: step3. Sync to real device + * @tc.steps: step3. Set return E_DISTRIBUTED_SCHEMA_NOT_FOUND when get sync data in device B * @tc.expected: step3.ok */ - Query query = Query::Select(tableSchema.name); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); - std::string sql = std::string("select count(*) from ").append(tableSchema.name); - int count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); + deviceB_->SetGetSyncDataResult(-E_DISTRIBUTED_SCHEMA_NOT_FOUND); /** - * @tc.steps: step4. Update date and sync again - * @tc.expected: step4.ok + * @tc.steps: step4. Sync + * @tc.expected: step4. return SCHEMA_MISMATCH */ - ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, 1, deviceB_, tableSchema), E_OK); - DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, OK, {deviceB_->GetDeviceId()}); - sql = std::string("select count(*) from ").append(tableSchema.name); - count = 0; - EXPECT_EQ(SQLiteUtils::GetCountBySql(db_, sql, count), E_OK); - EXPECT_EQ(count, 1); + Query query = Query::Select().FromTable({DEVICE_SYNC_TABLE}); + DistributedDBToolsUnitTest::BlockSync(*delegate_, query, SYNC_MODE_PULL_ONLY, SCHEMA_MISMATCH, + {deviceB_->GetDeviceId()}); } /** - * @tc.name: SetSchema013 - * @tc.desc: Test set tracker table for device table and check if timestamp has changed + * @tc.name: InvalidSync006 + * @tc.desc: Test FromTable and other predicates are combined when sync * @tc.type: FUNC * @tc.require: - * @tc.author: bty + * @tc.author: liaoyonghuang */ -HWTEST_F(DistributedDBRDBCollaborationTest, SetSchema013, TestSize.Level0) +HWTEST_F(DistributedDBRDBCollaborationTest, InvalidSync006, TestSize.Level0) { /** - * @tc.steps: step1. Create device table + * @tc.steps: step1. Create device table and cloud table in COLLABORATION * @tc.expected: step1.ok */ ASSERT_NO_FATAL_FAILURE(InitDelegate()); - DistributedSchema distributedSchema = GetDistributedSchema(DEVICE_SYNC_TABLE, {"pk", "int_field1"}); - EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); + auto schema = GetSchema(); + auto distributedSchema = RDBDataGenerator::ParseSchema(schema, true); + deviceB_->SetDistributedSchema(distributedSchema); EXPECT_EQ(delegate_->SetDistributedSchema(distributedSchema), OK); - TrackerSchema trackerSchema = { - .tableName = DEVICE_SYNC_TABLE, .extendColNames = {"int_field1"}, .trackerColNames = {"int_field1"} - }; + EXPECT_EQ(delegate_->CreateDistributedTable(DEVICE_SYNC_TABLE, TableSyncType::DEVICE_COOPERATION), OK); /** - * @tc.steps: step2. Insert one data and query timestamp + * @tc.steps: step2. Prepare device B * @tc.expected: step2.ok */ - EXPECT_EQ(RDBDataGenerator::InsertLocalDBData(0, 1, db_, GetTableSchema()), E_OK); - sqlite3_stmt *stmt = nullptr; - std::string sql = "select timestamp from " + DBCommon::GetLogTableName(DEVICE_SYNC_TABLE) + - " where data_key=0"; - EXPECT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); - EXPECT_EQ(SQLiteUtils::StepWithRetry(stmt), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); - int64_t timestamp1 = static_cast(sqlite3_column_int64(stmt, 0)); - int ret = E_OK; - SQLiteUtils::ResetStatement(stmt, true, ret); + int dataCount = 10; + auto tableSchema = GetTableSchema(); + ASSERT_EQ(RDBDataGenerator::InsertVirtualLocalDBData(0, dataCount, deviceB_, tableSchema), E_OK); + ASSERT_EQ(RDBDataGenerator::PrepareVirtualDeviceEnv(tableSchema.name, db_, deviceB_), E_OK); /** - * @tc.steps: step3. Set tracker table and query timestamp - * @tc.expected: step3.Equal + * @tc.steps: step3. Prepare query list + * @tc.expected: step3.ok */ - EXPECT_EQ(delegate_->SetTrackerTable(trackerSchema), WITH_INVENTORY_DATA); - EXPECT_EQ(SQLiteUtils::GetStatement(db_, sql, stmt), E_OK); - EXPECT_EQ(SQLiteUtils::StepWithRetry(stmt), SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)); - int64_t timestamp2 = static_cast(sqlite3_column_int64(stmt, 0)); - SQLiteUtils::ResetStatement(stmt, true, ret); - EXPECT_EQ(timestamp1, timestamp2); + std::set keys = {{1}, {2}, {3}}; + std::vector values = {1, 2, 3}; + std::vector queryList = { + Query::Select().FromTable({DEVICE_SYNC_TABLE}).BeginGroup().EqualTo("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).EqualTo("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).GreaterThan("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).GreaterThanOrEqualTo("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).In("pk", values), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).NotIn("pk", values), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).IsNotNull("pk"), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).LessThan("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).LessThanOrEqualTo("pk", 1), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).Like("pk", "abc"), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).NotLike("pk", "abc"), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).OrderByWriteTime(false), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).SuggestIndex("pk"), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).PrefixKey({1, 2, 3}), + Query::Select().FromTable({DEVICE_SYNC_TABLE}).InKeys(keys) + }; + /** + * @tc.steps: step4. Sync + * @tc.expected: step4. return NOT_SUPPORT + */ + for (const auto &query : queryList) { + DBStatus callStatus = delegate_->Sync({deviceB_->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, query, nullptr, true); + EXPECT_EQ(callStatus, NOT_SUPPORT); + } } } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_cloud_syncable_storage_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_cloud_syncable_storage_test.cpp index 70e66147..237415df 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_cloud_syncable_storage_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_cloud_syncable_storage_test.cpp @@ -731,7 +731,7 @@ HWTEST_F(DistributedDBRelationalCloudSyncableStorageTest, GetUploadCount004, Tes */ std::thread getUploadCountThread([&]() { for (int i = 0; i < 100; i++) { - if (g_storageProxy->StartTransaction() == E_OK) { + if(g_storageProxy->StartTransaction() == E_OK) { EXPECT_EQ(g_storageProxy->GetUploadCount(g_tableName, g_startTime, false, resCount), E_OK); EXPECT_EQ(resCount, insCount); EXPECT_EQ(g_storageProxy->Commit(), E_OK); @@ -1626,8 +1626,7 @@ HWTEST_F(DistributedDBRelationalCloudSyncableStorageTest, getAsset002, TestSize. gid = "11"; pk = "1"; hashKey.assign(pk.begin(), pk.end()); - EXPECT_EQ(g_storageProxy->GetAssetsByGidOrHashKey(g_tableName, false, gid, hashKey, assets).first, - -E_CLOUD_GID_MISMATCH); + EXPECT_EQ(g_storageProxy->GetAssetsByGidOrHashKey(g_tableName, false, gid, hashKey, assets).first, -E_CLOUD_GID_MISMATCH); CheckGetAsset(assets, static_cast(AssetStatus::NORMAL)); /** @@ -1675,6 +1674,7 @@ HWTEST_F(DistributedDBRelationalCloudSyncableStorageTest, GetCloudData007, TestS EXPECT_EQ(g_storageProxy->StartTransaction(TransactType::IMMEDIATE), E_OK); ASSERT_EQ(g_storageProxy->GetCloudData(g_tableName, g_startTime, token, cloudSyncData), E_OK); EXPECT_EQ(g_storageProxy->Rollback(), E_OK); + ASSERT_EQ(g_storageProxy->ReleaseContinueToken(token), E_OK); } /** @@ -1687,12 +1687,16 @@ HWTEST_F(DistributedDBRelationalCloudSyncableStorageTest, GetCloudData007, TestS HWTEST_F(DistributedDBRelationalCloudSyncableStorageTest, ContainAssetsTable001, TestSize.Level0) { ASSERT_NE(g_storageProxy, nullptr); - EXPECT_FALSE(g_storageProxy->IsContainAssetsTable()); + EXPECT_FALSE(g_storageProxy->IsExistTableContainAssets()); ASSERT_NE(g_cloudStore, nullptr); DataBaseSchema dataBaseSchema; dataBaseSchema.tables.push_back(g_tableSchema); EXPECT_EQ(g_cloudStore->SetCloudDbSchema(dataBaseSchema), E_OK); - EXPECT_TRUE(g_storageProxy->IsContainAssetsTable()); + EXPECT_FALSE(g_storageProxy->IsExistTableContainAssets()); + int64_t insCount = 100; + int64_t photoSize = 10; + InitUserDataForAssetTest(insCount, photoSize, g_localAsset); + EXPECT_TRUE(g_storageProxy->IsExistTableContainAssets()); } } #endif // RELATIONAL_STORE diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index 3b5bc65f..942d22b9 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -527,6 +527,8 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData3, TestSize.Level1) * @tc.expected: 2 data in the from deviceA are deleted and all data from deviceB are not deleted. */ ExpectCount(db, "SELECT count(*) FROM " + std::string(DBConstant::RELATIONAL_PREFIX) + g_tableName + + "_log WHERE flag&0x01=0x01;", 0U); // g_tableName has no data, therefore no delete log + ExpectCount(db, "SELECT count(*) FROM " + std::string(DBConstant::RELATIONAL_PREFIX) + tableName + "_log WHERE flag&0x01=0x01;", 2U); // 2 deleted log ExpectCount(db, "SELECT count(*) FROM " + std::string(DBConstant::RELATIONAL_PREFIX) + g_tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceA)) + ";", 3U); // 3 records in A diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_basic_kv_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_basic_kv_test.cpp new file mode 100644 index 00000000..416d4ead --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_basic_kv_test.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_general_ut.h" + +namespace DistributedDB { +using namespace testing::ext; +class DistributedDBBasicKVTest : public KVGeneralUt { +public: + void SetUp() override; +}; + +void DistributedDBBasicKVTest::SetUp() +{ + KVGeneralUt::SetUp(); + auto storeInfo1 = GetStoreInfo1(); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo1, "dev1"), E_OK); + auto storeInfo2 = GetStoreInfo2(); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo2, "dev2"), E_OK); +} + +/** + * @tc.name: ExampleSync001 + * @tc.desc: Test sync from dev1 to dev2. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBBasicKVTest, ExampleSync001, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync to dev2 + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockPush(storeInfo1, storeInfo2); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); +} +} // namespace DistributedDB \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_kv_data_status_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_kv_data_status_test.cpp new file mode 100644 index 00000000..71a331d6 --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/kv/distributeddb_kv_data_status_test.cpp @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_general_ut.h" + +namespace DistributedDB { +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +class DistributedDBKVDataStatusTest : public KVGeneralUt { +public: + void SetUp() override; +protected: + static constexpr const char *DEVICE_A = "DEVICE_A"; + static constexpr const char *DEVICE_B = "DEVICE_B"; + static constexpr const char *DEVICE_C = "DEVICE_C"; + static const uint32_t INVALID_DATA_OPERATOR = 0; +}; + +void DistributedDBKVDataStatusTest::SetUp() +{ + KVGeneralUt::SetUp(); + auto storeInfo1 = GetStoreInfo1(); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo1, DEVICE_A), E_OK); + auto storeInfo2 = GetStoreInfo2(); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo2, DEVICE_B), E_OK); +} + +/** + * @tc.name: OperateDataStatus001 + * @tc.desc: Test sync from dev1 to dev2 after operate valid data status. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, OperateDataStatus001, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync to dev2 + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockPush(storeInfo1, storeInfo2); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 modify deviceId and operate valid data status + * @tc.expected: step2. OK. + */ + ASSERT_EQ(KVGeneralUt::CloseDelegate(storeInfo1), E_OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo1, DEVICE_C), E_OK); + store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + EXPECT_EQ(store1->OperateDataStatus(static_cast(DataOperator::UPDATE_TIME) | + static_cast(DataOperator::RESET_UPLOAD_CLOUD)), OK); + /** + * @tc.steps: step3. dev1 sync to dev2 + * @tc.expected: step3. sync should return OK and dev2 exist (k,v). + */ + BlockPush(storeInfo1, storeInfo2); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), OK); + EXPECT_EQ(entries.size(), 1u); // 1 record + EXPECT_EQ(entries[0].value, expectValue); + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); +} + +/** + * @tc.name: OperateDataStatus002 + * @tc.desc: Test sync from dev1 to dev2 after operate invalid data status. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, OperateDataStatus002, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync to dev2 + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockPush(storeInfo1, storeInfo2); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 modify deviceId and operate invalid data status + * @tc.expected: step2. OK. + */ + ASSERT_EQ(KVGeneralUt::CloseDelegate(storeInfo1), E_OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo1, DEVICE_C), E_OK); + store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + EXPECT_EQ(store1->OperateDataStatus(INVALID_DATA_OPERATOR), OK); + /** + * @tc.steps: step3. dev1 sync to dev2 + * @tc.expected: step3. sync should return OK and dev2 get entries by device return NOT_FOUND. + */ + BlockPush(storeInfo1, storeInfo2); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), NOT_FOUND); + EXPECT_EQ(entries.size(), 0u); // 0 record +} + +/** + * @tc.name: OperateDataStatus003 + * @tc.desc: Test sync from dev1 to dev2 after operate data status RESET_UPLOAD_CLOUD. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, OperateDataStatus003, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync to dev2 + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockPush(storeInfo1, storeInfo2); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 modify deviceId and operate data status RESET_UPLOAD_CLOUD + * @tc.expected: step2. OK. + */ + ASSERT_EQ(KVGeneralUt::CloseDelegate(storeInfo1), E_OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(storeInfo1, DEVICE_C), E_OK); + store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + EXPECT_EQ(store1->OperateDataStatus(static_cast(DataOperator::RESET_UPLOAD_CLOUD)), OK); + /** + * @tc.steps: step3. dev1 sync to dev2 + * @tc.expected: step3. sync should return OK and dev2 get entries by device return NOT_FOUND. + */ + BlockPush(storeInfo1, storeInfo2); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), NOT_FOUND); + EXPECT_EQ(entries.size(), 0u); // 0 record +} + +#ifdef USE_DISTRIBUTEDDB_CLOUD +/** + * @tc.name: CloudOperateDataStatus001 + * @tc.desc: Test sync cloud after operate valid data status. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, CloudOperateDataStatus001, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync cloud and dev2 sync cloud + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + ASSERT_EQ(SetCloud(store1), OK); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + ASSERT_EQ(SetCloud(store2), OK); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockCloudSync(storeInfo1, DEVICE_A); + BlockCloudSync(storeInfo2, DEVICE_B); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 operate valid data status + * @tc.expected: step2. OK. + */ + EXPECT_EQ(store1->OperateDataStatus(static_cast(DataOperator::UPDATE_TIME) | + static_cast(DataOperator::RESET_UPLOAD_CLOUD)), OK); + /** + * @tc.steps: step3. dev1 modify deviceId and sync cloud and dev2 sync cloud + * @tc.expected: step3. sync should return OK and dev2 exist (k,v). + */ + BlockCloudSync(storeInfo1, DEVICE_C); + BlockCloudSync(storeInfo2, DEVICE_B); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), OK); + EXPECT_EQ(entries.size(), 1u); // 1 record + EXPECT_EQ(entries[0].value, expectValue); + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); +} + +/** + * @tc.name: CloudOperateDataStatus002 + * @tc.desc: Test sync cloud after operate invalid data status. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, CloudOperateDataStatus002, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync cloud and dev2 sync cloud + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + ASSERT_EQ(SetCloud(store1), OK); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + ASSERT_EQ(SetCloud(store2), OK); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockCloudSync(storeInfo1, DEVICE_A); + BlockCloudSync(storeInfo2, DEVICE_B); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 operate invalid data status + * @tc.expected: step2. OK. + */ + EXPECT_EQ(store1->OperateDataStatus(INVALID_DATA_OPERATOR), OK); + /** + * @tc.steps: step3. dev1 modify deviceId and sync cloud and dev2 sync cloud + * @tc.expected: step3. sync should return OK and dev2 get entries by device return NOT_FOUND. + */ + BlockCloudSync(storeInfo1, DEVICE_C); + BlockCloudSync(storeInfo2, DEVICE_B); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), NOT_FOUND); + EXPECT_EQ(entries.size(), 0u); // 0 record +} + +/** + * @tc.name: CloudOperateDataStatus003 + * @tc.desc: Test sync cloud after operate data status RESET_UPLOAD_CLOUD. + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBKVDataStatusTest, CloudOperateDataStatus003, TestSize.Level0) +{ + /** + * @tc.steps: step1. dev1 put (k,v) and sync cloud and dev2 sync cloud + * @tc.expected: step1. sync should return OK and dev2 exist (k,v). + */ + auto storeInfo1 = GetStoreInfo1(); + auto storeInfo2 = GetStoreInfo2(); + auto store1 = GetDelegate(storeInfo1); + ASSERT_NE(store1, nullptr); + ASSERT_EQ(SetCloud(store1), OK); + auto store2 = GetDelegate(storeInfo2); + ASSERT_NE(store2, nullptr); + ASSERT_EQ(SetCloud(store2), OK); + Value expectValue = {'v'}; + EXPECT_EQ(store1->Put({'k'}, expectValue), OK); + BlockCloudSync(storeInfo1, DEVICE_A); + BlockCloudSync(storeInfo2, DEVICE_B); + Value actualValue; + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); + /** + * @tc.steps: step2. dev1 operate data status RESET_UPLOAD_CLOUD + * @tc.expected: step2. OK. + */ + EXPECT_EQ(store1->OperateDataStatus(static_cast(DataOperator::RESET_UPLOAD_CLOUD)), OK); + /** + * @tc.steps: step3. dev1 modify deviceId and sync cloud and dev2 sync cloud + * @tc.expected: step3. sync should return OK and dev2 exist (k,v). + */ + BlockCloudSync(storeInfo1, DEVICE_C); + BlockCloudSync(storeInfo2, DEVICE_B); + std::vector entries; + EXPECT_EQ(KVGeneralUt::GetDeviceEntries(store2, std::string(DEVICE_C), false, entries), OK); + EXPECT_EQ(entries.size(), 1u); // 1 record + EXPECT_EQ(entries[0].value, expectValue); + EXPECT_EQ(store2->Get({'k'}, actualValue), OK); + EXPECT_EQ(actualValue, expectValue); +} +#endif // USE_DISTRIBUTEDDB_CLOUD +} // namespace DistributedDB \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_basic_rdb_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_basic_rdb_test.cpp new file mode 100644 index 00000000..ba3a7cbe --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_basic_rdb_test.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "rdb_general_ut.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +namespace { +class DistributedDBBasicRDBTest : public RDBGeneralUt { +public: + void SetUp() override; + void TearDown() override; + +protected: + static constexpr const char *DEVICE_SYNC_TABLE = "DEVICE_SYNC_TABLE"; + static constexpr const char *CLOUD_SYNC_TABLE = "CLOUD_SYNC_TABLE"; +}; + +void DistributedDBBasicRDBTest::SetUp() +{ + RDBGeneralUt::SetUp(); +} + +void DistributedDBBasicRDBTest::TearDown() +{ + RDBGeneralUt::TearDown(); +} + +/** + * @tc.name: InitDelegateExample001 + * @tc.desc: Test InitDelegate interface of RDBGeneralUt. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBBasicRDBTest, InitDelegateExample001, TestSize.Level0) +{ + /** + * @tc.steps: step1. Call InitDelegate interface with default data. + * @tc.expected: step1. Ok + */ + StoreInfo info1 = {USER_ID, APP_ID, STORE_ID_1}; + EXPECT_EQ(RDBGeneralUt::InitDelegate(info1), E_OK); + DataBaseSchema actualSchemaInfo = RDBGeneralUt::GetSchema(info1); + ASSERT_EQ(actualSchemaInfo.tables.size(), 2u); + EXPECT_EQ(actualSchemaInfo.tables[0].name, g_defaultTable1); + EXPECT_EQ(RDBGeneralUt::CloseDelegate(info1), E_OK); + + /** + * @tc.steps: step2. Call twice InitDelegate interface with the set data. + * @tc.expected: step2. Ok + */ + const std::vector filedInfo = { + {{"id", TYPE_INDEX, true, false}, true}, {{"name", TYPE_INDEX, false, true}, false}, + }; + UtDateBaseSchemaInfo schemaInfo = { + .tablesInfo = {{.name = DEVICE_SYNC_TABLE, .fieldInfo = filedInfo}} + }; + RDBGeneralUt::AddSchemaInfo(info1, schemaInfo); + RelationalStoreDelegate::Option option; + option.tableMode = DistributedTableMode::COLLABORATION; + SetOption(option); + EXPECT_EQ(RDBGeneralUt::InitDelegate(info1), E_OK); + + StoreInfo info2 = {USER_ID, APP_ID, STORE_ID_2}; + schemaInfo = { + .tablesInfo = { + {.name = DEVICE_SYNC_TABLE, .fieldInfo = filedInfo}, + {.name = CLOUD_SYNC_TABLE, .fieldInfo = filedInfo}, + } + }; + RDBGeneralUt::AddSchemaInfo(info2, schemaInfo); + EXPECT_EQ(RDBGeneralUt::InitDelegate(info2), E_OK); + actualSchemaInfo = RDBGeneralUt::GetSchema(info2); + ASSERT_EQ(actualSchemaInfo.tables.size(), schemaInfo.tablesInfo.size()); + EXPECT_EQ(actualSchemaInfo.tables[1].name, CLOUD_SYNC_TABLE); + TableSchema actualTableInfo = RDBGeneralUt::GetTableSchema(info2, CLOUD_SYNC_TABLE); + EXPECT_EQ(actualTableInfo.fields.size(), filedInfo.size()); +} +} \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_data_status_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_data_status_test.cpp new file mode 100644 index 00000000..920ece0a --- /dev/null +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/store_test/rdb/distributeddb_rdb_data_status_test.cpp @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "rdb_general_ut.h" + +namespace DistributedDB { +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +class DistributedDBRDBDataStatusTest : public RDBGeneralUt { +public: + void SetUp() override; +protected: + void PrepareTableBasicEnv(bool createWithTracker = false); + void DataStatusComplexTest(bool testWithTracker); + static UtDateBaseSchemaInfo GetDefaultSchema(); + static constexpr const char *DEVICE_SYNC_TABLE = "DEVICE_SYNC_TABLE"; + static constexpr const char *DEVICE_A = "DEVICE_A"; + static constexpr const char *DEVICE_B = "DEVICE_B"; + static constexpr const char *DEVICE_C = "DEVICE_C"; + StoreInfo info1_ = {USER_ID, APP_ID, STORE_ID_1}; + StoreInfo info2_ = {USER_ID, APP_ID, STORE_ID_2}; +}; + +void DistributedDBRDBDataStatusTest::SetUp() +{ + RDBGeneralUt::SetUp(); + AddSchemaInfo(info1_, GetDefaultSchema()); + AddSchemaInfo(info2_, GetDefaultSchema()); +} + +UtDateBaseSchemaInfo DistributedDBRDBDataStatusTest::GetDefaultSchema() +{ + UtDateBaseSchemaInfo info; + UtTableSchemaInfo table; + table.name = DEVICE_SYNC_TABLE; + UtFieldInfo field; + field.field.colName = "id"; + field.field.type = TYPE_INDEX; + field.field.primary = true; + table.fieldInfo.push_back(field); + info.tablesInfo.push_back(table); + return info; +} + +void DistributedDBRDBDataStatusTest::PrepareTableBasicEnv(bool createWithTracker) +{ + /** + * @tc.steps: step1. Call InitDelegate interface with default split table mode. + * @tc.expected: step1. Ok + */ + ASSERT_EQ(BasicUnitTest::InitDelegate(info1_, DEVICE_A), E_OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(info2_, DEVICE_B), E_OK); + /** + * @tc.steps: step2. Set distributed tables. + * @tc.expected: step2. Ok + */ + ASSERT_EQ(SetDistributedTables(info1_, {DEVICE_SYNC_TABLE}), E_OK); + ASSERT_EQ(SetDistributedTables(info2_, {DEVICE_SYNC_TABLE}), E_OK); + if (createWithTracker) { + ASSERT_EQ(SetTrackerTables(info1_, {DEVICE_SYNC_TABLE}), E_OK); + ASSERT_EQ(SetTrackerTables(info2_, {DEVICE_SYNC_TABLE}), E_OK); + } + /** + * @tc.steps: step3. Insert local data. + * @tc.expected: step3. Ok + */ + InsertLocalDBData(0, 1, info1_); + /** + * @tc.steps: step4. DEV_A sync to DEV_B. + * @tc.expected: step4. Ok + */ + BlockPush(info1_, info2_, DEVICE_SYNC_TABLE); + /** + * @tc.steps: step5. DEV_A update time + * @tc.expected: step5. Ok + */ + auto store = GetDelegate(info1_); + ASSERT_NE(store, nullptr); + EXPECT_EQ(store->OperateDataStatus(static_cast(DataOperator::UPDATE_TIME)), OK); +} + +void DistributedDBRDBDataStatusTest::DataStatusComplexTest(bool testWithTracker) +{ + RelationalStoreDelegate::Option option; + option.tableMode = DistributedTableMode::COLLABORATION; + SetOption(option); + ASSERT_NO_FATAL_FAILURE(PrepareTableBasicEnv(testWithTracker)); + /** + * @tc.steps: step6. Reopen store with DEV_C. + * @tc.expected: step6. Ok + */ + ASSERT_EQ(RDBGeneralUt::CloseDelegate(info1_), OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(info1_, DEVICE_C), E_OK); + /** + * @tc.steps: step7. DEV_C sync to DEV_B. + * @tc.expected: step7. Ok + */ + BlockPush(info1_, info2_, DEVICE_SYNC_TABLE); + /** + * @tc.steps: step8. Check 1 record with device_c. + * @tc.expected: step8. Ok + */ + EXPECT_EQ(CountTableDataByDev(info2_, DBCommon::GetLogTableName(DEVICE_SYNC_TABLE), DEVICE_C), 1); +} + +/** + * @tc.name: SplitTable001 + * @tc.desc: Test split table sync after update time. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBDataStatusTest, SplitTable001, TestSize.Level0) +{ + ASSERT_NO_FATAL_FAILURE(PrepareTableBasicEnv()); + /** + * @tc.steps: step6. DEV_A sync to DEV_B. + * @tc.expected: step6. Ok + */ + BlockPush(info1_, info2_, DEVICE_SYNC_TABLE); +} + + +/** + * @tc.name: SplitTable002 + * @tc.desc: Test split table sync with diff dev after update time. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBDataStatusTest, SplitTable002, TestSize.Level0) +{ + ASSERT_NO_FATAL_FAILURE(PrepareTableBasicEnv()); + /** + * @tc.steps: step6. Reopen store with DEV_C. + * @tc.expected: step6. Ok + */ + ASSERT_EQ(RDBGeneralUt::CloseDelegate(info1_), OK); + ASSERT_EQ(BasicUnitTest::InitDelegate(info1_, DEVICE_C), E_OK); + /** + * @tc.steps: step7. DEV_C sync to DEV_B. + * @tc.expected: step7. Ok + */ + BlockPush(info1_, info2_, DEVICE_SYNC_TABLE); + /** + * @tc.steps: step8. Check 1 record in device_c table. + * @tc.expected: step8. Ok + */ + EXPECT_EQ(CountTableData(info2_, DBCommon::GetDistributedTableName(DEVICE_C, DEVICE_SYNC_TABLE)), 1); +} + +/** + * @tc.name: CollaborationTable001 + * @tc.desc: Test collaboration table sync after update time. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBDataStatusTest, CollaborationTable001, TestSize.Level0) +{ + RelationalStoreDelegate::Option option; + option.tableMode = DistributedTableMode::COLLABORATION; + SetOption(option); + ASSERT_NO_FATAL_FAILURE(PrepareTableBasicEnv()); + /** + * @tc.steps: step6. DEV_A sync to DEV_B. + * @tc.expected: step6. Ok + */ + BlockPush(info1_, info2_, DEVICE_SYNC_TABLE); +} + +/** + * @tc.name: CollaborationTable002 + * @tc.desc: Test collaboration table sync after update time. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBDataStatusTest, CollaborationTable002, TestSize.Level0) +{ + DataStatusComplexTest(false); +} + +/** + * @tc.name: CollaborationTable003 + * @tc.desc: Test collaboration search table sync after update time. + * @tc.type: FUNC + * @tc.require: + * @tc.author: zqq + */ +HWTEST_F(DistributedDBRDBDataStatusTest, CollaborationTable003, TestSize.Level0) +{ + DataStatusComplexTest(true); +} +} \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/cloud_db_sync_utils_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/cloud_db_sync_utils_test.cpp index 7b207171..479520ad 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/cloud_db_sync_utils_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/cloud_db_sync_utils_test.cpp @@ -261,12 +261,15 @@ namespace DistributedDB { void CloudDBSyncUtilsTest::CloseDb(RelationalStoreObserverUnitTest *&observer, std::shared_ptr &virtualCloudDb, RelationalStoreDelegate *&delegate) { - delete observer; - virtualCloudDb = nullptr; if (delegate != nullptr) { EXPECT_EQ(g_mgr.CloseStore(delegate), DBStatus::OK); delegate = nullptr; } + if (observer != nullptr) { + delete observer; + observer = nullptr; + } + virtualCloudDb = nullptr; } int CloudDBSyncUtilsTest::QueryCountCallback(void *data, int count, char **colValue, char **colName) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp index ff25540a..e5a6d4c4 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_async_download_assets_test.cpp @@ -35,8 +35,11 @@ using namespace std; namespace { string g_testDir; -const std::string QUERY_INCONSISTENT_SQL = +const std::string QUERY_INCONSISTENT_SQL = "select count(*) from naturalbase_rdb_aux_AsyncDownloadAssetsTest_log where flag&0x20!=0;"; +IRelationalStore *g_store = nullptr; +ICloudSyncStorageHook *g_cloudStoreHook = nullptr; +RelationalStoreManager g_mgr(APP_ID, USER_ID); typedef struct SkipAssetTestParam { DBStatus downloadRes; bool useBatch; @@ -59,6 +62,8 @@ protected: void InitStore(); void CloseDb(); void DoSkipAssetDownload(SkipAssetTestParamT param); + void UpdateLocalData(sqlite3 *&db, const std::string &tableName, int32_t begin, int32_t end); + void UpdateLocalAndCheckUploadCount(const bool &isAsync, const int &dataCount, const int &expectCount); std::string storePath_; sqlite3 *db_ = nullptr; RelationalStoreDelegate *delegate_ = nullptr; @@ -94,6 +99,7 @@ void DistributedDBCloudAsyncDownloadAssetsTest::SetUp() void DistributedDBCloudAsyncDownloadAssetsTest::TearDown() { + RefObject::DecObjRef(g_store); CloseDb(); if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { LOGE("rm test db files error."); @@ -169,6 +175,19 @@ int DistributedDBCloudAsyncDownloadAssetsTest::GetAssetFieldCount() return count; } +const RelationalSyncAbleStorage *GetRelationalStore() +{ + RelationalDBProperties properties; + CloudDBSyncUtilsTest::InitStoreProp(g_testDir + "/" + STORE_ID_1 + ".db", APP_ID, USER_ID, STORE_ID_1, properties); + int errCode = E_OK; + g_store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (g_store == nullptr) { + LOGE("Get db failed:%d", errCode); + return nullptr; + } + return static_cast(g_store)->GetStorageEngine(); +} + void DistributedDBCloudAsyncDownloadAssetsTest::InitStore() { if (storePath_.empty()) { @@ -178,13 +197,14 @@ void DistributedDBCloudAsyncDownloadAssetsTest::InitStore() ASSERT_NE(db_, nullptr); auto schema = GetSchema(true); EXPECT_EQ(RDBDataGenerator::InitDatabase(schema, *db_), SQLITE_OK); - RelationalStoreManager mgr(APP_ID, USER_ID); - ASSERT_EQ(mgr.OpenStore(storePath_, STORE_ID_1, {}, delegate_), OK); + ASSERT_EQ(g_mgr.OpenStore(storePath_, STORE_ID_1, {}, delegate_), OK); ASSERT_NE(delegate_, nullptr); for (const auto &table : schema.tables) { EXPECT_EQ(delegate_->CreateDistributedTable(table.name, TableSyncType::CLOUD_COOPERATION), OK); LOGI("[DistributedDBCloudAsyncDownloadAssetsTest] CreateDistributedTable %s", table.name.c_str()); } + g_cloudStoreHook = (ICloudSyncStorageHook *) GetRelationalStore(); + ASSERT_NE(g_cloudStoreHook, nullptr); virtualCloudDb_ = make_shared(); ASSERT_NE(virtualCloudDb_, nullptr); ASSERT_EQ(delegate_->SetCloudDB(virtualCloudDb_), DBStatus::OK); @@ -209,8 +229,8 @@ void DistributedDBCloudAsyncDownloadAssetsTest::CloseDb() db_ = nullptr; } if (delegate_ != nullptr) { - RelationalStoreManager mgr(APP_ID, USER_ID); - EXPECT_EQ(mgr.CloseStore(delegate_), OK); + EXPECT_EQ(g_mgr.CloseStore(delegate_), OK); + delegate_ = nullptr; } } @@ -350,19 +370,19 @@ HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, FinishListener001, TestSize. * @tc.steps: step1. Begin download first time * @tc.expected: step1.ok */ - AssetsDownloadManager manager; + auto manager = RuntimeContext::GetInstance()->GetAssetsDownloadManager(); std::atomic finished = false; auto finishAction = [&finished](void *) { EXPECT_TRUE(finished); }; - auto [errCode, listener] = manager.BeginDownloadWithListener(finishAction); + auto [errCode, listener] = manager->BeginDownloadWithListener(finishAction); ASSERT_EQ(errCode, E_OK); ASSERT_EQ(listener, nullptr); /** * @tc.steps: step2. Begin download twice * @tc.expected: step2. -E_MAX_LIMITS because default one task */ - std::tie(errCode, listener) = manager.BeginDownloadWithListener(finishAction); + std::tie(errCode, listener) = manager->BeginDownloadWithListener(finishAction); EXPECT_EQ(errCode, -E_MAX_LIMITS); EXPECT_NE(listener, nullptr); /** @@ -370,7 +390,7 @@ HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, FinishListener001, TestSize. * @tc.expected: step3. finished is true in listener */ finished = true; - manager.FinishDownload(); + manager->FinishDownload(); listener->Drop(true); } @@ -976,6 +996,167 @@ HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, AsyncNormalDownload004, Test virtualAssetLoader_->ForkDownload(nullptr); } +/** + * @tc.name: AsyncAbnormalDownload006 + * @tc.desc: Test abnormal async download. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, AsyncAbnormalDownload006, TestSize.Level4) +{ + /** + * @tc.steps: step1. Set config and insert 70 cloud data + * @tc.expected: step1. ok + */ + AsyncDownloadAssetsConfig config; + config.maxDownloadTask = 5; + EXPECT_EQ(RuntimeConfig::SetAsyncDownloadAssetsConfig(config), OK); + const int cloudCount = 70; + auto schema = GetSchema(); + EXPECT_EQ(RDBDataGenerator::InsertCloudDBData(0, cloudCount, 0, schema, virtualCloudDb_), OK); + + /** + * @tc.steps: step2. Fork download abnormal for 0-10 records + * @tc.expected: step2. ok + */ + virtualAssetLoader_->SetDownloadStatus(DB_ERROR); + uint32_t failNum = 10; + const DownloadFailRange setRange = {.isAllFail = false, .failBeginIndex = 0, .failEndIndex = failNum}; + virtualAssetLoader_->SetDownloadFailRange(setRange); + + /** + * @tc.steps: step3. Async cloud data + * @tc.expected: step3. ok + */ + int count = 0; + std::mutex countMutex; + std::condition_variable cv; + virtualAssetLoader_->ForkDownload([&count, &countMutex, &cv](const std::string &tableName, + std::map &) { + std::lock_guard autoLock(countMutex); + count++; + if (count == 1) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + cv.notify_all(); + }); + + CloudSyncOption option = GetAsyncCloudSyncOption(); + RelationalTestUtils::CloudBlockSync(option, delegate_); + std::unique_lock uniqueLock(countMutex); + cv.wait_for(uniqueLock, std::chrono::milliseconds(DBConstant::MIN_TIMEOUT), [&count]() { + return count >= cloudCount; + }); + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto [status, downloadCount] = delegate_->GetDownloadingAssetsCount(); + EXPECT_EQ(status, OK); + EXPECT_EQ(downloadCount, static_cast(failNum * 2)); // 1 record has 2 asset + + virtualAssetLoader_->SetDownloadStatus(OK); + virtualAssetLoader_->Reset(); + virtualAssetLoader_->ForkDownload(nullptr); +} + +void DistributedDBCloudAsyncDownloadAssetsTest::UpdateLocalData( + sqlite3 *&db, const std::string &tableName, int32_t begin, int32_t end) +{ + const string sql = "update " + tableName + " set int_field = int_field+1 " + "where pk>=" + std::to_string(begin) + + " and pk<=" + std::to_string(end) + ";"; + EXPECT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + LOGW("update local data finished"); +} + +void DistributedDBCloudAsyncDownloadAssetsTest::UpdateLocalAndCheckUploadCount(const bool &isAsync, + const int &dataCount, const int &expectCount) +{ + /** + * @tc.steps: step1. Set async config + * @tc.expected: step1. ok + */ + AsyncDownloadAssetsConfig config; + config.maxDownloadTask = 12; + config.maxDownloadAssetsCount = 100; + EXPECT_EQ(RuntimeConfig::SetAsyncDownloadAssetsConfig(config), OK); + /** + * @tc.steps: step2. Init data, set download status false and sync + * @tc.expected: step2. async download will return OK and sync download will return CLOUD_ERROR + */ + const int cloudCount = dataCount; + auto schema = GetSchema(); + ASSERT_TRUE(!schema.tables.empty()); + EXPECT_EQ(RDBDataGenerator::InsertCloudDBData(0, cloudCount, 0, schema, virtualCloudDb_), OK); + virtualAssetLoader_->SetDownloadStatus(DB_ERROR); + CloudSyncOption option = GetAsyncCloudSyncOption(); + option.asyncDownloadAssets = isAsync; + DBStatus expectStatus = isAsync ? OK : CLOUD_ERROR; + RelationalTestUtils::CloudBlockSync(option, delegate_, OK, expectStatus); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step3. Modify all local data + * @tc.expected: step3. OK + */ + UpdateLocalData(db_, "AsyncDownloadAssetsTest", 0, cloudCount); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step4. Fork upload to count number of uploaded records + * @tc.expected: step4. OK + */ + int count = 0; + std::mutex countMutex; + std::condition_variable cv; + virtualCloudDb_->ForkUpload([&count, &countMutex, &cv](const std::string&, VBucket&) { + std::lock_guard autoLock(countMutex); + count++; + cv.notify_all(); + }); + virtualAssetLoader_->SetDownloadStatus(OK); + virtualAssetLoader_->Reset(); + /** + * @tc.steps: step5. Sync again and check upload count + * @tc.expected: step5. OK + */ + RelationalTestUtils::CloudBlockSync(option, delegate_); + std::unique_lock uniqueLock(countMutex); + cv.wait_for(uniqueLock, std::chrono::milliseconds(DBConstant::MIN_TIMEOUT), [&count, &cloudCount, &isAsync]() { + return !isAsync || count >= cloudCount; + }); + EXPECT_EQ(count, expectCount); + /** + * @tc.steps: step6. Release resources + * @tc.expected: step6. OK + */ + virtualAssetLoader_->Reset(); + virtualAssetLoader_->ForkDownload(nullptr); + virtualCloudDb_->ForkUpload(nullptr); +} + +/** + * @tc.name: AsyncAbnormalDownload007 + * @tc.desc: Test in async download mode and asset is not downloaded, the local data can be uploaded + * @tc.type: FUNC + * @tc.require: + * @tc.author: tankaisheng + */ +HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, AsyncAbnormalDownload007, TestSize.Level1) +{ + int cloudCount = 10; + UpdateLocalAndCheckUploadCount(true, cloudCount, cloudCount); +} + +/** + * @tc.name: AsyncAbnormalDownload009 + * @tc.desc: Test in sync download mode and asset is not downloaded, the local data will be ignored when upload + * @tc.type: FUNC + * @tc.require: + * @tc.author: liuhongyang + */ +HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, AsyncAbnormalDownload009, TestSize.Level1) +{ + int cloudCount = 10; + UpdateLocalAndCheckUploadCount(false, cloudCount, 0); +} + DBStatus ModifySkippedAsset(int rowIndex, std::map &assets, DBStatus fakeStatus) { if (rowIndex != 1) { @@ -1112,4 +1293,59 @@ HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, SkipAssetDownloadTest003, Te .startIndex = 0, .expectInconsistentCount = 5, .expectSyncRes = OK}; DoSkipAssetDownload(param); } + +/** + * @tc.name: AsyncAbnormalDownload008 + * @tc.desc: Test the total count of download after download failure + * @tc.type: FUNC + * @tc.require: + * @tc.author: bty + */ +HWTEST_F(DistributedDBCloudAsyncDownloadAssetsTest, AsyncAbnormalDownload008, TestSize.Level1) +{ + /** + * @tc.steps: step1. Set max download task 1 + * @tc.expected: step1. ok + */ + AsyncDownloadAssetsConfig config; + config.maxDownloadTask = 1; + config.maxDownloadAssetsCount = 2; + EXPECT_EQ(RuntimeConfig::SetAsyncDownloadAssetsConfig(config), OK); + /** + * @tc.steps: step2. Insert cloud data + * @tc.expected: step2. ok + */ + const int cloudCount = 4; + auto schema = GetSchema(); + EXPECT_EQ(RDBDataGenerator::InsertCloudDBData(0, cloudCount, 0, schema, virtualCloudDb_), OK); + /** + * @tc.steps: step3. Fork download return cloud error + * @tc.expected: step3. CLOUD_ERROR + */ + int downloadIndex = 0; + int failedCount = 2; + virtualAssetLoader_->ForkBatchDownload([&downloadIndex, failedCount]( + int rowIndex, std::map &assets) { + downloadIndex++; + if (downloadIndex <= failedCount) { + return CLOUD_ERROR; + } + return OK; + }); + /** + * @tc.steps: step4. Sync + * @tc.expected: step4. ok + */ + CloudSyncOption option = GetAsyncCloudSyncOption(); + RelationalTestUtils::CloudBlockSync(option, delegate_); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step5. Check download count + * @tc.expected: step5. ok + */ + auto [status, downloadCount] = delegate_->GetDownloadingAssetsCount(); + EXPECT_EQ(status, OK); + EXPECT_EQ(downloadCount, cloudCount); + virtualAssetLoader_->ForkBatchDownload(nullptr); +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kv_syncer_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kv_syncer_test.cpp index 9fc1014e..514201fd 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kv_syncer_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kv_syncer_test.cpp @@ -42,6 +42,7 @@ CloudSyncOption g_CloudSyncoption; const std::string USER_ID_2 = "user2"; const std::string USER_ID_3 = "user3"; const int64_t WAIT_TIME = 5; +DistributedDBToolsUnitTest g_tool; class DistributedDBCloudKvSyncerTest : public testing::Test { public: static void SetUpTestCase(); @@ -780,6 +781,53 @@ HWTEST_F(DistributedDBCloudKvSyncerTest, SyncWithMultipleUsers003, TestSize.Leve EXPECT_EQ(actualValue2, value2); } +/** + * @tc.name: SyncWithMultipleUsers004. + * @tc.desc: Test sync data with same users. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangxiangdong + */ +HWTEST_F(DistributedDBCloudKvSyncerTest, SyncWithMultipleUsers004, TestSize.Level0) +{ + /** + * @tc.steps: step1. put k v by user1 and sync to cloud. + * @tc.expected: step1. return ok. + */ + Key key = {'k', '1'}; + Value value = {'v', '1'}; + ASSERT_EQ(kvDelegatePtrS1_->Put(key, value), OK); + CloudSyncOption syncOption; + syncOption.mode = SyncMode::SYNC_MODE_CLOUD_MERGE; + syncOption.users.push_back(USER_ID); + syncOption.devices.push_back("cloud"); + BlockSync(kvDelegatePtrS1_, OK, syncOption); + /** + * @tc.steps: step2. data p2p to device B. + * @tc.expected: step2. return ok. + */ + std::vector devices; + devices.push_back(deviceB_->GetDeviceId()); + Query query = Query::Select().PrefixKey(key); + std::map result; + DBStatus status = g_tool.SyncTest(kvDelegatePtrS2_, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == OK); + /** + * @tc.steps: step3. sync by user1. + * @tc.expected: step3. return ok. + */ + syncOption.users.clear(); + syncOption.users.push_back(USER_ID); + BlockSync(kvDelegatePtrS2_, OK, syncOption); + /** + * @tc.steps: step4. get k1. + * @tc.expected: step4. get v2. + */ + Value actualValue; + EXPECT_EQ(kvDelegatePtrS2_->Get(key, actualValue), OK); + EXPECT_EQ(actualValue, value); +} + /** * @tc.name: AbnormalCloudKvExecutorTest001 * @tc.desc: Check SqliteCloudKvExecutorUtils interfaces abnormal scene. @@ -1013,4 +1061,54 @@ HWTEST_F(DistributedDBCloudKvSyncerTest, DeviceCollaborationTest003, TestSize.Le removeThread1.join(); removeThread2.join(); } + +/** + * @tc.name: SyncWithKvAndCloud001. + * @tc.desc: Test sync data with same key and different sync way. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangxiangdong + */ +HWTEST_F(DistributedDBCloudKvSyncerTest, SyncWithKvAndCloud001, TestSize.Level0) +{ + /** + * @tc.steps: step1. put k1 v1 by user1 and sync between devices. + * @tc.expected: step1. return ok. + */ + Key key = {'k', '1'}; + Value value = {'v', '1'}; + ASSERT_EQ(kvDelegatePtrS1_->Put(key, value), OK); + std::vector devices; + devices.push_back(deviceB_->GetDeviceId()); + Query query = Query::Select().PrefixKey(key); + std::map result; + DBStatus status = g_tool.SyncTest(kvDelegatePtrS2_, devices, SYNC_MODE_PUSH_PULL, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step2. insert k1 v2 and sync data to cloud. + * @tc.expected: step2. return ok. + */ + Value value2 = {'v', '2'}; + ASSERT_EQ(kvDelegatePtrS1_->Put(key, value2), OK); + CloudSyncOption syncOption; + syncOption.mode = SyncMode::SYNC_MODE_CLOUD_MERGE; + syncOption.users.push_back(USER_ID); + syncOption.devices.push_back("cloud"); + BlockSync(kvDelegatePtrS1_, OK, syncOption); + /** + * @tc.steps: step3. sync by device2. + * @tc.expected: step3. return ok. + */ + syncOption.users.clear(); + syncOption.users.push_back(USER_ID); + BlockSync(kvDelegatePtrS2_, OK, syncOption); + /** + * @tc.steps: step4. get k1. + * @tc.expected: step4. get v2. + */ + Value actualValue; + EXPECT_EQ(kvDelegatePtrS2_->Get(key, actualValue), OK); + EXPECT_EQ(actualValue, value2); +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kvstore_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kvstore_test.cpp index af3f7bbc..95797c58 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kvstore_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_kvstore_test.cpp @@ -1820,4 +1820,148 @@ HWTEST_F(DistributedDBCloudKvStoreTest, ConflictSync001, TestSize.Level0) } virtualCloudDb_->ForkInsertConflict(nullptr); } + +/** + * @tc.name: ObserverDataChangeTest001 + * @tc.desc: test RegisterObserver interface with OBSERVER_CHANGES_CLOUD mode. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudKvStoreTest, ObserverDataChangeTest001, TestSize.Level0) +{ + /** + * @tc.steps: step1. delegate1 insert two data and sync with cloud + * @tc.expected: step1. insert and sync ok + */ + Key k1 = {'k', '1'}; + Value v1 = {'v', '1'}; + Key k2 = {'k', '2'}; + Value v2 = {'v', '2'}; + ASSERT_EQ(kvDelegatePtrS1_->Put(k1, v1), OK); + ASSERT_EQ(kvDelegatePtrS1_->Put(k2, v2), OK); + Value actualValue; + EXPECT_EQ(kvDelegatePtrS1_->Get(k1, actualValue), OK); + EXPECT_EQ(actualValue, v1); + EXPECT_EQ(kvDelegatePtrS1_->Get(k2, actualValue), OK); + EXPECT_EQ(actualValue, v2); + BlockSync(kvDelegatePtrS1_, OK, g_CloudSyncoption); + + /** + * @tc.steps: step2. delegate2 call RegisterObserver with OBSERVER_CHANGES_CLOUD mode + * @tc.expected: step2. sync with cloud and check data change success + */ + auto *observer = new (std::nothrow) KvStoreObserverUnitTest; + EXPECT_EQ(kvDelegatePtrS2_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD, observer), OK); + BlockSync(kvDelegatePtrS2_, OK, g_CloudSyncoption); + EXPECT_EQ(kvDelegatePtrS2_->Get(k1, actualValue), OK); + EXPECT_EQ(actualValue, v1); + EXPECT_EQ(kvDelegatePtrS2_->Get(k2, actualValue), OK); + EXPECT_EQ(actualValue, v2); + EXPECT_EQ(static_cast(observer->GetCallCount()), 1); + auto changeData = observer->GetChangedData(); + ASSERT_EQ(changeData.size(), 1u); + EXPECT_EQ(changeData[CloudDbConstant::CLOUD_KV_TABLE_NAME].primaryData[OP_INSERT].size(), 2u); + EXPECT_EQ(changeData[CloudDbConstant::CLOUD_KV_TABLE_NAME].primaryData[OP_UPDATE].size(), 0u); + + observer->ResetToZero(); + EXPECT_EQ(kvDelegatePtrS2_->UnRegisterObserver(observer), OK); + delete observer; + observer = nullptr; +} + +/** + * @tc.name: ObserverDataChangeTest002 + * @tc.desc: test RegisterObserver interface with OBSERVER_CHANGES_CLOUD | OBSERVER_CHANGES_BRIEF mode. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudKvStoreTest, ObserverDataChangeTest002, TestSize.Level0) +{ + /** + * @tc.steps: step1. insert data and delegate1 sync with cloud + * @tc.expected: step1. insert and sync ok + */ + Key k1 = {'k', '1'}; + Value v1 = {'v', '1'}; + ASSERT_EQ(kvDelegatePtrS2_->Put(k1, v1), OK); + Value actualValue; + EXPECT_EQ(kvDelegatePtrS2_->Get(k1, actualValue), OK) ; + EXPECT_EQ(actualValue, v1); + + Key k2 = {'k', '2'}; + Value v2 = {'v', '2'}; + ASSERT_EQ(kvDelegatePtrS1_->Put(k1, v2), OK); + ASSERT_EQ(kvDelegatePtrS1_->Put(k2, v2), OK); + BlockSync(kvDelegatePtrS1_, OK, g_CloudSyncoption); + + /** + * @tc.steps: step2. delegate2 call RegisterObserver with OBSERVER_CHANGES_CLOUD | OBSERVER_CHANGES_BRIEF mode + * @tc.expected: step2. sync with cloud and check data change success + */ + auto *observer = new (std::nothrow) KvStoreObserverUnitTest; + EXPECT_EQ(kvDelegatePtrS2_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD | OBSERVER_CHANGES_BRIEF, observer), OK); + BlockSync(kvDelegatePtrS2_, OK, g_CloudSyncoption); + EXPECT_EQ(kvDelegatePtrS2_->Get(k1, actualValue), OK); + EXPECT_EQ(actualValue, v2); + EXPECT_EQ(kvDelegatePtrS2_->Get(k2, actualValue), OK); + EXPECT_EQ(actualValue, v2); + EXPECT_EQ(static_cast(observer->GetCallCount()), 1); + auto changeData = observer->GetChangedData(); + ASSERT_EQ(changeData.size(), 1u); + EXPECT_EQ(changeData[CloudDbConstant::CLOUD_KV_TABLE_NAME].primaryData[OP_INSERT].size(), 1u); + EXPECT_EQ(changeData[CloudDbConstant::CLOUD_KV_TABLE_NAME].primaryData[OP_UPDATE].size(), 1u); + + observer->ResetToZero(); + EXPECT_EQ(kvDelegatePtrS2_->UnRegisterObserver(observer), OK); + delete observer; + observer = nullptr; +} + +/** + * @tc.name: ObserverDataChangeTest003 + * @tc.desc: test RegisterObserver interface with err mode. + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudKvStoreTest, ObserverDataChangeTest003, TestSize.Level0) +{ + /** + * @tc.steps: step1. call RegisterObserver with err mode (out of enum range of 0x100 to 0xF00) + * @tc.expected: step1. return INVALID_ARGS + */ + auto *observer = new (std::nothrow) KvStoreObserverUnitTest; + unsigned int errMode = 0x1000; + EXPECT_EQ(kvDelegatePtrS1_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD | errMode, observer), INVALID_ARGS); + errMode = 0x110; + EXPECT_EQ(kvDelegatePtrS1_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD | errMode, observer), INVALID_ARGS); + errMode = OBSERVER_CHANGES_LOCAL_ONLY; + EXPECT_EQ(kvDelegatePtrS1_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD | errMode, observer), INVALID_ARGS); + delete observer; + observer = nullptr; +} + +/** + * @tc.name: ObserverDataChangeTest004 + * @tc.desc: test register cloud observer twice + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudKvStoreTest, ObserverDataChangeTest004, TestSize.Level0) +{ + /** + * @tc.steps: step1. register cloud observer twice + * @tc.expected: step1. return ALREADY_SET + */ + auto *observer = new (std::nothrow) KvStoreObserverUnitTest; + EXPECT_EQ(kvDelegatePtrS1_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD | OBSERVER_CHANGES_BRIEF, observer), OK); + EXPECT_EQ(kvDelegatePtrS1_->RegisterObserver({}, OBSERVER_CHANGES_CLOUD, observer), ALREADY_SET); + observer->ResetToZero(); + EXPECT_EQ(kvDelegatePtrS1_->UnRegisterObserver(observer), OK); + delete observer; + observer = nullptr; +} } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_strategy_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_strategy_test.cpp index c939121a..51efeed6 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_strategy_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_strategy_test.cpp @@ -417,6 +417,6 @@ HWTEST_F(DistributedDBCloudStrategyTest, TagOpTyeTest007, TestSize.Level0) * @tc.expected: step5. need UPDATE this record */ localInfo.flag = static_cast(LogInfoFlag::FLAG_CLOUD_WRITE); - EXPECT_EQ(strategy->TagSyncDataStatus(true, false, localInfo, cloudInfo), OpType::UPDATE); + EXPECT_EQ(strategy->TagSyncDataStatus(true, false, localInfo, cloudInfo), OpType::NOT_HANDLE); } } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_only_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_only_test.cpp index 8f8f361f..6af3557f 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_only_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_only_test.cpp @@ -988,7 +988,7 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly006, * @tc.steps:step2. Download assets which cloud no found * @tc.expected: step2. return ASSET_NOT_FOUND_FOR_DOWN_ONLY. */ - std::vector inValue = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::vector inValue = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};; std::map> assets; assets["assets"] = {ASSET_COPY.name + "10"}; Query query = Query::Select().From(ASSETS_TABLE_NAME).In("id", inValue).And().AssetsOnly(assets); @@ -1077,7 +1077,7 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly008, * @tc.steps:step2. Download assets which local no found * @tc.expected: step2. return ASSET_NOT_FOUND_FOR_DOWN_ONLY. */ - std::vector inValue = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::vector inValue = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};; std::map> assets; assets["assets"] = {ASSET_COPY.name + "0"}; std::map> assets1; @@ -1533,6 +1533,67 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly019, PriorityLevelSync(2, query, nullptr, SyncMode::SYNC_MODE_CLOUD_FORCE_PULL, DBStatus::ASSET_NOT_FOUND_FOR_DOWN_ONLY); } +/** + * @tc.name: DownloadAssetsOnly020 + * @tc.desc: Test the consistent flag after syncing without asset + * @tc.type: FUNC + * @tc.require: + * @tc.author: bty + */ +HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly020, TestSize.Level0) +{ + /** + * @tc.steps:step1. init data + * @tc.expected: step1. return OK. + */ + int dataCount = 30; + InsertLocalData(db, 0, dataCount, ASSETS_TABLE_NAME, true); + /** + * @tc.steps:step2. sync + * @tc.expected: step2. return OK. + */ + int upIdx = 0; + g_virtualCloudDb->ForkUpload([&upIdx](const std::string &tableName, VBucket &extend) { + upIdx++; + if (upIdx > 20 && upIdx <= 30) { + int64_t err = DBStatus::CLOUD_RECORD_EXIST_CONFLICT; + extend.insert_or_assign(CloudDbConstant::ERROR_FIELD, err); + } + }); + g_virtualAssetLoader->ForkDownload([](const std::string &tableName, std::map &) { + EXPECT_TRUE(false); + }); + int queryIdx = 0; + g_virtualCloudDb->ForkQuery([&queryIdx](const std::string &, VBucket &) { + queryIdx++; + if (queryIdx == 3) { + std::vector inValue = {5, 6, 7, 8, 9}; + Query query = Query::Select().From(ASSETS_TABLE_NAME).In("id", inValue); + CloudSyncOption option; + option.devices = {DEVICE_CLOUD}; + option.query = query; + option.priorityTask = true; + g_delegate->Sync(option, nullptr); // In order to pause compensate sync + } + }); + int callCount = 0; + g_cloudStoreHook->SetSyncFinishHook([&callCount]() { + callCount++; + g_processCondition.notify_one(); + }); + CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, DBStatus::OK); + WaitForSync(callCount); + /** + * @tc.steps:step3. check count + * @tc.expected: step3. return OK. + */ + CheckConsistentCount(db, dataCount); + g_virtualCloudDb->ForkUpload(nullptr); + g_cloudStoreHook->SetSyncFinishHook(nullptr); + g_virtualAssetLoader->ForkDownload(nullptr); + g_virtualCloudDb->ForkQuery(nullptr); +} + /** * @tc.name: DownloadAssetsOnly021 * @tc.desc: test force pull mode pull mode can forcibly pull assets. @@ -1564,5 +1625,65 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly021, EXPECT_EQ(data.size(), 1u); EXPECT_EQ(data[ASSETS_TABLE_NAME].type, ChangedDataType::ASSET); } + +/** + * @tc.name: DownloadAssetsOnly022 + * @tc.desc: test assets only without and. + * @tc.type: FUNC + * @tc.require: + * @tc.author: luoguo + */ +HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly022, TestSize.Level0) +{ + /** + * @tc.steps:step1. init data + * @tc.expected: step1. return OK. + */ + int dataCount = 10; + InsertCloudDBData(0, dataCount, 0, ASSETS_TABLE_NAME); + CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, DBStatus::OK); + + /** + * @tc.steps:step2. assets only sync. + * @tc.expected: step2. return INVALID_ARGS. + */ + std::map> assets; + assets["assets"] = {ASSET_COPY.name + "0"}; + std::map> assets1; + assets1["assets"] = {ASSET_COPY.name + "0_copy"}; + Query query = Query::Select().From(ASSETS_TABLE_NAME).BeginGroup().EqualTo("id", 0).AssetsOnly(assets). + EndGroup().Or().BeginGroup().EqualTo("id", 0).And().AssetsOnly(assets1).EndGroup(); + PriorityLevelSync(2, query, SyncMode::SYNC_MODE_CLOUD_FORCE_PULL, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: DownloadAssetsOnly023 + * @tc.desc: test assets only with group and. + * @tc.type: FUNC + * @tc.require: + * @tc.author: luoguo + */ +HWTEST_F(DistributedDBCloudSyncerDownloadAssetsOnlyTest, DownloadAssetsOnly023, TestSize.Level0) +{ + /** + * @tc.steps:step1. init data + * @tc.expected: step1. return OK. + */ + int dataCount = 10; + InsertCloudDBData(0, dataCount, 0, ASSETS_TABLE_NAME); + CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, DBStatus::OK); + + /** + * @tc.steps:step2. assets only sync. + * @tc.expected: step2. return Ok. + */ + std::map> assets; + assets["assets"] = {ASSET_COPY.name + "0"}; + std::map> assets1; + assets1["assets"] = {ASSET_COPY.name + "0_copy"}; + Query query = Query::Select().From(ASSETS_TABLE_NAME).BeginGroup().EqualTo("id", 0).And().AssetsOnly(assets). + EndGroup().And().BeginGroup().EqualTo("id", 0).And().AssetsOnly(assets1).EndGroup(); + PriorityLevelSync(2, query, SyncMode::SYNC_MODE_CLOUD_FORCE_PULL, DBStatus::OK); +} } // namespace #endif // RELATIONAL_STORE diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_test.cpp index 14f90be2..f47b0b18 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_download_assets_test.cpp @@ -2484,6 +2484,79 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsTest, DownloadAssetTest002, TestS EXPECT_EQ(errCode, E_OK); } +/** + * @tc.name: DownloadAssetTest003 + * @tc.desc: Test asset download after sync task recovery + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBCloudSyncerDownloadAssetsTest, DownloadAssetTest003, TestSize.Level0) +{ + /** + * @tc.steps:step1. init data + * @tc.expected: step1. return OK. + */ + int cloudCount = 10; // 10 is num of cloud + InsertCloudDBData(0, cloudCount, 0, ASSETS_TABLE_NAME); + CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK); + DeleteCloudDBData(0, cloudCount, ASSETS_TABLE_NAME); + InsertCloudDBData(0, cloudCount, 0, NO_PRIMARY_TABLE); + /** + * @tc.steps:step2. Set task interrupted before asset download + * @tc.expected: step2. return OK. + */ + int queryTime = 0; + g_virtualCloudDb->ForkQuery([&](const std::string &, VBucket &) { + queryTime++; + if (queryTime != 1) { + return; + } + Query query = Query::Select().FromTable({NO_PRIMARY_TABLE}); + CloudSyncOption option; + option.priorityTask = true; + option.devices = {DEVICE_CLOUD}; + option.mode = SYNC_MODE_CLOUD_MERGE; + option.query = query; + ASSERT_EQ(g_delegate->Sync(option, nullptr), OK); + }); + /** + * @tc.steps:step3. Sync + * @tc.expected: step3. return OK. + */ + int removeTime = 0; + g_virtualAssetLoader->SetRemoveLocalAssetsCallback([&](std::map &assets) { + removeTime++; + return OK; + }); + CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK); + /** + * @tc.steps:step4. Check fork asset download time and observer + * @tc.expected: step4. return OK. + */ + EXPECT_EQ(removeTime, cloudCount); + ChangedData expectedChangeData1; + ChangedData expectedChangeData2; + expectedChangeData1.tableName = NO_PRIMARY_TABLE; + expectedChangeData2.tableName = ASSETS_TABLE_NAME; + expectedChangeData1.type = ChangedDataType::ASSET; + expectedChangeData2.type = ChangedDataType::ASSET; + expectedChangeData1.field.push_back(std::string("rowid")); + expectedChangeData2.field.push_back(std::string("id")); + for (int i = 0; i < cloudCount; i++) { + expectedChangeData1.primaryData[ChangeType::OP_INSERT].push_back({(int64_t)i + 1}); + expectedChangeData2.primaryData[ChangeType::OP_DELETE].push_back({(int64_t)i}); + } + g_observer->SetExpectedResult(expectedChangeData1); + g_observer->SetExpectedResult(expectedChangeData2); + EXPECT_TRUE(g_observer->IsAllChangedDataEq()); + g_observer->ClearChangedData(); + + g_virtualCloudDb->ForkInsertConflict(nullptr); + g_virtualCloudDb->ForkQuery(nullptr); + g_virtualAssetLoader->SetRemoveLocalAssetsCallback(nullptr); +} + /** * @tc.name: RecordLockFuncTest001 * @tc.desc: UNLOCKING->UNLOCKING Synchronous download failure wholly. @@ -2685,7 +2758,7 @@ HWTEST_F(DistributedDBCloudSyncerDownloadAssetsTest, CloudTaskStatusTest001, Tes CallSync({ASSETS_TABLE_NAME}, SYNC_MODE_CLOUD_MERGE, DBStatus::OK, DBStatus::OK); }); std::this_thread::sleep_for(std::chrono::milliseconds(100)); - SyncProcess process1 = g_delegate->GetCloudTaskStatus(1); + SyncProcess process1 = g_delegate->GetCloudTaskStatus(UINT64_MAX); EXPECT_EQ(process1.errCode, OK); syncThread.join(); /** diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_lock_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_lock_test.cpp index 0b113f6a..5a900ecc 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_lock_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_lock_test.cpp @@ -93,12 +93,15 @@ void GetCloudDbSchema(DataBaseSchema &dataBaseSchema) void CloseDb() { - delete g_observer; - g_virtualCloudDb = nullptr; if (g_delegate != nullptr) { EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); g_delegate = nullptr; } + if (g_observer != nullptr) { + delete g_observer; + g_observer = nullptr; + } + g_virtualCloudDb = nullptr; } class DistributedDBCloudSyncerLockTest : public testing::Test { @@ -1394,5 +1397,100 @@ HWTEST_F(DistributedDBCloudSyncerLockTest, CompensateSyncTest001, TestSize.Level sleep(1); g_virtualCloudDb->ForkQuery(nullptr); } + +/** + * @tc.name: UnLockSyncTest001 + * @tc.desc: Test sync after unlock data + * @tc.type: FUNC + * @tc.require: + * @tc.author: suyue + */ +HWTEST_F(DistributedDBCloudSyncerLockTest, UnLockSyncTest001, TestSize.Level1) +{ + /** + * @tc.steps: step1. insert data and sync + * @tc.expected: step1. return ok. + */ + int localCount = 100; + InsertLocalData(0, localCount, ASSETS_TABLE_NAME, true); + CloudSyncOption option = PrepareOption(Query::Select().FromTable({ ASSETS_TABLE_NAME }), LockAction::INSERT); + CallSync(option); + + /** + * @tc.steps: step2. lock 0-70, update all data and unlock + * @tc.expected: step2. unlock return WAIT_COMPENSATED_SYNC. + */ + std::vector> hashKeys; + CloudDBSyncUtilsTest::GetHashKey(ASSETS_TABLE_NAME, " data_key < 70 ", db, hashKeys); + EXPECT_EQ(Lock(ASSETS_TABLE_NAME, hashKeys, db), OK); + std::string sql = "update " + ASSETS_TABLE_NAME + " set name = 'xxx';"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, sql.c_str()), SQLITE_OK); + + /** + * @tc.steps: step3. non-compensated sync for condition query and isPriorityTask is true. + * @tc.expected: step3. sync data that is not in the UNLOCKING state. + */ + EXPECT_EQ(UnLock(ASSETS_TABLE_NAME, hashKeys, db), WAIT_COMPENSATED_SYNC); + std::vector values; + for (int i = 50; i < 100; i++) { + values.push_back(i); + } + option = PrepareOption(Query::Select().From(ASSETS_TABLE_NAME).In("id", values), LockAction::INSERT, true, false); + CallSync(option); + for (const auto &table : g_syncProcess.tableProcess) { + EXPECT_EQ(table.second.upLoadInfo.total, 30u); + } + + /** + * @tc.steps: step4. compensate sync and check upLoadInfo + * @tc.expected: step4. synch all data to be compensated in the UNLOCKING state. + */ + option = PrepareOption(Query::Select().FromTable({ ASSETS_TABLE_NAME }), LockAction::INSERT, true, true); + CallSync(option); + for (const auto &table : g_syncProcess.tableProcess) { + EXPECT_EQ(table.second.upLoadInfo.total, 70u); + } +} + +/** + * @tc.name: TaskIdTest001 + * @tc.desc: Test sync with specific task id + * @tc.type: FUNC + * @tc.require: + * @tc.author: liaoyonghuang + */ +HWTEST_F(DistributedDBCloudSyncerLockTest, TaskIdTest001, TestSize.Level0) +{ + /** + * @tc.steps:step1. insert cloud and sync + * @tc.expected: step1. return ok. + */ + int cloudCount = 10; + InsertCloudDBData(0, cloudCount, 0, ASSETS_TABLE_NAME); + CloudSyncOption option = PrepareOption(Query::Select().FromTable({ ASSETS_TABLE_NAME }), LockAction::INSERT); + /** + * @tc.steps:step2. sync with specific task id(1) when query + * @tc.expected: step2. return ok. + */ + int queryTime = 0; + g_virtualCloudDb->ForkQuery([&](const std::string &, VBucket &extend) { + if (queryTime == 0) { + queryTime++; + EXPECT_EQ(g_delegate->Sync(option, nullptr, 1u), OK); + } + }); + CallSync(option); + /** + * @tc.steps:step3. sync without task id + * @tc.expected: step3. return ok. + */ + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + g_virtualCloudDb->ForkQuery([&](const std::string &, VBucket &extend) { + SyncProcess syncProcess = g_delegate->GetCloudTaskStatus(UINT64_MAX - 1); + EXPECT_EQ(syncProcess.errCode, OK); + }); + CallSync(option); + g_virtualCloudDb->ForkQuery(nullptr); +} } // namespace #endif // RELATIONAL_STORE \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_progress_manager_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_progress_manager_test.cpp index 1a92ff31..182583cd 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_progress_manager_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/distributeddb_cloud_syncer_progress_manager_test.cpp @@ -280,7 +280,7 @@ HWTEST_F(DistributedDBCloudSyncerProgressManagerTest, SyncerMgrCheck005, TestSiz cloudSyncer.InitCloudSyncer(0u, SYNC_MODE_CLOUD_FORCE_PUSH); int errCode = cloudSyncer.CreateCloudTaskInfoAndCallTryToAddSync(SYNC_MODE_CLOUD_FORCE_PUSH, tables, {}, 5000); - errCode = cloudSyncer.CallPrepareSync(1u); + errCode = cloudSyncer.CallPrepareSync(UINT64_MAX); EXPECT_EQ(errCode, E_OK); cloudSyncer.InitCloudSyncer(2u, SYNC_MODE_CLOUD_FORCE_PUSH); diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.cpp index f25aad41..896b785a 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.cpp @@ -21,7 +21,10 @@ DBStatus VirtualAssetLoader::Download(const std::string &tableName, const std::s { { std::lock_guard autoLock(dataMutex_); - if (downloadStatus_ != OK) { + downloadCount_++; + bool isNeedSetStatus = downloadFailRange_.isAllFail || (downloadCount_ >= downloadFailRange_.failBeginIndex + && downloadCount_ <= downloadFailRange_.failEndIndex); + if (downloadStatus_ != OK && isNeedSetStatus) { return downloadStatus_; } } @@ -91,6 +94,14 @@ void VirtualAssetLoader::SetDownloadStatus(DBStatus status) downloadStatus_ = status; } +void VirtualAssetLoader::SetDownloadFailRange(const DownloadFailRange &setRange) +{ + std::lock_guard autoLock(dataMutex_); + LOGD("[VirtualAssetLoader] set download fail range :isAllFail=%d, from %u to %u", + setRange.isAllFail, setRange.failBeginIndex, setRange.failEndIndex); + downloadFailRange_ = setRange; +} + void VirtualAssetLoader::SetRemoveStatus(DBStatus status) { std::lock_guard autoLock(dataMutex_); @@ -122,7 +133,7 @@ void VirtualAssetLoader::SetRemoveLocalAssetsCallback(const RemoveLocalAssetsCal void VirtualAssetLoader::BatchDownload(const std::string &tableName, std::vector &downloadAssets) { - downloadCount_++; + batchDownloadCount_++; int index = 0; for (auto &[gid, prefix, assets, status] : downloadAssets) { if (batchDownloadCallback_) { @@ -149,7 +160,7 @@ void VirtualAssetLoader::BatchRemoveLocalAssets(const std::string &tableName, uint32_t VirtualAssetLoader::GetBatchDownloadCount() { - return downloadCount_; + return batchDownloadCount_; } uint32_t VirtualAssetLoader::GetBatchRemoveCount() @@ -160,7 +171,9 @@ uint32_t VirtualAssetLoader::GetBatchRemoveCount() void VirtualAssetLoader::Reset() { removeCount_ = 0; + batchDownloadCount_ = 0; downloadCount_ = 0; + downloadFailRange_.isAllFail = true; } void VirtualAssetLoader::ForkBatchDownload(const BatchDownloadCallback &callback) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.h index 4f294054..cb438481 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/cloud/virtual_asset_loader.h @@ -24,6 +24,13 @@ using DownloadCallBack = std::function &assets)>; using RemoveLocalAssetsCallBack = std::function &assets)>; using BatchDownloadCallback = std::function &assets)>; + +struct DownloadFailRange { + bool isAllFail = true; + uint32_t failBeginIndex = 0; + uint32_t failEndIndex = 0; +}; + class VirtualAssetLoader : public IAssetLoader { public: VirtualAssetLoader() = default; @@ -64,6 +71,8 @@ public: DBStatus CancelDownload() override; uint32_t GetCancelCount() const; + + void SetDownloadFailRange(const DownloadFailRange &setRange); private: DBStatus RemoveLocalAssetsInner(const std::string &tableName, const std::string &gid, const Type &prefix, std::map &assets); @@ -72,13 +81,15 @@ private: DBStatus downloadStatus_ = OK; DBStatus removeStatus_ = OK; DBStatus batchRemoveStatus_ = OK; - std::atomic downloadCount_ = 0; + std::atomic batchDownloadCount_ = 0; std::atomic removeCount_ = 0; std::atomic cancelCount_ = 0; DownloadCallBack downloadCallBack_; RemoveAssetsCallBack removeAssetsCallBack_; RemoveLocalAssetsCallBack removeLocalAssetsCallBack_; BatchDownloadCallback batchDownloadCallback_; + DownloadFailRange downloadFailRange_; + std::atomic downloadCount_ = 0; }; } #endif // VIRTUAL_ASSETLOADER_H diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_meta_data_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_meta_data_test.cpp index dc8dab51..e8fd625c 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_meta_data_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_meta_data_test.cpp @@ -361,4 +361,24 @@ HWTEST_F(DistributedDBMetaDataTest, MetadataTest008, TestSize.Level0) EXPECT_EQ(res.first, E_OK); EXPECT_EQ(res.second, expectLocalMetaData.localSchemaVersion); } + +/** + * @tc.name: MetadataTest009 + * @tc.desc: Test initially saved metadata after time changed. + * @tc.type: FUNC + * @tc.require: + * @tc.author: chenghuitao + */ +HWTEST_F(DistributedDBMetaDataTest, MetadataTest009, TestSize.Level0) +{ + /** + * @tc.steps: step1. Check time changed after metadata is saved initially by SetDbCreateTime. + * @tc.expected: step1. B is change because of time change. + */ + RuntimeContext::GetInstance()->SetTimeChanged(true); + EXPECT_EQ(metadata_->SetDbCreateTime(DEVICE_B, 10u, true), E_OK); + EXPECT_TRUE(metadata_->IsTimeChange(DEVICE_B)); + RuntimeContext::GetInstance()->SetTimeChanged(false); + RuntimeContext::GetInstance()->StopTimeTickMonitorIfNeed(); +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_multi_user_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_multi_user_test.cpp index 8aaacac6..62b44139 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_multi_user_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_multi_user_test.cpp @@ -404,6 +404,16 @@ namespace { return sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr); } + void ClearAllDevicesData() + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db, g_storePath2), SQLITE_OK); + std::string dis_tableName = RelationalStoreManager::GetDistributedTableName(DEVICE_B, g_tableName); + std::string sql = "DELETE FROM " + dis_tableName; + EXPECT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + } + void CheckDataInRealDevice() { sqlite3 *db = nullptr; @@ -757,6 +767,7 @@ HWTEST_F(DistributedDBRelationalMultiUserTest, RdbMultiUser003, TestSize.Level3) RuntimeConfig::ReleaseAutoLaunch(USER_ID_2, APP_ID, STORE_ID, DBType::DB_RELATION); RuntimeConfig::ReleaseAutoLaunch(USER_ID_2, APP_ID, STORE_ID, DBType::DB_RELATION); g_currentStatus = 0; + ClearAllDevicesData(); CloseStore(); } @@ -1386,6 +1397,7 @@ HWTEST_F(DistributedDBRelationalMultiUserTest, SubUserAutoLaunchTest001, TestSiz RuntimeConfig::SetAutoLaunchRequestCallback(nullptr, DBType::DB_RELATION); RuntimeConfig::ReleaseAutoLaunch(USER_ID_1, SUB_USER_1, APP_ID, STORE_ID, DBType::DB_RELATION); + ClearAllDevicesData(); } /** diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 0c7d7d25..81936211 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -2764,22 +2764,6 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, EncryptedAlgoUpgrade001, TestSiz CheckVirtualData(dataMap); } -/** - * @tc.name: QueryParamCheck001 - * @tc.desc: Test sync query param - * @tc.type: FUNC - * @tc.require: - * @tc.author: zhangqiquan - */ -HWTEST_F(DistributedDBRelationalVerP2PSyncTest, QueryParamCheck001, TestSize.Level0) -{ - std::map dataMap; - PrepareEnvironment(dataMap, {g_deviceB}); - Query query = Query::Select().FromTable({ g_tableName }); - EXPECT_EQ(g_rdbDelegatePtr->Sync({DEVICE_B}, DistributedDB::SYNC_MODE_PUSH_ONLY, query, nullptr, false), - NOT_SUPPORT); -} - #ifndef OMIT_ENCRYPT /** * @tc.name: AutoLaunchSyncAfterRekey_001 diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_sub_user_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_sub_user_test.cpp index 54982e13..d21c470e 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_sub_user_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_sub_user_test.cpp @@ -133,7 +133,7 @@ namespace { } DBStatus OpenDelegate(const std::string &dlpPath, RelationalStoreDelegate *&rdbDelegatePtr, - RelationalStoreManager &mgr, sqlite3 *&db) + RelationalStoreObserverUnitTest *&observer, RelationalStoreManager &mgr, sqlite3 *&db) { if (g_testDir == nullptr) { return DB_ERROR; @@ -144,7 +144,7 @@ namespace { db = RelationalTestUtils::CreateDataBase(dbPath); EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); EXPECT_EQ(RelationalTestUtils::ExecSql(db, CREATE_SINGLE_PRIMARY_KEY_TABLE), SQLITE_OK); - RelationalStoreObserverUnitTest *observer = new (std::nothrow) RelationalStoreObserverUnitTest(); + observer = new (std::nothrow) RelationalStoreObserverUnitTest(); RelationalStoreDelegate::Option option = { .observer = observer }; return mgr.OpenStore(dbPath, STORE_ID_1, option, rdbDelegatePtr); } @@ -177,15 +177,20 @@ namespace { EXPECT_EQ(mgr.DeleteKvStore(storeID), OK); } - void CloseDelegate(RelationalStoreDelegate *&delegatePtr, RelationalStoreManager &mgr, sqlite3 *&db) + void CloseDelegate(RelationalStoreDelegate *&delegatePtr, RelationalStoreObserverUnitTest *&observer, + RelationalStoreManager &mgr, sqlite3 *&db) { EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); db = nullptr; - if (delegatePtr == nullptr) { + if (delegatePtr != nullptr) { + EXPECT_EQ(mgr.CloseStore(delegatePtr), OK); + delegatePtr = nullptr; + } + if (observer == nullptr) { return; } - EXPECT_EQ(mgr.CloseStore(delegatePtr), OK); - delegatePtr = nullptr; + delete observer; + observer = nullptr; } class DistributedDBSingleVerMultiSubUserTest : public testing::Test { @@ -508,24 +513,29 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, RDBDelegateInvalidParamTest001, RelationalStoreManager mgr1(APP_ID, USER_ID, subUser1, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr1 = nullptr; sqlite3 *db1; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, mgr1, db1), INVALID_ARGS); + RelationalStoreObserverUnitTest *observer1 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, observer1, mgr1, db1), INVALID_ARGS); ASSERT_EQ(rdbDelegatePtr1, nullptr); + CloseDelegate(rdbDelegatePtr1, observer1, mgr1, db1); std::string subUser2 = "subUser-1"; RelationalStoreManager mgr2(APP_ID, USER_ID, subUser2, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr2 = nullptr; sqlite3 *db2; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr2, mgr2, db2), INVALID_ARGS); + RelationalStoreObserverUnitTest *observer2 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr2, observer2, mgr2, db2), INVALID_ARGS); ASSERT_EQ(rdbDelegatePtr2, nullptr); + CloseDelegate(rdbDelegatePtr2, observer2, mgr2, db2); std::string subUser3 = std::string(128, 'a'); RelationalStoreManager mgr3(APP_ID, USER_ID, subUser3, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr3 = nullptr; sqlite3 *db3; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr3, mgr3, db3), OK); + RelationalStoreObserverUnitTest *observer3 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr3, observer3, mgr3, db3), OK); ASSERT_NE(rdbDelegatePtr3, nullptr); - CloseDelegate(rdbDelegatePtr3, mgr3, db3); + CloseDelegate(rdbDelegatePtr3, observer3, mgr3, db3); } /** @@ -579,13 +589,15 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SameDelegateTest002, TestSize.L RelationalStoreManager mgr1(APP_ID, USER_ID, SUB_USER_1, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr1 = nullptr; sqlite3 *db1; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, mgr1, db1), OK); + RelationalStoreObserverUnitTest *observer1 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, observer1, mgr1, db1), OK); ASSERT_NE(rdbDelegatePtr1, nullptr); RelationalStoreManager mgr2(APP_ID, USER_ID, SUB_USER_2, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr2 = nullptr; sqlite3 *db2; - EXPECT_EQ(OpenDelegate("/subUser2", rdbDelegatePtr2, mgr2, db2), OK); + RelationalStoreObserverUnitTest *observer2 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser2", rdbDelegatePtr2, observer2, mgr2, db2), OK); ASSERT_NE(rdbDelegatePtr2, nullptr); int localCount = 10; @@ -605,8 +617,8 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SameDelegateTest002, TestSize.L EXPECT_EQ(sqlite3_exec(db2, sql1.c_str(), CloudDBSyncUtilsTest::QueryCountCallback, reinterpret_cast(0), nullptr), SQLITE_OK); - CloseDelegate(rdbDelegatePtr1, mgr1, db1); - CloseDelegate(rdbDelegatePtr2, mgr2, db2); + CloseDelegate(rdbDelegatePtr1, observer1, mgr1, db1); + CloseDelegate(rdbDelegatePtr2, observer2, mgr2, db2); } /** @@ -621,7 +633,8 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SubUserDelegateCRUDTest001, Tes RelationalStoreManager mgr1(APP_ID, USER_ID, SUB_USER_1, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr1 = nullptr; sqlite3 *db1; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, mgr1, db1), OK); + RelationalStoreObserverUnitTest *observer1 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, observer1, mgr1, db1), OK); ASSERT_NE(rdbDelegatePtr1, nullptr); int localCount = 10; @@ -640,7 +653,7 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SubUserDelegateCRUDTest001, Tes EXPECT_EQ(sqlite3_exec(db1, sql1.c_str(), CloudDBSyncUtilsTest::QueryCountCallback, reinterpret_cast(0), nullptr), SQLITE_OK); - CloseDelegate(rdbDelegatePtr1, mgr1, db1); + CloseDelegate(rdbDelegatePtr1, observer1, mgr1, db1); } /** @@ -729,14 +742,16 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SubUserDelegateCloudSyncTest002 RelationalStoreManager mgr1(APP_ID, USER_ID, SUB_USER_1, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr1 = nullptr; sqlite3 *db1; - EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, mgr1, db1), OK); + RelationalStoreObserverUnitTest *observer1 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser1", rdbDelegatePtr1, observer1, mgr1, db1), OK); ASSERT_NE(rdbDelegatePtr1, nullptr); EXPECT_EQ(SetCloudDB(rdbDelegatePtr1), OK); RelationalStoreManager mgr2(APP_ID, USER_ID, SUB_USER_2, INSTANCE_ID_1); RelationalStoreDelegate *rdbDelegatePtr2 = nullptr; sqlite3 *db2; - EXPECT_EQ(OpenDelegate("/subUser2", rdbDelegatePtr2, mgr2, db2), OK); + RelationalStoreObserverUnitTest *observer2 = nullptr; + EXPECT_EQ(OpenDelegate("/subUser2", rdbDelegatePtr2, observer2, mgr2, db2), OK); ASSERT_NE(rdbDelegatePtr2, nullptr); EXPECT_EQ(SetCloudDB(rdbDelegatePtr2), OK); @@ -750,8 +765,8 @@ HWTEST_F(DistributedDBSingleVerMultiSubUserTest, SubUserDelegateCloudSyncTest002 EXPECT_EQ(sqlite3_exec(db2, sql.c_str(), CloudDBSyncUtilsTest::QueryCountCallback, reinterpret_cast(10), nullptr), SQLITE_OK); - CloseDelegate(rdbDelegatePtr1, mgr1, db1); - CloseDelegate(rdbDelegatePtr2, mgr2, db2); + CloseDelegate(rdbDelegatePtr1, observer1, mgr1, db1); + CloseDelegate(rdbDelegatePtr2, observer2, mgr2, db2); } #endif diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp index b76eebe2..89da1671 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp @@ -285,7 +285,9 @@ void TestSyncWithUserChange(bool wait) EXPECT_EQ(result.size(), devices.size()); for (const auto &pair : result) { LOGD("dev %s, status %d", pair.first.c_str(), pair.second); - EXPECT_EQ(pair.second, USER_CHANGED); + // If the ClearSyncOperations interface is scheduled to be executed first, syncer has been closed + // when AddSyncOperation is triggered and the returned status is OP_FAILED(DB_ERROR). + EXPECT_TRUE((pair.second == USER_CHANGED) || (pair.second == DB_ERROR)); } CloseStore(); } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp index 9d35a56d..6c5e788d 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp @@ -1182,7 +1182,7 @@ HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, GetQueryLastTimestamp002, TestS ASSERT_EQ(errCode, E_OK); for (auto &pair : idValueMap) { auto &queryId = pair.first; - auto &expectVal = pair.second.second; + auto &expectVal = pair.second.second; EXPECT_EQ(meta.GetQueryLastTimestamp("any", queryId), static_cast(expectVal)); } } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_simple_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_simple_sync_test.cpp index 2fb3f811..929482aa 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_simple_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_simple_sync_test.cpp @@ -193,7 +193,7 @@ void CheckWatermark(const std::string &dev, KvStoreNbDelegate *kvDelegatePtr, Wa * @tc.name: Normal Sync 001 * @tc.desc: Test normal push sync for add data. * @tc.type: FUNC - * @tc.require: AR000CQS3S SR000CQE0B + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync001, TestSize.Level1) @@ -244,7 +244,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync001, TestSize.Level1 * @tc.name: Normal Sync 002 * @tc.desc: Test normal push sync for update data. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync002, TestSize.Level1) @@ -297,7 +297,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync002, TestSize.Level1 * @tc.name: Normal Sync 003 * @tc.desc: Test normal push sync for delete data. * @tc.type: FUNC - * @tc.require: AR000CQS3S + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync003, TestSize.Level1) @@ -348,7 +348,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync003, TestSize.Level1 * @tc.name: Normal Sync 004 * @tc.desc: Test normal pull sync for add data. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync004, TestSize.Level1) @@ -405,7 +405,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync004, TestSize.Level1 * @tc.name: Normal Sync 005 * @tc.desc: Test normal pull sync for update data. * @tc.type: FUNC - * @tc.require: AR000CCPOM SR000CQE10 + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync005, TestSize.Level2) @@ -472,7 +472,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync005, TestSize.Level2 * @tc.name: Normal Sync 006 * @tc.desc: Test normal pull sync for delete data. * @tc.type: FUNC - * @tc.require: AR000CQS3S + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync006, TestSize.Level2) @@ -536,7 +536,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync006, TestSize.Level2 * @tc.name: Normal Sync 007 * @tc.desc: Test normal push_pull sync for add data. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync007, TestSize.Level1) @@ -611,7 +611,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync007, TestSize.Level1 * @tc.name: Normal Sync 008 * @tc.desc: Test normal push_pull sync for update data. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync008, TestSize.Level2) @@ -688,7 +688,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync008, TestSize.Level2 * @tc.name: Normal Sync 009 * @tc.desc: Test normal push_pull sync for delete data. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync009, TestSize.Level2) @@ -766,7 +766,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync009, TestSize.Level2 * @tc.name: Normal Sync 010 * @tc.desc: Test sync failed by invalid devices. * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync010, TestSize.Level1) @@ -902,7 +902,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, NormalSync012, TestSize.Level0 * @tc.name: Limit Data Sync 001 * @tc.desc: Test sync limit key and value data * @tc.type: FUNC - * @tc.require: AR000CCPOM + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, LimitDataSync001, TestSize.Level1) @@ -983,7 +983,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, LimitDataSync001, TestSize.Lev * @tc.name: Limit Data Sync 002 * @tc.desc: Test PutBatch with invalid entries and then call sync. * @tc.type: FUNC - * @tc.require: DTS2024012914038 + * @tc.require: * @tc.author: mazhao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, LimitDataSync002, TestSize.Level1) @@ -1028,7 +1028,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, LimitDataSync002, TestSize.Lev * @tc.name: Auto Sync 001 * @tc.desc: Verify auto sync enable function. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, AutoSync001, TestSize.Level1) @@ -1072,7 +1072,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, AutoSync001, TestSize.Level1) * @tc.name: Auto Sync 002 * @tc.desc: Verify auto sync disable function. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, AutoSync002, TestSize.Level1) @@ -1110,7 +1110,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, AutoSync002, TestSize.Level1) * @tc.name: Block Sync 001 * @tc.desc: Verify block push sync function. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync001, TestSize.Level1) @@ -1148,7 +1148,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync001, TestSize.Level1) * @tc.name: Block Sync 002 * @tc.desc: Verify block pull sync function. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync002, TestSize.Level1) @@ -1186,7 +1186,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync002, TestSize.Level1) * @tc.name: Block Sync 003 * @tc.desc: Verify block push and pull sync function. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync003, TestSize.Level1) @@ -1245,7 +1245,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync003, TestSize.Level1) * @tc.name: Block Sync 004 * @tc.desc: Verify block sync function invalid args. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync004, TestSize.Level2) @@ -1291,7 +1291,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync004, TestSize.Level2) * @tc.name: Block Sync 005 * @tc.desc: Verify block sync function busy. * @tc.type: FUNC - * @tc.require: AR000CKRTD AR000CQE0E + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync005, TestSize.Level2) @@ -1333,7 +1333,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, BlockSync005, TestSize.Level2) * @tc.desc: Invalid args check of Pragma GET_QUEUED_SYNC_SIZE SET_QUEUED_SYNC_LIMIT and * GET_QUEUED_SYNC_LIMIT, expect return INVALID_ARGS. * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: wangchuanqing */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue001, TestSize.Level3) @@ -1381,7 +1381,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue001, TestSize.Level3) * @tc.name: SyncQueue002 * @tc.desc: Pragma GET_QUEUED_SYNC_LIMIT and SET_QUEUED_SYNC_LIMIT * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: wangchuanqing */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue002, TestSize.Level3) @@ -1417,7 +1417,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue002, TestSize.Level3) * @tc.name: SyncQueue003 * @tc.desc: sync queue test * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: wangchuanqing */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue003, TestSize.Level3) @@ -1526,7 +1526,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue003, TestSize.Level3) * @tc.name: SyncQueue004 * @tc.desc: sync queue full test * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: wangchuanqing */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue004, TestSize.Level3) @@ -1569,7 +1569,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue004, TestSize.Level3) * @tc.name: SyncQueue005 * @tc.desc: block sync queue test * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: wangchuanqing */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue005, TestSize.Level3) @@ -1636,7 +1636,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, SyncQueue005, TestSize.Level3) * @tc.name: CalculateSyncData001 * @tc.desc: Test sync data whose device never synced before * @tc.type: FUNC - * @tc.require: AR000HI2JS + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData001, TestSize.Level3) @@ -1655,7 +1655,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData001, TestSize * @tc.name: CalculateSyncData002 * @tc.desc: Test sync data whose device synced before, but sync data is less than 1M * @tc.type: FUNC - * @tc.require: AR000HI2JS + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData002, TestSize.Level3) @@ -1685,7 +1685,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData002, TestSize * @tc.name: CalculateSyncData003 * @tc.desc: Test sync data whose device synced before, but sync data is larger than 1M * @tc.type: FUNC - * @tc.require: AR000HI2JS + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData003, TestSize.Level3) @@ -1714,7 +1714,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData003, TestSize * @tc.name: CalculateSyncData004 * @tc.desc: Test invalid device when call GetSyncDataSize interface * @tc.type: FUNC - * @tc.require: AR000HI2JS + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData004, TestSize.Level3) @@ -1728,7 +1728,7 @@ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData004, TestSize * @tc.name: CalculateSyncData005 * @tc.desc: Test CalculateSyncData and rekey Concurrently * @tc.type: FUNC - * @tc.require: AR000HI2JS + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerP2PSimpleSyncTest, CalculateSyncData005, TestSize.Level3) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subscribe_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subscribe_sync_test.cpp index c1d575e6..bb9b2db6 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subscribe_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subscribe_sync_test.cpp @@ -262,7 +262,7 @@ void InitLocalSubscribeMap(QuerySyncObject &queryCommonObj, std::map &processMap) { bool isAllCancel = true; - for (auto &process: processMap) { + for (auto &process : processMap) { syncId = process.second.syncId; if (process.second.errCode != COMM_FAILURE) { isAllCancel = false; @@ -973,7 +973,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncProcessCancel001, TestSize. } if (isAllCancel) { std::unique_lock lock(cancelMtx); - cancalFinished = true; + cancelFinished = true; cancelCv.notify_all(); } }; @@ -986,7 +986,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncProcessCancel001, TestSize. // Wait onProcess complete. { std::unique_lock lock2(cancelMtx); - cancelCv.wait(lock2, [&cancalFinished]() {return cancalFinished;}); + cancelCv.wait(lock2, [&cancelFinished]() {return cancelFinished;}); } // Wait until all the packets arrive. std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -1005,7 +1005,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncProcessCancel001, TestSize. * @tc.name: PushFinishedNotify 001 * @tc.desc: Test remote device push finished notify function. * @tc.type: FUNC - * @tc.require: AR000CQS3S + * @tc.require: * @tc.author: xushaohua */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize.Level1) @@ -1254,7 +1254,7 @@ void ReOpenDB() * @tc.name: AckSessionCheck 001 * @tc.desc: Test ack session check function. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSessionCheck001, TestSize.Level3) @@ -1303,7 +1303,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSessionCheck001, TestSize.Le * @tc.name: AckSafeCheck001 * @tc.desc: Test ack session check filter all bad ack in device offline scene. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSafeCheck001, TestSize.Level3) @@ -1354,7 +1354,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSafeCheck001, TestSize.Level * @tc.name: WaterMarkCheck001 * @tc.desc: Test waterMark work correct in lost package scene. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, WaterMarkCheck001, TestSize.Level1) @@ -1374,7 +1374,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, WaterMarkCheck001, TestSize.Lev * @tc.name: WaterMarkCheck002 * @tc.desc: Test pull work correct in error waterMark scene. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, WaterMarkCheck002, TestSize.Level1) @@ -1462,7 +1462,7 @@ void TestDifferentSyncMode(SyncMode mode) * @tc.desc: Test push sync task merge, task can not be merged when the two sync task is not in the queue * at the same time. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck001, TestSize.Level1) @@ -1475,7 +1475,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck001, TestSize.Lev * @tc.desc: Test push_pull sync task merge, task can not be merged when the two sync task is not in the queue * at the same time. * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck002, TestSize.Level1) @@ -1521,7 +1521,7 @@ void PrepareForSyncMergeTest(std::vector &devices, int &sendRequest * @tc.name: PushSyncMergeCheck003 * @tc.desc: Test push sync task merge, task can not be merged when there is change in db since last push sync * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck003, TestSize.Level3) @@ -1573,7 +1573,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck003, TestSize.Lev * @tc.name: PushSyncMergeCheck004 * @tc.desc: Test push sync task merge, task can be merged when there is no change in db since last push sync * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck004, TestSize.Level3) @@ -1625,7 +1625,7 @@ void RegOnDispatchWithInvalidMsgAndCnt(int &sendRequestCount, int sleepMs, bool * @tc.name: PushSyncMergeCheck005 * @tc.desc: Test push sync task merge, task cannot be merged when the last push sync is failed * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck005, TestSize.Level3) @@ -1746,7 +1746,7 @@ void PrePareForQuerySyncMergeTest(bool isQuerySync, std::vector &de * @tc.name: QuerySyncMergeCheck001 * @tc.desc: Test query push sync task merge, task can be merged when there is no change in db since last query sync * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck001, TestSize.Level3) @@ -1785,7 +1785,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck001, TestSiz * @tc.name: QuerySyncMergeCheck002 * @tc.desc: Test query push sync task merge, task can not be merged when there is change in db since last sync * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck002, TestSize.Level3) @@ -1839,7 +1839,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck002, TestSiz * @tc.name: QuerySyncMergeCheck003 * @tc.desc: Test query push sync task merge, task can not be merged when then query id is different * @tc.type: FUNC - * @tc.require: AR000F3OOV + * @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck003, TestSize.Level3) @@ -1885,7 +1885,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck003, TestSiz * @tc.name: QuerySyncMergeCheck004 * @tc.desc: Test query push sync task merge, task can be merged when there is no change in db since last push sync * @tc.type: FUNC -* @tc.require: AR000F3OOV +* @tc.require: * @tc.author: zhangshijie */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck004, TestSize.Level3) @@ -1925,7 +1925,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck004, TestSiz * @tc.name: GetDataNotify001 * @tc.desc: Test GetDataNotify function, delay < 30s should sync ok, > 36 should timeout * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, GetDataNotify001, TestSize.Level3) @@ -1985,7 +1985,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, GetDataNotify001, TestSize.Leve * @tc.name: GetDataNotify002 * @tc.desc: Test GetDataNotify function, two device sync each other at same time * @tc.type: FUNC - * @tc.require: AR000D4876 + * @tc.require: * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, GetDataNotify002, TestSize.Level3) @@ -2545,4 +2545,49 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, KVTimeChange002, TestSize.Level delegate2 = nullptr; ASSERT_TRUE(g_mgr.DeleteKvStore(STORE_ID_3) == OK); } + +/** + * @tc.name: InvalidSync001 + * @tc.desc: Test sync with empty tables + * @tc.type: FUNC + * @tc.require: + * @tc.author: caihaoting + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, InvalidSync001, TestSize.Level0) +{ + /** + * @tc.steps: step1. sync with empty tables + * @tc.expected: step1. NOT_SUPPORT + */ + Query query = Query::Select().FromTable({""}); + DBStatus callStatus = g_kvDelegatePtr->Sync({g_deviceB->GetDeviceId()}, SYNC_MODE_PUSH_ONLY, nullptr, query, true); + EXPECT_EQ(callStatus, NOT_SUPPORT); + + /** + * @tc.steps: step2. sync option with empty tables + * @tc.expected: step2. NOT_SUPPORT + */ + DeviceSyncOption option; + option.devices = {g_deviceB->GetDeviceId()}; + option.isQuery = true; + option.isWait = true; + option.query = query; + std::mutex cancelMtx; + bool cancelFinished = false; + DeviceSyncProcessCallback onProcess = [&](const std::map &processMap) { + bool isAllCancel = true; + for (auto &process : processMap) { + if (process.second.errCode != COMM_FAILURE) { + isAllCancel = false; + } + } + if (isAllCancel) { + std::unique_lock lock(cancelMtx); + cancelFinished = true; + } + }; + callStatus = OK; + callStatus = g_kvDelegatePtr->Sync(option, onProcess); + EXPECT_EQ(callStatus, NOT_SUPPORT); +} } \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index 306d796a..245e1074 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -121,7 +121,7 @@ void DistributedDBTimeSyncTest::TearDown(void) g_timeSyncB = nullptr; } if (g_virtualCommunicator != nullptr) { - delete g_virtualCommunicator; + RefObject::DecObjRef(g_virtualCommunicator); g_virtualCommunicator = nullptr; } } diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp index 9cdcae6d..96b5f541 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp @@ -64,5 +64,10 @@ void RelationalVirtualDevice::SetDistributedSchema(const DistributedDB::Distribu { static_cast(storage_)->SetDistributedSchema(schema); } + +void RelationalVirtualDevice::SetGetSyncDataResult(int errCode) +{ + static_cast(storage_)->SetGetSyncDataResult(errCode); +} } // DistributedDB #endif \ No newline at end of file diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h index bdc89991..45d0f548 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h @@ -36,6 +36,7 @@ public: int Sync(SyncMode mode, bool wait) override; void EraseSyncData(const std::string &tableName); void SetDistributedSchema(const DistributedSchema &schema); + void SetGetSyncDataResult(int errCode); template void PutDeviceData(const std::string &tableName, const std::vector &data) diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp index 66f1f924..ddc3d9c5 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp @@ -108,6 +108,7 @@ void VirtualCommunicatorAggregator::RunOnConnectCallback(const std::string &targ int VirtualCommunicatorAggregator::GetLocalIdentity(std::string &outTarget) const { + // no work with using virtual communicator std::lock_guard lock(localDeviceIdMutex_); if (localDeviceId_.empty()) { outTarget = "DEVICES_A"; @@ -404,4 +405,15 @@ void VirtualCommunicatorAggregator::MockDirectEndFlag(bool isDirectEnd) { isDirectEnd_ = isDirectEnd; } + +void VirtualCommunicatorAggregator::ClearOnlineLabel() +{ +} + +void VirtualCommunicatorAggregator::SetRemoteDeviceId(const std::string &dev) +{ + std::lock_guard autoLock(communicatorsLock_); + remoteDeviceId_ = dev; + LOGI("[VirtualCommunicatorAggregator] Set dev %s", dev.c_str()); +} } // namespace DistributedDB diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h index 4bd4ed58..b92e4884 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h @@ -103,6 +103,10 @@ public: void MockDirectEndFlag(bool isDirectEnd); + void ClearOnlineLabel() override; + + void SetRemoteDeviceId(const std::string &dev); + ~VirtualCommunicatorAggregator() override = default; VirtualCommunicatorAggregator() = default; diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index 946c8331..d2fea14c 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -125,6 +125,9 @@ int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, std::vector &entries) const { + if (getSyncDataResult_ != E_OK) { + return getSyncDataResult_; + } if (localData_.find(query.GetTableName()) == localData_.end()) { LOGD("[GetSyncData] No Data Return"); return E_OK; @@ -402,5 +405,10 @@ void VirtualRelationalVerSyncDBInterface::SetDistributedSchema(const Distributed schemaObj_.SetTableMode(DistributedTableMode::COLLABORATION); schemaObj_.SetDistributedSchema(schema); } + +void VirtualRelationalVerSyncDBInterface::SetGetSyncDataResult(int errCode) +{ + getSyncDataResult_ = errCode; +} } #endif diff --git a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h index db6ae341..d1ea9561 100644 --- a/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h +++ b/kv_store/frameworks/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -124,6 +124,8 @@ public: void ReleaseRemoteQueryContinueToken(ContinueToken &token) const override; void SetDistributedSchema(const DistributedSchema &schema); + + void SetGetSyncDataResult(int errCode); private: mutable std::map, std::vector> metadata_; std::map, CaseInsensitiveComparator> syncData_; @@ -138,6 +140,7 @@ private: uint64_t dbCreateTime_; mutable std::mutex remoteSchemaMutex_; std::map remoteSchema_; + int getSyncDataResult_ = E_OK; }; } #endif diff --git a/kv_store/interfaces/innerkits/distributeddata/BUILD.gn b/kv_store/interfaces/innerkits/distributeddata/BUILD.gn index e12f42c6..0cda2969 100644 --- a/kv_store/interfaces/innerkits/distributeddata/BUILD.gn +++ b/kv_store/interfaces/innerkits/distributeddata/BUILD.gn @@ -27,10 +27,7 @@ group("build_module") { config("distributeddatafwk_config") { visibility = [ ":*" ] - cflags = [ - "-Wno-multichar", - "-Wno-c99-designator", - ] + cflags = [ "-Wno-multichar" ] cflags_cc = [ "-fvisibility=hidden" ] @@ -137,7 +134,7 @@ ohos_shared_library("distributeddata_inner") { } configs = [ ":distributeddatafwk_config" ] public_configs = [ ":distributeddatafwk_public_config" ] - + cflags = [ "-Wno-c99-designator" ] deps = deps_config external_deps = external_deps_config if (dms_service_enable && qemu_disable) { @@ -161,7 +158,7 @@ ohos_shared_library("distributeddata_client_sync") { "${kv_store_base_path}/frameworks/innerkitsimpl/kvdb/src/process_communication_impl.cpp", "${kv_store_base_path}/frameworks/innerkitsimpl/kvdb/src/process_system_api_adapter_impl.cpp", ] - + cflags = [ "-Wno-c99-designator" ] configs = [ ":distributeddatafwk_config" ] deps = [ "${kv_store_base_path}/frameworks/libs/distributeddb:distributeddb" ] @@ -210,7 +207,7 @@ ohos_static_library("kvdb_inner_lite") { ] public_configs = [ ":kvdb_inner_lite_config" ] - + cflags = [ "-Wno-c99-designator" ] subsystem_name = "distributeddatamgr" part_name = "kv_store" } diff --git a/kv_store/interfaces/innerkits/distributeddata/include/distributed_kv_data_manager.h b/kv_store/interfaces/innerkits/distributeddata/include/distributed_kv_data_manager.h index f9b0a857..af304a78 100644 --- a/kv_store/interfaces/innerkits/distributeddata/include/distributed_kv_data_manager.h +++ b/kv_store/interfaces/innerkits/distributeddata/include/distributed_kv_data_manager.h @@ -66,11 +66,12 @@ public: * @brief Get all existed kvstore names. * @param appId The name of the application. * @param storeIds Output param, all existed kvstore names. + * @param subUser The user of the kvstore. * @return Will return: * if appId is invalid, PERMISSION_DENIED and empty vector, * otherwise, SUCCESS and the vector of kvstore names, will be returned. */ - API_EXPORT Status GetAllKvStoreId(const AppId &appId, std::vector &storeIds); + API_EXPORT Status GetAllKvStoreId(const AppId &appId, std::vector &storeIds, int32_t subUser = 0); /** * @brief Disconnect kvstore instance from kvstoreimpl with the given storeId. @@ -84,9 +85,10 @@ public: * @warning Try to close a KvStore while other thread(s) still using it may cause process crash. * @param appId The name of the application. * @param storeId The name of the kvstore. + * @param subUser The user of the kvstore. * @return Return SUCCESS for success, others for failure. */ - API_EXPORT Status CloseKvStore(const AppId &appId, const StoreId &storeId); + API_EXPORT Status CloseKvStore(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); /** * @brief Disconnect kvstore instance from kvstoreimpl. @@ -108,9 +110,10 @@ public: * @brief Close all opened kvstores for this appId. * @warning Try to close a KvStore while other thread(s) still using it may cause process crash. * @param appId The name of the application. + * @param subUser The user of the kvstore. * @return Return SUCCESS for success, others for failure. */ - API_EXPORT Status CloseAllKvStore(const AppId &appId); + API_EXPORT Status CloseAllKvStore(const AppId &appId, int32_t subUser = 0); /** * @brief Delete kvstore file with the given storeId. @@ -123,9 +126,11 @@ public: * @param appId The name of the application. * @param storeId The name of the kvstore. * @param path The base directory of the kvstore, it must be available. + * @param subUser The user of the kvstore. * @return Return SUCCESS for success, others for failure. */ - API_EXPORT Status DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path = ""); + API_EXPORT Status DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path = "", + int32_t subUser = 0); /** * @brief Delete all kvstore for this appId. @@ -134,9 +139,10 @@ public: * * @param appId The name of the application. * @param path The base directory of the application, it must be available. + * @param subUser The user of the kvstore. * @return Return SUCCESS for success, others for failure. */ - API_EXPORT Status DeleteAllKvStore(const AppId &appId, const std::string &path = ""); + API_EXPORT Status DeleteAllKvStore(const AppId &appId, const std::string &path = "", int32_t subUser = 0); /** * @brief Observer the kvstore service death. diff --git a/kv_store/interfaces/innerkits/distributeddata/include/kv_utils.h b/kv_store/interfaces/innerkits/distributeddata/include/kv_utils.h index a031086f..df01c30a 100644 --- a/kv_store/interfaces/innerkits/distributeddata/include/kv_utils.h +++ b/kv_store/interfaces/innerkits/distributeddata/include/kv_utils.h @@ -24,6 +24,13 @@ #include "result_set_bridge.h" #include "visibility.h" +#define KV_UTILS_PUSH_WARNING _Pragma("GCC diagnostic push") +#define KV_UTILS_POP_WARNING _Pragma("GCC diagnostic pop") +#define KV_UTILS_GNU_DISABLE_WARNING_INTERNAL2(warningName) #warningName +#define KV_UTILS_GNU_DISABLE_WARNING(warningName) \ + _Pragma( \ + KV_UTILS_GNU_DISABLE_WARNING_INTERNAL2(GCC diagnostic ignored warningName)) + namespace OHOS { namespace DistributedKv { class KvUtils { @@ -77,6 +84,8 @@ private: static const std::string KEY; static const std::string VALUE; using QueryHandler = void (*)(const DataShare::OperationItem &, DataQuery &); + KV_UTILS_PUSH_WARNING + KV_UTILS_GNU_DISABLE_WARNING("-Wc99-designator") static constexpr QueryHandler HANDLERS[DataShare::LAST_TYPE] = { [DataShare::INVALID_OPERATION] = &KvUtils::NoSupport, [DataShare::EQUAL_TO] = &KvUtils::EqualTo, @@ -111,6 +120,7 @@ private: [DataShare::NOTBETWEEN] = &KvUtils::NoSupport, [DataShare::KEY_PREFIX] = &KvUtils::KeyPrefix, }; + KV_UTILS_POP_WARNING }; } // namespace DistributedKv } // namespace OHOS diff --git a/kv_store/interfaces/innerkits/distributeddata/include/single_kvstore.h b/kv_store/interfaces/innerkits/distributeddata/include/single_kvstore.h index e504881f..3fe5de78 100644 --- a/kv_store/interfaces/innerkits/distributeddata/include/single_kvstore.h +++ b/kv_store/interfaces/innerkits/distributeddata/include/single_kvstore.h @@ -340,6 +340,15 @@ public: { return false; } + + /** + * @brief Get kvstore subUser of this kvstore instance. + * @return The kvstore subUser. + */ + virtual int32_t GetSubUser() + { + return 0; + } }; } // namespace DistributedKv } // namespace OHOS diff --git a/kv_store/interfaces/innerkits/distributeddatamgr/BUILD.gn b/kv_store/interfaces/innerkits/distributeddatamgr/BUILD.gn index a2ad46b1..79866023 100644 --- a/kv_store/interfaces/innerkits/distributeddatamgr/BUILD.gn +++ b/kv_store/interfaces/innerkits/distributeddatamgr/BUILD.gn @@ -47,7 +47,7 @@ ohos_shared_library("distributeddata_mgr") { "../../../frameworks/innerkitsimpl/distributeddatasvc/src/distributed_data_mgr.cpp", "../../../frameworks/innerkitsimpl/distributeddatasvc/src/kvstore_data_service_mgr.cpp", ] - + cflags = [ "-Wno-c99-designator" ] external_deps = [ "c_utils:utils", "hilog:libhilog", diff --git a/kv_store/interfaces/jskits/distributeddata/BUILD.gn b/kv_store/interfaces/jskits/distributeddata/BUILD.gn index 3481180e..0067ae49 100644 --- a/kv_store/interfaces/jskits/distributeddata/BUILD.gn +++ b/kv_store/interfaces/jskits/distributeddata/BUILD.gn @@ -89,7 +89,7 @@ ohos_shared_library("distributeddata") { "data_share:datashare_common", "data_share:datashare_provider", ] - + cflags = [ "-Wno-c99-designator" ] subsystem_name = "distributeddatamgr" relative_install_dir = "module/data" part_name = "kv_store" diff --git a/kv_store/interfaces/jskits/distributedkvstore/BUILD.gn b/kv_store/interfaces/jskits/distributedkvstore/BUILD.gn index 30fb7b48..67bc0b79 100644 --- a/kv_store/interfaces/jskits/distributedkvstore/BUILD.gn +++ b/kv_store/interfaces/jskits/distributedkvstore/BUILD.gn @@ -89,7 +89,7 @@ ohos_shared_library("distributedkvstore") { "data_share:datashare_common", "data_share:datashare_provider", ] - + cflags = [ "-Wno-c99-designator" ] subsystem_name = "distributeddatamgr" relative_install_dir = "module/data" part_name = "kv_store" diff --git a/kv_store/kvstoremock/distributeddb/BUILD.gn b/kv_store/kvstoremock/distributeddb/BUILD.gn index 98c04b59..a65a8765 100644 --- a/kv_store/kvstoremock/distributeddb/BUILD.gn +++ b/kv_store/kvstoremock/distributeddb/BUILD.gn @@ -139,7 +139,7 @@ ohos_shared_library("distributeddb_mock") { [ "//base/hiviewdfx/hilog/interfaces/native/innerkits:libhilog_mac" ] } configs += [ "//third_party/jsoncpp:jsoncpp_config" ] - + cflags = [ "-Wno-c99-designator" ] subsystem_name = "distributeddatamgr" part_name = "kv_store" } diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/include/distributed_kv_data_manager.h b/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/include/distributed_kv_data_manager.h index c6a0d6bf..a758081a 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/include/distributed_kv_data_manager.h +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/include/distributed_kv_data_manager.h @@ -50,7 +50,7 @@ public: std::shared_ptr &singleKvStore); // get all existed kvstore names. - API_EXPORT Status GetAllKvStoreId(const AppId &appId, std::vector &storeIds); + API_EXPORT Status GetAllKvStoreId(const AppId &appId, std::vector &storeIds, int32_t subUser = 0); // WARNING: try to close a KvStore while other thread(s) still using it may cause process crash. // Disconnect kvstore instance from kvstoreimpl with the given storeId, @@ -62,7 +62,7 @@ public: // Parameters: // appId: the name of the application. // storeId: the name of the kvstore. - API_EXPORT Status CloseKvStore(const AppId &appId, const StoreId &storeId); + API_EXPORT Status CloseKvStore(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); // WARNING: try to close a KvStore while other thread(s) still using it may cause process crash. // @@ -79,7 +79,7 @@ public: // WARNING: try to close a KvStore while other thread(s) still using it may cause process crash. // close all opened kvstores for this appId. - API_EXPORT Status CloseAllKvStore(const AppId &appId); + API_EXPORT Status CloseAllKvStore(const AppId &appId, int32_t subUser = 0); // delete kvstore file with the given storeId. // client should first close all connections to it and then delete it, @@ -89,10 +89,11 @@ public: // Parameters: // appId: the name of the application. // storeId: the name of the kvstore. - API_EXPORT Status DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path = ""); + API_EXPORT Status DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path = "", + int32_t subUser = 0); // delete all kvstore. - API_EXPORT Status DeleteAllKvStore(const AppId &appId, const std::string &path = ""); + API_EXPORT Status DeleteAllKvStore(const AppId &appId, const std::string &path = "", int32_t subUser = 0); API_EXPORT void RegisterKvStoreServiceDeathRecipient(std::shared_ptr deathRecipient); diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp b/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp index ff529a0d..d848292d 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/distributeddatafwk/src/distributed_kv_data_manager.cpp @@ -41,7 +41,7 @@ Status DistributedKvDataManager::GetSingleKvStore(const Options &options, const return SUCCESS; } -Status DistributedKvDataManager::GetAllKvStoreId(const AppId &appId, std::vector &storeIds) +Status DistributedKvDataManager::GetAllKvStoreId(const AppId &appId, std::vector &storeIds, int32_t subUser) { auto status = StoreManager::GetInstance().GetStoreIds(appId, storeIds); if (status == Status::SUCCESS) { @@ -50,7 +50,7 @@ Status DistributedKvDataManager::GetAllKvStoreId(const AppId &appId, std::vector return SUCCESS; } -Status DistributedKvDataManager::CloseKvStore(const AppId &appId, const StoreId &storeId) +Status DistributedKvDataManager::CloseKvStore(const AppId &appId, const StoreId &storeId, int32_t subUser) { auto status = StoreManager::GetInstance().CloseKVStore(appId, storeId); if (status == SUCCESS) { @@ -77,14 +77,15 @@ Status DistributedKvDataManager::CloseKvStore(const AppId &appId, std::shared_pt return SUCCESS; } -Status DistributedKvDataManager::CloseAllKvStore(const AppId &appId) +Status DistributedKvDataManager::CloseAllKvStore(const AppId &appId, int32_t subUser) { auto status = StoreManager::GetInstance().CloseAllKVStore(appId); return status; } -Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path) +Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId &storeId, const std::string &path, + int32_t subUser) { if (!path.empty()) { return StoreManager::GetInstance().Delete(appId, storeId, path); @@ -93,7 +94,7 @@ Status DistributedKvDataManager::DeleteKvStore(const AppId &appId, const StoreId return SUCCESS; } -Status DistributedKvDataManager::DeleteAllKvStore(const AppId &appId, const std::string &path) +Status DistributedKvDataManager::DeleteAllKvStore(const AppId &appId, const std::string &path, int32_t subUser) { if (!path.empty()) { std::vector storeIds; diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h index bda2fdcb..ac41b822 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/single_store_impl.h @@ -84,6 +84,7 @@ public: Status SetIdentifier(const std::string &accountId, const std::string &appId, const std::string &storeId, const std::vector &tagretDev) override; bool IsRebuild() override; + int32_t GetSubUser() override; private: static constexpr size_t MAX_VALUE_LENGTH = 4 * 1024 * 1024; static constexpr size_t MAX_OBSERVER_SIZE = 8; diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_factory.h b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_factory.h index 730fa780..0a9c8dca 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_factory.h +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_factory.h @@ -26,8 +26,8 @@ public: static StoreFactory &GetInstance(); std::shared_ptr GetOrOpenStore(const AppId &appId, const StoreId &storeId, const Options &options, Status &status, bool &isCreate); - Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path); - Status Close(const AppId &appId, const StoreId &storeId, bool isForce = false); + Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser = 0); + Status Close(const AppId &appId, const StoreId &storeId, int32_t subUser = 0, bool isForce = false); private: using DBManager = DistributedDB::KvStoreDelegateManager; @@ -36,7 +36,7 @@ private: using DBPassword = DistributedDB::CipherPassword; StoreFactory(); - std::shared_ptr GetDBManager(const std::string &path, const AppId &appId); + std::shared_ptr GetDBManager(const std::string &path, const AppId &appId, int32_t subUser = 0); DBOption GetDBOption(const Options &options, const DBPassword &password) const; ConcurrentMap> dbManagers_; ConcurrentMap>> stores_; diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_manager.h b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_manager.h index 04ee3a79..f1a4427b 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_manager.h +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/include/store_manager.h @@ -21,11 +21,11 @@ public: static StoreManager &GetInstance(); std::shared_ptr GetKVStore(const AppId &appId, const StoreId &storeId, const Options &options, Status &status); - Status CloseKVStore(const AppId &appId, const StoreId &storeId); + Status CloseKVStore(const AppId &appId, const StoreId &storeId, int32_t subUser = 0); Status CloseKVStore(const AppId &appId, std::shared_ptr &kvStore); - Status CloseAllKVStore(const AppId &appId); - Status GetStoreIds(const AppId &appId, std::vector &storeIds); - Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path); + Status CloseAllKVStore(const AppId &appId, int32_t subUser = 0); + Status GetStoreIds(const AppId &appId, std::vector &storeIds, int32_t subUser = 0); + Status Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser = 0); }; } diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp index 755aba0e..e48fe745 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/single_store_impl.cpp @@ -490,4 +490,8 @@ bool SingleStoreImpl::IsRebuild() { return false; } +int32_t SingleStoreImpl::GetSubUser() +{ + return 0; +} } // namespace OHOS::DistributedKv \ No newline at end of file diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp index ef5cdd96..24a55d58 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_factory.cpp @@ -75,16 +75,16 @@ std::shared_ptr StoreFactory::GetOrOpenStore(const AppId &appId, return kvStore; } -Status StoreFactory::Delete(const AppId &appId, const StoreId &storeId, const std::string &path) +Status StoreFactory::Delete(const AppId &appId, const StoreId &storeId, const std::string &path, int32_t subUser) { - Close(appId, storeId, true); + Close(appId, storeId, subUser, true); auto dbManager = GetDBManager(path, appId); auto status = dbManager->DeleteKvStore(storeId); SecurityManager::GetInstance().DelDBPassword(storeId.storeId, path); return StoreUtil::ConvertStatus(status); } -Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, bool isForce) +Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, int32_t subUser, bool isForce) { Status status = STORE_NOT_OPEN; stores_.ComputeIfPresent(appId, [&storeId, &status, isForce](auto &, auto &values) { @@ -107,7 +107,8 @@ Status StoreFactory::Close(const AppId &appId, const StoreId &storeId, bool isFo return status; } -std::shared_ptr StoreFactory::GetDBManager(const std::string &path, const AppId &appId) +std::shared_ptr StoreFactory::GetDBManager(const std::string &path, const AppId &appId, + int32_t subUser) { std::shared_ptr dbManager; dbManagers_.Compute(path, [&dbManager, &appId](const auto &path, std::shared_ptr &manager) { diff --git a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp index 5923d36f..922252d1 100644 --- a/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp +++ b/kv_store/kvstoremock/frameworks/innerkitsimpl/kvdb/src/store_manager.cpp @@ -46,7 +46,7 @@ std::shared_ptr StoreManager::GetKVStore(const AppId &appId, cons return kvStore; } -Status StoreManager::CloseKVStore(const AppId &appId, const StoreId &storeId) +Status StoreManager::CloseKVStore(const AppId &appId, const StoreId &storeId, int32_t subUser) { ZLOGD("appId:%{public}s, storeId:%{public}s", appId.appId.c_str(), StoreUtil::Anonymous(storeId.storeId).c_str()); if (!appId.IsValid() || !storeId.IsValid()) { @@ -67,17 +67,17 @@ Status StoreManager::CloseKVStore(const AppId &appId, std::shared_ptr &storeIds) +Status StoreManager::GetStoreIds(const AppId &appId, std::vector &storeIds, int32_t subUser) { ZLOGD("appId:%{public}s", appId.appId.c_str()); if (!appId.IsValid()) { diff --git a/kv_store/kvstoremock/interfaces/innerkits/distributeddata/BUILD.gn b/kv_store/kvstoremock/interfaces/innerkits/distributeddata/BUILD.gn index 3701747d..58ce9c9a 100644 --- a/kv_store/kvstoremock/interfaces/innerkits/distributeddata/BUILD.gn +++ b/kv_store/kvstoremock/interfaces/innerkits/distributeddata/BUILD.gn @@ -92,6 +92,8 @@ ohos_shared_library("distributeddata_inner_mock") { "//foundation/distributeddatamgr/kv_store/interfaces/innerkits/distributeddata/include", ] + cflags = [ "-Wno-c99-designator" ] + configs = [ ":distributeddatafwk_config" ] public_configs = [ ":distributeddatafwk_public_config" ] diff --git a/kv_store/kvstoremock/interfaces/jskits/distributeddata/BUILD.gn b/kv_store/kvstoremock/interfaces/jskits/distributeddata/BUILD.gn index 3506bdf6..43e78696 100644 --- a/kv_store/kvstoremock/interfaces/jskits/distributeddata/BUILD.gn +++ b/kv_store/kvstoremock/interfaces/jskits/distributeddata/BUILD.gn @@ -46,6 +46,9 @@ if (use_platform_win || use_platforn_mac) { "//commonlibrary/c_utils/base/include", "//foundation/distributeddatamgr/kv_store/frameworks/libs/distributeddb/include", ] + + cflags = [ "-Wno-c99-designator" ] + cflags_cc = [ "-std=c++17" ] sources = [ "../../../frameworks/innerkitsimpl/distributeddatafwk/src/kv_utils.cpp", diff --git a/kv_store/kvstoremock/interfaces/jskits/distributedkvstore/BUILD.gn b/kv_store/kvstoremock/interfaces/jskits/distributedkvstore/BUILD.gn index 6c0b9d29..440314fc 100644 --- a/kv_store/kvstoremock/interfaces/jskits/distributedkvstore/BUILD.gn +++ b/kv_store/kvstoremock/interfaces/jskits/distributedkvstore/BUILD.gn @@ -50,6 +50,9 @@ if (use_platform_win || use_platforn_mac) { "//foundation/ability/ability_runtime/interfaces/kits/native/ability/native", "//foundation/bundlemanager/bundle_framework/interfaces/inner_api/appexecfwk_base/include", ] + + cflags = [ "-Wno-c99-designator" ] + cflags_cc = [ "-std=c++17" ] sources = [ diff --git a/kv_store/test/distributedtest/single_kvstore_client/BUILD.gn b/kv_store/test/distributedtest/single_kvstore_client/BUILD.gn index 8af7a096..10750dcf 100644 --- a/kv_store/test/distributedtest/single_kvstore_client/BUILD.gn +++ b/kv_store/test/distributedtest/single_kvstore_client/BUILD.gn @@ -53,6 +53,8 @@ common_external_deps = [ ohos_distributedtest("DistributedTest") { module_out_path = module_output_path + cflags = [ "-Wno-c99-designator" ] + sources = [ "distributed_test.cpp" ] configs = [ ":module_private_config" ] @@ -65,6 +67,8 @@ ohos_distributedtest("DistributedTest") { ohos_distributedtest("DistributedTestAgent") { module_out_path = module_output_path + cflags = [ "-Wno-c99-designator" ] + sources = [ "distributed_test_agent.cpp" ] configs = [ ":module_private_config" ] diff --git a/kv_store/test/fuzztest/blob_fuzzer/BUILD.gn b/kv_store/test/fuzztest/blob_fuzzer/BUILD.gn index b32e46f3..125ed12e 100644 --- a/kv_store/test/fuzztest/blob_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/blob_fuzzer/BUILD.gn @@ -34,6 +34,7 @@ ohos_fuzztest("BlobFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ diff --git a/kv_store/test/fuzztest/distributedkvdatamanager_fuzzer/BUILD.gn b/kv_store/test/fuzztest/distributedkvdatamanager_fuzzer/BUILD.gn index 57c65b61..60c47156 100644 --- a/kv_store/test/fuzztest/distributedkvdatamanager_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/distributedkvdatamanager_fuzzer/BUILD.gn @@ -33,6 +33,7 @@ ohos_fuzztest("DistributedKvDataManagerFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ "distributedkvdatamanager_fuzzer.cpp" ] diff --git a/kv_store/test/fuzztest/distributedkvdataservice_fuzzer/BUILD.gn b/kv_store/test/fuzztest/distributedkvdataservice_fuzzer/BUILD.gn index d494dd5e..45dd0620 100644 --- a/kv_store/test/fuzztest/distributedkvdataservice_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/distributedkvdataservice_fuzzer/BUILD.gn @@ -33,6 +33,7 @@ ohos_fuzztest("DistributedKvDataServiceFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ "distributedkvdataservice_fuzzer.cpp" ] diff --git a/kv_store/test/fuzztest/singlekvstore_fuzzer/BUILD.gn b/kv_store/test/fuzztest/singlekvstore_fuzzer/BUILD.gn index 534acacf..16fe25c3 100644 --- a/kv_store/test/fuzztest/singlekvstore_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/singlekvstore_fuzzer/BUILD.gn @@ -33,6 +33,7 @@ ohos_fuzztest("SingleKvStoreFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ "singlekvstore_fuzzer.cpp" ] diff --git a/kv_store/test/fuzztest/singlekvstorestub_fuzzer/BUILD.gn b/kv_store/test/fuzztest/singlekvstorestub_fuzzer/BUILD.gn index 39487e58..d735c3c1 100644 --- a/kv_store/test/fuzztest/singlekvstorestub_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/singlekvstorestub_fuzzer/BUILD.gn @@ -33,6 +33,7 @@ ohos_fuzztest("SingleKvStoreStubFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ "singlekvstorestub_fuzzer.cpp" ] diff --git a/kv_store/test/fuzztest/typesutil_fuzzer/BUILD.gn b/kv_store/test/fuzztest/typesutil_fuzzer/BUILD.gn index 09c3750c..feee1440 100644 --- a/kv_store/test/fuzztest/typesutil_fuzzer/BUILD.gn +++ b/kv_store/test/fuzztest/typesutil_fuzzer/BUILD.gn @@ -35,6 +35,7 @@ ohos_fuzztest("TypesUtilFuzzTest") { "-O0", "-Wno-unused-variable", "-fno-omit-frame-pointer", + "-Wno-c99-designator", ] sources = [ diff --git a/kv_store/test/unittest/distributedKVStore/BUILD.gn b/kv_store/test/unittest/distributedKVStore/BUILD.gn index 22c46fcf..5ae93040 100644 --- a/kv_store/test/unittest/distributedKVStore/BUILD.gn +++ b/kv_store/test/unittest/distributedKVStore/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "kv_store/distributedkvstore" +module_output_path = "kv_store/kv_store/distributedkvstore" ohos_js_unittest("KvStoreJsTest") { module_out_path = module_output_path diff --git a/kv_store/test/unittest/distributedKVStore/SingleKvStoreKVPromiseJsTest.js b/kv_store/test/unittest/distributedKVStore/SingleKvStoreKVPromiseJsTest.js index dcd1f01c..829fc8f1 100644 --- a/kv_store/test/unittest/distributedKVStore/SingleKvStoreKVPromiseJsTest.js +++ b/kv_store/test/unittest/distributedKVStore/SingleKvStoreKVPromiseJsTest.js @@ -2604,4 +2604,29 @@ describe('SingleKvStorePromiseTest', function () { }); done(); }) + + /** + * @tc.name SingleKvStoreOffSyncCompleteTest + * @tc.desc Test OffSyncComplete with mismatched callback function + * @tc.type: FUNC + * @tc.require: + */ + it('SingleKvStoreOffSyncCompleteTest', 0, async function (done) { + function syncCallback1(data) { + console.info('SingleKvStoreOffSyncCompleteTest callback1'); + } + function syncCallback2(data) { + console.info('SingleKvStoreOffSyncCompleteTest callback2'); + } + try { + console.info('SingleKvStoreOffSyncCompleteTest start'); + kvStore.on('syncComplete', syncCallback1); + kvStore.off('syncComplete', syncCallback2); + kvStore.off('syncComplete', syncCallback1); + expect(true).assertTrue(); + } catch (e) { + expect(null).assertFail(); + } + done(); + }) }) diff --git a/kv_store/test/unittest/distributedKVStore/config.json b/kv_store/test/unittest/distributedKVStore/config.json index a98b5dc3..72fa5d4c 100644 --- a/kv_store/test/unittest/distributedKVStore/config.json +++ b/kv_store/test/unittest/distributedKVStore/config.json @@ -74,4 +74,4 @@ } ] } -} +} \ No newline at end of file diff --git a/kv_store/test/unittest/distributeddata/BUILD.gn b/kv_store/test/unittest/distributeddata/BUILD.gn index 8401b344..a97cdba2 100644 --- a/kv_store/test/unittest/distributeddata/BUILD.gn +++ b/kv_store/test/unittest/distributeddata/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "kv_store/distributeddata" +module_output_path = "kv_store/kv_store/distributeddata" ohos_js_unittest("KvStoreDataJsTest") { module_out_path = module_output_path diff --git a/kv_store/test/unittest/distributeddata/config.json b/kv_store/test/unittest/distributeddata/config.json index a98b5dc3..72fa5d4c 100644 --- a/kv_store/test/unittest/distributeddata/config.json +++ b/kv_store/test/unittest/distributeddata/config.json @@ -74,4 +74,4 @@ } ] } -} +} \ No newline at end of file diff --git a/mock/CMakeLists.txt b/mock/CMakeLists.txt index 8fffca19..bf8b4867 100644 --- a/mock/CMakeLists.txt +++ b/mock/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.2) project(mock) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) #set(CMAKE_CXX_FLAGS "-std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "-std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fpic -fdata-sections -ffunction-sections -D_GLIBC_MOCK") @@ -12,16 +12,20 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-incompatible-pointer-types") add_definitions(-DNDEBUG=1 -DHAVE_USLEEP=1 -DSQLITE_HAVE_ISNAN -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576) add_definitions(-DSQLITE_THREADSAFE=2 -DSQLITE_TEMP_STORE=3 -DSQLITE_POWERSAFE_OVERWRITE=1) add_definitions(-DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_DEFAULT_AUTOVACUUM=1 -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1) -add_definitions(-DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS4 -DSQLITE_OMIT_COMPILEOPTION_DIAGS) -add_definitions(-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_FILE_PERMISSIONS=0600 -DSQLITE_SECURE_DELETE) +add_definitions(-DSQLITE_ENABLE_LOAD_EXTENSION -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_TOKENIZER -DSQLITE_ENABLE_FTS4) +add_definitions(-DSQLITE_ENABLE_FTS5 -DSQLITE_OMIT_COMPILEOPTION_DIAGS) +add_definitions(-DSQLITE_DEFAULT_FILE_PERMISSIONS=0600 -DSQLITE_SECURE_DELETE) add_definitions(-DSQLITE_ENABLE_BATCH_ATOMIC_WRITE -DUSE_PREAD64 -Dfdatasync=fdatasync -DSQLITE_CODEC_ATTACH_CHANGED) add_definitions(-DHAVE_MALLOC_H=1 -DHAVE_MALLOC_USABLE_SIZE -DSQLITE_DIRECT_OVERFLOW_READ -DSQLITE_HAS_CODEC -DSQLITE_EXPORT_SYMBOLS) -add_definitions(-DSQLITE_HAS_CODEC -DSQLITE3_HW_EXPORT_SYMBOLS -DSQLITE_HW_SHARED_BLOCK_OPTIMIZATION) +add_definitions(-DSQLITE_SHARED_BLOCK_OPTIMIZATION -DSQLITE_HAS_CODEC -DSQLITE3_HW_EXPORT_SYMBOLS -DSQLITE_HW_SHARED_BLOCK_OPTIMIZATION) add_definitions(-DOPENSSL_USE_NODELETE -DOPENSSL_PIC -DNDEBUG -DOPENSSL_NO_MD2 -DOPENSSL_NO_RC5 -DOPENSSL_NO_RIPEMD) add_definitions(-DL_ENDIAN -DUNICODE -D_UNICODE -DWIN32_LEAN_AND_MEAN -D_MT -DWINDOWS_PLATFORM -DSQLITE_ENABLE_DROPTABLE_CALLBACK) +add_definitions(-DOPENSSL_SUPPRESS_DEPRECATED -DLOG_DUMP -DFDSAN_ENABLE -DHARMONY_OS) +add_definitions(-DSQLITE_HDR_CHECK -DSQLITE_META_DWR -DSQLITE_CHECK_PAGES) add_definitions(-DNAPI_EXPERIMENTAL -DSQLITE_DISTRIBUTE_RELATIONAL) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src mockSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/sqlite/src mockSrc) +list(REMOVE_ITEM mockSrc "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/src/sqlite3icu.c") set(MOCK_DIR ${CMAKE_CURRENT_SOURCE_DIR}) include(${CMAKE_CURRENT_SOURCE_DIR}/include/CMakeLists.txt OPTIONAL) diff --git a/mock/innerkits/ability_runtime/dataobs_manager/include/dataobs_mgr_client.h b/mock/innerkits/ability_runtime/dataobs_manager/include/dataobs_mgr_client.h index 317a121d..ecc23543 100644 --- a/mock/innerkits/ability_runtime/dataobs_manager/include/dataobs_mgr_client.h +++ b/mock/innerkits/ability_runtime/dataobs_manager/include/dataobs_mgr_client.h @@ -1,36 +1,36 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright (c) 2021 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ #ifndef OHOS_ABILITY_RUNTIME_DATAOBS_MGR_CLIENT_H #define OHOS_ABILITY_RUNTIME_DATAOBS_MGR_CLIENT_H #include +//#include "concurrent_map.h" #include "data_ability_observer_interface.h" #include "dataobs_mgr_errors.h" #include "dataobs_mgr_interface.h" -#include "uri.h" - #include "iremote_object.h" +#include "uri.h" namespace OHOS { namespace AAFwk { /** - * @class DataObsMgrClient - * DataObsMgrClient is used to access dataobs manager services. - */ +* @class DataObsMgrClient +* DataObsMgrClient is used to access dataobs manager services. +*/ class DataObsMgrClient { public: DataObsMgrClient(); @@ -38,84 +38,89 @@ public: static std::shared_ptr GetInstance(); /** - * Registers an observer to DataObsMgr specified by the given Uri. - * - * @param uri, Indicates the path of the data to operate. - * @param dataObserver, Indicates the IDataAbilityObserver object. - * - * @return Returns ERR_OK on success, others on failure. - */ + * Registers an observer to DataObsMgr specified by the given Uri. + * + * @param uri, Indicates the path of the data to operate. + * @param dataObserver, Indicates the IDataAbilityObserver object. + * + * @return Returns ERR_OK on success, others on failure. + */ ErrCode RegisterObserver(const Uri &uri, sptr dataObserver); /** - * Deregisters an observer used for DataObsMgr specified by the given Uri. - * - * @param uri, Indicates the path of the data to operate. - * @param dataObserver, Indicates the IDataAbilityObserver object. - * - * @return Returns ERR_OK on success, others on failure. - */ + * Deregisters an observer used for DataObsMgr specified by the given Uri. + * + * @param uri, Indicates the path of the data to operate. + * @param dataObserver, Indicates the IDataAbilityObserver object. + * + * @return Returns ERR_OK on success, others on failure. + */ ErrCode UnregisterObserver(const Uri &uri, sptr dataObserver); /** - * Notifies the registered observers of a change to the data resource specified by Uri. - * - * @param uri, Indicates the path of the data to operate. - * - * @return Returns ERR_OK on success, others on failure. - */ + * Notifies the registered observers of a change to the data resource specified by Uri. + * + * @param uri, Indicates the path of the data to operate. + * + * @return Returns ERR_OK on success, others on failure. + */ ErrCode NotifyChange(const Uri &uri); /** - * Registers an observer to DataObsMgr specified by the given Uri. - * - * @param uri, Indicates the path of the data to operate. - * @param dataObserver, Indicates the IDataAbilityObserver object. - * - * @return Returns SUCCESS on success, others on failure. - */ + * Registers an observer to DataObsMgr specified by the given Uri. + * + * @param uri, Indicates the path of the data to operate. + * @param dataObserver, Indicates the IDataAbilityObserver object. + * + * @return Returns SUCCESS on success, others on failure. + */ Status RegisterObserverExt(const Uri &uri, sptr dataObserver, bool isDescendants); /** - * Deregisters an observer used for DataObsMgr specified by the given Uri. - * - * @param uri, Indicates the path of the data to operate. - * @param dataObserver, Indicates the IDataAbilityObserver object. - * - * @return Returns SUCCESS on success, others on failure. - */ + * Deregisters an observer used for DataObsMgr specified by the given Uri. + * + * @param uri, Indicates the path of the data to operate. + * @param dataObserver, Indicates the IDataAbilityObserver object. + * + * @return Returns SUCCESS on success, others on failure. + */ Status UnregisterObserverExt(const Uri &uri, sptr dataObserver); /** - * Deregisters observers used for DataObsMgr specified. - * - * @param dataObserver, Indicates the IDataAbilityObserver object. - * - * @return Returns SUCCESS on success, others on failure. - */ + * Deregisters observers used for DataObsMgr specified. + * + * @param dataObserver, Indicates the IDataAbilityObserver object. + * + * @return Returns SUCCESS on success, others on failure. + */ Status UnregisterObserverExt(sptr dataObserver); /** - * Notifies the registered observers of a change to the data resource specified by Uris. - * - * @param changeInfo Indicates the info of the data to operate. - * - * @return Returns SUCCESS on success, others on failure. - */ + * Notifies the registered observers of a change to the data resource specified by Uris. + * + * @param changeInfo Indicates the info of the data to operate. + * + * @return Returns SUCCESS on success, others on failure. + */ Status NotifyChangeExt(const ChangeInfo &changeInfo); -private: /** - * Connect dataobs manager service. - * - * @return Returns SUCCESS on success, others on failure. - */ - Status Connect(); + * Notifies the process observer with the given progress key and cancel observer. + * + * @param key Identifies the progress of a specific task. + + * @param observer Observer for monitoring the ongoing process. + * + * @return Returns SUCCESS on success, others on failure. + */ + Status NotifyProcessObserver(const std::string &key, const sptr &observer); + +private: static std::mutex mutex_; static std::shared_ptr instance_; - sptr dataObsManger_; +// sptr dataObsManger_; }; -} // namespace AAFwk -} // namespace OHOS -#endif // OHOS_ABILITY_RUNTIME_DATAOBS_MGR_CLIENT_H +} // namespace AAFwk +} // namespace OHOS +#endif // OHOS_ABILITY_RUNTIME_DATAOBS_MGR_CLIENT_H diff --git a/mock/innerkits/access_token/libaccesstoken_sdk/include/tokenid_kit.h b/mock/innerkits/access_token/libaccesstoken_sdk/include/tokenid_kit.h index 1560692d..1b877702 100644 --- a/mock/innerkits/access_token/libaccesstoken_sdk/include/tokenid_kit.h +++ b/mock/innerkits/access_token/libaccesstoken_sdk/include/tokenid_kit.h @@ -17,6 +17,7 @@ #define TOKENID_KIT_H #include +#include "ipc_skeleton.h" namespace OHOS { namespace Security { diff --git a/mock/innerkits/filemanagement/file_api/interfaces/kits/security_label.h b/mock/innerkits/filemanagement/file_api/interfaces/kits/security_label.h index 470a68fe..6dd3b9c2 100644 --- a/mock/innerkits/filemanagement/file_api/interfaces/kits/security_label.h +++ b/mock/innerkits/filemanagement/file_api/interfaces/kits/security_label.h @@ -20,61 +20,76 @@ #include #include #include +#include #include namespace OHOS { namespace FileManagement { namespace ModuleSecurityLabel { -const char XATTR_KEY[] = {"user.security"}; +const char XATTR_KEY[] = { "user.security" }; const std::string DEFAULT_DATA_LEVEL = "s3"; -const std::set DATA_LEVEL = {"s0", "s1", "s2", "s3", "s4"}; +const int DEFAULT_DATA_LENGTH = 2; +const std::set DATA_LEVEL = { "s0", "s1", "s2", "s3", "s4" }; class SecurityLabel { public: - static bool SetSecurityLabel(const std::string &path, const std::string &dataLevel) - { - if (DATA_LEVEL.count(dataLevel) != 1) { - return false; - } + static bool SetSecurityLabel(const std::string &path, const std::string &dataLevel) + { + if (DATA_LEVEL.count(dataLevel) != 1) { + errno = EINVAL; + return false; + } #ifdef IOS_PLATFORM - if (setxattr(path.c_str(), XATTR_KEY, dataLevel.c_str(), dataLevel.size(), 0, 0) < 0) { + auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0, 0, 0); #else - if (setxattr(path.c_str(), XATTR_KEY, dataLevel.c_str(), dataLevel.size(), 0) < 0) { + auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0); #endif - return false; - } - return true; - } + if (xattrValueSize == static_cast(DEFAULT_DATA_LENGTH)) { + char xattrValue[DEFAULT_DATA_LENGTH + 1]; +#ifdef IOS_PLATFORM + xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue, xattrValueSize, 0, 0); +#else + xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue, xattrValueSize); +#endif + xattrValue[DEFAULT_DATA_LENGTH] = '\0'; + if (std::string(xattrValue) > dataLevel) { + errno = EINVAL; + return false; + } + } +#ifdef IOS_PLATFORM + if (setxattr(path.c_str(), XATTR_KEY, dataLevel.c_str(), dataLevel.size(), 0, 0) < 0) { +#else + if (setxattr(path.c_str(), XATTR_KEY, dataLevel.c_str(), dataLevel.size(), 0) < 0) { +#endif + return false; + } + return true; + } - static std::string GetSecurityLabel(const std::string &path) - { + static std::string GetSecurityLabel(const std::string &path) + { #ifdef IOS_PLATFORM - auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0, 0, 0); + auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0, 0, 0); #else - auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0); + auto xattrValueSize = getxattr(path.c_str(), XATTR_KEY, nullptr, 0); #endif - if (xattrValueSize == -1 || errno == ENOTSUP) { - return ""; - } - if (xattrValueSize <= 0) { - return DEFAULT_DATA_LEVEL; - } - std::unique_ptr xattrValue = std::make_unique((long)xattrValueSize + 1); - if (xattrValue == nullptr) { - return ""; - } + if (xattrValueSize == -1 || xattrValueSize == 0) { + return DEFAULT_DATA_LEVEL; + } + std::unique_ptr xattrValue = std::make_unique((long)xattrValueSize + 1); + if (xattrValue == nullptr) { + return DEFAULT_DATA_LEVEL; + } #ifdef IOS_PLATFORM - xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue.get(), xattrValueSize, 0, 0); + xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue.get(), xattrValueSize, 0, 0); #else - xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue.get(), xattrValueSize); + xattrValueSize = getxattr(path.c_str(), XATTR_KEY, xattrValue.get(), xattrValueSize); #endif - if (xattrValueSize == -1 || errno == ENOTSUP) { - return ""; - } - if (xattrValueSize <= 0) { - return DEFAULT_DATA_LEVEL; - } - return std::string(xattrValue.get()); - } + if (xattrValueSize == -1 || xattrValueSize == 0) { + return DEFAULT_DATA_LEVEL; + } + return std::string(xattrValue.get()); + } }; } // namespace ModuleSecurityLabel } // namespace FileManagement diff --git a/mock/innerkits/huks/libhukssdk/include/hks_type.h b/mock/innerkits/huks/libhukssdk/include/hks_type.h index 8c314aad..9d70569a 100644 --- a/mock/innerkits/huks/libhukssdk/include/hks_type.h +++ b/mock/innerkits/huks/libhukssdk/include/hks_type.h @@ -16,7 +16,7 @@ /** * @file hks_type.h * -* @brief Declares huks struct and enum. +* @brief Declares huks type. * * @since 8 */ @@ -28,6 +28,8 @@ #include #include +#include "hks_type_enum.h" + #ifdef __cplusplus extern "C" { #endif @@ -56,7 +58,7 @@ extern "C" { #define MAX_KEY_SIZE 2048 #define HKS_AE_TAG_LEN 16 #define HKS_AE_NONCE_LEN 12 -#define HKS_MAX_KEY_ALIAS_LEN 64 +#define HKS_MAX_KEY_ALIAS_LEN 128 #define HKS_MAX_PROCESS_NAME_LEN 50 #define HKS_MAX_RANDOM_LEN 1024 #define HKS_KEY_BYTES(keySize) (((keySize) + HKS_BITS_PER_BYTE - 1) / HKS_BITS_PER_BYTE) @@ -70,7 +72,7 @@ extern "C" { #define UDID_LEN 64 #define SHA256_SIGN_LEN 32 #define TOKEN_SIZE 32 -#define MAX_AUTH_TIMEOUT_SECOND 60 +#define MAX_AUTH_TIMEOUT_SECOND 600 #define SECURE_SIGN_VERSION 0x01000001 #define HKS_CERT_COUNT 4 @@ -85,600 +87,14 @@ extern "C" { #define HKS_KEY_BLOB_AT_KEY_BYTES 32 #define HKS_MAX_KEY_ALIAS_COUNT 2048 - -/** -* @brief hks key type -*/ -enum HksKeyType { - HKS_KEY_TYPE_RSA_PUBLIC_KEY = 0x01001000, - HKS_KEY_TYPE_RSA_KEYPAIR = 0x01002000, - - HKS_KEY_TYPE_ECC_P256_PUBLIC_KEY = 0x02021000, - HKS_KEY_TYPE_ECC_P256_KEYPAIR = 0x02022000, - HKS_KEY_TYPE_ECC_P384_PUBLIC_KEY = 0x02031000, - HKS_KEY_TYPE_ECC_P384_KEYPAIR = 0x02032000, - HKS_KEY_TYPE_ECC_P521_PUBLIC_KEY = 0x02051000, - HKS_KEY_TYPE_ECC_P521_KEYPAIR = 0x02052000, - - HKS_KEY_TYPE_ED25519_PUBLIC_KEY = 0x02101000, - HKS_KEY_TYPE_ED25519_KEYPAIR = 0x02102000, - HKS_KEY_TYPE_X25519_PUBLIC_KEY = 0x02111000, - HKS_KEY_TYPE_X25519_KEYPAIR = 0x02112000, - - HKS_KEY_TYPE_AES = 0x03000000, - HKS_KEY_TYPE_CHACHA20 = 0x04010000, - HKS_KEY_TYPE_CHACHA20_POLY1305 = 0x04020000, - - HKS_KEY_TYPE_HMAC = 0x05000000, - HKS_KEY_TYPE_HKDF = 0x06000000, - HKS_KEY_TYPE_PBKDF2 = 0x07000000, -}; - -/** -* @brief hks key purpose -*/ -enum HksKeyPurpose { - HKS_KEY_PURPOSE_ENCRYPT = 1, /* Usable with RSA, EC, AES, SM2, and SM4 keys. */ - HKS_KEY_PURPOSE_DECRYPT = 2, /* Usable with RSA, EC, AES, SM2, and SM4 keys. */ - HKS_KEY_PURPOSE_SIGN = 4, /* Usable with RSA, EC keys. */ - HKS_KEY_PURPOSE_VERIFY = 8, /* Usable with RSA, EC keys. */ - HKS_KEY_PURPOSE_DERIVE = 16, /* Usable with EC keys. */ - HKS_KEY_PURPOSE_WRAP = 32, /* Usable with wrap key. */ - HKS_KEY_PURPOSE_UNWRAP = 64, /* Usable with unwrap key. */ - HKS_KEY_PURPOSE_MAC = 128, /* Usable with mac. */ - HKS_KEY_PURPOSE_AGREE = 256, /* Usable with agree. */ -}; - -/** -* @brief hks key digest -*/ -enum HksKeyDigest { - HKS_DIGEST_NONE = 0, - HKS_DIGEST_MD5 = 1, - HKS_DIGEST_SM3 = 2, - HKS_DIGEST_SHA1 = 10, - HKS_DIGEST_SHA224 = 11, - HKS_DIGEST_SHA256 = 12, - HKS_DIGEST_SHA384 = 13, - HKS_DIGEST_SHA512 = 14, -}; - -/** -* @brief hks key padding -*/ -enum HksKeyPadding { - HKS_PADDING_NONE = 0, - HKS_PADDING_OAEP = 1, - HKS_PADDING_PSS = 2, - HKS_PADDING_PKCS1_V1_5 = 3, - HKS_PADDING_PKCS5 = 4, - HKS_PADDING_PKCS7 = 5, -}; - -/** -* @brief hks cipher mode -*/ -enum HksCipherMode { - HKS_MODE_ECB = 1, - HKS_MODE_CBC = 2, - HKS_MODE_CTR = 3, - HKS_MODE_OFB = 4, - HKS_MODE_CFB = 5, - HKS_MODE_CCM = 31, - HKS_MODE_GCM = 32, -}; - -/** -* @brief hks key size -*/ -enum HksKeySize { - HKS_RSA_KEY_SIZE_512 = 512, - HKS_RSA_KEY_SIZE_768 = 768, - HKS_RSA_KEY_SIZE_1024 = 1024, - HKS_RSA_KEY_SIZE_2048 = 2048, - HKS_RSA_KEY_SIZE_3072 = 3072, - HKS_RSA_KEY_SIZE_4096 = 4096, - - HKS_ECC_KEY_SIZE_224 = 224, - HKS_ECC_KEY_SIZE_256 = 256, - HKS_ECC_KEY_SIZE_384 = 384, - HKS_ECC_KEY_SIZE_521 = 521, - - HKS_AES_KEY_SIZE_128 = 128, - HKS_AES_KEY_SIZE_192 = 192, - HKS_AES_KEY_SIZE_256 = 256, - HKS_AES_KEY_SIZE_512 = 512, - - HKS_CURVE25519_KEY_SIZE_256 = 256, - - HKS_DH_KEY_SIZE_2048 = 2048, - HKS_DH_KEY_SIZE_3072 = 3072, - HKS_DH_KEY_SIZE_4096 = 4096, - - HKS_SM2_KEY_SIZE_256 = 256, - HKS_SM4_KEY_SIZE_128 = 128, -}; - -/** -* @brief hks key algorithm -*/ -enum HksKeyAlg { - HKS_ALG_RSA = 1, - HKS_ALG_ECC = 2, - HKS_ALG_DSA = 3, - - HKS_ALG_AES = 20, - HKS_ALG_HMAC = 50, - HKS_ALG_HKDF = 51, - HKS_ALG_PBKDF2 = 52, - HKS_ALG_GMKDF = 53, - - HKS_ALG_ECDH = 100, - HKS_ALG_X25519 = 101, - HKS_ALG_ED25519 = 102, - HKS_ALG_DH = 103, - - HKS_ALG_SM2 = 150, - HKS_ALG_SM3 = 151, - HKS_ALG_SM4 = 152, -}; - -/** -* @brief hks algorithm suite -*/ -enum HuksAlgSuite { - /* Algorithm suites of unwrapping wrapped-key by huks */ - /* Unwrap suite of key agreement type */ - /* WrappedData format(Bytes Array): - * | x25519_plain_pubkey_length (4 Byte) | x25519_plain_pubkey | agreekey_aad_length (4 Byte) | agreekey_aad - * | agreekey_nonce_length (4 Byte) | agreekey_nonce | agreekey_aead_tag_len(4 Byte) | agreekey_aead_tag - * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_aad_length (4 Byte) | kek_aad - * | kek_nonce_length (4 Byte) | kek_nonce | kek_aead_tag_len (4 Byte) | kek_aead_tag - * | key_material_size_len (4 Byte) | key_material_size | key_mat_enc_length (4 Byte) | key_mat_enc_data - */ - HKS_UNWRAP_SUITE_X25519_AES_256_GCM_NOPADDING = 1, - - /* WrappedData format(Bytes Array): - * | ECC_plain_pubkey_length (4 Byte) | ECC_plain_pubkey | agreekey_aad_length (4 Byte) | agreekey_aad - * | agreekey_nonce_length (4 Byte) | agreekey_nonce | agreekey_aead_tag_len(4 Byte) | agreekey_aead_tag - * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_aad_length (4 Byte) | kek_aad - * | kek_nonce_length (4 Byte) | kek_nonce | kek_aead_tag_len (4 Byte) | kek_aead_tag - * | key_material_size_len (4 Byte) | key_material_size | key_mat_enc_length (4 Byte) | key_mat_enc_data - */ - HKS_UNWRAP_SUITE_ECDH_AES_256_GCM_NOPADDING = 2, - - /* WrappedData format(Bytes Array): - * | SM2_plain_pubkey_length (4 Byte) | SM2_plain_pubkey | signData_size_length (4 Byte) | signData_size - * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_material_size_len(4 Byte) | kek_material_size - * | factor1_data_len (4 Byte) | factor1_data | factor2_data_len (4 Byte) | factor2_data - * | mac_data_length (4 Byte) | mac_data | key_mat_enc_length (4 Byte) | key_mat_enc_data - * | iv_data_length (4 Byte) | iv_data |key_material_size_len (4 Byte) | key_material_size - */ - HKS_UNWRAP_SUITE_SM2_SM4_128_CBC_PKCS7_WITH_VERIFY_DIG_SM3 = 3, - - /* WrappedData format(Bytes Array): - * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_material_size_len(4 Byte) | kek_material_size - * | factor1_data_len (4 Byte) | factor1_data | factor2_data_len (4 Byte) | factor2_data - * | mac_data_length (4 Byte) | mac_data | key_mat_enc_length (4 Byte) | key_mat_enc_data - * | iv_data_length (4 Byte) | iv_data |key_material_size_len (4 Byte) | key_material_size - */ - HKS_UNWRAP_SUITE_SM2_SM4_128_CBC_PKCS7 = 4, -}; - -/** -* @brief hks key generate type -*/ -enum HksKeyGenerateType { - HKS_KEY_GENERATE_TYPE_DEFAULT = 0, - HKS_KEY_GENERATE_TYPE_DERIVE = 1, - HKS_KEY_GENERATE_TYPE_AGREE = 2, -}; - -/** -* @brief hks key flag -*/ -enum HksKeyFlag { - HKS_KEY_FLAG_IMPORT_KEY = 1, - HKS_KEY_FLAG_GENERATE_KEY = 2, - HKS_KEY_FLAG_AGREE_KEY = 3, - HKS_KEY_FLAG_DERIVE_KEY = 4, -}; - -/** -* @brief hks key storage type -*/ -enum HksKeyStorageType { - HKS_STORAGE_TEMP = 0, - HKS_STORAGE_PERSISTENT = 1, - HKS_STORAGE_ONLY_USED_IN_HUKS = 2, - HKS_STORAGE_ALLOW_KEY_EXPORTED = 3, -}; - -/** -* @brief hks import key type -*/ -enum HksImportKeyType { - HKS_KEY_TYPE_PUBLIC_KEY = 0, - HKS_KEY_TYPE_PRIVATE_KEY = 1, - HKS_KEY_TYPE_KEY_PAIR = 2, -}; - -/** -* @brief hks rsa pss salt len type -*/ -enum HksRsaPssSaltLenType { - HKS_RSA_PSS_SALTLEN_DIGEST = 0, /* Salt length matches digest */ - HKS_RSA_PSS_SALTLEN_MAX = 1, /* Set salt length to maximum possible, default type */ -}; - -/** -* @brief hks error code -*/ -enum HksErrorCode { - HKS_SUCCESS = 0, - HKS_FAILURE = -1, - HKS_ERROR_BAD_STATE = -2, - HKS_ERROR_INVALID_ARGUMENT = -3, - HKS_ERROR_NOT_SUPPORTED = -4, - HKS_ERROR_NO_PERMISSION = -5, - HKS_ERROR_INSUFFICIENT_DATA = -6, - HKS_ERROR_BUFFER_TOO_SMALL = -7, - HKS_ERROR_INSUFFICIENT_MEMORY = -8, - HKS_ERROR_COMMUNICATION_FAILURE = -9, - HKS_ERROR_STORAGE_FAILURE = -10, - HKS_ERROR_HARDWARE_FAILURE = -11, - HKS_ERROR_ALREADY_EXISTS = -12, - HKS_ERROR_NOT_EXIST = -13, - HKS_ERROR_NULL_POINTER = -14, - HKS_ERROR_FILE_SIZE_FAIL = -15, - HKS_ERROR_READ_FILE_FAIL = -16, - HKS_ERROR_INVALID_PUBLIC_KEY = -17, - HKS_ERROR_INVALID_PRIVATE_KEY = -18, - HKS_ERROR_INVALID_KEY_INFO = -19, - HKS_ERROR_HASH_NOT_EQUAL = -20, - HKS_ERROR_MALLOC_FAIL = -21, - HKS_ERROR_WRITE_FILE_FAIL = -22, - HKS_ERROR_REMOVE_FILE_FAIL = -23, - HKS_ERROR_OPEN_FILE_FAIL = -24, - HKS_ERROR_CLOSE_FILE_FAIL = -25, - HKS_ERROR_MAKE_DIR_FAIL = -26, - HKS_ERROR_INVALID_KEY_FILE = -27, - HKS_ERROR_IPC_MSG_FAIL = -28, - HKS_ERROR_REQUEST_OVERFLOWS = -29, - HKS_ERROR_PARAM_NOT_EXIST = -30, - HKS_ERROR_CRYPTO_ENGINE_ERROR = -31, - HKS_ERROR_COMMUNICATION_TIMEOUT = -32, - HKS_ERROR_IPC_INIT_FAIL = -33, - HKS_ERROR_IPC_DLOPEN_FAIL = -34, - HKS_ERROR_EFUSE_READ_FAIL = -35, - HKS_ERROR_NEW_ROOT_KEY_MATERIAL_EXIST = -36, - HKS_ERROR_UPDATE_ROOT_KEY_MATERIAL_FAIL = -37, - HKS_ERROR_VERIFICATION_FAILED = -38, - HKS_ERROR_SESSION_REACHED_LIMIT = -39, - - HKS_ERROR_GET_USERIAM_SECINFO_FAILED = -40, - HKS_ERROR_GET_USERIAM_AUTHINFO_FAILED = -41, - HKS_ERROR_USER_AUTH_TYPE_NOT_SUPPORT = -42, - HKS_ERROR_KEY_AUTH_FAILED = -43, - HKS_ERROR_DEVICE_NO_CREDENTIAL = -44, - HKS_ERROR_API_NOT_SUPPORTED = -45, - HKS_ERROR_KEY_AUTH_PERMANENTLY_INVALIDATED = -46, - HKS_ERROR_KEY_AUTH_VERIFY_FAILED = -47, - HKS_ERROR_KEY_AUTH_TIME_OUT = -48, - - HKS_ERROR_CREDENTIAL_NOT_EXIST = -49, - - HKS_ERROR_CHECK_GET_ALG_FAIL = -100, - HKS_ERROR_CHECK_GET_KEY_SIZE_FAIL = -101, - HKS_ERROR_CHECK_GET_PADDING_FAIL = -102, - HKS_ERROR_CHECK_GET_PURPOSE_FAIL = -103, - HKS_ERROR_CHECK_GET_DIGEST_FAIL = -104, - HKS_ERROR_CHECK_GET_MODE_FAIL = -105, - HKS_ERROR_CHECK_GET_NONCE_FAIL = -106, - HKS_ERROR_CHECK_GET_AAD_FAIL = -107, - HKS_ERROR_CHECK_GET_IV_FAIL = -108, - HKS_ERROR_CHECK_GET_AE_TAG_FAIL = -109, - HKS_ERROR_CHECK_GET_SALT_FAIL = -110, - HKS_ERROR_CHECK_GET_ITERATION_FAIL = -111, - HKS_ERROR_INVALID_ALGORITHM = -112, - HKS_ERROR_INVALID_KEY_SIZE = -113, - HKS_ERROR_INVALID_PADDING = -114, - HKS_ERROR_INVALID_PURPOSE = -115, - HKS_ERROR_INVALID_MODE = -116, - HKS_ERROR_INVALID_DIGEST = -117, - HKS_ERROR_INVALID_SIGNATURE_SIZE = -118, - HKS_ERROR_INVALID_IV = -119, - HKS_ERROR_INVALID_AAD = -120, - HKS_ERROR_INVALID_NONCE = -121, - HKS_ERROR_INVALID_AE_TAG = -122, - HKS_ERROR_INVALID_SALT = -123, - HKS_ERROR_INVALID_ITERATION = -124, - HKS_ERROR_INVALID_OPERATION = -125, - HKS_ERROR_INVALID_WRAPPED_FORMAT = -126, - HKS_ERROR_INVALID_USAGE_OF_KEY = -127, - HKS_ERROR_CHECK_GET_AUTH_TYP_FAILED = -128, - HKS_ERROR_CHECK_GET_CHALLENGE_TYPE_FAILED = -129, - HKS_ERROR_CHECK_GET_ACCESS_TYPE_FAILED = -130, - HKS_ERROR_CHECK_GET_AUTH_TOKEN_FAILED = -131, - HKS_ERROR_INVALID_TIME_OUT = -132, - HKS_ERROR_INVALID_AUTH_TYPE = -133, - HKS_ERROR_INVALID_CHALLENGE_TYPE = -134, - HKS_ERROR_INVALID_ACCESS_TYPE = -135, - HKS_ERROR_INVALID_AUTH_TOKEN = -136, - HKS_ERROR_INVALID_SECURE_SIGN_TYPE = -137, - HKS_ERROR_NEED_SKIP_ACCESS_CONTROL = -138, - HKS_ERROR_DEVICE_PASSWORD_UNSET = -139, - HKS_ERROR_NOT_SYSTEM_APP = -140, - HKS_ERROR_EXCEED_LIMIT = -141, - HKS_ERROR_UPGRADE_FAIL = -142, - - HKS_ERROR_LOAD_PLUGIN_FAIL = -998, - HKS_ERROR_INTERNAL_ERROR = -999, - HKS_ERROR_UNKNOWN_ERROR = -1000, -}; - -/** -* @brief hks err code -*/ -enum HksErrCode { - HUKS_ERR_CODE_PERMISSION_FAIL = 201, - HUKS_ERR_CODE_NOT_SYSTEM_APP = 202, - HUKS_ERR_CODE_ILLEGAL_ARGUMENT = 401, - HUKS_ERR_CODE_NOT_SUPPORTED_API = 801, - - HUKS_ERR_CODE_FEATURE_NOT_SUPPORTED = 12000001, - HUKS_ERR_CODE_MISSING_CRYPTO_ALG_ARGUMENT = 12000002, - HUKS_ERR_CODE_INVALID_CRYPTO_ALG_ARGUMENT = 12000003, - HUKS_ERR_CODE_FILE_OPERATION_FAIL = 12000004, - HUKS_ERR_CODE_COMMUNICATION_FAIL = 12000005, - HUKS_ERR_CODE_CRYPTO_FAIL = 12000006, - HUKS_ERR_CODE_KEY_AUTH_PERMANENTLY_INVALIDATED = 12000007, - HUKS_ERR_CODE_KEY_AUTH_VERIFY_FAILED = 12000008, - HUKS_ERR_CODE_KEY_AUTH_TIME_OUT = 12000009, - HUKS_ERR_CODE_SESSION_LIMIT = 12000010, - HUKS_ERR_CODE_ITEM_NOT_EXIST = 12000011, - HUKS_ERR_CODE_EXTERNAL_ERROR = 12000012, - HUKS_ERR_CODE_CREDENTIAL_NOT_EXIST = 12000013, - HUKS_ERR_CODE_INSUFFICIENT_MEMORY = 12000014, - HUKS_ERR_CODE_CALL_SERVICE_FAILED = 12000015, - HUKS_ERR_CODE_DEVICE_PASSWORD_UNSET = 12000016, -}; - -/** -* @brief hks tag type -*/ -enum HksTagType { - HKS_TAG_TYPE_INVALID = 0 << 28, - HKS_TAG_TYPE_INT = 1 << 28, - HKS_TAG_TYPE_UINT = 2 << 28, - HKS_TAG_TYPE_ULONG = 3 << 28, - HKS_TAG_TYPE_BOOL = 4 << 28, - HKS_TAG_TYPE_BYTES = 5 << 28, -}; - -/** -* @brief hks send type -*/ -enum HksSendType { - HKS_SEND_TYPE_ASYNC = 0, - HKS_SEND_TYPE_SYNC, -}; - -/** -* @brief hks user auth type -* @see `enum AuthType` in `drivers/interface/user_auth/v2_0/UserAuthTypes.idl` -*/ -enum HksUserAuthType { - HKS_USER_AUTH_TYPE_FINGERPRINT = 1 << 0, - HKS_USER_AUTH_TYPE_FACE = 1 << 1, - HKS_USER_AUTH_TYPE_PIN = 1 << 2, -}; - -/** -* @brief hks auth access type -*/ -enum HksAuthAccessType { - HKS_AUTH_ACCESS_INVALID_CLEAR_PASSWORD = 1 << 0, - HKS_AUTH_ACCESS_INVALID_NEW_BIO_ENROLL = 1 << 1, - HKS_AUTH_ACCESS_ALWAYS_VALID = 1 << 2, -}; - -/** -* @brief hks challenge type -*/ -enum HksChallengeType { - HKS_CHALLENGE_TYPE_NORMAL = 0, - HKS_CHALLENGE_TYPE_CUSTOM = 1, - HKS_CHALLENGE_TYPE_NONE = 2, -}; - -/** -* @brief hks challenge position -*/ -enum HksChallengePosition { - HKS_CHALLENGE_POS_0 = 0, - HKS_CHALLENGE_POS_1, - HKS_CHALLENGE_POS_2, - HKS_CHALLENGE_POS_3, -}; - -/** -* @brief hks secure sign type -*/ -enum HksSecureSignType { - HKS_SECURE_SIGN_WITH_AUTHINFO = 1, -}; - -/** -* @brief hks attestation type -*/ -enum HksAttestationMode { - HKS_ATTESTATION_MODE_DEFAULT = 0, - HKS_ATTESTATION_MODE_ANONYMOUS -}; - -/** -* @brief hks attestation Caller Type -*/ -enum HksCallerType { - HKS_HAP_TYPE = 0x1, - HKS_SA_TYPE, - HKS_UNIFIED_TYPE, -}; - -#define HKS_ASSIGN_ENUM_VALUE(x, y) x = y, - -#define HKS_ASSIGN_PARAM_ALG_ENUM \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_INVALID, HKS_TAG_TYPE_INVALID | 0) \ - /* Base algrithom TAG: 1 - 200 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ALGORITHM, HKS_TAG_TYPE_UINT | 1) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PURPOSE, HKS_TAG_TYPE_UINT | 2) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_SIZE, HKS_TAG_TYPE_UINT | 3) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DIGEST, HKS_TAG_TYPE_UINT | 4) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PADDING, HKS_TAG_TYPE_UINT | 5) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BLOCK_MODE, HKS_TAG_TYPE_UINT | 6) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_TYPE, HKS_TAG_TYPE_UINT | 7) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASSOCIATED_DATA, HKS_TAG_TYPE_BYTES | 8) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_NONCE, HKS_TAG_TYPE_BYTES | 9) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IV, HKS_TAG_TYPE_BYTES | 10) \ - /* Key derivation TAG */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_INFO, HKS_TAG_TYPE_BYTES | 11) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SALT, HKS_TAG_TYPE_BYTES | 12) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PWD, HKS_TAG_TYPE_BYTES | 13) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ITERATION, HKS_TAG_TYPE_UINT | 14) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_GENERATE_TYPE, HKS_TAG_TYPE_UINT | 15) /* choose from enum HksKeyGenerateType */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_MAIN_KEY, HKS_TAG_TYPE_BYTES | 16) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_FACTOR, HKS_TAG_TYPE_BYTES | 17) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_ALG, HKS_TAG_TYPE_UINT | 18) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_ALG, HKS_TAG_TYPE_UINT | 19) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PUBLIC_KEY_IS_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 20) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PRIVATE_KEY_ALIAS, HKS_TAG_TYPE_BYTES | 21) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PUBLIC_KEY, HKS_TAG_TYPE_BYTES | 22) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ALIAS, HKS_TAG_TYPE_BYTES | 23) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_KEY_SIZE, HKS_TAG_TYPE_UINT | 24) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IMPORT_KEY_TYPE, HKS_TAG_TYPE_UINT | 25) /* choose from enum HksImportKeyType */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_UNWRAP_ALGORITHM_SUITE, HKS_TAG_TYPE_UINT | 26) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CIPHER_TEXT, HKS_TAG_TYPE_BYTES | 27) \ - /* parameters required by HuksCoreChipsetPlatformDecrypt */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PEER_PUBLIC_KEY, HKS_TAG_TYPE_BYTES | 28) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_AGREE_KEY_STORAGE_FLAG, HKS_TAG_TYPE_UINT | 29) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_RSA_PSS_SALT_LEN_TYPE, HKS_TAG_TYPE_UINT | 30) /* only supported for PSS padding */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_MGF_DIGEST, HKS_TAG_TYPE_UINT | 31) \ - /* Key authentication related TAG: 201 - 300 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACTIVE_DATETIME, HKS_TAG_TYPE_ULONG | 201) \ - /* Date when new "messages" should not be created. */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ORIGINATION_EXPIRE_DATETIME, HKS_TAG_TYPE_ULONG | 202) \ - /* Date when existing "messages" should not be used. */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USAGE_EXPIRE_DATETIME, HKS_TAG_TYPE_ULONG | 203) \ - /* Key creation time */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CREATION_DATETIME, HKS_TAG_TYPE_ULONG | 204) \ - /* Other authentication related TAG: 301 - 500 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ALL_USERS, HKS_TAG_TYPE_BOOL | 301) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_ID, HKS_TAG_TYPE_UINT | 302) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_NO_AUTH_REQUIRED, HKS_TAG_TYPE_BOOL | 303) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_TYPE, HKS_TAG_TYPE_UINT | 304) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_TIMEOUT, HKS_TAG_TYPE_UINT | 305) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_TOKEN, HKS_TAG_TYPE_BYTES | 306) \ - /* Key secure access control and user auth TAG */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_ACCESS_TYPE, HKS_TAG_TYPE_UINT | 307) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_SECURE_SIGN_TYPE, HKS_TAG_TYPE_UINT | 308) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CHALLENGE_TYPE, HKS_TAG_TYPE_UINT | 309) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CHALLENGE_POS, HKS_TAG_TYPE_UINT | 310) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_PURPOSE, HKS_TAG_TYPE_UINT | 311) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_FRONT_USER_ID, HKS_TAG_TYPE_INT | 312) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BATCH_PURPOSE, HKS_TAG_TYPE_UINT | 313) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_BATCH_OPERATION, HKS_TAG_TYPE_BOOL | 314) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BATCH_OPERATION_TIMEOUT, HKS_TAG_TYPE_UINT | 315) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_STORAGE_LEVEL, HKS_TAG_TYPE_UINT | 316) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SPECIFIC_USER_ID, HKS_TAG_TYPE_INT | 317) \ - /* Attestation related TAG: 501 - 600 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_CHALLENGE, HKS_TAG_TYPE_BYTES | 501) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_APPLICATION_ID, HKS_TAG_TYPE_BYTES | 502) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_BRAND, HKS_TAG_TYPE_BYTES | 503) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_DEVICE, HKS_TAG_TYPE_BYTES | 504) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_PRODUCT, HKS_TAG_TYPE_BYTES | 505) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SERIAL, HKS_TAG_TYPE_BYTES | 506) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_IMEI, HKS_TAG_TYPE_BYTES | 507) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MEID, HKS_TAG_TYPE_BYTES | 508) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MANUFACTURER, HKS_TAG_TYPE_BYTES | 509) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MODEL, HKS_TAG_TYPE_BYTES | 510) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_ALIAS, HKS_TAG_TYPE_BYTES | 511) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SOCID, HKS_TAG_TYPE_BYTES | 512) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_UDID, HKS_TAG_TYPE_BYTES | 513) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SEC_LEVEL_INFO, HKS_TAG_TYPE_BYTES | 514) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_VERSION_INFO, HKS_TAG_TYPE_BYTES | 515) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_BASE64, HKS_TAG_TYPE_BOOL | 516) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_MODE, HKS_TAG_TYPE_UINT | 517) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_APPLICATION_ID_TYPE, HKS_TAG_TYPE_UINT | 518) \ - /* Extention TAG: 1001 - 9999 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 1001) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_STORAGE_FLAG, HKS_TAG_TYPE_UINT | 1002) /* choose from enum HksKeyStorageType */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_ALLOWED_WRAP, HKS_TAG_TYPE_BOOL | 1003) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_WRAP_TYPE, HKS_TAG_TYPE_UINT | 1004) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_ID, HKS_TAG_TYPE_BYTES | 1005) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ROLE, HKS_TAG_TYPE_UINT | 1006) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_FLAG, HKS_TAG_TYPE_UINT | 1007) /* choose from enum HksKeyFlag */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_ASYNCHRONIZED, HKS_TAG_TYPE_UINT | 1008) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SECURE_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 1009) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SECURE_KEY_UUID, HKS_TAG_TYPE_BYTES | 1010) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_DOMAIN, HKS_TAG_TYPE_UINT | 1011) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_DEVICE_PASSWORD_SET, HKS_TAG_TYPE_BOOL | 1012) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_EXT_INFO, HKS_TAG_TYPE_BYTES | 1013) \ - /* Inner-use TAG: 10001 - 10999 */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PACKAGE_NAME, HKS_TAG_TYPE_BYTES | 10002) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACCESS_TIME, HKS_TAG_TYPE_UINT | 10003) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USES_TIME, HKS_TAG_TYPE_UINT | 10004) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CRYPTO_CTX, HKS_TAG_TYPE_ULONG | 10005) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PAYLOAD_LEN, HKS_TAG_TYPE_UINT | 10008) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AE_TAG, HKS_TAG_TYPE_BYTES | 10009) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_KEY_HANDLE, HKS_TAG_TYPE_ULONG | 10010) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_INIT_CHALLENGE, HKS_TAG_TYPE_BYTES | 10011) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_USER_AUTH_ACCESS, HKS_TAG_TYPE_BOOL | 10012) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_CHALLENGE, HKS_TAG_TYPE_BYTES | 10013) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_ENROLL_ID_INFO, HKS_TAG_TYPE_BYTES | 10014) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_SECURE_UID, HKS_TAG_TYPE_BYTES | 10015) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_RESULT, HKS_TAG_TYPE_INT | 10016) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IF_NEED_APPEND_AUTH_INFO, HKS_TAG_TYPE_BOOL | 10017) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_VERIFIED_AUTH_TOKEN, HKS_TAG_TYPE_BYTES | 10018) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_APPEND_UPDATE_DATA, HKS_TAG_TYPE_BOOL | 10019) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ACCESS_TIME, HKS_TAG_TYPE_ULONG | 10020) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OWNER_ID, HKS_TAG_TYPE_BYTES | 10021) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OWNER_TYPE, HKS_TAG_TYPE_UINT | 10022) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACCOUNT_ID, HKS_TAG_TYPE_BYTES | 10023) \ - /* TAGs used for paramSetOut */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SYMMETRIC_KEY_DATA, HKS_TAG_TYPE_BYTES | 20001) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASYMMETRIC_PUBLIC_KEY_DATA, HKS_TAG_TYPE_BYTES | 20002) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASYMMETRIC_PRIVATE_KEY_DATA, HKS_TAG_TYPE_BYTES | 20003) - -#define HKS_ASSIGN_PARAM_FILE_ENUM \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PROCESS_NAME, HKS_TAG_TYPE_BYTES | 10001) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY, HKS_TAG_TYPE_BYTES | 10006) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_VERSION, HKS_TAG_TYPE_UINT | 10007) \ - /* Os version related TAG */ \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OS_VERSION, HKS_TAG_TYPE_UINT | 10101) \ - HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OS_PATCHLEVEL, HKS_TAG_TYPE_UINT | 10102) - -/** -* @brief hks Tag -*/ -enum HksTag { - /** - * HUKS tags for alg enum - */ - HKS_ASSIGN_PARAM_ALG_ENUM - - /** - * HUKS tags for key file enum - */ - HKS_ASSIGN_PARAM_FILE_ENUM -}; +#define MAX_ERROR_MESSAGE_LEN 512 /** * @brief hks blob */ struct HksBlob { uint32_t size; - uint8_t* data; + uint8_t *data; }; /** @@ -708,7 +124,7 @@ struct HksParamSet { * @brief hks certificate chain */ struct HksCertChain { - struct HksBlob* certs; + struct HksBlob *certs; uint32_t certsCount; }; @@ -717,7 +133,7 @@ struct HksCertChain { */ struct HksKeyInfo { struct HksBlob alias; - struct HksParamSet* paramSet; + struct HksParamSet *paramSet; }; /** @@ -848,10 +264,6 @@ typedef struct __attribute__((__packed__)) HksSecureSignAuthInfo { uint64_t credentialId; } __attribute__((__packed__)) HksSecureSignAuthInfo; -enum HksUserIamType { - HKS_AUTH_TYPE = 0, -}; - struct EnrolledInfoWrap { enum HksUserAuthType authType; uint64_t enrolledId; @@ -860,7 +272,7 @@ struct EnrolledInfoWrap { struct SecInfoWrap { uint64_t secureUid; uint32_t enrolledInfoLen; - struct EnrolledInfoWrap* enrolledInfo; + struct EnrolledInfoWrap *enrolledInfo; }; /** @@ -868,7 +280,7 @@ struct SecInfoWrap { */ struct HksKeyAliasSet { uint32_t aliasesCnt; - struct HksBlob* aliases; + struct HksBlob *aliases; }; #define HKS_DERIVE_DEFAULT_SALT_LEN 16 @@ -912,6 +324,18 @@ struct HksStoreKeyInfo { uint8_t authIdSize; }; +struct ErrorInfoHead { + int32_t version; + int32_t errorType; + int32_t innerErrCode; + int32_t extErrCode; +}; + +struct ErrorInfo { + struct ErrorInfoHead head; + char erMsg[MAX_ERROR_MESSAGE_LEN + 1]; +}; + /** * @brief hks 25519 key pair */ @@ -930,7 +354,7 @@ static inline bool IsInvalidLength(uint32_t length) return (length == 0) || (length > MAX_OUT_BLOB_SIZE); } -static inline int32_t CheckBlob(const struct HksBlob* blob) +static inline int32_t CheckBlob(const struct HksBlob *blob) { if ((blob == NULL) || (blob->data == NULL) || (blob->size == 0)) { return HKS_ERROR_INVALID_ARGUMENT; @@ -938,22 +362,6 @@ static inline int32_t CheckBlob(const struct HksBlob* blob) return HKS_SUCCESS; } -/** -* @brief hks chipset platform decrypt scene -*/ -enum HksChipsetPlatformDecryptScene { - HKS_CHIPSET_PLATFORM_DECRYPT_SCENE_TA_TO_TA = 1, -}; - -/** -* @brief hks auth storage level -*/ -enum HksAuthStorageLevel { - HKS_AUTH_STORAGE_LEVEL_DE = 0, - HKS_AUTH_STORAGE_LEVEL_CE = 1, - HKS_AUTH_STORAGE_LEVEL_ECE = 2, -}; - #ifdef __cplusplus } #endif diff --git a/mock/innerkits/huks/libhukssdk/include/hks_type_enum.h b/mock/innerkits/huks/libhukssdk/include/hks_type_enum.h new file mode 100644 index 00000000..21f72298 --- /dev/null +++ b/mock/innerkits/huks/libhukssdk/include/hks_type_enum.h @@ -0,0 +1,697 @@ +/* + * Copyright (c) 2021-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file hks_type_enum.h + * + * @brief Declares huks type enum. + * + * @since 8 + */ + +#ifndef HKS_TYPE_ENUM_H +#define HKS_TYPE_ENUM_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief hks key type + */ +enum HksKeyType { + HKS_KEY_TYPE_RSA_PUBLIC_KEY = 0x01001000, + HKS_KEY_TYPE_RSA_KEYPAIR = 0x01002000, + + HKS_KEY_TYPE_ECC_P256_PUBLIC_KEY = 0x02021000, + HKS_KEY_TYPE_ECC_P256_KEYPAIR = 0x02022000, + HKS_KEY_TYPE_ECC_P384_PUBLIC_KEY = 0x02031000, + HKS_KEY_TYPE_ECC_P384_KEYPAIR = 0x02032000, + HKS_KEY_TYPE_ECC_P521_PUBLIC_KEY = 0x02051000, + HKS_KEY_TYPE_ECC_P521_KEYPAIR = 0x02052000, + + HKS_KEY_TYPE_ED25519_PUBLIC_KEY = 0x02101000, + HKS_KEY_TYPE_ED25519_KEYPAIR = 0x02102000, + HKS_KEY_TYPE_X25519_PUBLIC_KEY = 0x02111000, + HKS_KEY_TYPE_X25519_KEYPAIR = 0x02112000, + + HKS_KEY_TYPE_AES = 0x03000000, + HKS_KEY_TYPE_CHACHA20 = 0x04010000, + HKS_KEY_TYPE_CHACHA20_POLY1305 = 0x04020000, + + HKS_KEY_TYPE_HMAC = 0x05000000, + HKS_KEY_TYPE_HKDF = 0x06000000, + HKS_KEY_TYPE_PBKDF2 = 0x07000000, +}; + +/** + * @brief hks key purpose + */ +enum HksKeyPurpose { + HKS_KEY_PURPOSE_ENCRYPT = 1, /* Usable with RSA, EC, AES, SM2, and SM4 keys. */ + HKS_KEY_PURPOSE_DECRYPT = 2, /* Usable with RSA, EC, AES, SM2, and SM4 keys. */ + HKS_KEY_PURPOSE_SIGN = 4, /* Usable with RSA, EC keys. */ + HKS_KEY_PURPOSE_VERIFY = 8, /* Usable with RSA, EC keys. */ + HKS_KEY_PURPOSE_DERIVE = 16, /* Usable with EC keys. */ + HKS_KEY_PURPOSE_WRAP = 32, /* Usable with wrap key. */ + HKS_KEY_PURPOSE_UNWRAP = 64, /* Usable with unwrap key. */ + HKS_KEY_PURPOSE_MAC = 128, /* Usable with mac. */ + HKS_KEY_PURPOSE_AGREE = 256, /* Usable with agree. */ +}; + +/** + * @brief hks key digest + */ +enum HksKeyDigest { + HKS_DIGEST_NONE = 0, + HKS_DIGEST_MD5 = 1, + HKS_DIGEST_SM3 = 2, + HKS_DIGEST_SHA1 = 10, + HKS_DIGEST_SHA224 = 11, + HKS_DIGEST_SHA256 = 12, + HKS_DIGEST_SHA384 = 13, + HKS_DIGEST_SHA512 = 14, +}; + +/** + * @brief hks key padding + */ +enum HksKeyPadding { + HKS_PADDING_NONE = 0, + HKS_PADDING_OAEP = 1, + HKS_PADDING_PSS = 2, + HKS_PADDING_PKCS1_V1_5 = 3, + HKS_PADDING_PKCS5 = 4, + HKS_PADDING_PKCS7 = 5, + HKS_PADDING_ISO_IEC_9796_2 = 6, + HKS_PADDING_ISO_IEC_9797_1 = 7, +}; + +/** + * @brief hks cipher mode + */ +enum HksCipherMode { + HKS_MODE_ECB = 1, + HKS_MODE_CBC = 2, + HKS_MODE_CTR = 3, + HKS_MODE_OFB = 4, + HKS_MODE_CFB = 5, + HKS_MODE_CCM = 31, + HKS_MODE_GCM = 32, +}; + +/** + * @brief hks key size + */ +enum HksKeySize { + HKS_RSA_KEY_SIZE_512 = 512, + HKS_RSA_KEY_SIZE_768 = 768, + HKS_RSA_KEY_SIZE_1024 = 1024, + HKS_RSA_KEY_SIZE_2048 = 2048, + HKS_RSA_KEY_SIZE_3072 = 3072, + HKS_RSA_KEY_SIZE_4096 = 4096, + + HKS_ECC_KEY_SIZE_224 = 224, + HKS_ECC_KEY_SIZE_256 = 256, + HKS_ECC_KEY_SIZE_384 = 384, + HKS_ECC_KEY_SIZE_521 = 521, + + HKS_AES_KEY_SIZE_128 = 128, + HKS_AES_KEY_SIZE_192 = 192, + HKS_AES_KEY_SIZE_256 = 256, + HKS_AES_KEY_SIZE_512 = 512, + + HKS_CURVE25519_KEY_SIZE_256 = 256, + + HKS_DH_KEY_SIZE_2048 = 2048, + HKS_DH_KEY_SIZE_3072 = 3072, + HKS_DH_KEY_SIZE_4096 = 4096, + + HKS_SM2_KEY_SIZE_256 = 256, + HKS_SM4_KEY_SIZE_128 = 128, + + HKS_DES_KEY_SIZE_64 = 64, + HKS_3DES_KEY_SIZE_128 = 128, + HKS_3DES_KEY_SIZE_192 = 192, +}; + +/** + * @brief hks key algorithm + */ +enum HksKeyAlg { + HKS_ALG_RSA = 1, + HKS_ALG_ECC = 2, + HKS_ALG_DSA = 3, + + HKS_ALG_AES = 20, + HKS_ALG_HMAC = 50, + HKS_ALG_HKDF = 51, + HKS_ALG_PBKDF2 = 52, + HKS_ALG_GMKDF = 53, + + HKS_ALG_ECDH = 100, + HKS_ALG_X25519 = 101, + HKS_ALG_ED25519 = 102, + HKS_ALG_DH = 103, + + HKS_ALG_SM2 = 150, + HKS_ALG_SM3 = 151, + HKS_ALG_SM4 = 152, + + HKS_ALG_DES = 160, + HKS_ALG_3DES = 161, + HKS_ALG_CMAC = 162, +}; + +/** + * @brief hks algorithm suite + */ +enum HuksAlgSuite { + /* Algorithm suites of unwrapping wrapped-key by huks */ + /* Unwrap suite of key agreement type */ + /* WrappedData format(Bytes Array): + * | x25519_plain_pubkey_length (4 Byte) | x25519_plain_pubkey | agreekey_aad_length (4 Byte) | agreekey_aad + * | agreekey_nonce_length (4 Byte) | agreekey_nonce | agreekey_aead_tag_len(4 Byte) | agreekey_aead_tag + * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_aad_length (4 Byte) | kek_aad + * | kek_nonce_length (4 Byte) | kek_nonce | kek_aead_tag_len (4 Byte) | kek_aead_tag + * | key_material_size_len (4 Byte) | key_material_size | key_mat_enc_length (4 Byte) | key_mat_enc_data + */ + HKS_UNWRAP_SUITE_X25519_AES_256_GCM_NOPADDING = 1, + + /* WrappedData format(Bytes Array): + * | ECC_plain_pubkey_length (4 Byte) | ECC_plain_pubkey | agreekey_aad_length (4 Byte) | agreekey_aad + * | agreekey_nonce_length (4 Byte) | agreekey_nonce | agreekey_aead_tag_len(4 Byte) | agreekey_aead_tag + * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_aad_length (4 Byte) | kek_aad + * | kek_nonce_length (4 Byte) | kek_nonce | kek_aead_tag_len (4 Byte) | kek_aead_tag + * | key_material_size_len (4 Byte) | key_material_size | key_mat_enc_length (4 Byte) | key_mat_enc_data + */ + HKS_UNWRAP_SUITE_ECDH_AES_256_GCM_NOPADDING = 2, + + /* WrappedData format(Bytes Array): + * | SM2_plain_pubkey_length (4 Byte) | SM2_plain_pubkey | signData_size_length (4 Byte) | signData_size + * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_material_size_len(4 Byte) | kek_material_size + * | factor1_data_len (4 Byte) | factor1_data | factor2_data_len (4 Byte) | factor2_data + * | mac_data_length (4 Byte) | mac_data | key_mat_enc_length (4 Byte) | key_mat_enc_data + * | iv_data_length (4 Byte) | iv_data |key_material_size_len (4 Byte) | key_material_size + */ + HKS_UNWRAP_SUITE_SM2_SM4_128_CBC_PKCS7_WITH_VERIFY_DIG_SM3 = 3, + + /* WrappedData format(Bytes Array): + * | kek_enc_data_length (4 Byte) | kek_enc_data | kek_material_size_len(4 Byte) | kek_material_size + * | factor1_data_len (4 Byte) | factor1_data | factor2_data_len (4 Byte) | factor2_data + * | mac_data_length (4 Byte) | mac_data | key_mat_enc_length (4 Byte) | key_mat_enc_data + * | iv_data_length (4 Byte) | iv_data |key_material_size_len (4 Byte) | key_material_size + */ + HKS_UNWRAP_SUITE_SM2_SM4_128_CBC_PKCS7 = 4, +}; + +/** + * @brief hks key generate type + */ +enum HksKeyGenerateType { + HKS_KEY_GENERATE_TYPE_DEFAULT = 0, + HKS_KEY_GENERATE_TYPE_DERIVE = 1, + HKS_KEY_GENERATE_TYPE_AGREE = 2, +}; + +/** + * @brief hks key flag + */ +enum HksKeyFlag { + HKS_KEY_FLAG_IMPORT_KEY = 1, + HKS_KEY_FLAG_GENERATE_KEY = 2, + HKS_KEY_FLAG_AGREE_KEY = 3, + HKS_KEY_FLAG_DERIVE_KEY = 4, +}; + +/** + * @brief hks key storage type + */ +enum HksKeyStorageType { + HKS_STORAGE_TEMP = 0, + HKS_STORAGE_PERSISTENT = 1, + HKS_STORAGE_ONLY_USED_IN_HUKS = 2, + HKS_STORAGE_ALLOW_KEY_EXPORTED = 3, +}; + +/** + * @brief hks import key type + */ +enum HksImportKeyType { + HKS_KEY_TYPE_PUBLIC_KEY = 0, + HKS_KEY_TYPE_PRIVATE_KEY = 1, + HKS_KEY_TYPE_KEY_PAIR = 2, +}; + +/** + * @brief hks rsa pss salt len type + */ +enum HksRsaPssSaltLenType { + HKS_RSA_PSS_SALTLEN_DIGEST = 0, /* Salt length matches digest */ + HKS_RSA_PSS_SALTLEN_MAX = 1, /* Set salt length to maximum possible, default type */ +}; + +/** + * @brief hks error code + */ +enum HksErrorCode { + HKS_SUCCESS = 0, + HKS_FAILURE = -1, + HKS_ERROR_BAD_STATE = -2, + HKS_ERROR_INVALID_ARGUMENT = -3, + HKS_ERROR_NOT_SUPPORTED = -4, + HKS_ERROR_NO_PERMISSION = -5, + HKS_ERROR_INSUFFICIENT_DATA = -6, + HKS_ERROR_BUFFER_TOO_SMALL = -7, + HKS_ERROR_INSUFFICIENT_MEMORY = -8, + HKS_ERROR_COMMUNICATION_FAILURE = -9, + HKS_ERROR_STORAGE_FAILURE = -10, + HKS_ERROR_HARDWARE_FAILURE = -11, + HKS_ERROR_ALREADY_EXISTS = -12, + HKS_ERROR_NOT_EXIST = -13, + HKS_ERROR_NULL_POINTER = -14, + HKS_ERROR_FILE_SIZE_FAIL = -15, + HKS_ERROR_READ_FILE_FAIL = -16, + HKS_ERROR_INVALID_PUBLIC_KEY = -17, + HKS_ERROR_INVALID_PRIVATE_KEY = -18, + HKS_ERROR_INVALID_KEY_INFO = -19, + HKS_ERROR_HASH_NOT_EQUAL = -20, + HKS_ERROR_MALLOC_FAIL = -21, + HKS_ERROR_WRITE_FILE_FAIL = -22, + HKS_ERROR_REMOVE_FILE_FAIL = -23, + HKS_ERROR_OPEN_FILE_FAIL = -24, + HKS_ERROR_CLOSE_FILE_FAIL = -25, + HKS_ERROR_MAKE_DIR_FAIL = -26, + HKS_ERROR_INVALID_KEY_FILE = -27, + HKS_ERROR_IPC_MSG_FAIL = -28, + HKS_ERROR_REQUEST_OVERFLOWS = -29, + HKS_ERROR_PARAM_NOT_EXIST = -30, + HKS_ERROR_CRYPTO_ENGINE_ERROR = -31, + HKS_ERROR_COMMUNICATION_TIMEOUT = -32, + HKS_ERROR_IPC_INIT_FAIL = -33, + HKS_ERROR_IPC_DLOPEN_FAIL = -34, + HKS_ERROR_EFUSE_READ_FAIL = -35, + HKS_ERROR_NEW_ROOT_KEY_MATERIAL_EXIST = -36, + HKS_ERROR_UPDATE_ROOT_KEY_MATERIAL_FAIL = -37, + HKS_ERROR_VERIFICATION_FAILED = -38, + HKS_ERROR_SESSION_REACHED_LIMIT = -39, + + HKS_ERROR_GET_USERIAM_SECINFO_FAILED = -40, + HKS_ERROR_GET_USERIAM_AUTHINFO_FAILED = -41, + HKS_ERROR_USER_AUTH_TYPE_NOT_SUPPORT = -42, + HKS_ERROR_KEY_AUTH_FAILED = -43, + HKS_ERROR_DEVICE_NO_CREDENTIAL = -44, + HKS_ERROR_API_NOT_SUPPORTED = -45, + HKS_ERROR_KEY_AUTH_PERMANENTLY_INVALIDATED = -46, + HKS_ERROR_KEY_AUTH_VERIFY_FAILED = -47, + HKS_ERROR_KEY_AUTH_TIME_OUT = -48, + + HKS_ERROR_CREDENTIAL_NOT_EXIST = -49, + + HKS_ERROR_CHECK_GET_ALG_FAIL = -100, + HKS_ERROR_CHECK_GET_KEY_SIZE_FAIL = -101, + HKS_ERROR_CHECK_GET_PADDING_FAIL = -102, + HKS_ERROR_CHECK_GET_PURPOSE_FAIL = -103, + HKS_ERROR_CHECK_GET_DIGEST_FAIL = -104, + HKS_ERROR_CHECK_GET_MODE_FAIL = -105, + HKS_ERROR_CHECK_GET_NONCE_FAIL = -106, + HKS_ERROR_CHECK_GET_AAD_FAIL = -107, + HKS_ERROR_CHECK_GET_IV_FAIL = -108, + HKS_ERROR_CHECK_GET_AE_TAG_FAIL = -109, + HKS_ERROR_CHECK_GET_SALT_FAIL = -110, + HKS_ERROR_CHECK_GET_ITERATION_FAIL = -111, + HKS_ERROR_INVALID_ALGORITHM = -112, + HKS_ERROR_INVALID_KEY_SIZE = -113, + HKS_ERROR_INVALID_PADDING = -114, + HKS_ERROR_INVALID_PURPOSE = -115, + HKS_ERROR_INVALID_MODE = -116, + HKS_ERROR_INVALID_DIGEST = -117, + HKS_ERROR_INVALID_SIGNATURE_SIZE = -118, + HKS_ERROR_INVALID_IV = -119, + HKS_ERROR_INVALID_AAD = -120, + HKS_ERROR_INVALID_NONCE = -121, + HKS_ERROR_INVALID_AE_TAG = -122, + HKS_ERROR_INVALID_SALT = -123, + HKS_ERROR_INVALID_ITERATION = -124, + HKS_ERROR_INVALID_OPERATION = -125, + HKS_ERROR_INVALID_WRAPPED_FORMAT = -126, + HKS_ERROR_INVALID_USAGE_OF_KEY = -127, + HKS_ERROR_CHECK_GET_AUTH_TYP_FAILED = -128, + HKS_ERROR_CHECK_GET_CHALLENGE_TYPE_FAILED = -129, + HKS_ERROR_CHECK_GET_ACCESS_TYPE_FAILED = -130, + HKS_ERROR_CHECK_GET_AUTH_TOKEN_FAILED = -131, + HKS_ERROR_INVALID_TIME_OUT = -132, + HKS_ERROR_INVALID_AUTH_TYPE = -133, + HKS_ERROR_INVALID_CHALLENGE_TYPE = -134, + HKS_ERROR_INVALID_ACCESS_TYPE = -135, + HKS_ERROR_INVALID_AUTH_TOKEN = -136, + HKS_ERROR_INVALID_SECURE_SIGN_TYPE = -137, + HKS_ERROR_NEED_SKIP_ACCESS_CONTROL = -138, + HKS_ERROR_DEVICE_PASSWORD_UNSET = -139, + HKS_ERROR_NOT_SYSTEM_APP = -140, + HKS_ERROR_EXCEED_LIMIT = -141, + HKS_ERROR_UPGRADE_FAIL = -142, + HKS_ERROR_INVALID_ACCOUNT_INFO = -143, + HKS_ERROR_CORRUPT_FILE = -144, + HKS_ERROR_KEY_NODE_NOT_FOUND = -145, + HKS_ERROR_KEY_NODE_IN_USE = -146, + HKS_ERROR_RETRYABLE_ERROR = -147, + HKS_ERROR_TRUST_RING_DECRYPT_FAILED = -148, + HKS_ERROR_KEY_CLEAR_FAILED = -149, + HKS_ERROR_KEY_CONFLICT = -150, + HKS_ERROR_EMPTY_BASE64_STRING = -151, + HKS_ERROR_ACCESS_OTHER_USER_KEY = -152, + + HKS_ERROR_LOAD_PLUGIN_FAIL = -998, + HKS_ERROR_INTERNAL_ERROR = -999, + HKS_ERROR_UNKNOWN_ERROR = -1000, +}; + +/** + * @brief hks err code + */ +enum HksErrCode { + HUKS_ERR_CODE_PERMISSION_FAIL = 201, + HUKS_ERR_CODE_NOT_SYSTEM_APP = 202, + HUKS_ERR_CODE_ILLEGAL_ARGUMENT = 401, + HUKS_ERR_CODE_NOT_SUPPORTED_API = 801, + + HUKS_ERR_CODE_FEATURE_NOT_SUPPORTED = 12000001, + HUKS_ERR_CODE_MISSING_CRYPTO_ALG_ARGUMENT = 12000002, + HUKS_ERR_CODE_INVALID_CRYPTO_ALG_ARGUMENT = 12000003, + HUKS_ERR_CODE_FILE_OPERATION_FAIL = 12000004, + HUKS_ERR_CODE_COMMUNICATION_FAIL = 12000005, + HUKS_ERR_CODE_CRYPTO_FAIL = 12000006, + HUKS_ERR_CODE_KEY_AUTH_PERMANENTLY_INVALIDATED = 12000007, + HUKS_ERR_CODE_KEY_AUTH_VERIFY_FAILED = 12000008, + HUKS_ERR_CODE_KEY_AUTH_TIME_OUT = 12000009, + HUKS_ERR_CODE_SESSION_LIMIT = 12000010, + HUKS_ERR_CODE_ITEM_NOT_EXIST = 12000011, + HUKS_ERR_CODE_EXTERNAL_ERROR = 12000012, + HUKS_ERR_CODE_CREDENTIAL_NOT_EXIST = 12000013, + HUKS_ERR_CODE_INSUFFICIENT_MEMORY = 12000014, + HUKS_ERR_CODE_CALL_SERVICE_FAILED = 12000015, + HUKS_ERR_CODE_DEVICE_PASSWORD_UNSET = 12000016, +}; + +/** + * @brief hks tag type + */ +enum HksTagType { + HKS_TAG_TYPE_INVALID = 0 << 28, + HKS_TAG_TYPE_INT = 1 << 28, + HKS_TAG_TYPE_UINT = 2 << 28, + HKS_TAG_TYPE_ULONG = 3 << 28, + HKS_TAG_TYPE_BOOL = 4 << 28, + HKS_TAG_TYPE_BYTES = 5 << 28, +}; + +/** + * @brief hks send type + */ +enum HksSendType { + HKS_SEND_TYPE_ASYNC = 0, + HKS_SEND_TYPE_SYNC, +}; + +/** + * @brief hks user auth type + * @see `enum AuthType` in `drivers/interface/user_auth/v3_0/UserAuthTypes.idl` + */ +enum HksUserAuthType { + HKS_USER_AUTH_TYPE_FINGERPRINT = 1 << 0, + HKS_USER_AUTH_TYPE_FACE = 1 << 1, + HKS_USER_AUTH_TYPE_PIN = 1 << 2, +}; + +/** + * @brief hks auth access type + */ +enum HksAuthAccessType { + HKS_AUTH_ACCESS_INVALID_CLEAR_PASSWORD = 1 << 0, + HKS_AUTH_ACCESS_INVALID_NEW_BIO_ENROLL = 1 << 1, + HKS_AUTH_ACCESS_ALWAYS_VALID = 1 << 2, +}; + +/** + * @brief hks challenge type + */ +enum HksChallengeType { + HKS_CHALLENGE_TYPE_NORMAL = 0, + HKS_CHALLENGE_TYPE_CUSTOM = 1, + HKS_CHALLENGE_TYPE_NONE = 2, +}; + +/** + * @brief hks auth mode + */ +enum HksUserAuthMode { + HKS_USER_AUTH_MODE_LOCAL = 0, + HKS_USER_AUTH_MODE_COAUTH = 1, +}; + +/** + * @brief hks challenge position + */ +enum HksChallengePosition { + HKS_CHALLENGE_POS_0 = 0, + HKS_CHALLENGE_POS_1, + HKS_CHALLENGE_POS_2, + HKS_CHALLENGE_POS_3, +}; + +/** + * @brief hks secure sign type + */ +enum HksSecureSignType { + HKS_SECURE_SIGN_WITH_AUTHINFO = 1, +}; + +/** + * @brief hks attestation type + */ +enum HksAttestationMode { + HKS_ATTESTATION_MODE_DEFAULT = 0, + HKS_ATTESTATION_MODE_ANONYMOUS +}; + +/** + * @brief hks attestation cert type + */ +enum HksAttestationCertType { + HKS_ATTESTATION_CERT_TYPE_PROVISION = 0, + HKS_ATTESTATION_CERT_TYPE_HARDWARE_BOUND = 1, + HKS_ATTESTATION_CERT_TYPE_RSA = 2, +}; + +/** + * @brief hks attestation Caller Type + */ +enum HksCallerType { + HKS_HAP_TYPE = 0x1, + HKS_SA_TYPE, + HKS_UNIFIED_TYPE, +}; + +#define HKS_ASSIGN_ENUM_VALUE(x, y) x = y, + +#define HKS_ASSIGN_PARAM_ALG_ENUM \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_INVALID, HKS_TAG_TYPE_INVALID | 0) \ + /* Base algrithom TAG: 1 - 200 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ALGORITHM, HKS_TAG_TYPE_UINT | 1) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PURPOSE, HKS_TAG_TYPE_UINT | 2) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_SIZE, HKS_TAG_TYPE_UINT | 3) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DIGEST, HKS_TAG_TYPE_UINT | 4) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PADDING, HKS_TAG_TYPE_UINT | 5) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BLOCK_MODE, HKS_TAG_TYPE_UINT | 6) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_TYPE, HKS_TAG_TYPE_UINT | 7) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASSOCIATED_DATA, HKS_TAG_TYPE_BYTES | 8) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_NONCE, HKS_TAG_TYPE_BYTES | 9) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IV, HKS_TAG_TYPE_BYTES | 10) \ + /* Key derivation TAG */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_INFO, HKS_TAG_TYPE_BYTES | 11) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SALT, HKS_TAG_TYPE_BYTES | 12) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PWD, HKS_TAG_TYPE_BYTES | 13) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ITERATION, HKS_TAG_TYPE_UINT | 14) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_GENERATE_TYPE, HKS_TAG_TYPE_UINT | 15) /* choose from enum HksKeyGenerateType */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_MAIN_KEY, HKS_TAG_TYPE_BYTES | 16) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_FACTOR, HKS_TAG_TYPE_BYTES | 17) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_ALG, HKS_TAG_TYPE_UINT | 18) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_ALG, HKS_TAG_TYPE_UINT | 19) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PUBLIC_KEY_IS_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 20) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PRIVATE_KEY_ALIAS, HKS_TAG_TYPE_BYTES | 21) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PUBLIC_KEY, HKS_TAG_TYPE_BYTES | 22) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ALIAS, HKS_TAG_TYPE_BYTES | 23) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_KEY_SIZE, HKS_TAG_TYPE_UINT | 24) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IMPORT_KEY_TYPE, HKS_TAG_TYPE_UINT | 25) /* choose from enum HksImportKeyType */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_UNWRAP_ALGORITHM_SUITE, HKS_TAG_TYPE_UINT | 26) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CIPHER_TEXT, HKS_TAG_TYPE_BYTES | 27) \ + /* parameters required by HuksCoreChipsetPlatformDecrypt */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PEER_PUBLIC_KEY, HKS_TAG_TYPE_BYTES | 28) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_AGREE_KEY_STORAGE_FLAG, HKS_TAG_TYPE_UINT | 29) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_RSA_PSS_SALT_LEN_TYPE, HKS_TAG_TYPE_UINT | 30) /* only supported for PSS padding */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_MGF_DIGEST, HKS_TAG_TYPE_UINT | 31) \ + /* Key authentication related TAG: 201 - 300 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACTIVE_DATETIME, HKS_TAG_TYPE_ULONG | 201) \ + /* Date when new "messages" should not be created. */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ORIGINATION_EXPIRE_DATETIME, HKS_TAG_TYPE_ULONG | 202) \ + /* Date when existing "messages" should not be used. */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USAGE_EXPIRE_DATETIME, HKS_TAG_TYPE_ULONG | 203) \ + /* Key creation time */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CREATION_DATETIME, HKS_TAG_TYPE_ULONG | 204) \ + /* Other authentication related TAG: 301 - 500 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ALL_USERS, HKS_TAG_TYPE_BOOL | 301) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_ID, HKS_TAG_TYPE_UINT | 302) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_NO_AUTH_REQUIRED, HKS_TAG_TYPE_BOOL | 303) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_TYPE, HKS_TAG_TYPE_UINT | 304) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_TIMEOUT, HKS_TAG_TYPE_UINT | 305) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_TOKEN, HKS_TAG_TYPE_BYTES | 306) \ + /* Key secure access control and user auth TAG */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_ACCESS_TYPE, HKS_TAG_TYPE_UINT | 307) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_SECURE_SIGN_TYPE, HKS_TAG_TYPE_UINT | 308) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CHALLENGE_TYPE, HKS_TAG_TYPE_UINT | 309) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CHALLENGE_POS, HKS_TAG_TYPE_UINT | 310) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_PURPOSE, HKS_TAG_TYPE_UINT | 311) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_FRONT_USER_ID, HKS_TAG_TYPE_INT | 312) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BATCH_PURPOSE, HKS_TAG_TYPE_UINT | 313) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_BATCH_OPERATION, HKS_TAG_TYPE_BOOL | 314) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_BATCH_OPERATION_TIMEOUT, HKS_TAG_TYPE_UINT | 315) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AUTH_STORAGE_LEVEL, HKS_TAG_TYPE_UINT | 316) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SPECIFIC_USER_ID, HKS_TAG_TYPE_INT | 317) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_MODE, HKS_TAG_TYPE_UINT | 319) \ + /* Attestation related TAG: 501 - 600 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_CHALLENGE, HKS_TAG_TYPE_BYTES | 501) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_APPLICATION_ID, HKS_TAG_TYPE_BYTES | 502) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_BRAND, HKS_TAG_TYPE_BYTES | 503) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_DEVICE, HKS_TAG_TYPE_BYTES | 504) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_PRODUCT, HKS_TAG_TYPE_BYTES | 505) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SERIAL, HKS_TAG_TYPE_BYTES | 506) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_IMEI, HKS_TAG_TYPE_BYTES | 507) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MEID, HKS_TAG_TYPE_BYTES | 508) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MANUFACTURER, HKS_TAG_TYPE_BYTES | 509) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_MODEL, HKS_TAG_TYPE_BYTES | 510) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_ALIAS, HKS_TAG_TYPE_BYTES | 511) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SOCID, HKS_TAG_TYPE_BYTES | 512) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_UDID, HKS_TAG_TYPE_BYTES | 513) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_SEC_LEVEL_INFO, HKS_TAG_TYPE_BYTES | 514) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_ID_VERSION_INFO, HKS_TAG_TYPE_BYTES | 515) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_BASE64, HKS_TAG_TYPE_BOOL | 516) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_MODE, HKS_TAG_TYPE_UINT | 517) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_APPLICATION_ID_TYPE, HKS_TAG_TYPE_UINT | 518) \ + /* Extention TAG: 1001 - 9999 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 1001) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_STORAGE_FLAG, HKS_TAG_TYPE_UINT | 1002) /* choose from enum HksKeyStorageType */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_ALLOWED_WRAP, HKS_TAG_TYPE_BOOL | 1003) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_WRAP_TYPE, HKS_TAG_TYPE_UINT | 1004) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_ID, HKS_TAG_TYPE_BYTES | 1005) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ROLE, HKS_TAG_TYPE_UINT | 1006) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_FLAG, HKS_TAG_TYPE_UINT | 1007) /* choose from enum HksKeyFlag */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_ASYNCHRONIZED, HKS_TAG_TYPE_UINT | 1008) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SECURE_KEY_ALIAS, HKS_TAG_TYPE_BOOL | 1009) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SECURE_KEY_UUID, HKS_TAG_TYPE_BYTES | 1010) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_DOMAIN, HKS_TAG_TYPE_UINT | 1011) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_DEVICE_PASSWORD_SET, HKS_TAG_TYPE_BOOL | 1012) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_EXT_INFO, HKS_TAG_TYPE_BYTES | 1013) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DATA_WRAP_TYPE, HKS_TAG_TYPE_UINT | 1014) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_ALLOWED_DATA_WRAP, HKS_TAG_TYPE_BOOL | 1015) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_WRAP_KEY_VERSION, HKS_TAG_TYPE_UINT | 1016) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AGREE_PUBKEY_TYPE, HKS_TAG_TYPE_UINT | 1017) \ + /* Inner-use TAG: 10001 - 10999 */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PACKAGE_NAME, HKS_TAG_TYPE_BYTES | 10002) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACCESS_TIME, HKS_TAG_TYPE_UINT | 10003) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USES_TIME, HKS_TAG_TYPE_UINT | 10004) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_CRYPTO_CTX, HKS_TAG_TYPE_ULONG | 10005) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PAYLOAD_LEN, HKS_TAG_TYPE_UINT | 10008) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_AE_TAG, HKS_TAG_TYPE_BYTES | 10009) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_KEY_HANDLE, HKS_TAG_TYPE_ULONG | 10010) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_INIT_CHALLENGE, HKS_TAG_TYPE_BYTES | 10011) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_USER_AUTH_ACCESS, HKS_TAG_TYPE_BOOL | 10012) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_CHALLENGE, HKS_TAG_TYPE_BYTES | 10013) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_ENROLL_ID_INFO, HKS_TAG_TYPE_BYTES | 10014) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_USER_AUTH_SECURE_UID, HKS_TAG_TYPE_BYTES | 10015) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_AUTH_RESULT, HKS_TAG_TYPE_INT | 10016) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IF_NEED_APPEND_AUTH_INFO, HKS_TAG_TYPE_BOOL | 10017) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_VERIFIED_AUTH_TOKEN, HKS_TAG_TYPE_BYTES | 10018) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_APPEND_UPDATE_DATA, HKS_TAG_TYPE_BOOL | 10019) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_ACCESS_TIME, HKS_TAG_TYPE_ULONG | 10020) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OWNER_ID, HKS_TAG_TYPE_BYTES | 10021) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OWNER_TYPE, HKS_TAG_TYPE_UINT | 10022) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ACCOUNT_ID, HKS_TAG_TYPE_BYTES | 10023) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_CHANGE_STORAGE_LEVEL, HKS_TAG_TYPE_BOOL | 10024) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_IS_COPY_NEW_KEY, HKS_TAG_TYPE_BOOL | 10025) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_DERIVE_MAIN_KEY_MODE, HKS_TAG_TYPE_UINT | 10026) \ + /* TAGs used for paramSetOut */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_SYMMETRIC_KEY_DATA, HKS_TAG_TYPE_BYTES | 20001) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASYMMETRIC_PUBLIC_KEY_DATA, HKS_TAG_TYPE_BYTES | 20002) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ASYMMETRIC_PRIVATE_KEY_DATA, HKS_TAG_TYPE_BYTES | 20003) \ + /* TAGs used for huksExt */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_ATTESTATION_CERT_TYPE, HKS_TAG_TYPE_UINT | 100001) + +#define HKS_ASSIGN_PARAM_FILE_ENUM \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_PROCESS_NAME, HKS_TAG_TYPE_BYTES | 10001) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY, HKS_TAG_TYPE_BYTES | 10006) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_KEY_VERSION, HKS_TAG_TYPE_UINT | 10007) \ + /* Os version related TAG */ \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OS_VERSION, HKS_TAG_TYPE_UINT | 10101) \ + HKS_ASSIGN_ENUM_VALUE(HKS_TAG_OS_PATCHLEVEL, HKS_TAG_TYPE_UINT | 10102) + +/** + * @brief hks Tag + */ +enum HksTag { + /** + * HUKS tags for alg enum + */ + HKS_ASSIGN_PARAM_ALG_ENUM + + /** + * HUKS tags for key file enum + */ + HKS_ASSIGN_PARAM_FILE_ENUM +}; + +enum HksUserIamType { + HKS_AUTH_TYPE = 0, +}; + +/** + * @brief hks chipset platform decrypt scene + */ +enum HksChipsetPlatformDecryptScene { + HKS_CHIPSET_PLATFORM_DECRYPT_SCENE_TA_TO_TA = 1, +}; + +/** + * @brief hks auth storage level + */ +enum HksAuthStorageLevel { + HKS_AUTH_STORAGE_LEVEL_DE = 0, + HKS_AUTH_STORAGE_LEVEL_CE = 1, + HKS_AUTH_STORAGE_LEVEL_ECE = 2, +}; + +enum HksAgreePubKeyType { + HKS_PUBKEY_DEFAULT = 0 +}; + +#ifdef __cplusplus +} +#endif + +#endif /* HKS_TYPE_ENUM_H */ diff --git a/mock/innerkits/resource_management/global_resmgr/include/resource_manager.h b/mock/innerkits/resource_management/global_resmgr/include/resource_manager.h index 24535467..b3644405 100644 --- a/mock/innerkits/resource_management/global_resmgr/include/resource_manager.h +++ b/mock/innerkits/resource_management/global_resmgr/include/resource_manager.h @@ -1,41 +1,50 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Copyright (c) 2021-2023 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ #ifndef OHOS_RESOURCE_MANAGER_RESOURCEMANAGER_H #define OHOS_RESOURCE_MANAGER_RESOURCEMANAGER_H #include #include #include +#include #include + #include "res_config.h" namespace OHOS { namespace Global { namespace Resource { +enum FunctionType { + SYNC = 0, + ASYNC = 1 +}; class ResourceManager { public: + std::pair bundleInfo; + + int32_t userId = 0; typedef struct { /** the raw file fd */ int fd; /** the offset from where the raw file starts in the HAP */ - long offset; + int64_t offset; /** the length of the raw file in the HAP. */ - long length; + int64_t length; } RawFileDescriptor; struct Resource { @@ -46,17 +55,58 @@ public: std::string moduleName; /** the resource id in hap */ - int32_t id; + uint32_t id; }; - virtual ~ResourceManager() = 0; + struct ResData { + /** the resource type */ + ResType resType; - virtual bool AddResource(const char *path) = 0; + /** the resource value */ + std::string value; + }; - virtual RState UpdateResConfig(ResConfig &resConfig) = 0; + struct Quantity { + /** the indication of whether quantity is an integer. */ + bool isInteger = false; + + /** the integer representation of the quantity value. */ + int intValue = 0; + + /** the double representation of the quantity value. */ + double doubleValue = 0.0; + }; + + enum class NapiValueType { + NAPI_NUMBER = 0, + NAPI_STRING = 1 + }; + + virtual ~ResourceManager() = 0; + + /** + * Add resource of hap. + * + * @param path The path of hap. + * @param selectedTypes If this param is setted, will only add resource of types specified by this param, + * it will be faster than add all resource, you can set this param by combining flags + * named SELECT_XXX defined in res_common.h. What's more, if you call UpdateResConfig() + * before calling this method, will only add resource that matching the config, for example, + * only add resource of current language, which means it will be further faster. + * @return true if init success, else false + */ + virtual bool AddResource(const char *path, const uint32_t &selectedTypes = 0, + bool forceReload = false) = 0; + + virtual RState UpdateResConfig(ResConfig &resConfig, bool isUpdateTheme = false) = 0; virtual void GetResConfig(ResConfig &resConfig) = 0; + virtual RState GetResConfigById(uint32_t resId, ResConfig &resConfig, uint32_t density = 0) = 0; + + virtual RState GetResConfigByName(const std::string &name, const ResType type, ResConfig &resConfig, + uint32_t density = 0) = 0; + virtual RState GetStringById(uint32_t id, std::string &outValue) = 0; virtual RState GetStringByName(const char *name, std::string &outValue) = 0; @@ -113,24 +163,174 @@ public: virtual RState GetProfileByName(const char *name, std::string &outValue) = 0; - virtual RState GetProfileDataByName(const char *name, size_t len, std::unique_ptr &outValue) = 0; - - virtual RState GetMediaById(uint32_t id, std::string &outValue) = 0; - - virtual RState GetMediaById(uint32_t id, uint32_t density, std::string &outValue) = 0; + virtual RState GetMediaById(uint32_t id, std::string &outValue, uint32_t density = 0) = 0; - virtual RState GetMediaByName(const char *name, std::string &outValue) = 0; - - virtual RState GetMediaByName(const char *name, uint32_t density, std::string &outValue) = 0; + virtual RState GetMediaByName(const char *name, std::string &outValue, uint32_t density = 0) = 0; virtual RState GetRawFilePathByName(const std::string &name, std::string &outValue) = 0; virtual RState GetRawFileDescriptor(const std::string &name, RawFileDescriptor &descriptor) = 0; virtual RState CloseRawFileDescriptor(const std::string &name) = 0; + + virtual RState GetMediaDataById(uint32_t id, size_t &len, std::unique_ptr &outValue, + uint32_t density = 0) = 0; + + virtual RState GetMediaDataByName(const char *name, size_t &len, std::unique_ptr &outValue, + uint32_t density = 0) = 0; + + virtual RState GetMediaBase64DataById(uint32_t id, std::string &outValue, uint32_t density = 0) = 0; + + virtual RState GetMediaBase64DataByName(const char *name, std::string &outValue, uint32_t density = 0) = 0; + + virtual RState GetProfileDataById(uint32_t id, size_t &len, std::unique_ptr &outValue) = 0; + + virtual RState GetProfileDataByName(const char *name, size_t &len, std::unique_ptr &outValue) = 0; + + virtual RState GetRawFileFromHap(const std::string &rawFileName, size_t &len, + std::unique_ptr &outValue) = 0; + + virtual RState GetRawFileDescriptorFromHap(const std::string &rawFileName, RawFileDescriptor &descriptor) = 0; + + virtual RState IsLoadHap(std::string &hapPath) = 0; + + virtual RState GetRawFileList(const std::string &rawDirPath, std::vector &rawfileList) = 0; + + virtual RState GetDrawableInfoById(uint32_t id, std::string &type, size_t &len, + std::unique_ptr &outValue, uint32_t density = 0) = 0; + + virtual RState GetDrawableInfoByName(const char *name, std::string &type, size_t &len, + std::unique_ptr &outValue, uint32_t density = 0) = 0; + + virtual bool AddResource(const std::string &path, const std::vector &overlayPaths) = 0; + + virtual bool RemoveResource(const std::string &path, const std::vector &overlayPaths) = 0; + + virtual RState GetStringFormatById(uint32_t id, std::string &outValue, + std::vector> &jsParams) = 0; + + virtual RState GetStringFormatByName(const char *name, std::string &outValue, + std::vector> &jsParams) = 0; + + virtual uint32_t GetResourceLimitKeys() = 0; + + virtual bool AddAppOverlay(const std::string &path) = 0; + + virtual bool RemoveAppOverlay(const std::string &path) = 0; + + virtual RState GetRawFdNdkFromHap(const std::string &rawFileName, RawFileDescriptor &descriptor) = 0; + + virtual RState GetResId(const std::string &resTypeName, uint32_t &resId) = 0; + + virtual void GetLocales(std::vector &outValue, bool includeSystem = false) = 0; + + virtual RState GetDrawableInfoById(uint32_t id, std::tuple &drawableInfo, + std::unique_ptr &outValue, uint32_t iconType, uint32_t density = 0) = 0; + + virtual RState GetDrawableInfoByName(const char *name, std::tuple &drawableInfo, + std::unique_ptr &outValue, uint32_t iconType, uint32_t density = 0) = 0; + + virtual RState GetSymbolById(uint32_t id, uint32_t &outValue) = 0; + + virtual RState GetSymbolByName(const char *name, uint32_t &outValue) = 0; + + virtual RState GetThemeIcons(uint32_t resId, std::pair, size_t> &foregroundInfo, + std::pair, size_t> &backgroundInfo, uint32_t density = 0, + const std::string &abilityName = "") = 0; + + virtual std::string GetThemeMask() = 0; + + virtual bool HasIconInTheme(const std::string &bundleName) = 0; + + virtual RState GetOtherIconsInfo(const std::string &iconName, std::unique_ptr &outValue, size_t &len, + bool isGlobalMask) = 0; + + virtual RState IsRawDirFromHap(const std::string &pathName, bool &outValue) = 0; + + virtual std::shared_ptr GetOverrideResourceManager( + std::shared_ptr overrideResConfig) = 0; + + virtual RState UpdateOverrideResConfig(ResConfig &resConfig) = 0; + + virtual void GetOverrideResConfig(ResConfig &resConfig) = 0; + + virtual RState GetDynamicIcon(const std::string &resName, std::pair, size_t> &iconInfo, + uint32_t density = 0) = 0; + + virtual RState GetStringFormatById(std::string &outValue, uint32_t id, va_list args) = 0; + + virtual RState GetStringFormatByName(std::string &outValue, const char *name, va_list args) = 0; + + virtual RState GetFormatPluralStringById(std::string &outValue, uint32_t id, int quantity, + std::vector> &jsParams) = 0; + + virtual RState GetFormatPluralStringByName(std::string &outValue, const char *name, int quantity, + std::vector> &jsParams) = 0; + + virtual bool AddPatchResource(const char *path, const char *patchPath) = 0; + + virtual RState GetThemeDataByName(const char *name, std::map &outValue) = 0; + + virtual RState GetThemeDataById(uint32_t id, std::map &outValue) = 0; + + virtual RState GetPatternDataById(uint32_t id, std::map &outValue) = 0; + + virtual RState GetPatternDataByName(const char *name, std::map &outValue) = 0; + + virtual RState GetFormatPluralStringById(std::string &outValue, uint32_t id, Quantity quantity, va_list args) = 0; + + virtual RState GetFormatPluralStringById(std::string &outValue, uint32_t id, Quantity quantity, + std::vector> &jsParams) = 0; + + virtual RState GetFormatPluralStringByName(std::string &outValue, const char *name, Quantity quantity, + va_list args) = 0; + + virtual RState GetFormatPluralStringByName(std::string &outValue, const char *name, Quantity quantity, + std::vector> &jsParams) = 0; }; EXPORT_FUNC ResourceManager *CreateResourceManager(); + +/** +* Get system resource manager, the added system resource is sandbox path. This method should call +* after the sandbox mount. +* +* @return pointer of system resource manager +* @deprecated since 14 +* @useinstead CreateResourceManager +*/ +EXPORT_FUNC ResourceManager *GetSystemResourceManager(); + +/** +* Get system resource manager, the added system resource is no sandbox path. This method should call +* before the sandbox mount, for example appspawn. +* +* @return pointer of system resource manager +* @deprecated since 14 +* @useinstead CreateResourceManager +*/ +EXPORT_FUNC ResourceManager *GetSystemResourceManagerNoSandBox(); + +/** +* Create app resource manager. +* +* @param bundleName the hap bundleName +* @param moduleName the hap moduleName +* @param hapPath the hap resource path +* @param overlayPath the hap overlay resource path +* @param resConfig the device resConfig +* @param appType the app type +* @return pointer of app resource manager +*/ +EXPORT_FUNC std::shared_ptr CreateResourceManager(const std::string &bundleName, + const std::string &moduleName, const std::string &hapPath, const std::vector &overlayPath, + ResConfig &resConfig, int32_t appType = 0, int32_t userId = 100); + +/** +* Release system resource manager. This object may be held by multiple objects, and once released, the +* ResourceManagers of all holders are released. When using this interface, pay attention to the impact on all holders. +*/ +EXPORT_FUNC void ReleaseSystemResourceManager(); } // namespace Resource } // namespace Global } // namespace OHOS diff --git a/mock/sqlite/BUILD.gn b/mock/sqlite/BUILD.gn index d65b05e5..3b03c379 100644 --- a/mock/sqlite/BUILD.gn +++ b/mock/sqlite/BUILD.gn @@ -1,40 +1,89 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2019-2019. All rights reserved. +# Copyright (C) 2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import("//build/ohos.gni") +import("//third_party/sqlite/sqlite.gni") + +is_cross_platform_build = defined(is_arkui_x) && is_arkui_x +sqlite_patched_dir = root_out_dir + "/patched_sqlite" # Lets callers do '#include ' config("sqlite_config") { - include_dirs = [ - "include", - "//utils/native/base/include", - ] + include_dirs = [ "$sqlite_patched_dir/include" ] } # This is the configuration used to build sqlite itself. # It should not be needed outside of this library. config("sqlite3_private_config") { visibility = [ ":*" ] - include_dirs = [ - "include", - ] + include_dirs = [ "$sqlite_patched_dir/include" ] } group("libsqlite") { - public_deps = [ ":sqlite" ] + public_deps = [ + ":sqlite", + ":sqliteicu", + ] } -ohos_shared_library("sqlite") { - visibility = [ "//foundation/distributeddatamgr/*" ] - visibility += [ "//foundation/multimodalinput/*" ] - visibility += [ "//hit/fuzzing_test/projects/multimodal_InputService_fuzzer/*" ] - visibility += [ "//foundation/communication/*" ] - visibility += [ "//build/common/*" ] - visibility += [ ":*" ] - visibility += [ "//base/security/permission/*" ] +ohos_shared_library("sqliteicu") { + branch_protector_ret = "pac_ret" + sources = [ "$sqlite_patched_dir/src/sqlite3icu.c" ] - sources = [ - "src/sqlite3.c", + defines = [ + "NDEBUG=1", + "SQLITE_EXPORT_SYMBOLS", + "SQLITE_ENABLE_FTS3", + "HARMONY_OS", + "SQLITE_ENABLE_ICU", ] + cflags_c = [ + "-fvisibility=hidden", + "-Wno-implicit-fallthrough", + ] + if (target_os != "ios") { + ldflags = [ "-Wl,--exclude-libs,ALL" ] + } + deps = [ + ":sqlite", + "//third_party/sqlite/patch:apply_patch", + ] + public_configs = [ ":sqlite_config" ] + configs = [ ":sqlite3_private_config" ] + innerapi_tags = [ "platformsdk_indirect" ] + part_name = "sqlite" + subsystem_name = "thirdparty" + install_images = [ system_base_dir ] + relative_install_dir = "platformsdk" + external_deps = [] + if (is_cross_platform_build) { + if (target_os == "ios") { + deps += [ "//third_party/bounds_checking_function:libsec_shared" ] + } else { + deps += [ "//commonlibrary/c_utils/base:utils" ] + } + } else { + external_deps += [ "c_utils:utils" ] + } + external_deps += [ + "icu:shared_icui18n", + "icu:shared_icuuc", + ] +} + +ohos_shared_library("sqlite") { + branch_protector_ret = "pac_ret" + sources = [ "$sqlite_patched_dir/src/sqlite3.c" ] defines = [ "NDEBUG=1", @@ -47,11 +96,13 @@ ohos_shared_library("sqlite") { "SQLITE_DEFAULT_FILE_FORMAT=4", "SQLITE_DEFAULT_AUTOVACUUM=1", "SQLITE_ENABLE_MEMORY_MANAGEMENT=1", + "SQLITE_ENABLE_LOAD_EXTENSION", "SQLITE_ENABLE_FTS3", + "SQLITE_ENABLE_FTS3_TOKENIZER", "SQLITE_ENABLE_FTS4", + "SQLITE_ENABLE_FTS5", "SQLITE_OMIT_COMPILEOPTION_DIAGS", - "SQLITE_OMIT_LOAD_EXTENSION", - "SQLITE_DEFAULT_FILE_PERMISSIONS=0600", + "SQLITE_DEFAULT_FILE_PERMISSIONS=0660", "SQLITE_SECURE_DELETE", "SQLITE_ENABLE_BATCH_ATOMIC_WRITE", "USE_PREAD64", @@ -60,29 +111,56 @@ ohos_shared_library("sqlite") { "HAVE_MALLOC_USABLE_SIZE", "SQLITE_DIRECT_OVERFLOW_READ", "SQLITE_HAS_CODEC", - "SQLITE3_HW_EXPORT_SYMBOLS", - "SQLITE_HW_SHARED_BLOCK_OPTIMIZATION", + "SQLITE_EXPORT_SYMBOLS", + "SQLITE_SHARED_BLOCK_OPTIMIZATION", "SQLITE_CODEC_ATTACH_CHANGED", + "SQLITE_ENABLE_DROPTABLE_CALLBACK", + "OPENSSL_SUPPRESS_DEPRECATED", + "LOG_DUMP", + "FDSAN_ENABLE", + "HARMONY_OS", + "SQLITE_HDR_CHECK", + "SQLITE_ENABLE_ICU", + "SQLITE_META_DWR", ] + if (sqlite_support_check_pages) { + defines += [ "SQLITE_CHECK_PAGES" ] + } cflags_c = [ "-fvisibility=hidden", + "-Wno-implicit-fallthrough", ] - ldflags = [ "-Wl,--exclude-libs,ALL" ] - deps = [ - "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", - ] + if (target_os != "ios") { + ldflags = [ "-Wl,--exclude-libs,ALL" ] + } + deps = [ "//third_party/sqlite/patch:apply_patch" ] public_configs = [ ":sqlite_config" ] + public_external_deps = [ "c_utils:utils" ] configs = [ ":sqlite3_private_config" ] + innerapi_tags = [ "platformsdk_indirect" ] + part_name = "sqlite" + subsystem_name = "thirdparty" + install_images = [ system_base_dir ] + relative_install_dir = "platformsdk" + if (is_cross_platform_build) { + if (target_os == "ios") { + deps += [ "//third_party/bounds_checking_function:libsec_shared" ] + } else { + external_deps = [ "c_utils:utils" ] + } + } else { + external_deps = [ + "c_utils:utils", + "openssl:libcrypto_shared", + ] + } } ohos_executable("sqlite3") { - include_dirs = [ - "include", - ] + include_dirs = [ "$sqlite_patched_dir/include" ] sources = [ - "src/sqlite3.c", - "src/shell.c", + "$sqlite_patched_dir/src/shell.c", + "$sqlite_patched_dir/src/sqlite3.c", ] defines = [ @@ -99,17 +177,140 @@ ohos_executable("sqlite3") { "SQLITE_ENABLE_FTS3", "SQLITE_ENABLE_FTS4", "SQLITE_OMIT_COMPILEOPTION_DIAGS", - "SQLITE_OMIT_LOAD_EXTENSION", "SQLITE_DEFAULT_FILE_PERMISSIONS=0600", "SQLITE_SECURE_DELETE", "SQLITE_ENABLE_BATCH_ATOMIC_WRITE", "USE_PREAD64", "fdatasync=fdatasync", "SQLITE_DIRECT_OVERFLOW_READ", - "SQLITE_HW_SHARED_BLOCK_OPTIMIZATION", + "SQLITE_SHARED_BLOCK_OPTIMIZATION", + "OPENSSL_SUPPRESS_DEPRECATED", + "LOG_DUMP", + "FDSAN_ENABLE", + "HARMONY_OS", ] cflags = [ - "-Wno-error=implicit-function-declaration" + "-Wno-error=implicit-function-declaration", + "-Wno-implicit-fallthrough", ] + + deps = [ "//third_party/sqlite/patch:apply_patch" ] +} + +if (is_mingw || is_mac) { + ohos_shared_library("sqlite_sdk") { + include_dirs = [ + "$sqlite_patched_dir/include", + "//third_party/bounds_checking_function/include", + ] + + sources = [ "$sqlite_patched_dir/src/sqlite3.c" ] + + defines = [ + "NDEBUG=1", + "HAVE_USLEEP=1", + "SQLITE_HAVE_ISNAN", + "SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576", + "SQLITE_THREADSAFE=2", + "SQLITE_TEMP_STORE=3", + "SQLITE_POWERSAFE_OVERWRITE=1", + "SQLITE_DEFAULT_FILE_FORMAT=4", + "SQLITE_DEFAULT_AUTOVACUUM=1", + "SQLITE_ENABLE_MEMORY_MANAGEMENT=1", + "SQLITE_ENABLE_FTS3", + "SQLITE_ENABLE_FTS4", + "SQLITE_OMIT_COMPILEOPTION_DIAGS", + "SQLITE_DEFAULT_FILE_PERMISSIONS=0600", + "SQLITE_SECURE_DELETE", + "SQLITE_ENABLE_BATCH_ATOMIC_WRITE", + "USE_PREAD64", + "fdatasync=fdatasync", + "SQLITE_DIRECT_OVERFLOW_READ", + "SQLITE_HAS_CODEC", + "SQLITE_EXPORT_SYMBOLS", + "SQLITE_SHARED_BLOCK_OPTIMIZATION", + "OPENSSL_SUPPRESS_DEPRECATED", + ] + remove_configs = [ "//build/config/compiler:chromium_code" ] + deps = [ + "//third_party/bounds_checking_function:libsec_shared", + "//third_party/sqlite/patch:apply_patch", + ] + external_deps = [ "openssl:libcrypto_restool" ] + if (is_mingw) { + libs = [ "//prebuilts/mingw-w64/ohos/linux-x86_64/clang-mingw/x86_64-w64-mingw32/lib/libws2_32.a" ] + } + cflags = [ + "-Wno-error=implicit-function-declaration", + "-Wno-implicit-fallthrough", + ] + part_name = "sqlite" + subsystem_name = "thirdparty" + } +} + +if (is_cross_platform_build) { + ohos_static_library("sqlite_static") { + visibility = [ "//foundation/distributeddatamgr/*" ] + visibility += [ "//foundation/multimodalinput/*" ] + visibility += + [ "//hit/fuzzing_test/projects/multimodal_InputService_fuzzer/*" ] + visibility += [ "//foundation/communication/*" ] + visibility += [ "//build/common/*" ] + visibility += [ ":*" ] + visibility += [ "//base/security/*" ] + visibility += [ "//third_party/libsoup/*" ] + visibility += + [ "//foundation/resourceschedule/resource_schedule_service/*" ] + visibility += [ "//foundation/bundlemanager/ecological_rule_mgr/*" ] + + sources = [ "$sqlite_patched_dir/src/sqlite3.c" ] + + defines = [ + "NDEBUG=1", + "HAVE_USLEEP=1", + "SQLITE_HAVE_ISNAN", + "SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576", + "SQLITE_THREADSAFE=2", + "SQLITE_TEMP_STORE=3", + "SQLITE_POWERSAFE_OVERWRITE=1", + "SQLITE_DEFAULT_FILE_FORMAT=4", + "SQLITE_DEFAULT_AUTOVACUUM=1", + "SQLITE_ENABLE_MEMORY_MANAGEMENT=1", + "SQLITE_ENABLE_FTS3", + "SQLITE_ENABLE_FTS4", + "SQLITE_ENABLE_FTS5", + "SQLITE_OMIT_COMPILEOPTION_DIAGS", + "SQLITE_DEFAULT_FILE_PERMISSIONS=0660", + "SQLITE_SECURE_DELETE", + "SQLITE_ENABLE_BATCH_ATOMIC_WRITE", + "USE_PREAD64", + "fdatasync=fdatasync", + "HAVE_MALLOC_H=1", + "HAVE_MALLOC_USABLE_SIZE", + "SQLITE_DIRECT_OVERFLOW_READ", + "SQLITE_HAS_CODEC", + "SQLITE_EXPORT_SYMBOLS", + "SQLITE_SHARED_BLOCK_OPTIMIZATION", + "SQLITE_CODEC_ATTACH_CHANGED", + "SQLITE_ENABLE_DROPTABLE_CALLBACK", + "OPENSSL_SUPPRESS_DEPRECATED", + ] + + cflags_c = [ + "-fvisibility=hidden", + "-Wno-implicit-fallthrough", + ] + + deps = [ + "//third_party/bounds_checking_function:libsec_static", + "//third_party/openssl:libcrypto_static", + "//third_party/sqlite/patch:apply_patch", + ] + public_configs = [ ":sqlite_config" ] + configs = [ ":sqlite3_private_config" ] + part_name = "sqlite" + subsystem_name = "thirdparty" + } } diff --git a/mock/sqlite/OAT.xml b/mock/sqlite/OAT.xml new file mode 100644 index 00000000..c74351d2 --- /dev/null +++ b/mock/sqlite/OAT.xml @@ -0,0 +1,111 @@ + + + + + + LICENSE + + + + + + + + + + + + + + + + + + + + + diff --git a/mock/sqlite/README.OpenSource b/mock/sqlite/README.OpenSource index b3c91a2b..27e78426 100644 --- a/mock/sqlite/README.OpenSource +++ b/mock/sqlite/README.OpenSource @@ -1,10 +1,11 @@ [ { - "Name": "sqlite", + "Name": "SQLite", "License": "Public Domain", "License File": "LICENSE", - "Version Number": "3.32.3", - "Upstream URL": "https://www.sqlite.org/2020/sqlite-amalgamation-3320300.zip", + "Version Number": "3.40.1", + "Owner": "wangbingquan@huawei.com", + "Upstream URL": "https://www.sqlite.org/2022/sqlite-amalgamation-3400100.zip", "Description": "SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine." } ] diff --git a/mock/sqlite/README.md b/mock/sqlite/README.md index 3a2a6601..cf39f4cf 100644 --- a/mock/sqlite/README.md +++ b/mock/sqlite/README.md @@ -1,5 +1,5 @@ Name: sqlite -URL: http://sqlite.org/ -Version: 3032003 +URL: "https://www.sqlite.org/" +Version: 3400100 This is a shared library build of sqlite3 and its shell configured for ohos based off of the autoconf build flags. diff --git a/mock/sqlite/bundle.json b/mock/sqlite/bundle.json new file mode 100644 index 00000000..b733d959 --- /dev/null +++ b/mock/sqlite/bundle.json @@ -0,0 +1,54 @@ +{ + "name": "@ohos/sqlite", + "description": "SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine.", + "version": "3.40.1", + "homePage": "https://www.sqlite.org/", + "license": "Public Domain", + "publishAs": "code-segment", + "segment": { + "destPath": "third_party/sqlite" + }, + "dirs": {}, + "scripts": {}, + "component": { + "name": "sqlite", + "subsystem": "thirdparty", + "syscap": [], + "features": [ "sqlite_support_check_pages" ], + "adapted_system_type": [ "standard" ], + "rom": "2200KB", + "ram": "1024KB", + "deps": { + "components": [ + "c_utils", + "icu", + "openssl" + ], + "third_party": [ + "openssl" + ] + }, + "build": { + "sub_component": [ + "//third_party/sqlite:sqlite", + "//third_party/sqlite:sqliteicu" + ], + "inner_kits": [ + { + "name": "//third_party/sqlite:sqlite", + "header": { + "header_files": [ + "sqlite3ext.h", + "sqlite3sym.h" + ], + "header_base": "//third_party/sqlite/include" + } + }, + { + "name": "//third_party/sqlite:sqliteicu" + } + ], + "test": [] + } + } +} diff --git a/mock/sqlite/ext/misc/cksumvfs.c b/mock/sqlite/ext/misc/cksumvfs.c new file mode 100644 index 00000000..6f4c55c2 --- /dev/null +++ b/mock/sqlite/ext/misc/cksumvfs.c @@ -0,0 +1,916 @@ +/* +** 2020-04-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a checksum on each page +** of an SQLite database file. When reading pages, the checksum is verified +** and an error is raised if the checksum is incorrect. +** +** COMPILING +** +** This extension requires SQLite 3.32.0 or later. It uses the +** sqlite3_database_file_object() interface which was added in +** version 3.32.0, so it will not link with an earlier version of +** SQLite. +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so +** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib +** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** If you want to statically link this extension with your product, +** then compile it like any other C-language module but add the +** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that +** it is being statically linked rather than dynamically linked +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extension(db, "./cksumvfs"); +** sqlite3_close(db); +** +** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and +** statically linked against the application, initialize it using +** a single API call as follows: +** +** sqlite3_register_cksumvfs(); +** +** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, in +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that cksumvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a checksum) will operate normally. Databases with +** checksums will return an SQLITE_IOERR_DATA error if a page is +** encountered that contains an invalid checksum. +** +** Checksumming only works on databases that have a reserve-bytes +** value of exactly 8. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the checksum by +** default. To create a database that includes a checksum, change +** the reserve-bytes value to 8 by runing: +** +** int n = 8; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** If the database is in WAL mode, you should shutdown and +** reopen all database connections before continuing. +** +** From the CLI, use the ".filectrl reserve_bytes 8" command, +** followed by "VACUUM;". +** +** Note that SQLite allows the number of reserve-bytes to be +** increased but not decreased. So if a database file already +** has a reserve-bytes value greater than 8, there is no way to +** activate checksumming on that database, other than to dump +** and restore the database file. Note also that other extensions +** might also make use of the reserve-bytes. Checksumming will +** be incompatible with those other extensions. +** +** VERIFICATION OF CHECKSUMS +** +** If any checksum is incorrect, the "PRAGMA quick_check" command +** will find it. To verify that checksums are actually enabled +** and running, use the following query: +** +** SELECT count(*), verify_checksum(data) +** FROM sqlite_dbpage +** GROUP BY 2; +** +** There are three possible outputs form the verify_checksum() +** function: 1, 0, and NULL. 1 is returned if the checksum is +** correct. 0 is returned if the checksum is incorrect. NULL +** is returned if the page is unreadable. If checksumming is +** enabled, the read will fail if the checksum is wrong, so the +** usual result from verify_checksum() on a bad checksum is NULL. +** +** If everything is OK, the query above should return a single +** row where the second column is 1. Any other result indicates +** either that there is a checksum error, or checksum validation +** is disabled. +** +** CONTROLLING CHECKSUM VERIFICATION +** +** The cksumvfs extension implements a new PRAGMA statement that can +** be used to disable, re-enable, or query the status of checksum +** verification: +** +** PRAGMA checksum_verification; -- query status +** PRAGMA checksum_verification=OFF; -- disable verification +** PRAGMA checksum_verification=ON; -- re-enable verification +** +** The "checksum_verification" pragma will return "1" (true) or "0" +** (false) if checksum verification is enabled or disabled, respectively. +** "Verification" in this context means the feature that causes +** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while +** reading. Checksums are always kept up-to-date as long as the +** reserve-bytes value of the database is 8, regardless of the setting +** of this pragma. Checksum verification can be disabled (for example) +** to do forensic analysis of a database that has previously reported +** a checksum error. +** +** The "checksum_verification" pragma will always respond with "0" if +** the database file does not have a reserve-bytes value of 8. The +** pragma will return no rows at all if the cksumvfs extension is +** not loaded. +** +** IMPLEMENTATION NOTES +** +** The checksum is stored in the last 8 bytes of each page. This +** module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 8. If +** the reserved-space value is not 8, this module is a no-op. +*/ +#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_CKSUMVFS_STATIC) +# define SQLITE_CKSUMVFS_STATIC +#endif +#ifdef SQLITE_CKSUMVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include + +// hw export the symbols +#ifdef SQLITE_EXPORT_SYMBOLS +#if defined(__GNUC__) +# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) +#elif defined(_MSC_VER) +# define EXPORT_SYMBOLS __declspec(dllexport) +#else +# define EXPORT_SYMBOLS +#endif +#endif + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs CksmVfs; +typedef struct CksmFile CksmFile; + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_AMALGAMATION) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) + +/* An open file */ +struct CksmFile { + sqlite3_file base; /* IO methods */ + const char *zFName; /* Original name of the file */ + char computeCksm; /* True to compute checksums. + ** Always true if reserve size is 8. */ + char verifyCksm; /* True to verify checksums */ + char isWal; /* True if processing a WAL file */ + char inCkpt; /* Currently doing a checkpoint */ + CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ +}; + +/* +** Methods for CksmFile +*/ +static int cksmClose(sqlite3_file*); +static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); +static int cksmSync(sqlite3_file*, int flags); +static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int cksmLock(sqlite3_file*, int); +static int cksmUnlock(sqlite3_file*, int); +static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); +static int cksmFileControl(sqlite3_file*, int op, void *pArg); +static int cksmSectorSize(sqlite3_file*); +static int cksmDeviceCharacteristics(sqlite3_file*); +static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); +static void cksmShmBarrier(sqlite3_file*); +static int cksmShmUnmap(sqlite3_file*, int deleteFlag); +static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for CksmVfs +*/ +static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); +static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void cksmDlClose(sqlite3_vfs*, void*); +static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int cksmSleep(sqlite3_vfs*, int microseconds); +static int cksmCurrentTime(sqlite3_vfs*, double*); +static int cksmGetLastError(sqlite3_vfs*, int, char *); +static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); +static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs cksm_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "cksmvfs", /* zName */ + 0, /* pAppData (set when registered) */ + cksmOpen, /* xOpen */ + cksmDelete, /* xDelete */ + cksmAccess, /* xAccess */ + cksmFullPathname, /* xFullPathname */ + cksmDlOpen, /* xDlOpen */ + cksmDlError, /* xDlError */ + cksmDlSym, /* xDlSym */ + cksmDlClose, /* xDlClose */ + cksmRandomness, /* xRandomness */ + cksmSleep, /* xSleep */ + cksmCurrentTime, /* xCurrentTime */ + cksmGetLastError, /* xGetLastError */ + cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ + cksmSetSystemCall, /* xSetSystemCall */ + cksmGetSystemCall, /* xGetSystemCall */ + cksmNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods cksm_io_methods = { + 3, /* iVersion */ + cksmClose, /* xClose */ + cksmRead, /* xRead */ + cksmWrite, /* xWrite */ + cksmTruncate, /* xTruncate */ + cksmSync, /* xSync */ + cksmFileSize, /* xFileSize */ + cksmLock, /* xLock */ + cksmUnlock, /* xUnlock */ + cksmCheckReservedLock, /* xCheckReservedLock */ + cksmFileControl, /* xFileControl */ + cksmSectorSize, /* xSectorSize */ + cksmDeviceCharacteristics, /* xDeviceCharacteristics */ + cksmShmMap, /* xShmMap */ + cksmShmLock, /* xShmLock */ + cksmShmBarrier, /* xShmBarrier */ + cksmShmUnmap, /* xShmUnmap */ + cksmFetch, /* xFetch */ + cksmUnfetch /* xUnfetch */ +}; + +/* Do byte swapping on a unsigned 32-bit integer */ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* Compute a checksum on a buffer */ +static void cksmCompute( + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + u8 *aOut /* OUT: Final 8-byte checksum value output */ +){ + u32 s1 = 0, s2 = 0; + u32 *aData = (u32*)a; + u32 *aEnd = (u32*)&a[nByte]; + u32 x = 1; + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( 1 == *(u8*)&x ){ + /* Little-endian */ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aData65536 || (nByte & (nByte-1))!=0 ) return; + cksmCompute(data, nByte-8, cksum); + sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); +} + +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME +/* +** SQL function: initialize_cksumvfs(SCHEMANAME) +** +** This SQL functions (whose name is actually determined at compile-time +** by the value of the SQLITE_CKSUMVFS_INIT_FUNCNAME macro) invokes: +** +** sqlite3_file_control(db, SCHEMANAME, SQLITE_FCNTL_RESERVE_BYTE, &n); +** +** In order to set the reserve bytes value to 8, so that cksumvfs will +** operation. This feature is provided (if and only if the +** SQLITE_CKSUMVFS_INIT_FUNCNAME compile-time option is set to a string +** which is the name of the SQL function) so as to provide the ability +** to invoke the file-control in programming languages that lack +** direct access to the sqlite3_file_control() interface (ex: Java). +** +** This interface is undocumented, apart from this comment. Usage +** example: +** +** 1. Compile with -DSQLITE_CKSUMVFS_INIT_FUNCNAME="ckvfs_init" +** 2. Run: "SELECT cksum_init('main'); VACUUM;" +*/ +static void cksmInitFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nByte = 8; + const char *zSchemaName = (const char*)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_file_control(db, zSchemaName, SQLITE_FCNTL_RESERVE_BYTES, &nByte); + /* Return NULL */ +} +#endif /* SQLITE_CKSUMBFS_INIT_FUNCNAME */ + +/* +** Close a cksm-file. +*/ +static int cksmClose(sqlite3_file *pFile){ + CksmFile *p = (CksmFile *)pFile; + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Set the computeCkSm and verifyCksm flags, if they need to be +** changed. +*/ +static void cksmSetFlags(CksmFile *p, int hasCorrectReserveSize){ + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + if( p->pPartner ){ + p->pPartner->verifyCksm = hasCorrectReserveSize; + p->pPartner->computeCksm = hasCorrectReserveSize; + } + } +} + +static void EncodeReservedBytesIntoBase16(const u8 *reserved, int len, char *encodeStr, int maxLen){ + static const char baseCode[] = "0123456789ABCDEF"; + for(int i=0; i> 4) & 0x0F]; + *encodeStr++ = baseCode[reserved[i] & 0x0F]; + } + *encodeStr = '0'; +} + +/* +** Read data from a cksm-file. +*/ +static int cksmRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* Verify the checksum if + ** (1) the size indicates that we are dealing with a complete + ** database page, only support pageSize:4K + ** (2) checksum verification is enabled + ** (3) we are not in the middle of checkpoint + */ + if( iAmt==4096 /* (1) */ + && p->verifyCksm /* (2) */ + && !p->inCkpt /* (3) */ + ){ + u8 cksum[8]; + cksmCompute((u8*)zBuf, iAmt-8, cksum); + if( memcmp((u8*)zBuf+iAmt-8, cksum, 8)!=0 ){ + char expect[18] = {0}; + char actual[18] = {0}; + EncodeReservedBytesIntoBase16((u8 *)zBuf+iAmt-8, 8, expect, 18); + EncodeReservedBytesIntoBase16(cksum, 8, actual, 18); + sqlite3_log(SQLITE_IOERR_DATA, "checksum fault offset %lld of \"%s\", amt:%d, expect:%s, actual:%s", + iOfst, p->zFName, iAmt, expect, actual); + rc = SQLITE_IOERR_DATA; + } + } + } + return rc; +} + +/* +** Write data to a cksm-file. +*/ +static int cksmWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* If the write size is appropriate for a database page and if + ** checksums where ever enabled, then it will be safe to compute + ** the checksums. The reserve byte size might have increased, but + ** it will never decrease. And because it cannot decrease, the + ** checksum will not overwrite anything. + */ + if( iAmt==4096 + && p->computeCksm + && !p->inCkpt + ){ + cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); +} + +/* +** Truncate a cksm-file. +*/ +static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a cksm-file. +*/ +static int cksmSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a cksm-file. +*/ +static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a cksm-file. +*/ +static int cksmLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a cksm-file. +*/ +static int cksmUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a cksm-file. +*/ +static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a cksm-file. +*/ +static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + CksmFile *p = (CksmFile*)pFile; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **azArg = (char**)pArg; + assert( azArg[1]!=0 ); + if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ + char *zArg = azArg[2]; + if( zArg!=0 ){ + if( (zArg[0]>='1' && zArg[0]<='9') + || sqlite3_strlike("enable%",zArg,0)==0 + || sqlite3_stricmp("yes",zArg)==0 + || sqlite3_stricmp("on",zArg)==0 + ){ + p->verifyCksm = p->computeCksm; + }else{ + p->verifyCksm = 0; + } + if( p->pPartner ) p->pPartner->verifyCksm = p->verifyCksm; + } + azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); + return SQLITE_OK; + }else if( p->computeCksm && azArg[2]!=0 + && sqlite3_stricmp(azArg[1], "page_size")==0 ){ + /* Do not allow page size changes on a checksum database */ + return SQLITE_OK; + } + }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ + p->inCkpt = op==SQLITE_FCNTL_CKPT_START; + if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; + }else if( op==SQLITE_FCNTL_CKSM_FILE ){ + /* This VFS needs to obtain a pointer to the corresponding database + ** file handle from within xOpen() calls to open wal files. To do this, + ** it uses the sqlite3_database_file_object() API to obtain a pointer + ** to the file-handle used by SQLite to access the db file. This is + ** fine if cksmvfs happens to be the top-level VFS, but not if there + ** are one or more wrapper VFS. To handle this case, this file-control + ** is used to extract the cksmvfs file-handle from any wrapper file + ** handle. */ + sqlite3_file **ppFile = (sqlite3_file**)pArg; + *ppFile = (sqlite3_file*)p; + return SQLITE_OK; + } + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for a cksm-file. +*/ +static int cksmSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a cksm-file. +*/ +static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int cksmShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void cksmShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int cksmFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + CksmFile *p = (CksmFile *)pFile; + if( p->computeCksm ){ + *pp = 0; + return SQLITE_OK; + } + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xFetch ){ + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); + } + *pp = 0; + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xUnfetch ){ + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); + } + return SQLITE_OK; +} + +/* +** Open a cksm file handle. +*/ +static int cksmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + CksmFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + p = (CksmFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + pFile->pMethods = &cksm_io_methods; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto cksm_open_done; + if( flags & SQLITE_OPEN_WAL ){ + sqlite3_file *pDb = sqlite3_database_file_object(zName); + rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); + assert( rc==SQLITE_OK ); + p->pPartner = (CksmFile*)pDb; + assert( p->pPartner->pPartner==0 ); + p->pPartner->pPartner = p; + p->isWal = 1; + p->computeCksm = p->pPartner->computeCksm; + }else{ + p->isWal = 0; + p->computeCksm = 0; + } + p->zFName = zName; +cksm_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int cksmAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int cksmFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + sqlite3_vfs *pOrig = ORIGVFS(pVfs); + int rc; + assert( pOrig->iVersion>=2 ); + if( pOrig->xCurrentTimeInt64 ){ + rc = pOrig->xCurrentTimeInt64(pOrig, p); + }else{ + double r; + rc = pOrig->xCurrentTime(pOrig, &r); + *p = (sqlite3_int64)(r*86400000.0); + } + return rc; +} +static int cksmSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr cksmGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + +/* Register the verify_checksum() SQL function. +*/ +static int cksmRegisterFunc( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + if( db==0 ) return SQLITE_OK; + rc = sqlite3_create_function(db, "verify_checksum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, cksmVerifyFunc, 0, 0); +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME + (void)sqlite3_create_function(db, SQLITE_CKSUMVFS_INIT_FUNCNAME, 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, + 0, cksmInitFunc, 0, 0); +#endif + return rc; +} + +/* +** Register the cksum VFS as the default VFS for the system. +** Also make arrangements to automatically register the "verify_checksum()" +** SQL function on each new database connection. +*/ +static int cksmRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); + rc = sqlite3_vfs_register(&cksm_vfs, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } + return rc; +} + +#if defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_cksumvfs(const char *NotUsed){ + (void)NotUsed; + return cksmRegisterVfs(); +} +int sqlite3_unregister_cksumvfs(void){ + if( sqlite3_vfs_find("cksmvfs") ){ + sqlite3_vfs_unregister(&cksm_vfs); + sqlite3_cancel_auto_extension((void(*)(void))cksmRegisterFunc); + } + return SQLITE_OK; +} +#endif /* defined(SQLITE_CKSUMVFS_STATIC */ + +#if !defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_cksumvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + rc = cksmRegisterFunc(db, 0, 0); + if( rc==SQLITE_OK ){ + rc = cksmRegisterVfs(); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ + +#ifdef SQLITE_CKSUMVFS_STATIC +struct sqlite3_api_routines_cksumvfs { + int (*register_cksumvfs)(const char *); + int (*unregister_cksumvfs)(); +}; +typedef struct sqlite3_api_routines_cksumvfs sqlite3_api_routines_cksumvfs; +static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { + sqlite3_register_cksumvfs, + sqlite3_unregister_cksumvfs +}; + +EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; +#endif diff --git a/mock/sqlite/include/sqlite3.h b/mock/sqlite/include/sqlite3.h index 92bf7156..3b1a37a5 100644 --- a/mock/sqlite/include/sqlite3.h +++ b/mock/sqlite/include/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.37.2" -#define SQLITE_VERSION_NUMBER 3037002 -#define SQLITE_SOURCE_ID "2022-01-06 13:25:41 872ba256cbf61d9290b571c0e6d82a20c224ca3ad82971edc46b29818d5d17a0" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -420,6 +420,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( @@ -441,38 +443,38 @@ SQLITE_API int sqlite3_exec( ** ** See also: [extended result code definitions] */ -#define SQLITE_OK 0 /* �ɹ� | Successful result */ -/* �����뿪ʼ */ -#define SQLITE_ERROR 1 /* SQL���� �� ��ʧ���ݿ� | SQL error or missing database */ -#define SQLITE_INTERNAL 2 /* SQLite �ڲ��߼����� | Internal logic error in SQLite */ -#define SQLITE_PERM 3 /* �ܾ����� | Access permission denied */ -#define SQLITE_ABORT 4 /* �ص���������ȡ������ | Callback routine requested an abort */ -#define SQLITE_BUSY 5 /* ���ݿ��ļ������� | The database file is locked */ -#define SQLITE_LOCKED 6 /* ���ݿ��е�һ�������� | A table in the database is locked */ -#define SQLITE_NOMEM 7 /* ij�� malloc() ��������ʧ�� | A malloc() failed */ -#define SQLITE_READONLY 8 /* ����д��һ��ֻ�����ݿ� | Attempt to write a readonly database */ -#define SQLITE_INTERRUPT 9 /* ������ sqlite3_interupt() �����ж� | Operation terminated by sqlite3_interrupt() */ -#define SQLITE_IOERR 10 /* ����ijЩ���� I/O ���� | Some kind of disk I/O error occurred */ -#define SQLITE_CORRUPT 11 /* ���ݿ����ӳ����ȷ | The database disk image is malformed */ -#define SQLITE_NOTFOUND 12 /* sqlite3_file_control() �г���δ֪������ | Unknown opcode in sqlite3_file_control() */ -#define SQLITE_FULL 13 /* ��Ϊ���ݿ������²���ʧ�� | Insertion failed because database is full */ -#define SQLITE_CANTOPEN 14 /* �޷������ݿ��ļ� | Unable to open the database file */ -#define SQLITE_PROTOCOL 15 /* ���ݿ�����Э����� | Database lock protocol error */ -#define SQLITE_EMPTY 16 /* ���ݿ�Ϊ�� | Database is empty */ -#define SQLITE_SCHEMA 17 /* ���ݽṹ�����ı� | The database schema changed */ -#define SQLITE_TOOBIG 18 /* �ַ�������������ݳ�����С���� | String or BLOB exceeds size limit */ -#define SQLITE_CONSTRAINT 19 /* ����Լ��Υ����ȡ�� | Abort due to constraint violation */ -#define SQLITE_MISMATCH 20 /* �������Ͳ�ƥ�� | Data type mismatch */ -#define SQLITE_MISUSE 21 /* ����ȷ�Ŀ�ʹ�� | Library used incorrectly */ -#define SQLITE_NOLFS 22 /* ʹ���˲���ϵͳ��֧�ֵĹ��� | Uses OS features not supported on host */ -#define SQLITE_AUTH 23 /* ��Ȩʧ�� | Authorization denied */ -#define SQLITE_FORMAT 24 /* �������ݿ��ʽ���� | Auxiliary database format error */ -#define SQLITE_RANGE 25 /* ���ݸ�sqlite3_bind()�ĵڶ�������������Χ | 2nd parameter to sqlite3_bind out of range */ -#define SQLITE_NOTADB 26 /* ���򿪵��ļ�����һ�����ݿ��ļ� | File opened that is not a database file */ +#define SQLITE_OK 0 /* Successful result */ +/* beginning-of-error-codes */ +#define SQLITE_ERROR 1 /* Generic error */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ +#define SQLITE_PERM 3 /* Access permission denied */ +#define SQLITE_ABORT 4 /* Callback routine requested an abort */ +#define SQLITE_BUSY 5 /* The database file is locked */ +#define SQLITE_LOCKED 6 /* A table in the database is locked */ +#define SQLITE_NOMEM 7 /* A malloc() failed */ +#define SQLITE_READONLY 8 /* Attempt to write a readonly database */ +#define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ +#define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ +#define SQLITE_CORRUPT 11 /* The database disk image is malformed */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ +#define SQLITE_FULL 13 /* Insertion failed because database is full */ +#define SQLITE_CANTOPEN 14 /* Unable to open the database file */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Internal use only */ +#define SQLITE_SCHEMA 17 /* The database schema changed */ +#define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ +#define SQLITE_MISMATCH 20 /* Data type mismatch */ +#define SQLITE_MISUSE 21 /* Library used incorrectly */ +#define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ +#define SQLITE_AUTH 23 /* Authorization denied */ +#define SQLITE_FORMAT 24 /* Not used */ +#define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ +#define SQLITE_NOTADB 26 /* File opened that is not a database file */ #define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ #define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ -#define SQLITE_ROW 100 /* sqlite3_step() �Ѿ�����һ���н�� | sqlite3_step() has another row ready */ -#define SQLITE_DONE 101 /* sqlite3_step() ���ִ�в��� | sqlite3_step() has finished executing */ +#define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ +#define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ /* @@ -528,6 +530,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -563,10 +566,11 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ /* ** CAPI3REF: Flags For File Open Operations @@ -670,13 +674,17 @@ SQLITE_API int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -754,7 +762,14 @@ struct sqlite3_file { **
  • [SQLITE_LOCK_PENDING], or **
  • [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +** If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -859,9 +874,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1165,7 +1179,6 @@ struct sqlite3_io_methods { ** in wal mode after the client has finished copying pages from the wal ** file to the database file, but before the *-shm file is updated to ** record the fact that the pages have been checkpointed. -** ** **
  • [[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect @@ -1178,10 +1191,16 @@ struct sqlite3_io_methods { ** the database is not a wal-mode db, or if there is no such connection in any ** other process. This opcode cannot be used to detect transactions opened ** by clients within the current process, only within other processes. -** ** **
  • [[SQLITE_FCNTL_CKSM_FILE]] -** Used by the cksmvfs VFS module only. +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the +** [checksum VFS shim] only. +** +**
  • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control +** purges the contents of the in-memory page cache. If there is an open +** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1224,6 +1243,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKPT_START 39 #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1253,6 +1273,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; */ typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
      +**
    • sqlite3_filename_database() +**
    • sqlite3_filename_journal() +**
    • sqlite3_filename_wal() +**
    • sqlite3_uri_parameter() +**
    • sqlite3_uri_boolean() +**
    • sqlite3_uri_int64() +**
    • sqlite3_uri_key() +**
    +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1431,7 +1471,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -1618,20 +1658,23 @@ SQLITE_API int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -1739,6 +1782,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
      +**
    • SQLITE_CONFIG_LOG +**
    • SQLITE_CONFIG_PCACHE_HDRSZ +**
    +** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -2069,7 +2129,7 @@ struct sqlite3_mem_methods { ** is stored in each sorted record and the required column values loaded ** from the database as records are returned in sorted order. The default ** value for this option is to never use this optimization. Specifying a -** negative value for this option restores the default behaviour. +** negative value for this option restores the default behavior. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** @@ -2083,30 +2143,61 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. +** +** [[SQLITE_CONFIG_CORRUPTION]]
    SQLITE_CONFIG_CORRUPTION
    +**
    The SQLITE_CONFIG_CORRUPTION option is used to configure the SQLite +** global [ corruption error]. +** (^The SQLITE_CONFIG_CORRUPTION option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,const void*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked to process each data corruption event. ^If the +** function pointer is NULL, no=op will do when corruption detect. +** ^The void pointer that is the second argument to SQLITE_CONFIG_CORRUPTION is +** passed through as the first parameter to the application-defined corruption +** function whenever that function is invoked. ^The second parameter to +** the corruption function is a corruption message after formatting via [sqlite3_snprintf()]. +** In a multi-threaded application, the application-defined corruption +** function must be threadsafe.
    ** */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ @@ -2114,8 +2205,9 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ -#define SQLITE_CONFIG_CORRUPTION 30 /* xCorruption */ -#define SQLITE_CONFIG_ENABLE_ICU 31 /* boolean */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ +#define SQLITE_CONFIG_CORRUPTION 31 /* xCorruption */ +#define SQLITE_CONFIG_ENABLE_ICU 32 /* boolean */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2149,7 +2241,7 @@ struct sqlite3_mem_methods { ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words ** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^ @@ -2246,7 +2338,7 @@ struct sqlite3_mem_methods { ** database handle, SQLite checks if this will mean that there are now no ** connections at all to the database. If so, it performs a checkpoint ** operation before closing the connection. This option may be used to -** override this behaviour. The first parameter passed to this operation +** override this behavior. The first parameter passed to this operation ** is an integer - positive to disable checkpoints-on-close, or zero (the ** default) to enable them, and negative to leave the setting unchanged. ** The second parameter is a pointer to an integer @@ -2299,8 +2391,12 @@ struct sqlite3_mem_methods { **
  • sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); ** ** Because resetting a database is destructive and irreversible, the -** process requires the use of this obscure API and multiple steps to help -** ensure that it does not happen by accident. +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. ** ** [[SQLITE_DBCONFIG_DEFENSIVE]]
    SQLITE_DBCONFIG_DEFENSIVE
    **
    The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the @@ -2311,6 +2407,7 @@ struct sqlite3_mem_methods { **
      **
    • The [PRAGMA writable_schema=ON] statement. **
    • The [PRAGMA journal_mode=OFF] statement. +**
    • The [PRAGMA schema_version=N] statement. **
    • Writes to the [sqlite_dbpage] virtual table. **
    • Direct writes to [shadow tables]. **
    @@ -2338,7 +2435,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
    SQLITE_DBCONFIG_DQS_DML +**
    SQLITE_DBCONFIG_DQS_DML
    **
    The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2347,7 +2444,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
    SQLITE_DBCONFIG_DQS_DDL +**
    SQLITE_DBCONFIG_DQS_DDL
    **
    The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2356,7 +2453,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA
    **
    The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2376,7 +2473,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
    **
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2385,7 +2482,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2394,8 +2491,40 @@ struct sqlite3_mem_methods { ** the [VACUUM] command will fail with an obscure error when attempting to ** process a table with generated columns and a descending index. This is ** not considered a bug since SQLite versions 3.3.0 and earlier do not support -** either generated columns or decending indexes. +** either generated columns or descending indexes. +**
    +** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
    SQLITE_DBCONFIG_STMT_SCANSTATUS
    +**
    The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
    +** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
    SQLITE_DBCONFIG_REVERSE_SCANORDER
    +**
    The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. **
    +** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2416,7 +2545,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2638,8 +2769,13 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); ** ^A call to sqlite3_interrupt(D) that occurs when there are no running ** SQL statements is a no-op and has no effect on SQL statements ** that are started after the sqlite3_interrupt() call returns. +** +** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether +** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API int sqlite3_is_interrupted(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -3120,7 +3256,7 @@ SQLITE_API int sqlite3_set_droptable_handle( sqlite3 *db, void (*xFunc)(sqlite3*,const char*,const char*) ); -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ /* ** CAPI3REF: Authorizer Return Codes @@ -3193,8 +3329,8 @@ SQLITE_API int sqlite3_set_droptable_handle( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -3264,8 +3400,8 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, **
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same ** information as is provided by the [sqlite3_profile()] callback. ** ^The P argument is a pointer to the [prepared statement] and the -** X argument points to a 64-bit integer which is the estimated of -** the number of nanosecond that the prepared statement took to run. +** X argument points to a 64-bit integer which is approximately +** the number of nanoseconds that the prepared statement took to run. ** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. ** ** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    @@ -3297,8 +3433,10 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, ** M argument should be the bitwise OR-ed combination of ** zero or more [SQLITE_TRACE] constants. ** -** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides -** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P) +** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or +** sqlite3_trace_v2(D,M,X,P) for the [database connection] D. Each +** database connection may have at most one trace callback. ** ** ^The X callback is invoked whenever any of the events identified by ** mask M occur. ^The integer return value from the callback is currently @@ -3328,7 +3466,7 @@ SQLITE_API int sqlite3_trace_v2( ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to -** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** [sqlite3_step()] and [sqlite3_prepare()] and similar for ** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** @@ -3353,6 +3491,13 @@ SQLITE_API int sqlite3_trace_v2( ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** +** The progress handler callback would originally only be invoked from the +** bytecode engine. It still might be invoked during [sqlite3_prepare()] +** and similar because those routines might force a reparse of the schema +** which involves running the bytecode engine. However, beginning with +** SQLite version 3.41.0, the progress handler callback might also be +** invoked directly from [sqlite3_prepare()] while analyzing and generating +** code for complex queries. */ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); @@ -3389,13 +3534,18 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** **
    ** ^(
    [SQLITE_OPEN_READONLY]
    -**
    The database is opened in read-only mode. If the database does not -** already exist, an error is returned.
    )^ +**
    The database is opened in read-only mode. If the database does +** not already exist, an error is returned.
    )^ ** ** ^(
    [SQLITE_OPEN_READWRITE]
    -**
    The database is opened for reading and writing if possible, or reading -** only if the file is write protected by the operating system. In either -** case the database must already exist, otherwise an error is returned.
    )^ +**
    The database is opened for reading and writing if possible, or +** reading only if the file is write protected by the operating +** system. In either case the database must already exist, otherwise +** an error is returned. For historical reasons, if opening in +** read-write mode fails due to OS-level permissions, an attempt is +** made to open it in read-only mode. [sqlite3_db_readonly()] can be +** used to determine whether the database is actually +** read-write.
    )^ ** ** ^(
    [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
    **
    The database is opened for reading and writing, and is created if @@ -3433,6 +3583,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
    The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    **
    The database is opened [shared cache] disabled, overriding @@ -3448,7 +3601,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code.
    ** ** [[OPEN_NOFOLLOW]] ^(
    [SQLITE_OPEN_NOFOLLOW]
    -**
    The database filename is not allowed to be a symbolic link
    +**
    The database filename is not allowed to contain a symbolic link
    **
    )^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the @@ -3652,7 +3805,7 @@ SQLITE_API int sqlite3_open_v2( ** as F) must be one of: **
      **
    • A database filename pointer created by the SQLite core and -** passed into the xOpen() method of a VFS implemention, or +** passed into the xOpen() method of a VFS implementation, or **
    • A filename obtained from [sqlite3_db_filename()], or **
    • A new filename constructed using [sqlite3_create_filename()]. **
    @@ -3707,10 +3860,10 @@ SQLITE_API int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -3739,9 +3892,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -SQLITE_API const char *sqlite3_filename_database(const char*); -SQLITE_API const char *sqlite3_filename_journal(const char*); -SQLITE_API const char *sqlite3_filename_wal(const char*); +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -3765,7 +3918,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); /* ** CAPI3REF: Create and Destroy VFS Filenames ** -** These interfces are provided for use by [VFS shim] implementations and +** These interfaces are provided for use by [VFS shim] implementations and ** are not useful outside of that context. ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of @@ -3807,14 +3960,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -SQLITE_API void sqlite3_free_filename(char*); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -3833,27 +3986,38 @@ SQLITE_API void sqlite3_free_filename(char*); ** sqlite3_extended_errcode() might change with each API call. ** Except, there are some interfaces that are guaranteed to never ** change the value of the error code. The error-code preserving -** interfaces are: +** interfaces include the following: ** **
      **
    • sqlite3_errcode() **
    • sqlite3_extended_errcode() **
    • sqlite3_errmsg() **
    • sqlite3_errmsg16() +**
    • sqlite3_error_offset() **
    ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language -** text that describes the error, as either UTF-8 or UTF-16 respectively. +** text that describes the error, as either UTF-8 or UTF-16 respectively, +** or NULL if no error message is available. +** (See how SQLite handles [invalid UTF] for exceptions to this rule.) ** ^(Memory to hold the error message string is managed internally. ** The application does not need to worry about freeing the result. ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** -** ^The sqlite3_errstr() interface returns the English-language text -** that describes the [result code], as UTF-8. +** ^The sqlite3_errstr(E) interface returns the English-language text +** that describes the [result code] E, as UTF-8, or NULL if E is not an +** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. ** +** ^If the most recent error references a specific token in the input +** SQL, the sqlite3_error_offset() interface returns the byte offset +** of the start of that token. ^The byte offset returned by +** sqlite3_error_offset() assumes that the input SQL is UTF8. +** ^If the most recent error does not reference a specific token in the input +** SQL, then the sqlite3_error_offset() function returns -1. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -3873,6 +4037,7 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int sqlite3_error_offset(sqlite3 *db); /* ** CAPI3REF: Prepared Statement Object @@ -4284,6 +4449,10 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); ** be false. ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a ** read-only no-op if the table already exists, but ** sqlite3_stmt_readonly() still returns false for such a statement. +** +** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN] +** statement, then sqlite3_stmt_readonly(X) returns the same value as +** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted. */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); @@ -4299,6 +4468,41 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for [prepared statement] S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if +** its SQL text began with "[EXPLAIN QUERY PLAN]". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using [sqlite3_prepare()] instead of +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4352,6 +4556,8 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); ** ** ^The sqlite3_value objects that are passed as parameters into the ** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()] +** are protected. ** ^The sqlite3_value object returned by ** [sqlite3_column_value()] is unprotected. ** Unprotected sqlite3_value objects may only be used as arguments @@ -4460,7 +4666,7 @@ typedef struct sqlite3_context sqlite3_context; ** with it may be passed. ^It is called to dispose of the BLOB or string even ** if the call to the bind API fails, except the destructor is not called if ** the third parameter is a NULL pointer or the fourth parameter is negative. -** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that +** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that ** the application remains responsible for disposing of the object. ^In this ** case, the object and the provided pointer to it must remain valid until ** either the prepared statement is finalized or the same SQL parameter is @@ -4973,6 +5179,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** +** ^Strings returned by sqlite3_column_text16() always have the endianness +** which is native to the platform, regardless of the text encoding set +** for the database. +** ** Warning: ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. In a multithreaded environment, ** an unprotected sqlite3_value object may only be used safely with @@ -4986,7 +5196,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** [application-defined SQL functions] or [virtual tables], not within ** top-level application code. ** -** The these routines may attempt to convert the datatype of the result. +** These routines may attempt to convert the datatype of the result. ** ^For example, if the internal representation is FLOAT and a text result ** is requested, [sqlite3_snprintf()] is used internally to perform the ** conversion automatically. ^(The following table details the conversions @@ -5011,7 +5221,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** TEXT BLOB No change ** BLOB INTEGER [CAST] to INTEGER ** BLOB FLOAT [CAST] to REAL -** BLOB TEXT Add a zero terminator if needed +** BLOB TEXT [CAST] to TEXT, ensure zero terminator ** ** )^ ** @@ -5135,20 +5345,33 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S ** back to the beginning of its program. ** -** ^If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. ** ** ^If the most recent call to [sqlite3_step(S)] for the ** [prepared statement] S indicated an error, then ** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. ** ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} @@ -5354,10 +5577,21 @@ SQLITE_API int sqlite3_create_window_function( ** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in ** schema structures such as [CHECK constraints], [DEFAULT clauses], ** [expression indexes], [partial indexes], or [generated columns]. -** The SQLITE_DIRECTONLY flags is a security feature which is recommended -** for all [application-defined SQL functions], and especially for functions -** that have side-effects or that could potentially leak sensitive -** information. +**

    +** The SQLITE_DIRECTONLY flag is recommended for any +** [application-defined SQL function] +** that has side-effects or that could potentially leak sensitive information. +** This will prevent attacks in which an application is tricked +** into using a database file that has had its schema surreptitiously +** modified to invoke the application-defined function in ways that are +** harmful. +**

    +** Some people say it is good practice to set SQLITE_DIRECTONLY on all +** [application-defined SQL functions], regardless of whether or not they +** are security sensitive, as doing so prevents those functions from being used +** inside of the database schema, and thus ensures that the database +** can be inspected and modified using generic tools (such as the [CLI]) +** that do not have access to the application-defined functions. ** ** ** [[SQLITE_INNOCUOUS]]

    SQLITE_INNOCUOUS
    @@ -5384,13 +5618,27 @@ SQLITE_API int sqlite3_create_window_function( **
    ** ** [[SQLITE_SUBTYPE]]
    SQLITE_SUBTYPE
    -** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. -** Specifying this flag makes no difference for scalar or aggregate user -** functions. However, if it is not specified for a user-defined window -** function, then any sub-types belonging to arguments passed to the window -** function may be discarded before the window function is called (i.e. -** sqlite3_value_subtype() will always return 0). +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** SQL functions that invokes [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
    SQLITE_RESULT_SUBTYPE
    +** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. **
    ** */ @@ -5398,6 +5646,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_DIRECTONLY 0x000080000 #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 /* ** CAPI3REF: Deprecated Functions @@ -5563,6 +5812,28 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); SQLITE_API int sqlite3_value_frombind(sqlite3_value*); +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); + /* ** CAPI3REF: Finding The Subtype Of SQL Values ** METHOD: sqlite3_value @@ -5572,6 +5843,12 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** information can be used to pass a limited amount of context from ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. +** +** Every [application-defined SQL function] that invoke this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. */ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); @@ -5583,7 +5860,8 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); ** object D and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a -** memory allocation fails. +** memory allocation fails. ^If V is a [pointer value], then the result +** of sqlite3_value_dup(V) is a NULL value. ** ** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object ** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer @@ -5614,7 +5892,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -5669,48 +5947,56 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to -** associate metadata with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. An example -** of where this might be useful is in a regular-expression matching -** function. The compiled version of the regular expression can be stored as -** metadata associated with the pattern string. +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. ** Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument ** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no metadata +** function argument. ^If there is no auxiliary data ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface ** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th -** argument of the application-defined function. ^Subsequent +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent ** calls to sqlite3_get_auxdata(C,N) return P from the most recent -** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or -** NULL if the metadata has been discarded. +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, ** SQLite will invoke the destructor function X with parameter P exactly -** once, when the metadata is discarded. -** SQLite is free to discard the metadata at any time, including:
      +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
        **
      • ^(when the corresponding function parameter changes)^, or **
      • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the ** SQL statement)^, or **
      • ^(when sqlite3_set_auxdata() is invoked again on the same ** parameter)^, or **
      • ^(during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.)^
      +** allocation error occurs.)^ +**
    • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^
    ** -** Note the last bullet in particular. The destructor X in +** Note the last two bullets in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the ** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() ** should be called near the end of the function implementation and the ** function implementation should not make any use of P after -** sqlite3_set_auxdata() has been called. -** -** ^(In practice, metadata is preserved between function calls for +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. +** +** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal ** values and [parameters] and expressions composed from the same.)^ ** @@ -5720,10 +6006,67 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** ** These routines must be called from the same thread in which ** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
      +**
    • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
    • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
    • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
    +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior @@ -5819,9 +6162,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -5924,6 +6268,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); ** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. */ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); @@ -6105,7 +6463,7 @@ SQLITE_API int sqlite3_rekey_v2( const void *pKey, int nKey /* The new key */ ); -#endif +#endif /* SQLITE_HAS_CODEC */ #ifdef SQLITE_ENABLE_CEROD /* @@ -6133,6 +6491,13 @@ SQLITE_API void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ SQLITE_API int sqlite3_sleep(int); @@ -6303,6 +6668,28 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); */ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +/* +** CAPI3REF: Return The Schema Name For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name +** for the N-th database on database connection D, or a NULL pointer of N is +** out of range. An N value of 0 means the main database file. An N of 1 is +** the "temp" schema. Larger values of N correspond to various ATTACH-ed +** databases. +** +** Space to hold the string that is returned by sqlite3_db_name() is managed +** by SQLite itself. The string might be deallocated by any operation that +** changes the schema, including [ATTACH] or [DETACH] or calls to +** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that +** occur on a different thread. Applications that need to +** remember the string long-term should make their own copy. Applications that +** are accessing the same database connection simultaneously on multiple +** threads should mutex-protect calls to this API and should make their own +** private copy of the result prior to releasing the mutex. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); + /* ** CAPI3REF: Return The Filename For A Database Connection ** METHOD: sqlite3 @@ -6333,7 +6720,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); **
  • [sqlite3_filename_wal()] ** */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -6364,7 +6751,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); /* -** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** CAPI3REF: Allowed return values from sqlite3_txn_state() ** KEYWORDS: {transaction state} ** ** These constants define the current transaction state of a database file. @@ -6470,7 +6857,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6496,7 +6883,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all ** previous invocations for that database connection. ^If the callback ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, -** then the autovacuum steps callback is cancelled. The return value +** then the autovacuum steps callback is canceled. The return value ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might ** be some other error code if something goes wrong. The current ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other @@ -6562,6 +6949,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -6591,6 +6984,11 @@ SQLITE_API void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, @@ -6689,7 +7087,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -6950,15 +7348,6 @@ SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); */ SQLITE_API void sqlite3_reset_auto_extension(void); -/* -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** Structures used by the virtual table interface */ @@ -7019,6 +7408,10 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); + /* The methods above are in versions 1 through 3 of the sqlite_module object. + ** Those below are for version 4 and greater. */ + int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema, + const char *zTabName, int mFlags, char **pzErr); }; /* @@ -7077,10 +7470,10 @@ struct sqlite3_module { ** when the omit flag is true there is no guarantee that the constraint will ** not be checked again using byte code.)^ ** -** ^The idxNum and idxPtr values are recorded and passed into the +** ^The idxNum and idxStr values are recorded and passed into the ** [xFilter] method. -** ^[sqlite3_free()] is used to free idxPtr if and only if -** needToFreeIdxPtr is true. +** ^[sqlite3_free()] is used to free idxStr if and only if +** needToFreeIdxStr is true. ** ** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in ** the correct order to satisfy the ORDER BY clause so that no separate @@ -7169,24 +7562,56 @@ struct sqlite3_index_info { ** ** These macros define the allowed values for the ** [sqlite3_index_info].aConstraint[].op field. Each value represents -** an operator that is part of a constraint term in the wHERE clause of +** an operator that is part of a constraint term in the WHERE clause of ** a query that uses a [virtual table]. -*/ -#define SQLITE_INDEX_CONSTRAINT_EQ 2 -#define SQLITE_INDEX_CONSTRAINT_GT 4 -#define SQLITE_INDEX_CONSTRAINT_LE 8 -#define SQLITE_INDEX_CONSTRAINT_LT 16 -#define SQLITE_INDEX_CONSTRAINT_GE 32 -#define SQLITE_INDEX_CONSTRAINT_MATCH 64 -#define SQLITE_INDEX_CONSTRAINT_LIKE 65 -#define SQLITE_INDEX_CONSTRAINT_GLOB 66 -#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 -#define SQLITE_INDEX_CONSTRAINT_NE 68 -#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 -#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 -#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 -#define SQLITE_INDEX_CONSTRAINT_IS 72 -#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 +** +** ^The left-hand operand of the operator is given by the corresponding +** aConstraint[].iColumn field. ^An iColumn of -1 indicates the left-hand +** operand is the rowid. +** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET +** operators have no left-hand operand, and so for those operators the +** corresponding aConstraint[].iColumn is meaningless and should not be +** used. +** +** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through +** value 255 are reserved to represent functions that are overloaded +** by the [xFindFunction|xFindFunction method] of the virtual table +** implementation. +** +** The right-hand operands for each constraint might be accessible using +** the [sqlite3_vtab_rhs_value()] interface. Usually the right-hand +** operand is only available if it appears as a single constant literal +** in the input SQL. If the right-hand operand is another column or an +** expression (even a constant expression) or a parameter, then the +** sqlite3_vtab_rhs_value() probably will not be able to extract it. +** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and +** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand +** and hence calls to sqlite3_vtab_rhs_value() for those operators will +** always return SQLITE_NOTFOUND. +** +** The collating sequence to be used for comparison can be found using +** the [sqlite3_vtab_collation()] interface. For most real-world virtual +** tables, the collating sequence of constraints does not matter (for example +** because the constraints are numeric) and so the sqlite3_vtab_collation() +** interface is not commonly needed. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 +#define SQLITE_INDEX_CONSTRAINT_NE 68 +#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 +#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 +#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 +#define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_LIMIT 73 +#define SQLITE_INDEX_CONSTRAINT_OFFSET 74 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -7215,7 +7640,7 @@ struct sqlite3_index_info { ** destructor. ** ** ^If the third parameter (the pointer to the sqlite3_module object) is -** NULL then no new module is create and any existing modules with the +** NULL then no new module is created and any existing modules with the ** same name are dropped. ** ** See also: [sqlite3_drop_modules()] @@ -7327,16 +7752,6 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); */ SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); -/* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} @@ -7484,7 +7899,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); ** code is returned and the transaction rolled back. ** ** Calling this function with an argument that is not a NULL pointer or an -** open blob handle results in undefined behaviour. ^Calling this routine +** open blob handle results in undefined behavior. ^Calling this routine ** with a null pointer (such as would be returned by a failed call to ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function ** is passed a valid open blob handle, the values returned by the @@ -7711,18 +8126,20 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable -** behavior.)^ +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was ** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ @@ -7964,6 +8381,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 #define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -7971,6 +8389,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ @@ -7991,7 +8410,9 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SEEK_COUNT 30 #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 -#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_LOGEST 33 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -8004,7 +8425,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -8514,6 +8935,16 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** The counter is incremented on the first [sqlite3_step()] call of each ** cycle. ** +** [[SQLITE_STMTSTATUS_FILTER_MISS]] +** [[SQLITE_STMTSTATUS_FILTER HIT]] +**
    SQLITE_STMTSTATUS_FILTER_HIT
    +** SQLITE_STMTSTATUS_FILTER_MISS
    +**
    ^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join +** step was bypassed because a Bloom filter returned not-found. The +** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of +** times that the Bloom filter returned a find, and thus the join step +** had to be processed as normal. +** ** [[SQLITE_STMTSTATUS_MEMUSED]]
    SQLITE_STMTSTATUS_MEMUSED
    **
    ^This is the approximate number of bytes of heap memory ** used to store the prepared statement. ^This value is not actually @@ -8528,6 +8959,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); #define SQLITE_STMTSTATUS_VM_STEP 4 #define SQLITE_STMTSTATUS_REPREPARE 5 #define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_FILTER_MISS 7 +#define SQLITE_STMTSTATUS_FILTER_HIT 8 #define SQLITE_STMTSTATUS_MEMUSED 99 /* @@ -8939,7 +9372,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9367,7 +9800,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -9435,7 +9868,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_DIRECTONLY]]
    SQLITE_VTAB_DIRECTONLY
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** prohibits that virtual table from being used from within triggers and ** views. **
    @@ -9443,18 +9876,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
    SQLITE_VTAB_INNOCUOUS
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
    +** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
    SQLITE_VTAB_USES_ALL_SCHEMAS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
    ** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -9496,18 +9939,295 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); /* ** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** METHOD: sqlite3_index_info ** ** This function may only be called from within a call to the [xBestIndex] -** method of a [virtual table]. +** method of a [virtual table]. This function returns a pointer to a string +** that is the name of the appropriate collation sequence to use for text +** comparisons on the constraint identified by its arguments. ** -** The first argument must be the sqlite3_index_info object that is the -** first parameter to the xBestIndex() method. The second argument must be -** an index into the aConstraint[] array belonging to the sqlite3_index_info -** structure passed to xBestIndex. This function returns a pointer to a buffer -** containing the name of the collation sequence for the corresponding -** constraint. +** The first argument must be the pointer to the [sqlite3_index_info] object +** that is the first parameter to the xBestIndex() method. The second argument +** must be an index into the aConstraint[] array belonging to the +** sqlite3_index_info structure passed to xBestIndex. +** +** Important: +** The first parameter must be the same pointer that is passed into the +** xBestMethod() method. The first parameter may not be a pointer to a +** different [sqlite3_index_info] object, even an exact copy. +** +** The return value is computed as follows: +** +**
      +**

    1. If the constraint comes from a WHERE clause expression that contains +** a [COLLATE operator], then the name of the collation specified by +** that COLLATE operator is returned. +**

    2. If there is no COLLATE operator, but the column that is the subject +** of the constraint specifies an alternative collating sequence via +** a [COLLATE clause] on the column definition within the CREATE TABLE +** statement that was passed into [sqlite3_declare_vtab()], then the +** name of that alternative collating sequence is returned. +**

    3. Otherwise, "BINARY" is returned. +**

    +*/ +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + +/* +** CAPI3REF: Determine if a virtual table query is DISTINCT +** METHOD: sqlite3_index_info +** +** This API may only be used from within an [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this +** interface from outside of xBestIndex() is undefined and probably harmful. +** +** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and +** 3. The integer returned by sqlite3_vtab_distinct() +** gives the virtual table additional information about how the query +** planner wants the output to be ordered. As long as the virtual table +** can meet the ordering requirements of the query planner, it may set +** the "orderByConsumed" flag. +** +**

    1. +** ^If the sqlite3_vtab_distinct() interface returns 0, that means +** that the query planner needs the virtual table to return all rows in the +** sort order defined by the "nOrderBy" and "aOrderBy" fields of the +** [sqlite3_index_info] object. This is the default expectation. If the +** virtual table outputs all rows in sorted order, then it is always safe for +** the xBestIndex method to set the "orderByConsumed" flag, regardless of +** the return value from sqlite3_vtab_distinct(). +**

    2. +** ^(If the sqlite3_vtab_distinct() interface returns 1, that means +** that the query planner does not need the rows to be returned in sorted order +** as long as all rows with the same values in all columns identified by the +** "aOrderBy" field are adjacent.)^ This mode is used when the query planner +** is doing a GROUP BY. +**

    3. +** ^(If the sqlite3_vtab_distinct() interface returns 2, that means +** that the query planner does not need the rows returned in any particular +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for a DISTINCT query. +**

    4. +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries +** that have both DISTINCT and ORDER BY clauses. +**

    +** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** +** ^For the purposes of comparing virtual table output values to see if the +** values are same value for sorting purposes, two NULL values are considered +** to be the same. In other words, the comparison operator is "IS" +** (or "IS NOT DISTINCT FROM") and not "==". +** +** If a virtual table implementation is unable to meet the requirements +** specified above, then it must not set the "orderByConsumed" flag in the +** [sqlite3_index_info] object or an incorrect answer may result. +** +** ^A virtual table implementation is always free to return rows in any order +** it wants, as long as the "orderByConsumed" flag is not set. ^When the +** the "orderByConsumed" flag is unset, the query planner will add extra +** [bytecode] to ensure that the final results returned by the SQL query are +** ordered correctly. The use of the "orderByConsumed" flag and the +** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful +** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed" +** flag might help queries against a virtual table to run faster. Being +** overly aggressive and setting the "orderByConsumed" flag when it is not +** valid to do so, on the other hand, might cause SQLite to return incorrect +** results. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); + +/* +** CAPI3REF: Identify and handle IN constraints in xBestIndex +** +** This interface may only be used from within an +** [xBestIndex|xBestIndex() method] of a [virtual table] implementation. +** The result of invoking this interface from any other context is +** undefined and probably harmful. +** +** ^(A constraint on a virtual table of the form +** "[IN operator|column IN (...)]" is +** communicated to the xBestIndex method as a +** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use +** this constraint, it must set the corresponding +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under +** the usual mode of handling IN operators, SQLite generates [bytecode] +** that invokes the [xFilter|xFilter() method] once for each value +** on the right-hand side of the IN operator.)^ Thus the virtual table +** only sees a single value from the right-hand side of the IN operator +** at a time. +** +** In some cases, however, it would be advantageous for the virtual +** table to see all values on the right-hand of the IN operator all at +** once. The sqlite3_vtab_in() interfaces facilitates this in two ways: +** +**

      +**

    1. +** ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero) +** if and only if the [sqlite3_index_info|P->aConstraint][N] constraint +** is an [IN operator] that can be processed all at once. ^In other words, +** sqlite3_vtab_in() with -1 in the third argument is a mechanism +** by which the virtual table can ask SQLite if all-at-once processing +** of the IN operator is even possible. +** +**

    2. +** ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates +** to SQLite that the virtual table does or does not want to process +** the IN operator all-at-once, respectively. ^Thus when the third +** parameter (F) is non-negative, this interface is the mechanism by +** which the virtual table tells SQLite how it wants to process the +** IN operator. +**

    +** +** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times +** within the same xBestIndex method call. ^For any given P,N pair, +** the return value from sqlite3_vtab_in(P,N,F) will always be the same +** within the same xBestIndex call. ^If the interface returns true +** (non-zero), that means that the constraint is an IN operator +** that can be processed all-at-once. ^If the constraint is not an IN +** operator or cannot be processed all-at-once, then the interface returns +** false. +** +** ^(All-at-once processing of the IN operator is selected if both of the +** following conditions are met: +** +**
      +**

    1. The P->aConstraintUsage[N].argvIndex value is set to a positive +** integer. This is how the virtual table tells SQLite that it wants to +** use the N-th constraint. +** +**

    2. The last call to sqlite3_vtab_in(P,N,F) for which F was +** non-negative had F>=1. +**

    )^ +** +** ^If either or both of the conditions above are false, then SQLite uses +** the traditional one-at-a-time processing strategy for the IN constraint. +** ^If both conditions are true, then the argvIndex-th parameter to the +** xFilter method will be an [sqlite3_value] that appears to be NULL, +** but which can be passed to [sqlite3_vtab_in_first()] and +** [sqlite3_vtab_in_next()] to find all values on the right-hand side +** of the IN constraint. */ -SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); + +/* +** CAPI3REF: Find all elements on the right-hand side of an IN constraint. +** +** These interfaces are only useful from within the +** [xFilter|xFilter() method] of a [virtual table] implementation. +** The result of invoking these interfaces from any other context +** is undefined and probably harmful. +** +** The X parameter in a call to sqlite3_vtab_in_first(X,P) or +** sqlite3_vtab_in_next(X,P) should be one of the parameters to the +** xFilter method which invokes these routines, and specifically +** a parameter that was previously selected for all-at-once IN constraint +** processing use the [sqlite3_vtab_in()] interface in the +** [xBestIndex|xBestIndex method]. ^(If the X parameter is not +** an xFilter argument that was selected for all-at-once IN constraint +** processing, then these routines return [SQLITE_ERROR].)^ +** +** ^(Use these routines to access all values on the right-hand side +** of the IN constraint using code like the following: +** +** 
    +**    for(rc=sqlite3_vtab_in_first(pList, &pVal);
+**        rc==SQLITE_OK && pVal;
+**        rc=sqlite3_vtab_in_next(pList, &pVal)
+**    ){
+**      // do something with pVal
+**    }
+**    if( rc!=SQLITE_OK ){
+**      // an error has occurred
+**    }
+**
    )^ +** +** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P) +** routines return SQLITE_OK and set *P to point to the first or next value +** on the RHS of the IN constraint. ^If there are no more values on the +** right hand side of the IN constraint, then *P is set to NULL and these +** routines return [SQLITE_DONE]. ^The return value might be +** some other value, such as SQLITE_NOMEM, in the event of a malfunction. +** +** The *ppOut values returned by these routines are only valid until the +** next call to either of these routines or until the end of the xFilter +** method from which these routines were called. If the virtual table +** implementation needs to retain the *ppOut values for longer, it must make +** copies. The *ppOut values are [protected sqlite3_value|protected]. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut); +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); + +/* +** CAPI3REF: Constraint values in xBestIndex() +** METHOD: sqlite3_index_info +** +** This API may only be used from within the [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this interface +** from outside of an xBestIndex method are undefined and probably harmful. +** +** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within +** the [xBestIndex] method of a [virtual table] implementation, with P being +** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and +** J being a 0-based index into P->aConstraint[], then this routine +** attempts to set *V to the value of the right-hand operand of +** that constraint if the right-hand operand is known. ^If the +** right-hand operand is not known, then *V is set to a NULL pointer. +** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if +** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) +** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th +** constraint is not available. ^The sqlite3_vtab_rhs_value() interface +** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** something goes wrong. +** +** The sqlite3_vtab_rhs_value() interface is usually only successful if +** the right-hand operand of a constraint is a literal value in the original +** SQL statement. If the right-hand operand is an expression or a reference +** to some other column or a [host parameter], then sqlite3_vtab_rhs_value() +** will probably return [SQLITE_NOTFOUND]. +** +** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and +** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand. For such +** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^ +** +** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value +** and remains valid for the duration of the xBestIndex method call. +** ^When xBestIndex returns, the sqlite3_value object returned by +** sqlite3_vtab_rhs_value() is automatically deallocated. +** +** The "_rhs_" in the name of this routine is an abbreviation for +** "Right-Hand Side". +*/ +SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); /* ** CAPI3REF: Conflict resolution modes @@ -9539,6 +10259,10 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    **
    ^The [sqlite3_int64] variable pointed to by the V parameter will be @@ -9566,12 +10290,24 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECTID
    **
    ^The "int" variable pointed to by the V parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_PARENTID]]
    SQLITE_SCANSTAT_PARENTID
    +**
    The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
    SQLITE_SCANSTAT_NCYCLE
    +**
    The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. **
    */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -9580,12 +10316,14 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. @@ -9596,19 +10334,25 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. -** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. +** +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ @@ -9618,6 +10362,19 @@ SQLITE_API int sqlite3_stmt_scanstatus( int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ ); +SQLITE_API int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 /* ** CAPI3REF: Zero Scan-Status Counters @@ -9708,6 +10465,10 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** function is not defined for operations on WITHOUT ROWID tables, or for ** DELETE operations on rowid tables. ** +** ^The sqlite3_preupdate_hook(D,C,P) function returns the P argument from +** the previous call on the same [database connection] D, or NULL for +** the first call on D. +** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces ** provide additional information about a preupdate event. These routines @@ -9747,7 +10508,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** When the [sqlite3_blob_write()] API is used to update a blob column, ** the pre-update hook is invoked with SQLITE_DELETE. This is because the ** in this case the new values are not available. In this case, when a -** callback made with op==SQLITE_DELETE is actuall a write using the +** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the ** pre-update hook is being invoked for some other reason, including a @@ -10008,6 +10769,13 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy ** of the database exists. ** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory ** allocation error occurs. @@ -10056,6 +10824,9 @@ SQLITE_API unsigned char *sqlite3_serialize( ** SQLite will try to increase the buffer size using sqlite3_realloc64() ** if writes on the database cause it to grow larger than M bytes. ** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the ** database is currently in a read transaction or is involved in a backup ** operation. @@ -10064,6 +10835,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** ** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then ** [sqlite3_free()] is invoked on argument P prior to returning. @@ -10113,6 +10891,19 @@ SQLITE_API int sqlite3_deserialize( # undef double #endif +#if defined(__wasi__) +# undef SQLITE_WASI +# define SQLITE_WASI 1 +# undef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ +# ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION +# endif +# ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +# endif +#endif + #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif @@ -10319,16 +11110,20 @@ SQLITE_API int sqlite3session_create( SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** **
    SQLITE_SESSION_OBJCONFIG_SIZE
    @@ -10344,12 +11139,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +**
    SQLITE_SESSION_OBJCONFIG_ROWID
    +** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -11110,6 +11914,18 @@ SQLITE_API int sqlite3changeset_concat( ); +/* +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + /* ** CAPI3REF: Changegroup Handle ** @@ -11156,6 +11972,38 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; */ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); +/* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
      +**
    • The name identified by the changeset, and +**
    • at least as many columns as recorded in the changeset, and +**
    • the primary key columns in the same position as recorded in +** the changeset. +**
    +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + /* ** CAPI3REF: Add A Changeset To A Changegroup ** METHOD: sqlite3_changegroup @@ -11224,16 +12072,45 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); ** If the new changeset contains changes to a table that is already present ** in the changegroup, then the number of columns and the position of the ** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the state -** of the final contents of the changegroup is undefined. +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. ** -** If no error occurs, SQLITE_OK is returned. +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. +** +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -11482,9 +12359,30 @@ SQLITE_API int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
    SQLITE_CHANGESETAPPLY_IGNORENOOP
    +** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
      +**
    • a delete change if the row being deleted cannot be found, +**
    • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
    • an insert change if all fields of the conflicting row match +** the row being inserted. +**
    +** +**
    SQLITE_CHANGESETAPPLY_FKNOACTION
    +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -12017,8 +12915,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -12050,8 +12948,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -12061,8 +12962,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -12078,12 +12981,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -12109,6 +13013,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -12223,6 +13131,39 @@ struct Fts5PhraseIter { ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ @@ -12260,6 +13201,13 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); }; /* @@ -12454,8 +13402,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -12503,7 +13451,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -12512,7 +13460,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -12520,7 +13468,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); diff --git a/mock/sqlite/include/sqlite3ext.h b/mock/sqlite/include/sqlite3ext.h index 7c1373fe..437cd3eb 100644 --- a/mock/sqlite/include/sqlite3ext.h +++ b/mock/sqlite/include/sqlite3ext.h @@ -331,9 +331,9 @@ struct sqlite3_api_routines { const char *(*filename_journal)(const char*); const char *(*filename_wal)(const char*); /* Version 3.32.0 and later */ - char *(*create_filename)(const char*,const char*,const char*, + const char *(*create_filename)(const char*,const char*,const char*, int,const char**); - void (*free_filename)(char*); + void (*free_filename)(const char*); sqlite3_file *(*database_file_object)(const char*); /* Version 3.34.0 and later */ int (*txn_state)(sqlite3*,const char*); @@ -344,9 +344,30 @@ struct sqlite3_api_routines { int (*autovacuum_pages)(sqlite3*, unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int), void*, void(*)(void*)); -#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK + /* Version 3.38.0 and later */ + int (*error_offset)(sqlite3*); + int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**); + int (*vtab_distinct)(sqlite3_index_info*); + int (*vtab_in)(sqlite3_index_info*,int,int); + int (*vtab_in_first)(sqlite3_value*,sqlite3_value**); + int (*vtab_in_next)(sqlite3_value*,sqlite3_value**); + /* Version 3.39.0 and later */ + int (*deserialize)(sqlite3*,const char*,unsigned char*, + sqlite3_int64,sqlite3_int64,unsigned); + unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, + unsigned int); + const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); + /* Version 3.41.0 and later */ + int (*is_interrupted)(sqlite3*); + /* Version 3.43.0 and later */ + int (*stmt_explain)(sqlite3_stmt*,int); + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); + /* handle after drop table done */ int (*set_droptable_handle)(sqlite3*,void(*)(sqlite3*,const char*,const char*)); -#endif }; /* @@ -374,7 +395,7 @@ typedef int (*sqlite3_loadext_entry)( #ifdef SQLITE3_EXPORT_SYMBOLS extern const sqlite3_api_routines *sqlite3_export_symbols; #define sqlite3_api sqlite3_export_symbols -#endif +#endif /* SQLITE3_EXPORT_SYMBOLS */ #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -662,11 +683,31 @@ extern const sqlite3_api_routines *sqlite3_export_symbols; #define sqlite3_total_changes64 sqlite3_api->total_changes64 /* Version 3.37.0 and later */ #define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages - -#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK -#define sqlite3_set_droptable_handle sqlite3_api->set_droptable_handle +/* Version 3.38.0 and later */ +#define sqlite3_error_offset sqlite3_api->error_offset +#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value +#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct +#define sqlite3_vtab_in sqlite3_api->vtab_in +#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first +#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next +/* Version 3.39.0 and later */ +#ifndef SQLITE_OMIT_DESERIALIZE +#define sqlite3_deserialize sqlite3_api->deserialize +#define sqlite3_serialize sqlite3_api->serialize #endif -#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ +#define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding +/* Version 3.41.0 and later */ +#define sqlite3_is_interrupted sqlite3_api->is_interrupted +/* Version 3.43.0 and later */ +#define sqlite3_stmt_explain sqlite3_api->stmt_explain +/* Version 3.44.0 and later */ +#define sqlite3_get_clientdata sqlite3_api->get_clientdata +#define sqlite3_set_clientdata sqlite3_api->set_clientdata +/* handle after drop table done */ +#define sqlite3_set_droptable_handle sqlite3_api->set_droptable_handle +#endif /* !defined(SQLITE_CORE) && (!defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE3_EXPORT_SYMBOLS)) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable diff --git a/mock/sqlite/include/sqlite3icu.h b/mock/sqlite/include/sqlite3icu.h new file mode 100644 index 00000000..be58299e --- /dev/null +++ b/mock/sqlite/include/sqlite3icu.h @@ -0,0 +1,50 @@ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26` +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +#include "sqlite3sym.h" +#include "sqlite3tokenizer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SQLITE_API int sqlite3IcuInit(sqlite3 *db); + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif + +#ifdef SQLITE_ENABLE_ICU +SQLITE_API void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif \ No newline at end of file diff --git a/mock/sqlite/include/sqlite3sym.h b/mock/sqlite/include/sqlite3sym.h index bd459b3e..e63a497d 100644 --- a/mock/sqlite/include/sqlite3sym.h +++ b/mock/sqlite/include/sqlite3sym.h @@ -39,4 +39,12 @@ extern const struct sqlite3_api_routines_hw *sqlite3_export_hw_symbols; #define sqlite3_rekey sqlite3_export_hw_symbols->rekey #define sqlite3_rekey_v2 sqlite3_export_hw_symbols->rekey_v2 +struct sqlite3_api_routines_cksumvfs { + int (*register_cksumvfs)(const char *); + int (*unregister_cksumvfs)(); +}; +extern const struct sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols; +#define sqlite3_register_cksumvfs sqlite3_export_cksumvfs_symbols->register_cksumvfs +#define sqlite3_unregister_cksumvfs sqlite3_export_cksumvfs_symbols->unregister_cksumvfs + #endif diff --git a/mock/sqlite/include/sqlite3tokenizer.h b/mock/sqlite/include/sqlite3tokenizer.h new file mode 100644 index 00000000..a7f10ca4 --- /dev/null +++ b/mock/sqlite/include/sqlite3tokenizer.h @@ -0,0 +1,145 @@ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +/* #include "sqlite3.h" */ + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; \ No newline at end of file diff --git a/mock/sqlite/patch/0001-BaselineWithHistoryOH.patch b/mock/sqlite/patch/0001-BaselineWithHistoryOH.patch new file mode 100644 index 00000000..ef9b8cf8 --- /dev/null +++ b/mock/sqlite/patch/0001-BaselineWithHistoryOH.patch @@ -0,0 +1,3639 @@ +From f2e21aa63fb875f21e963ba046850e8f9dd4a86d Mon Sep 17 00:00:00 2001 +From: linzhuobin1 +Date: Mon, 24 Feb 2025 18:36:42 +0800 +Subject: [PATCH] IssueNo:#IBNXAW Description:Sqlite baseline update to 3.46.1 + with patch-1-1 Sig: SIG_DataManagement Feature or Bugfix:Feature Binary + Source:No/Yes TDD:Pass/Fail/NA XTS:Pass/Fail/NA Pretest:Pass/Fail/NA + +Signed-off-by: linzhuobin1 +--- + src/sqlite3.c | 2841 ++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 2824 insertions(+), 17 deletions(-) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 946815f..d2bfc23 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -881,6 +881,7 @@ SQLITE_API int sqlite3_exec( + #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) + #define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) + #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) ++#define SQLITE_WARNING_DUMP (SQLITE_WARNING | (2<<8)) + #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) + #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) + #define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +@@ -2845,6 +2846,11 @@ struct sqlite3_mem_methods { + #define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ + #define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ + ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++#define SQLITE_DBCONFIG_SET_SHAREDBLOCK 2004 ++#define SQLITE_DBCONFIG_USE_SHAREDBLOCK 2005 ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + /* + ** CAPI3REF: Enable Or Disable Extended Result Codes + ** METHOD: sqlite3 +@@ -5316,6 +5322,10 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); + */ + SQLITE_API int sqlite3_step(sqlite3_stmt*); + ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++SQLITE_API int sqlite3_set_droptable_handle(sqlite3*, void (*xFunc)(sqlite3*,const char*,const char*)); ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ ++ + /* + ** CAPI3REF: Number of columns in a result set + ** METHOD: sqlite3_stmt +@@ -6716,6 +6726,44 @@ SQLITE_API int sqlite3_collation_needed16( + void(*)(void*,sqlite3*,int eTextRep,const void*) + ); + ++#ifdef SQLITE_HAS_CODEC ++/* ++** Specify the key for an encrypted database. This routine should be ++** called right after sqlite3_open(). ++** ++** The code to implement this API is not available in the public release ++** of SQLite. ++*/ ++SQLITE_API int sqlite3_key( ++ sqlite3 *db, /* Database to be rekeyed */ ++ const void *pKey, int nKey /* The key */ ++); ++SQLITE_API int sqlite3_key_v2( ++ sqlite3 *db, /* Database to be rekeyed */ ++ const char *zDbName, /* Name of the database */ ++ const void *pKey, int nKey /* The key */ ++); ++ ++/* ++** Change the key on an open database. If the current database is not ++** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the ++** database is decrypted. ++** ++** The code to implement this API is not available in the public release ++** of SQLite. ++*/ ++SQLITE_API int sqlite3_rekey( ++ sqlite3 *db, /* Database to be rekeyed */ ++ const void *pKey, int nKey /* The new key */ ++); ++SQLITE_API int sqlite3_rekey_v2( ++ sqlite3 *db, /* Database to be rekeyed */ ++ const char *zDbName, /* Name of the database */ ++ const void *pKey, int nKey /* The new key */ ++); ++ ++#endif /* SQLITE_HAS_CODEC */ ++ + #ifdef SQLITE_ENABLE_CEROD + /* + ** Specify the activation key for a CEROD database. Unless +@@ -10162,6 +10210,27 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + */ + SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++typedef struct Sqlite3SharedBlockMethods Sqlite3SharedBlockMethods; ++struct Sqlite3SharedBlockMethods { ++ int iVersion; ++ void* pContext; ++ int countAllRows; ++ int startPos; ++ int requiredPos; ++ int (*xAddRow)(void* pCtx, int addedRows); ++ int (*xReset)(void* pCtx, int startPos); ++ int (*xFinish)(void* pCtx, int addedRows, int totalRows); ++ int (*xPutString)(void *pCtx, int addedRows, int column, const char* text, int len); ++ int (*xPutLong)(void *pCtx, int addedRows, int column, sqlite3_int64 value); ++ int (*xPutDouble)(void *pCtx, int addedRows, int column, double value); ++ int (*xPutBlob)(void *pCtx, int addedRows, int column, const void* blob, int len); ++ int (*xPutNull)(void *pCtx, int addedRows, int column); ++ int (*xPutOther)(void *pCtx, int addedRows, int column); ++ /* Additional methods may be added in future releases */ ++}; ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + /* + ** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE + ** +@@ -10221,7 +10290,7 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); + **

  • Otherwise, "BINARY" is returned. + ** + */ +-SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ++SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + + /* + ** CAPI3REF: Determine if a virtual table query is DISTINCT +@@ -15896,6 +15965,9 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); + /* Functions used to configure a Pager object. */ + SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); + SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); ++#ifdef SQLITE_HAS_CODEC ++SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*, Pager*); ++#endif /* SQLITE_HAS_CODEC */ + SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager*, Pgno); + SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); + SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); +@@ -15992,6 +16064,10 @@ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); + + SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); + ++#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) ++SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); ++#endif /* defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) */ ++ + /* Functions to support testing and debugging. */ + #if !defined(NDEBUG) || defined(SQLITE_TEST) + SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); +@@ -17632,6 +17708,21 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + */ + #define SQLITE_MAX_DB (SQLITE_MAX_ATTACHED+2) + ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++typedef void (*sqlite3_xDropTableHandle)(sqlite3*, const char*, const char*); ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ ++ ++#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) ++typedef struct CodecParameter { ++ int kdfIter; ++ int pageSize; ++ u8 cipher; ++ u8 hmacAlgo; ++ u8 kdfAlgo; ++ u8 reserved; ++} CodecParameter; ++#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ ++ + /* + ** Each database connection is an instance of the following structure. + */ +@@ -17776,6 +17867,15 @@ struct sqlite3 { + #ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ + #endif ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ unsigned int isDropTable; ++ char *mDropTableName; ++ char *mDropSchemaName; ++ sqlite3_xDropTableHandle xDropTableHandle; /* User drop table callback */ ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ ++#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) ++ CodecParameter codecParm; ++#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ + }; + + /* +@@ -20889,7 +20989,14 @@ SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u32,Select*); + SQLITE_PRIVATE void sqlite3AddReturning(Parse*,ExprList*); + SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); ++#ifndef SQLITE_HAS_CODEC + #define sqlite3CodecQueryParameters(A,B,C) 0 ++#else ++SQLITE_PRIVATE int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); ++#endif /* !SQLITE_HAS_CODEC */ ++#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) ++SQLITE_PRIVATE void sqlite3CodecResetParameters(CodecParameter *p); ++#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ + SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); + + #ifdef SQLITE_UNTESTABLE +@@ -21873,6 +21980,10 @@ SQLITE_API extern int sqlite3_open_file_count; + #define OpenCounter(X) + #endif /* defined(SQLITE_TEST) */ + ++#ifdef FDSAN_ENABLE ++#define SQLITE_FDSAN_TAG 5351 ++#endif /* FDSAN_ENABLE */ ++ + #endif /* !defined(_OS_COMMON_H_) */ + + /************** End of os_common.h *******************************************/ +@@ -22258,6 +22369,9 @@ static const char * const sqlite3azCompileOpt[] = { + #ifdef SQLITE_FTS5_NO_WITHOUT_ROWID + "FTS5_NO_WITHOUT_ROWID", + #endif ++#if SQLITE_HAS_CODEC ++ "HAS_CODEC", ++#endif + #if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", + #endif +@@ -22266,6 +22380,9 @@ static const char * const sqlite3azCompileOpt[] = { + "HOMEGROWN_RECURSIVE_MUTEX=" CTIMEOPT_VAL(SQLITE_HOMEGROWN_RECURSIVE_MUTEX), + # endif + #endif ++#if SQLITE_SHARED_BLOCK_OPTIMIZATION ++ "SHARED_BLOCK_OPTIMIZATION", ++#endif + #ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", + #endif +@@ -22841,9 +22958,16 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { + ** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally + ** disabled. The default value may be changed by compiling with the + ** SQLITE_USE_URI symbol defined. ++** ++** URI filenames are enabled by default if SQLITE_HAS_CODEC is ++** enabled. + */ + #ifndef SQLITE_USE_URI +-# define SQLITE_USE_URI 0 ++#ifndef SQLITE_HAS_CODEC ++#define SQLITE_USE_URI 0 ++#else ++#define SQLITE_USE_URI 1 ++#endif /* !SQLITE_HAS_CODEC */ + #endif + + /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +@@ -23615,6 +23739,13 @@ struct Vdbe { + int nScan; /* Entries in aScan[] */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ + #endif ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ Sqlite3SharedBlockMethods *pSharedBlock; ++ int totalRows; ++ int blockFull; ++ int startPos; ++ int addedRows; ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + }; + + /* +@@ -32452,7 +32583,11 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + */ + static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ + StrAccum acc; /* String accumulator */ ++#ifndef LOG_DUMP + char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ ++#else ++char zMsg[SQLITE_PRINT_BUF_SIZE*10]; /* Complete log message */ ++#endif /* !LOG_DUMP */ + + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); + sqlite3_str_vappendf(&acc, zFormat, ap); +@@ -36425,7 +36560,7 @@ SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ + return (u8)(h & 0xf); + } + +-#if !defined(SQLITE_OMIT_BLOB_LITERAL) ++#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) + /* + ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary + ** value. Return a pointer to its binary value. Space to hold the +@@ -36446,7 +36581,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ + } + return zBlob; + } +-#endif /* !SQLITE_OMIT_BLOB_LITERAL */ ++#endif /* !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) */ + + /* + ** Log an error that is an API call on a connection pointer that should +@@ -38754,6 +38889,29 @@ static pid_t randomnessPid = 0; + #define F2FS_FEATURE_ATOMIC_WRITE 0x0004 + #endif /* __linux__ */ + ++#ifdef HARMONY_OS ++#define HMFS_MONITOR_FL 0x00000002 ++#define HMFS_IOCTL_HW_GET_FLAGS _IOR(0xf5, 70, unsigned int) ++#define HMFS_IOCTL_HW_SET_FLAGS _IOR(0xf5, 71, unsigned int) ++ ++static void enableDbFileDelMonitor(int32_t fd) ++{ ++ unsigned int flags = 0; ++ int ret = ioctl(fd, HMFS_IOCTL_HW_GET_FLAGS, &flags); ++ if (ret < 0) { ++ return; ++ } ++ if (flags & HMFS_MONITOR_FL) { ++ return; ++ } ++ flags |= HMFS_MONITOR_FL; ++ ret = ioctl(fd, HMFS_IOCTL_HW_SET_FLAGS, &flags); ++ if (ret < 0) { ++ sqlite3_log(SQLITE_WARNING, "Fd %d enable del monitor go wrong, errno = %d", fd, errno); ++ } ++} ++ ++#endif /* HARMONY_OS */ + + /* + ** Different Unix systems declare open() in different ways. Same use +@@ -38764,7 +38922,29 @@ static pid_t randomnessPid = 0; + ** which always has the same well-defined interface. + */ + static int posixOpen(const char *zFile, int flags, int mode){ +- return open(zFile, flags, mode); ++ int fd = open(zFile, flags, mode); ++#ifdef FDSAN_ENABLE ++ if( fd >= 0 ){ ++ fdsan_exchange_owner_tag(fd, 0, fdsan_create_owner_tag(FDSAN_OWNER_TYPE_FILE, SQLITE_FDSAN_TAG)); ++ } ++#endif /* FDSAN_ENABLE */ ++#ifdef HARMONY_OS ++ if( fd >= 0 ){ ++ enableDbFileDelMonitor(fd); ++ } ++#endif /* HARMONY_OS */ ++ return fd; ++} ++ ++/* ++** Change close to posixClose, use fdsan_close_with_tag when fdsan enable. ++*/ ++static int posixClose(int fd) { ++#ifdef FDSAN_ENABLE ++ return fdsan_close_with_tag(fd, fdsan_create_owner_tag(FDSAN_OWNER_TYPE_FILE, SQLITE_FDSAN_TAG)); ++#else ++ return close(fd); ++#endif /* FDSAN_ENABLE */ + } + + /* Forward reference */ +@@ -38785,7 +38965,7 @@ static struct unix_syscall { + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, + #define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) + +- { "close", (sqlite3_syscall_ptr)close, 0 }, ++ { "close", (sqlite3_syscall_ptr)posixClose, 0 }, + #define osClose ((int(*)(int))aSyscall[1].pCurrent) + + { "access", (sqlite3_syscall_ptr)access, 0 }, +@@ -39789,6 +39969,9 @@ static int findInodeInfo( + #if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; + #endif ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR, "findInodeInfo-osFstat, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR; + } + +@@ -39807,11 +39990,17 @@ static int findInodeInfo( + do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + if( rc!=1 ){ + storeLastErrno(pFile, errno); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR, "findInodeInfo-osWrite, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR; + } + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + storeLastErrno(pFile, errno); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR, "findInodeInfo-msdos-osFstat, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR; + } + } +@@ -40973,6 +41162,9 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { + #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + return SQLITE_OK; + #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_UNLOCK, "IOERR_UNLOCK fd[%d], eFileLock[%d]", pFile->h, eFileLock); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_UNLOCK; + }else{ + pFile->eFileLock = NO_LOCK; +@@ -41785,8 +41977,14 @@ static int unixRead( + #ifdef EDEVERR + case EDEVERR: + #endif ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_CORRUPTFS, "unixRead-EDEVERR, fd:[%d], amt[%d], got[%d], offset[%lld]", pFile->h, amt, got, offset); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_CORRUPTFS; + } ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_READ, "unixRead-got<0, fd: [%d], amt[%d], got[%d], offset[%lld]", pFile->h, amt, got, offset); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_READ; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ +@@ -41928,9 +42126,19 @@ static int unixWrite( + if( amt>wrote ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ + /* lastErrno set by seekAndWrite */ ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_WRITE, ++ "unixWrite, lastErrno set by seekAndWrite, fd[%d], offset[%lld], wrote[%d], amt[%d], lastErrno[%d]", ++ pFile->h, offset, wrote, amt, pFile->lastErrno); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_WRITE; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_FULL, ++ "unixWrite, not a system error, fd[%d], offset[%lld], wrote[%d], amt[%d], lastErrno[%d]", ++ pFile->h, offset, wrote, amt, pFile->lastErrno); ++#endif /* LOG_DUMP */ + return SQLITE_FULL; + } + } +@@ -42284,7 +42492,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + do{ + err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); + }while( err==EINTR ); +- if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; ++ if( err && err!=EINVAL ) { ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_WRITE, ++ "fcntlSizeHint-fallocate, fd[%d], bufSize[%lld], nSize[%lld] err[%d]", ++ pFile->h, buf.st_size, nSize, err); ++#endif /* LOG_DUMP */ ++ return SQLITE_IOERR_WRITE; ++ } + #else + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely +@@ -42303,7 +42518,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; + nWrite = seekAndWrite(pFile, iWrite, "", 1); +- if( nWrite!=1 ) return SQLITE_IOERR_WRITE; ++ if( nWrite!=1 ) { ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_WRITE, ++ "fcntlSizeHint-seekAndWrite, fd[%d], nWrite[%d], nSize[%d], nBlk[%d], iWrite[%lld]", ++ pFile->h, nWrite, nSize, nBlk, iWrite); ++#endif /* LOG_DUMP */ ++ return SQLITE_IOERR_WRITE; ++ } + } + #endif + } +@@ -42358,14 +42580,29 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); ++#ifdef LOG_DUMP ++ if( rc ){ ++ sqlite3_log(SQLITE_IOERR_BEGIN_ATOMIC, "unixFileControl-begin, fd[%d], op[%d]", pFile->h, op); ++ } ++#endif /* LOG_DUMP */ + return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); ++#ifdef LOG_DUMP ++ if( rc ){ ++ sqlite3_log(SQLITE_IOERR_COMMIT_ATOMIC, "unixFileControl-commit, fd[%d], op[%d]", pFile->h, op); ++ } ++#endif /* LOG_DUMP */ + return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); ++#ifdef LOG_DUMP ++ if( rc ){ ++ sqlite3_log(SQLITE_IOERR_ROLLBACK_ATOMIC, "unixFileControl-rollback, fd[%d], op[%d]", pFile->h, op); ++ } ++#endif /* LOG_DUMP */ + return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; + } + #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ +@@ -43378,9 +43615,19 @@ static int unixShmLock( + int *aLock; + + p = pDbFd->pShm; +- if( p==0 ) return SQLITE_IOERR_SHMLOCK; ++ if( p==0 ) { ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_SHMLOCK, "unixShmLock-pShm, fd[%d], ofst[%d], n[%d], flags[%d]", pDbFd->h, ofst, n, flags); ++#endif ++ return SQLITE_IOERR_SHMLOCK; ++ } + pShmNode = p->pShmNode; +- if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; ++ if( NEVER(pShmNode==0) ) { ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_SHMLOCK, "unixShmLock-pShmNode, fd[%d], ofst[%d], n[%d], flags[%d]", pDbFd->h, ofst, n, flags); ++#endif ++ return SQLITE_IOERR_SHMLOCK; ++ } + aLock = pShmNode->aLock; + + assert( pShmNode==pDbFd->pInode->pShmNode ); +@@ -44827,6 +45074,9 @@ static int unixOpen( + if( fstatfs(fd, &fsInfo) == -1 ){ + storeLastErrno(p, errno); + robust_close(p, fd, __LINE__); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_ACCESS, "unixOpen, fd[%d], flags[%d], errno[%d]", fd, errno, flags); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { +@@ -44967,6 +45217,8 @@ static int unixAccess( + return SQLITE_OK; + } + ++#ifndef HARMONY_OS ++ + /* + ** A pathname under construction + */ +@@ -45097,6 +45349,187 @@ static int unixFullPathname( + return SQLITE_OK; + } + ++#else ++ ++/* ++** If the last component of the pathname in z[0]..z[j-1] is something ++** other than ".." then back it out and return true. If the last ++** component is empty or if it is ".." then return false. ++*/ ++static int unixBackupDir(const char *z, int *pJ){ ++ int j = *pJ; ++ int i; ++ if( j<=0 ) return 0; ++ for(i=j-1; i>0 && z[i-1]!='/'; i--){} ++ if( i==0 ) return 0; ++ if( z[i]=='.' && i==j-2 && z[i+1]=='.' ) return 0; ++ *pJ = i-1; ++ return 1; ++} ++ ++/* ++** Convert a relative pathname into a full pathname. Also ++** simplify the pathname as follows: ++** ++** Remove all instances of /./ ++** Remove all isntances of /X/../ for any X ++*/ ++static int mkFullPathname( ++ const char *zPath, /* Input path */ ++ char *zOut, /* Output buffer */ ++ int nOut /* Allocated size of buffer zOut */ ++){ ++ int nPath = sqlite3Strlen30(zPath); ++ int iOff = 0; ++ int i, j; ++ if( zPath[0]!='/' ){ ++ if( osGetcwd(zOut, nOut-2)==0 ){ ++ return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); ++ } ++ iOff = sqlite3Strlen30(zOut); ++ zOut[iOff++] = '/'; ++ } ++ if( (iOff+nPath+1)>nOut ){ ++ /* SQLite assumes that xFullPathname() nul-terminates the output buffer ++ ** even if it returns an error. */ ++ zOut[iOff] = '\0'; ++ return SQLITE_CANTOPEN_BKPT; ++ } ++ sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); ++ ++ /* Remove duplicate '/' characters. Except, two // at the beginning ++ ** of a pathname is allowed since this is important on windows. */ ++ for(i=j=1; zOut[i]; i++){ ++ zOut[j++] = zOut[i]; ++ while( zOut[i]=='/' && zOut[i+1]=='/' ) i++; ++ } ++ zOut[j] = 0; ++ ++ assert( zOut[0]=='/' ); ++ for(i=j=0; zOut[i]; i++){ ++ if( zOut[i]=='/' ){ ++ /* Skip over internal "/." directory components */ ++ if( zOut[i+1]=='.' && zOut[i+2]=='/' ){ ++ i += 1; ++ continue; ++ } ++ ++ /* If this is a "/.." directory component then back out the ++ ** previous term of the directory if it is something other than "..". ++ */ ++ if( zOut[i+1]=='.' ++ && zOut[i+2]=='.' ++ && zOut[i+3]=='/' ++ && unixBackupDir(zOut, &j) ++ ){ ++ i += 2; ++ continue; ++ } ++ } ++ if( ALWAYS(j>=0) ) zOut[j] = zOut[i]; ++ j++; ++ } ++ if( NEVER(j==0) ) zOut[j++] = '/'; ++ zOut[j] = 0; ++ return SQLITE_OK; ++} ++ ++/* ++** Turn a relative pathname into a full pathname. The relative path ++** is stored as a nul-terminated string in the buffer pointed to by ++** zPath. ++** ++** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes ++** (in this case, MAX_PATHNAME bytes). The full-path is written to ++** this buffer before returning. ++*/ ++static int unixFullPathname( ++ sqlite3_vfs *pVfs, /* Pointer to vfs object */ ++ const char *zPath, /* Possibly relative input path */ ++ int nOut, /* Size of output buffer in bytes */ ++ char *zOut /* Output buffer */ ++){ ++#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) ++ return mkFullPathname(zPath, zOut, nOut); ++#else ++ int rc = SQLITE_OK; ++ int nByte; ++ int nLink = 0; /* Number of symbolic links followed so far */ ++ const char *zIn = zPath; /* Input path for each iteration of loop */ ++ char *zDel = 0; ++ ++ assert( pVfs->mxPathname==MAX_PATHNAME ); ++ UNUSED_PARAMETER(pVfs); ++ ++ /* It's odd to simulate an io-error here, but really this is just ++ ** using the io-error infrastructure to test that SQLite handles this ++ ** function failing. This function could fail if, for example, the ++ ** current working directory has been unlinked. ++ */ ++ SimulateIOError( return SQLITE_ERROR ); ++ ++ do { ++ ++ /* Call stat() on path zIn. Set bLink to true if the path is a symbolic ++ ** link, or false otherwise. */ ++ int bLink = 0; ++ struct stat buf; ++ if( osLstat(zIn, &buf)!=0 ){ ++ if( errno!=ENOENT ){ ++ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); ++ } ++ }else{ ++ bLink = S_ISLNK(buf.st_mode); ++ } ++ ++ if( bLink ){ ++ nLink++; ++ if( zDel==0 ){ ++ zDel = sqlite3_malloc(nOut); ++ if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; ++ }else if( nLink>=SQLITE_MAX_SYMLINKS ){ ++ rc = SQLITE_CANTOPEN_BKPT; ++ } ++ ++ if( rc==SQLITE_OK ){ ++ nByte = osReadlink(zIn, zDel, nOut-1); ++ if( nByte<0 ){ ++ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); ++ }else{ ++ if( zDel[0]!='/' ){ ++ int n; ++ for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); ++ if( nByte+n+1>nOut ){ ++ rc = SQLITE_CANTOPEN_BKPT; ++ }else{ ++ memmove(&zDel[n], zDel, nByte+1); ++ memcpy(zDel, zIn, n); ++ nByte += n; ++ } ++ } ++ zDel[nByte] = '\0'; ++ } ++ } ++ ++ zIn = zDel; ++ } ++ ++ assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); ++ if( rc==SQLITE_OK && zIn!=zOut ){ ++ rc = mkFullPathname(zIn, zOut, nOut); ++ } ++ if( bLink==0 ) break; ++ zIn = zOut; ++ }while( rc==SQLITE_OK ); ++ ++ sqlite3_free(zDel); ++ if( rc==SQLITE_OK && nLink ) rc = SQLITE_OK_SYMLINK; ++ return rc; ++#endif /* HAVE_READLINK && HAVE_LSTAT */ ++} ++ ++#endif /* !HARMONY_OS */ ++ + #ifndef SQLITE_OMIT_LOAD_EXTENSION + /* + ** Interfaces for opening a shared library, finding entry points +@@ -45507,6 +45940,9 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", + lPath, errno, osGetpid(0))); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_LOCK, "proxyGetLockPath len[%d], dbLen[%d], i[%d]", len, dbLen, i); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_LOCK; + } + len = strlcat(lPath, "sqliteplocks", maxLen); +@@ -45625,6 +46061,9 @@ static int proxyCreateUnixFile( + case EACCES: + return SQLITE_PERM; + case EIO: ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_LOCK, "proxyCreateUnixFile-EIO, fd[%d]", fd); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; +@@ -45793,6 +46232,9 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + struct stat buf; + if( osFstat(conchFile->h, &buf) ){ + storeLastErrno(pFile, errno); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_LOCK, "proxyConchLock pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, errno); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_LOCK; + } + +@@ -45813,6 +46255,9 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + storeLastErrno(pFile, errno); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_LOCK, "proxyConchLock tries 2, pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, errno); ++#endif /* LOG_DUMP */ + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ +@@ -45887,6 +46332,9 @@ static int proxyTakeConch(unixFile *pFile){ + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + storeLastErrno(pFile, conchFile->lastErrno); ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_IOERR_READ, "proxyTakeConch pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, conchFile->lastErrno); ++#endif /* LOG_DUMP */ + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || +@@ -57514,6 +57962,20 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ + */ + #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + ++#ifdef SQLITE_HAS_CODEC ++/* ++** A macro used for invoking the codec if there is one ++*/ ++# define CODEC1(P,D,N,X,E) \ ++ if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } ++# define CODEC2(P,D,N,X,E,O) \ ++ if( P->xCodec==0 ){ O=(char*)D; }else \ ++ if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } ++#else ++# define CODEC1(P,D,N,X,E) /* NO-OP */ ++# define CODEC2(P,D,N,X,E,O) O=(char*)D ++#endif /* SQLITE_HAS_CODEC */ ++ + /* + ** The maximum allowed sector size. 64KiB. If the xSectorsize() method + ** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +@@ -57802,6 +58264,12 @@ struct Pager { + #endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ ++#ifdef SQLITE_HAS_CODEC ++ void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ ++ void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ ++ void (*xCodecFree)(void*); /* Destructor for the codec */ ++ void *pCodec; /* First argument to xCodec... methods */ ++#endif /* SQLITE_HAS_CODEC */ + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ + #ifndef SQLITE_OMIT_WAL +@@ -57923,6 +58391,9 @@ static const unsigned char aJournalMagic[] = { + SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; ++#ifdef SQLITE_HAS_CODEC ++ if( pPager->xCodec!=0 ) return 0; ++#endif /* SQLITE_HAS_CODEC */ + #ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; +@@ -58155,7 +58626,11 @@ static void setGetterMethod(Pager *pPager){ + if( pPager->errCode ){ + pPager->xGet = getPageError; + #if SQLITE_MAX_MMAP_SIZE>0 +- }else if( USEFETCH(pPager) ){ ++ }else if( USEFETCH(pPager) ++#ifdef SQLITE_HAS_CODEC ++ && pPager->xCodec==0 ++#endif /* SQLITE_HAS_CODEC */ ++ ){ + pPager->xGet = getPageMMap; + #endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ +@@ -59347,6 +59822,32 @@ static u32 pager_cksum(Pager *pPager, const u8 *aData){ + return cksum; + } + ++#ifdef SQLITE_HAS_CODEC ++/* ++** Report the current page size and number of reserved bytes back ++** to the codec. ++*/ ++static void pagerReportSize(Pager *pPager){ ++ if( pPager->xCodecSizeChng ){ ++ pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, ++ (int)pPager->nReserve); ++ } ++} ++/* ++** Make sure the number of reserved bits is the same in the destination ++** pager as it is in the source. This comes up when a VACUUM changes the ++** number of reserved bits to the "optimal" amount. ++*/ ++SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ ++ if( pDest->nReserve!=pSrc->nReserve ){ ++ pDest->nReserve = pSrc->nReserve; ++ pagerReportSize(pDest); ++ } ++} ++#else ++# define pagerReportSize(X) /* No-op if we do not support a codec */ ++#endif ++ + /* + ** Read a single page from either the journal file (if isMainJrnl==1) or + ** from the sub-journal (if isMainJrnl==0) and playback that page. +@@ -59398,6 +59899,11 @@ static int pager_playback_one_page( + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ ++#ifdef SQLITE_HAS_CODEC ++ /* The jrnlEnc flag is true if Journal pages should be passed through ++ ** the codec. It is false for pure in-memory journals. */ ++ const int jrnlEnc = (isMainJrnl || pPager->subjInMemory==0); ++#endif /* SQLITE_HAS_CODEC */ + + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ +@@ -59527,12 +60033,26 @@ static int pager_playback_one_page( + ** is if the data was just read from an in-memory sub-journal. In that + ** case it must be encrypted here before it is copied into the database + ** file. */ ++#ifdef SQLITE_HAS_CODEC ++ if( !jrnlEnc ){ ++ CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); ++ rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); ++ CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); ++ }else ++#endif /* SQLITE_HAS_CODEC */ + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ ++#ifdef SQLITE_HAS_CODEC ++ if( jrnlEnc ){ ++ CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); ++ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); ++ CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT,aData); ++ }else ++#endif /* SQLITE_HAS_CODEC */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + } + }else if( !isMainJrnl && pPg==0 ){ +@@ -59583,6 +60103,10 @@ static int pager_playback_one_page( + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); + } ++#if SQLITE_HAS_CODEC ++ /* Decode the page just read from disk */ ++ if( jrnlEnc ){ CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); } ++#endif /* SQLITE_HAS_CODEC */ + sqlite3PcacheRelease(pPg); + } + return rc; +@@ -60161,6 +60685,8 @@ static int readDbPage(PgHdr *pPg){ + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } ++ CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM_BKPT); ++ + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); +@@ -61306,6 +61832,10 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); ++#ifdef SQLITE_HAS_CODEC ++ if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); ++#endif /* SQLITE_HAS_CODEC */ ++ + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + +@@ -61556,7 +62086,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); + +- pData = pList->pData; ++ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); + + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); +@@ -61645,6 +62175,11 @@ static int subjournalPage(PgHdr *pPg){ + void *pData = pPg->pData; + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); + char *pData2; ++#if SQLITE_HAS_CODEC ++ if( !pPager->subjInMemory ){ ++ CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); ++ }else ++#endif /* SQLITE_HAS_CODEC */ + pData2 = pData; + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); +@@ -62737,6 +63272,9 @@ static int getPageMMap( + ); + + assert( USEFETCH(pPager) ); ++#ifdef SQLITE_HAS_CODEC ++ assert( pPager->xCodec==0 ); ++#endif /* SQLITE_HAS_CODEC */ + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" +@@ -63081,7 +63619,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + assert( pPg->pgno!=PAGER_SJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); +- pData2 = pPg->pData; ++ CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the +@@ -63446,7 +63984,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); +- zBuf = pPgHdr->pData; ++ CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; +@@ -64209,6 +64747,48 @@ SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; + } + ++#ifdef SQLITE_HAS_CODEC ++/* ++** Set or retrieve the codec for this pager ++*/ ++SQLITE_PRIVATE void sqlite3PagerSetCodec( ++ Pager *pPager, ++ void *(*xCodec)(void*,void*,Pgno,int), ++ void (*xCodecSizeChng)(void*,int,int), ++ void (*xCodecFree)(void*), ++ void *pCodec ++){ ++ if( pPager->xCodecFree ){ ++ pPager->xCodecFree(pPager->pCodec); ++ }else{ ++ pager_reset(pPager); ++ } ++ pPager->xCodec = pPager->memDb ? 0 : xCodec; ++ pPager->xCodecSizeChng = xCodecSizeChng; ++ pPager->xCodecFree = xCodecFree; ++ pPager->pCodec = pCodec; ++ setGetterMethod(pPager); ++ pagerReportSize(pPager); ++} ++SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ ++ return pPager->pCodec; ++} ++ ++/* ++** This function is called by the wal module when writing page content ++** into the log file. ++** ++** This function returns a pointer to a buffer containing the encrypted ++** page content. If a malloc fails, this function may return NULL. ++*/ ++SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ ++ void *aData = 0; ++ CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); ++ return aData; ++} ++ ++#endif /* SQLITE_HAS_CODEC */ ++ + #ifndef SQLITE_OMIT_AUTOVACUUM + /* + ** Move the page pPg to location pgno in the file. +@@ -67063,6 +67643,11 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ + assert( pInfo->aReadMark[0]==0 ); + } + ++#ifdef SQLITE_HDR_CHECK ++static int checkHeaderValid(Pager *pager, u8 *zBuf, const char *logStr); ++static int checkDbHeaderValid(sqlite3 *db, int iDbpage, u8 *zBuf); ++#endif /* SQLITE_HDR_CHECK */ ++ + /* + ** Copy as much content as we can from the WAL back into the database file + ** in response to an sqlite3_wal_checkpoint() request or the equivalent. +@@ -67203,6 +67788,10 @@ static int walCheckpoint( + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); ++#ifdef SQLITE_HDR_CHECK ++ rc = checkDbHeaderValid(db, iDbpage, zBuf); ++ if( rc!=SQLITE_OK ) break; ++#endif /* SQLITE_HDR_CHECK */ + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } +@@ -68833,7 +69422,18 @@ static int walWriteOneFrame( + int rc; /* Result code from subfunctions */ + void *pData; /* Data actually written */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ ++#ifdef SQLITE_HDR_CHECK ++ if( pPage->pgno==1 ){ ++ rc = checkHeaderValid(pPage->pPager, pPage->pData, "walWrite"); ++ if( rc!=SQLITE_OK ) return rc; ++ } ++#endif /* SQLITE_HDR_CHECK */ ++ ++#ifdef SQLITE_HAS_CODEC ++ if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; ++#else + pData = pPage->pData; ++#endif /* SQLITE_HAS_CODEC */ + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); + if( rc ) return rc; +@@ -69018,7 +69618,11 @@ static int walFrames( + if( pWal->iReCksum==0 || iWriteiReCksum ){ + pWal->iReCksum = iWrite; + } ++#ifdef SQLITE_HAS_CODEC ++ if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; ++#else + pData = p->pData; ++#endif /* SQLITE_HAS_CODEC */ + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + if( rc ) return rc; + p->flags &= ~PGHDR_WAL_APPEND; +@@ -69129,6 +69733,10 @@ static int walFrames( + return rc; + } + ++#ifdef LOG_DUMP ++static sqlite3_int64 g_lastCkptTime = 0; ++#endif /* LOG_DUMP */ ++ + /* + ** Write a set of frames to the log. The caller must hold the write-lock + ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +@@ -69282,6 +69890,12 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + } ++#ifdef LOG_DUMP ++ if( rc ){ ++ sqlite3_log(SQLITE_NOTICE, "ckpt rc[%d]", rc); ++ } ++ sqlite3OsCurrentTimeInt64(db->pVfs, &g_lastCkptTime); ++#endif /* LOG_DUMP */ + WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); + #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +@@ -70666,6 +71280,32 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ + } + #endif + ++#ifdef SQLITE_HDR_CHECK ++static int checkHeaderValid(Pager *pager, u8 *zBuf, const char *logStr){ ++#ifdef SQLITE_HAS_CODEC ++ if( pager==NULL || pager->pCodec ){ ++ return SQLITE_OK; ++ } ++#endif /* SQLITE_HAS_CODEC */ ++ if( zBuf && strncmp((const char *)zBuf, zMagicHeader, 16)!=0 ){ ++ sqlite3_log(SQLITE_NOTADB, "[%s]wrong header format, memory might be overwritten!", logStr); ++ return SQLITE_NOTADB; ++ } ++ return SQLITE_OK; ++} ++ ++static int checkDbHeaderValid(sqlite3 *db, int iDbpage, u8 *zBuf){ ++ if( iDbpage==1 && db->aDb ){ ++ Btree *p = db->aDb[0].pBt; ++ if( p && p->pBt ){ ++ Pager *pager = sqlite3BtreePager(p); ++ return checkHeaderValid(pager, zBuf, "ckpt"); ++ } ++ } ++ return SQLITE_OK; ++} ++#endif /* SQLITE_HDR_CHECK */ ++ + /* + ** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single + ** (MemPage*) as an argument. The (MemPage*) must not be NULL. +@@ -72591,6 +73231,13 @@ static int decodeFlags(MemPage *pPage, int flagByte){ + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_CORRUPT, ++ "database corruption at page[%u], flagByte[%x], isInit[%u], intKey[%u], intKeyLeaf[%u], leaf[%u], " ++ "childPtrSize[%u], cellOffset[%u], nCell[%u], hdrOffset[%u], minLocal[%u], maxLocal[%u], last ckpt time[%lld]", ++ pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, ++ pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); ++#endif /* LOG_DUMP */ + return SQLITE_CORRUPT_PAGE(pPage); + } + }else{ +@@ -72615,6 +73262,13 @@ static int decodeFlags(MemPage *pPage, int flagByte){ + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; ++#ifdef LOG_DUMP ++ sqlite3_log(SQLITE_CORRUPT, ++ "database corruption at page[%u], flagByte[%x], isInit[%u], intKey[%u], intKeyLeaf[%u], leaf[%u], " ++ "childPtrSize[%u], cellOffset[%u], nCell[%u], hdrOffset[%u], minLocal[%u], maxLocal[%u], last ckpt time[%lld]", ++ pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, ++ pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); ++#endif /* LOG_DUMP */ + return SQLITE_CORRUPT_PAGE(pPage); + } + } +@@ -82246,6 +82900,13 @@ static int backupOnePage( + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; ++#ifdef SQLITE_HAS_CODEC ++ /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is ++ ** guaranteed that the shared-mutex is held by this thread, handle ++ ** p->pSrc may not actually be the owner. */ ++ int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); ++ int nDestReserve = sqlite3BtreeGetRequestedReserve(p->pDest); ++#endif /* SQLITE_HAS_CODEC */ + int rc = SQLITE_OK; + i64 iOff; + +@@ -82256,6 +82917,26 @@ static int backupOnePage( + assert( zSrcData ); + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); + ++#ifdef SQLITE_HAS_CODEC ++ /* Backup is not possible if the page size of the destination is changing ++ ** and a codec is in use. ++ */ ++ if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ ++ rc = SQLITE_READONLY; ++ } ++ ++ /* Backup is not possible if the number of bytes of reserve space differ ++ ** between source and destination. If there is a difference, try to ++ ** fix the destination to agree with the source. If that is not possible, ++ ** then the backup cannot proceed. ++ */ ++ if( nSrcReserve!=nDestReserve ){ ++ u32 newPgsz = nSrcPgsz; ++ rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); ++ if( rc==SQLITE_OK && newPgsz!=(u32)nSrcPgsz ) rc = SQLITE_READONLY; ++ } ++#endif /* SQLITE_HAS_CODEC */ ++ + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. +@@ -82754,6 +83435,10 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + b.pDest = pTo; + b.iNext = 1; + ++#ifdef SQLITE_HAS_CODEC ++ sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); ++#endif /* SQLITE_HAS_CODEC */ ++ + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes +@@ -91264,6 +91949,15 @@ end_of_step: + return (rc&db->errMask); + } + ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++SQLITE_API int sqlite3_set_droptable_handle(sqlite3 *db, void (*xFunc)(sqlite3*, const char*, const char*)){ ++ sqlite3_mutex_enter(db->mutex); ++ db->xDropTableHandle = xFunc; ++ sqlite3_mutex_leave(db->mutex); ++ return 0; ++} ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ ++ + /* + ** This is the top-level implementation of sqlite3_step(). Call + ** sqlite3Step() to do most of the work. If a schema error occurs, +@@ -91283,6 +91977,13 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + while( (rc = sqlite3Step(v))==SQLITE_SCHEMA + && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ + int savedPc = v->pc; ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ Sqlite3SharedBlockMethods *pSharedBlock = v->pSharedBlock; ++ int totalRows = v->totalRows; ++ int blockFull = v->blockFull; ++ int startPos = v->startPos; ++ int addedRows = v->addedRows; ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK ){ + /* This case occurs after failing to recompile an sql statement. +@@ -91304,6 +92005,15 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + } + break; + } ++ ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ v->pSharedBlock = pSharedBlock; ++ v->totalRows = totalRows; ++ v->blockFull = blockFull; ++ v->startPos = startPos; ++ v->addedRows = addedRows; ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + sqlite3_reset(pStmt); + if( savedPc>=0 ){ + /* Setting minWriteFileFormat to 254 is a signal to the OP_Init and +@@ -91314,6 +92024,20 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + } + assert( v->expired==0 ); + } ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ if( rc==SQLITE_DONE && db->xDropTableHandle!=NULL && db->isDropTable==1 ){ ++ db->isDropTable = 0; ++ db->xDropTableHandle(db, db->mDropTableName, db->mDropSchemaName); ++ } ++ if( db->mDropTableName!=NULL ){ ++ sqlite3_free(db->mDropTableName); ++ db->mDropTableName = NULL; ++ } ++ if( db->mDropSchemaName!=NULL ){ ++ sqlite3_free(db->mDropSchemaName); ++ db->mDropSchemaName = NULL; ++ } ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + sqlite3_mutex_leave(db->mutex); + return rc; + } +@@ -93761,6 +94485,61 @@ static int checkSavepointCount(sqlite3 *db){ + } + #endif + ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++static int copySharedBlockRow( ++ Vdbe *p, /* The VDBE */ ++ Op *pOp, /* Current operation */ ++ Mem *pMem, ++ void *pCtx ++){ ++ int i = 0; ++ ++ int rc = p->pSharedBlock->xAddRow(pCtx, p->addedRows); ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ ++ for(i=0; ip2; i++){ ++ switch (sqlite3_value_type(&pMem[i])) { ++ case SQLITE_INTEGER:{ ++ rc = p->pSharedBlock->xPutLong(pCtx, p->addedRows, i, (sqlite3_int64)pMem[i].u.i); ++ break; ++ } ++ case SQLITE_FLOAT: { ++ rc = p->pSharedBlock->xPutDouble(pCtx, p->addedRows, i, pMem[i].u.r); ++ break; ++ } ++ case SQLITE_TEXT: { ++ Deephemeralize(&pMem[i]); ++ sqlite3VdbeMemNulTerminate(&pMem[i]); ++ sqlite3VdbeChangeEncoding(&pMem[i],SQLITE_UTF8); ++ rc = p->pSharedBlock->xPutString(pCtx, p->addedRows, i, pMem[i].z, pMem[i].n+1); ++ break; ++ } ++ case SQLITE_BLOB: { ++ rc = p->pSharedBlock->xPutBlob(pCtx, p->addedRows, i, pMem[i].z, pMem[i].n); ++ break; ++ } ++ case SQLITE_NULL: { ++ rc = p->pSharedBlock->xPutNull(pCtx, p->addedRows, i); ++ break; ++ } ++ default: ++ rc = p->pSharedBlock->xPutOther(pCtx, p->addedRows, i); ++ break; ++ } ++ if( rc!=SQLITE_OK ){ ++ return rc; ++ } ++ } ++ ++ return rc; ++no_mem: ++ rc = SQLITE_NOMEM; ++ return rc; ++} ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + /* + ** Return the register of pOp->p2 after first preparing it to be + ** overwritten with an integer value. +@@ -94361,6 +95140,12 @@ case OP_Halt: { + VdbeFrame *pFrame; + int pcx; + ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( p->pSharedBlock!=NULL ){ ++ p->pSharedBlock->xFinish(p->pSharedBlock->pContext, p->addedRows, p->totalRows); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + #ifdef SQLITE_DEBUG + if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } + #endif +@@ -94806,6 +95591,43 @@ case OP_ResultRow: { + assert( pOp->p1>0 || CORRUPT_DB ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( p->pSharedBlock!=NULL ){ ++ void *pCtx = p->pSharedBlock->pContext; ++ p->totalRows++; ++ if( p->totalRows<=p->startPos || p->blockFull ){ ++ break; ++ } ++ Mem *pMem = &aMem[pOp->p1]; ++ rc = copySharedBlockRow(p, pOp, pMem, pCtx); ++ if( rc==SQLITE_FULL && p->addedRows && (p->startPos + p->addedRows) <= p->pSharedBlock->requiredPos ){ ++ p->startPos += p->addedRows; ++ p->addedRows = 0; ++ p->pSharedBlock->xReset(pCtx,p->startPos); ++ p->blockFull = 0; ++ rc = copySharedBlockRow(p, pOp, pMem, pCtx); ++ } ++ ++ if( rc==SQLITE_OK ){ ++ p->addedRows++; ++ }else if( rc==SQLITE_FULL ){ ++ p->blockFull = 1; ++ }else{ ++ //SQLITE_NOMEM ++ goto no_mem; ++ } ++ ++ if( p->blockFull && p->pSharedBlock->countAllRows==0 ){ ++ p->pSharedBlock->xFinish(pCtx, p->addedRows, p->totalRows); ++ rc = SQLITE_DONE; ++ goto vdbe_return; ++ }else if( p->blockFull && p->pSharedBlock->countAllRows==1 ){ ++ rc = SQLITE_OK; ++ } ++ break; ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ ++ + p->cacheCtr = (p->cacheCtr + 2)|1; + p->pResultRow = &aMem[pOp->p1]; + #ifdef SQLITE_DEBUG +@@ -95612,6 +96434,17 @@ case OP_Compare: { + ** This opcode must immediately follow an OP_Compare opcode. + */ + case OP_Jump: { /* jump */ ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( pOp->p5&0x80 ){ ++ if( p->pSharedBlock!=NULL ){ ++ if( p->totalRows < p->startPos || p->blockFull ){ ++ p->totalRows++; ++ pOp = &aOp[pOp->p2 - 1]; ++ } ++ } ++ break; ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + assert( pOp>aOp && pOp[-1].opcode==OP_Compare ); + assert( iCompareIsInit ); + if( iCompare<0 ){ +@@ -100719,6 +101552,20 @@ case OP_IfNotZero: { /* jump, in1 */ + ** and jump to P2 if the new value is exactly zero. + */ + case OP_DecrJumpZero: { /* jump, in1 */ ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( pOp->p5&0x80 ){ ++ if( p->pSharedBlock!=NULL ){ ++ if( p->totalRows < p->startPos || p->blockFull ){ ++ pIn1 = &aMem[pOp->p1]; ++ assert( pIn1->flags&MEM_Int ); ++ if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--; ++ VdbeBranchTaken(pIn1->u.i==-1, 2); ++ if( pIn1->u.i==-1 ) goto jump_to_p2; ++ } ++ } ++ break; ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--; +@@ -120686,6 +121533,40 @@ static void attachFunc( + if( rc==SQLITE_OK && pNew->zDbSName==0 ){ + rc = SQLITE_NOMEM_BKPT; + } ++#ifdef SQLITE_HAS_CODEC ++ if( rc==SQLITE_OK ){ ++ extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); ++ extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); ++ int nKey; ++ char *zKey; ++ int t = sqlite3_value_type(argv[2]); ++ switch( t ){ ++ case SQLITE_INTEGER: ++ case SQLITE_FLOAT: ++ zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); ++ rc = SQLITE_ERROR; ++ break; ++ ++ case SQLITE_TEXT: ++ case SQLITE_BLOB: ++ nKey = sqlite3_value_bytes(argv[2]); ++ zKey = (char *)sqlite3_value_blob(argv[2]); ++ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); ++ break; ++ ++ case SQLITE_NULL: ++ /* No key specified. Use the key from URI filename, or if none, ++ ** use the key from the main database. */ ++ if( sqlite3CodecQueryParameters(db, zName, zPath)==0 ){ ++ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); ++ if( nKey || sqlite3BtreeGetRequestedReserve(db->aDb[0].pBt)>0 ){ ++ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); ++ } ++ } ++ break; ++ } ++ } ++#endif /* SQLITE_HAS_CODEC */ + sqlite3_free_filename( zPath ); + + /* If the file was opened successfully, read the schema for the new database. +@@ -122260,8 +123141,24 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ + pDb = &db->aDb[iDb]; + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ u8 tableType = p->eTabType; ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + sqlite3DeleteTable(db, p); + db->mDbFlags |= DBFLAG_SchemaChange; ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ if( tableType!=TABTYP_VIEW ){ ++ db->isDropTable = 1; ++ db->mDropTableName = sqlite3_malloc(strlen(zTabName) + 1); ++ if( db->mDropTableName!=NULL ){ ++ memcpy(db->mDropTableName, zTabName, strlen(zTabName) + 1); ++ } ++ db->mDropSchemaName = sqlite3_malloc(strlen(db->aDb[iDb].zDbSName) + 1); ++ if( db->mDropSchemaName!=NULL ){ ++ memcpy(db->mDropSchemaName, db->aDb[iDb].zDbSName, strlen(db->aDb[iDb].zDbSName) + 1); ++ } ++ } ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + } + + /* +@@ -130873,10 +131770,16 @@ static void groupConcatValue(sqlite3_context *context){ + */ + SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ + int rc = sqlite3_overload_function(db, "MATCH", 2); ++#ifdef SQLITE_HAS_CODEC ++ extern void sqlite3CodecExportData(sqlite3_context *, int, sqlite3_value **); ++#endif /* SQLITE_HAS_CODEC */ + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } ++#ifdef SQLITE_HAS_CODEC ++ sqlite3CreateFunc(db, "export_database", 1, SQLITE_TEXT, 0, sqlite3CodecExportData, 0, 0, 0, 0, 0); ++#endif /* SQLITE_HAS_CODEC */ + } + + /* +@@ -136799,6 +137702,10 @@ struct sqlite3_api_routines { + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ /* handle after drop table done */ ++ int (*set_droptable_handle)(sqlite3*,void(*)(sqlite3*,const char*,const char*)); ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + }; + + /* +@@ -137132,6 +138039,10 @@ typedef int (*sqlite3_loadext_entry)( + /* Version 3.44.0 and later */ + #define sqlite3_get_clientdata sqlite3_api->get_clientdata + #define sqlite3_set_clientdata sqlite3_api->set_clientdata ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++/* handle after drop table done */ ++#define sqlite3_set_droptable_handle sqlite3_api->set_droptable_handle ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + + #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +@@ -137155,7 +138066,7 @@ typedef int (*sqlite3_loadext_entry)( + /************** Continuing where we left off in loadext.c ********************/ + /* #include "sqliteInt.h" */ + +-#ifndef SQLITE_OMIT_LOAD_EXTENSION ++#if !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) + /* + ** Some API routines are omitted when various features are + ** excluded from a build of SQLite. Substitute a NULL pointer +@@ -137169,7 +138080,9 @@ typedef int (*sqlite3_loadext_entry)( + # define sqlite3_column_origin_name 0 + # define sqlite3_column_origin_name16 0 + #endif ++#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) */ + ++#ifndef SQLITE_OMIT_LOAD_EXTENSION + #ifdef SQLITE_OMIT_AUTHORIZATION + # define sqlite3_set_authorizer 0 + #endif +@@ -137250,6 +138163,7 @@ typedef int (*sqlite3_loadext_entry)( + #if defined(SQLITE_OMIT_TRACE) + # define sqlite3_trace_v2 0 + #endif ++#endif /* !SQLITE_OMIT_LOAD_EXTENSION */ + + /* + ** The following structure contains pointers to all SQLite API routines. +@@ -137266,6 +138180,7 @@ typedef int (*sqlite3_loadext_entry)( + ** also check to make sure that the pointer to the function is + ** not NULL before calling it. + */ ++#if !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) + static const sqlite3_api_routines sqlite3Apis = { + sqlite3_aggregate_context, + #ifndef SQLITE_OMIT_DEPRECATED +@@ -137535,7 +138450,11 @@ static const sqlite3_api_routines sqlite3Apis = { + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, ++#ifndef SQLITE_OMIT_LOAD_EXTENSION + sqlite3_load_extension, ++#else ++ 0, ++#endif /* !SQLITE_OMIT_LOAD_EXTENSION */ + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, +@@ -137653,7 +138572,12 @@ static const sqlite3_api_routines sqlite3Apis = { + sqlite3_stmt_explain, + /* Version 3.44.0 and later */ + sqlite3_get_clientdata, +- sqlite3_set_clientdata ++ sqlite3_set_clientdata, ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ sqlite3_set_droptable_handle ++#else ++ 0 ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + }; + + /* True if x is the directory separator character +@@ -137664,6 +138588,9 @@ static const sqlite3_api_routines sqlite3Apis = { + # define DirSep(X) ((X)=='/') + #endif + ++#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) */ ++ ++#ifndef SQLITE_OMIT_LOAD_EXTENSION + /* + ** Attempt to load an SQLite extension library contained in the file + ** zFile. The entry point is zProc. zProc may be 0 in which case a +@@ -138143,6 +139070,9 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ + #define PragTyp_WAL_CHECKPOINT 43 + #define PragTyp_LOCK_STATUS 44 + #define PragTyp_STATS 45 ++#ifdef SQLITE_HAS_CODEC ++#define PragTyp_KEY 255 ++#endif /* SQLITE_HAS_CODEC */ + + /* Property flags associated with various pragma. */ + #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ +@@ -138433,6 +139363,18 @@ static const PragmaName aPragmaName[] = { + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, ++#if defined(SQLITE_HAS_CODEC) ++ {/* zName: */ "hexkey", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 2 }, ++ {/* zName: */ "hexrekey", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 3 }, ++#endif + #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + #if !defined(SQLITE_OMIT_CHECK) + {/* zName: */ "ignore_check_constraints", +@@ -138485,6 +139427,13 @@ static const PragmaName aPragmaName[] = { + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + #endif ++#if defined(SQLITE_HAS_CODEC) ++ {/* zName: */ "key", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 0 }, ++#endif + #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "legacy_alter_table", + /* ePragTyp: */ PragTyp_FLAG, +@@ -138592,6 +139541,15 @@ static const PragmaName aPragmaName[] = { + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_RecTriggers }, ++#endif ++#if defined(SQLITE_HAS_CODEC) ++ {/* zName: */ "rekey", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 1 }, ++#endif ++#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, +@@ -138680,6 +139638,18 @@ static const PragmaName aPragmaName[] = { + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, ++#endif ++#if defined(SQLITE_HAS_CODEC) ++ {/* zName: */ "textkey", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 4 }, ++ {/* zName: */ "textrekey", ++ /* ePragTyp: */ PragTyp_KEY, ++ /* ePragFlg: */ 0, ++ /* ColNames: */ 0, 0, ++ /* iArg: */ 5 }, + #endif + {/* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, +@@ -139149,6 +140119,10 @@ SQLITE_PRIVATE void sqlite3Pragma( + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + const PragmaName *pPragma; /* The pragma */ + ++#ifdef SQLITE_HAS_CODEC ++ extern int sqlite3CodecPragma(sqlite3*, int, Parse *, const char *, const char *); ++#endif /* SQLITE_HAS_CODEC */ ++ + if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); + pParse->nMem = 2; +@@ -139218,6 +140192,13 @@ SQLITE_PRIVATE void sqlite3Pragma( + goto pragma_out; + } + ++#ifdef SQLITE_HAS_CODEC ++ if(sqlite3CodecPragma(db, iDb, pParse, zLeft, zRight)) { ++ /* sqlite3CodecPragma executes internal */ ++ goto pragma_out; ++ } ++#endif /* SQLITE_HAS_CODEC */ ++ + /* Locate the pragma in the lookup table */ + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ){ +@@ -141459,6 +142440,48 @@ SQLITE_PRIVATE void sqlite3Pragma( + } + #endif + ++#ifdef SQLITE_HAS_CODEC ++ /* Pragma iArg ++ ** ---------- ------ ++ ** key 0 ++ ** rekey 1 ++ ** hexkey 2 ++ ** hexrekey 3 ++ ** textkey 4 ++ ** textrekey 5 ++ */ ++ case PragTyp_KEY: { ++ if( zRight ){ ++ char zBuf[40]; ++ const char *zKey = zRight; ++ int n; ++ if( pPragma->iArg==2 || pPragma->iArg==3 ){ ++ u8 iByte; ++ int i; ++ for(i=0, iByte=0; i<(int)(sizeof(zBuf)*2) && sqlite3Isxdigit(zRight[i]); i++){ ++ iByte = (iByte<<4) + sqlite3HexToInt(zRight[i]); ++ if( (i&1)!=0 ) zBuf[i/2] = iByte; ++ } ++ zKey = zBuf; ++ n = i/2; ++ }else{ ++ n = pPragma->iArg<4 ? sqlite3Strlen30(zRight) : -1; ++ } ++ if( (pPragma->iArg & 1)==0 ){ ++ rc = sqlite3_key_v2(db, zDb, zKey, n); ++ }else{ ++ rc = sqlite3_rekey_v2(db, zDb, zKey, n); ++ } ++ if( rc==SQLITE_OK && n!=0 ){ ++ sqlite3VdbeSetNumCols(v, 1); ++ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "ok", SQLITE_STATIC); ++ returnSingleText(v, "ok"); ++ } ++ } ++ break; ++ } ++#endif /* SQLITE_HAS_CODEC */ ++ + #if defined(SQLITE_ENABLE_CEROD) + case PragTyp_ACTIVATE_EXTENSIONS: if( zRight ){ + if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ +@@ -144023,9 +145046,25 @@ static void selectInnerLoop( + assert( p->pEList!=0 ); + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; + if( pSort && pSort->pOrderBy==0 ) pSort = 0; ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( hasDistinct && (pDistinct->eTnctType==WHERE_DISTINCT_UNIQUE) ){ ++ hasDistinct = WHERE_DISTINCT_NOOP; ++ sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); + codeOffset(v, p->iOffset, iContinue); ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( eDest==SRT_Output ){ ++ if( p->iLimit ){ ++ sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++ sqlite3VdbeAddOp2(v, OP_Jump, 0, iContinue); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + } + + /* Pull the requested columns. +@@ -144154,6 +145193,16 @@ static void selectInnerLoop( + fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct); + if( pSort==0 ){ + codeOffset(v, p->iOffset, iContinue); ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( eDest==SRT_Output ){ ++ if( p->iLimit ){ ++ sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++ sqlite3VdbeAddOp2(v, OP_Jump, 0, iContinue); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + } + } + +@@ -144623,11 +145672,23 @@ static void generateSortTail( + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); + assert( p->iLimit==0 && p->iOffset==0 ); ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( eDest==SRT_Output ){ ++ sqlite3VdbeAddOp2(v, OP_Jump, 0, addrContinue); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ if( eDest==SRT_Output ){ ++ sqlite3VdbeAddOp2(v, OP_Jump, 0, addrContinue); ++ sqlite3VdbeChangeP5(v, 128); ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + iSortTab = iTab; + bSeq = 1; + if( p->iOffset>0 ){ +@@ -155217,6 +156278,17 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( + } + nRes = sqlite3BtreeGetRequestedReserve(pMain); + ++#ifdef SQLITE_HAS_CODEC ++ /* A VACUUM cannot change the pagesize of an encrypted database. */ ++ if( db->nextPagesize ){ ++ extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); ++ int nKey; ++ char *zKey; ++ sqlite3CodecGetKey(db, iDb, (void**)&zKey, &nKey); ++ if( nKey ) db->nextPagesize = 0; ++ } ++#endif /* SQLITE_HAS_CODEC */ ++ + sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); + sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); +@@ -180744,6 +181816,29 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } ++#ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION ++ case SQLITE_DBCONFIG_SET_SHAREDBLOCK: { ++ Vdbe *pVdbe = (Vdbe *)va_arg(ap, sqlite3_stmt*); ++ Sqlite3SharedBlockMethods *pSharedBlock = va_arg(ap, Sqlite3SharedBlockMethods*); ++ if( pVdbe==NULL ){ ++ rc = SQLITE_MISUSE; ++ break; ++ } ++ pVdbe->pSharedBlock = pSharedBlock; ++ if( pSharedBlock!=NULL ){ ++ pVdbe->startPos = pSharedBlock->startPos; ++ } ++ pVdbe->totalRows = 0; ++ pVdbe->blockFull = 0; ++ pVdbe->addedRows = 0; ++ rc = SQLITE_OK; ++ break; ++ } ++ case SQLITE_DBCONFIG_USE_SHAREDBLOCK: { ++ rc = SQLITE_OK; ++ break; ++ } ++#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + default: { + static const struct { + int op; /* The opcode */ +@@ -182998,7 +184093,41 @@ static const char *uriParameter(const char *zFilename, const char *zParam){ + return 0; + } + +- ++#ifdef SQLITE_HAS_CODEC ++/* ++** Process URI filename query parameters relevant to the SQLite Encryption ++** Extension. Return true if any of the relevant query parameters are ++** seen and return false if not. ++*/ ++SQLITE_PRIVATE int sqlite3CodecQueryParameters( ++ sqlite3 *db, /* Database connection */ ++ const char *zDb, /* Which schema is being created/attached */ ++ const char *zUri /* URI filename */ ++){ ++ const char *zKey; ++ if( zUri==0 ){ ++ return 0; ++ }else if( (zKey = uriParameter(zUri, "hexkey"))!=0 && zKey[0] ){ ++ u8 iByte; ++ int i; ++ char zDecoded[40]; ++ for(i=0, iByte=0; i<(int)(sizeof(zDecoded)*2) && sqlite3Isxdigit(zKey[i]); i++){ ++ iByte = (iByte<<4) + sqlite3HexToInt(zKey[i]); ++ if( (i&1)!=0 ) zDecoded[i/2] = iByte; ++ } ++ sqlite3_key_v2(db, zDb, zDecoded, i/2); ++ return 1; ++ }else if( (zKey = uriParameter(zUri, "key"))!=0 ){ ++ sqlite3_key_v2(db, zDb, zKey, sqlite3Strlen30(zKey)); ++ return 1; ++ }else if( (zKey = uriParameter(zUri, "textkey"))!=0 ){ ++ sqlite3_key_v2(db, zDb, zKey, -1); ++ return 1; ++ }else{ ++ return 0; ++ } ++} ++#endif /* SQLITE_HAS_CODEC */ + + /* + ** This routine does the work of opening a database on behalf of +@@ -183346,6 +184475,12 @@ opendb_out: + }else if( rc!=SQLITE_OK ){ + db->eOpenState = SQLITE_STATE_SICK; + } ++#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK ++ db->isDropTable = 0; ++ db->mDropTableName = NULL; ++ db->mDropSchemaName = NULL; ++ db->xDropTableHandle = NULL; ++#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + *ppDb = db; + #ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ +@@ -183354,6 +184489,14 @@ opendb_out: + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } + #endif ++#ifdef SQLITE_HAS_CODEC ++ if( rc==SQLITE_OK ) { ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++ sqlite3CodecResetParameters(&db->codecParm); ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ sqlite3CodecQueryParameters(db, 0, zOpen); ++ } ++#endif /* SQLITE_HAS_CODEC */ + sqlite3_free_filename(zOpen); + return rc; + } +@@ -257677,3 +258820,1667 @@ SQLITE_API int sqlite3_stmt_init( + /* Return the source-id for this library */ + SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + /************************** End of sqlite3.c ******************************/ ++ ++#ifdef SQLITE_HAS_CODEC ++/************** Begin file hw_codec_openssl.h *******************************/ ++#ifndef EXPOSE_INTERNAL_FUNC ++#define CODEC_STATIC static ++#else ++#define CODEC_STATIC ++#endif ++ ++#define DEFAULT_CIPHER "aes-256-gcm" ++ ++typedef struct{ ++ unsigned char *buffer; ++ int bufferSize; ++}Buffer; ++/************** End file hw_codec_openssl.h *********************************/ ++/************** Begin file hw_codec.h ***************************************/ ++#define DEFAULT_PAGE_SIZE 1024 ++#define DEFAULT_ITER 10000 ++#define FILE_HEADER_SIZE 16 ++#define SALT_SIZE FILE_HEADER_SIZE ++#define HMAC_SALT_MASK 0x3a ++#define HMAC_ITER 2 ++#define MAX_HMAC_SIZE 64 ++#define MAX_INIT_VECTOR_SIZE 16 ++#define MIN_BLOCK_SIZE 16 ++ ++#define CODEC_OPERATION_ENCRYPT 1 ++#define CODEC_OPERATION_DECRYPT 0 ++ ++#define KEY_CONTEXT_HEAD_SIZE (sizeof(CodecConstant) + 3 * sizeof(int)) ++ ++typedef struct{ ++ void *cipher; ++ int keySize; ++ int keyInfoSize; ++ int cipherPageSize; ++ int initVectorSize; ++ int hmacSize; ++ int reserveSize; ++ int hmacAlgo; ++ int kdfAlgo; ++ int rekeyHmacAlgo; ++}CodecConstant; ++ ++typedef struct{ ++ CodecConstant codecConst; ++ int deriveFlag; ++ int iter; ++ int passwordSize; ++ unsigned char *password; ++ unsigned char *key; ++ unsigned char *hmacKey; ++ unsigned char *keyInfo; ++}KeyContext; ++ ++typedef struct{ ++ Btree *pBt; ++ int savePassword; ++ unsigned char salt[SALT_SIZE]; ++ unsigned char hmacSalt[SALT_SIZE]; ++ unsigned char *buffer; ++ KeyContext *readCtx; ++ KeyContext *writeCtx; ++}CodecContext; ++ ++/************** End file hw_codec.h *****************************************/ ++/************** Begin file hw_codec_openssl.c *******************************/ ++#include ++#include ++#include ++#include ++ ++unsigned int openssl_init_count = 0; ++unsigned int openssl_external_init_flag = 0; ++sqlite3_mutex *openssl_random_mutex = NULL; ++ ++CODEC_STATIC void opensslActive(){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ ++ if(openssl_init_count == 0){ ++ if(EVP_get_cipherbyname(DEFAULT_CIPHER) == NULL){ ++ OpenSSL_add_all_algorithms(); ++ }else{ ++ openssl_external_init_flag = 1; ++ } ++ } ++ ++ if(openssl_random_mutex == NULL){ ++ openssl_random_mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); ++ } ++ openssl_init_count++; ++ ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return; ++} ++ ++CODEC_STATIC void opensslDeactive(){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ ++ openssl_init_count--; ++ if(openssl_init_count == 0){ ++ if(openssl_external_init_flag){ ++ openssl_external_init_flag = 0; ++ }else{ ++ EVP_cleanup(); ++ } ++ } ++ ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return; ++} ++ ++CODEC_STATIC int opensslGetRandom(Buffer *buffer){ ++ sqlite3_mutex_enter(openssl_random_mutex); ++ int rc = RAND_bytes(buffer->buffer, buffer->bufferSize); ++ sqlite3_mutex_leave(openssl_random_mutex); ++ if(rc != 1){ ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC void *opensslGetCipher(const char *cipherName){ ++ return (void *)EVP_get_cipherbyname(cipherName); ++} ++ ++CODEC_STATIC int opensslFreeCipher(void *cipher){ ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC const char *opensslGetCipherName(void *cipher){ ++ return EVP_CIPHER_name((EVP_CIPHER *)cipher); ++} ++ ++CODEC_STATIC int opensslGetKeySize(void *cipher){ ++ return EVP_CIPHER_key_length((EVP_CIPHER *)cipher); ++} ++ ++CODEC_STATIC int opensslGetInitVectorSize(void *cipher){ ++ return EVP_CIPHER_iv_length((EVP_CIPHER *)cipher); ++} ++ ++#define CIPHER_HMAC_ALGORITHM_SHA1 1 ++#define CIPHER_HMAC_ALGORITHM_SHA256 2 ++#define CIPHER_HMAC_ALGORITHM_SHA512 3 ++ ++#define DEFAULT_HMAC_ALGORITHM CIPHER_HMAC_ALGORITHM_SHA1 ++ ++#define CIPHER_HMAC_ALGORITHM_NAME_SHA1 "SHA1" ++#define CIPHER_HMAC_ALGORITHM_NAME_SHA256 "SHA256" ++#define CIPHER_HMAC_ALGORITHM_NAME_SHA512 "SHA512" ++ ++#define CIPHER_KDF_ALGORITHM_SHA1 1 ++#define CIPHER_KDF_ALGORITHM_SHA256 2 ++#define CIPHER_KDF_ALGORITHM_SHA512 3 ++ ++#define DEFAULT_KDF_ALGORITHM CIPHER_KDF_ALGORITHM_SHA1 ++ ++#define CIPHER_KDF_ALGORITHM_NAME_SHA1 "KDF_SHA1" ++#define CIPHER_KDF_ALGORITHM_NAME_SHA256 "KDF_SHA256" ++#define CIPHER_KDF_ALGORITHM_NAME_SHA512 "KDF_SHA512" ++ ++ ++CODEC_STATIC int opensslGetHmacSize(KeyContext *keyCtx){ ++ if( keyCtx==NULL ){ ++ return 0; ++ } ++ if( keyCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ ++ return EVP_MD_size(EVP_sha1()); ++ }else if( keyCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ ++ return EVP_MD_size(EVP_sha256()); ++ }else if( keyCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ ++ return EVP_MD_size(EVP_sha512()); ++ } ++ return 0; ++} ++ ++CODEC_STATIC int opensslGetBlockSize(void *cipher){ ++ return EVP_CIPHER_block_size((EVP_CIPHER *)cipher); ++} ++ ++CODEC_STATIC void *opensslGetCtx(void *cipher, int mode, unsigned char *key, unsigned char *initVector){ ++ EVP_CIPHER_CTX *tmpCtx = EVP_CIPHER_CTX_new(); ++ if(tmpCtx == NULL){ ++ return (void *)tmpCtx; ++ } ++ EVP_CipherInit_ex(tmpCtx, (EVP_CIPHER *)cipher, NULL, NULL, NULL, mode); ++ EVP_CIPHER_CTX_set_padding(tmpCtx, 0); ++ EVP_CipherInit_ex(tmpCtx, NULL, NULL, key, initVector, mode); ++ return (void *)tmpCtx; ++} ++ ++CODEC_STATIC int opensslCipher(void *iCtx, Buffer *input, unsigned char *output){ ++ int outputLength = 0; ++ int cipherLength; ++ EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)iCtx; ++ EVP_CipherUpdate(ctx, output, &cipherLength, input->buffer, input->bufferSize); ++ outputLength += cipherLength; ++ output += cipherLength; ++ EVP_CipherFinal_ex(ctx, output, &cipherLength); ++ outputLength += cipherLength; ++ if(outputLength != input->bufferSize){ ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC void opensslFreeCtx(void *ctx){ ++ EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)ctx); ++} ++ ++CODEC_STATIC int opensslHmac(Buffer *key, Buffer *input1, Buffer *input2, Buffer *output, int hmacAlgo){ ++ HMAC_CTX *ctx = HMAC_CTX_new(); ++ if(ctx == NULL){ ++ return SQLITE_ERROR; ++ } ++ if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ ++ HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha1(), NULL); ++ }else if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ ++ HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha256(), NULL); ++ }else if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ ++ HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha512(), NULL); ++ } ++ HMAC_Update(ctx, input1->buffer, input1->bufferSize); ++ HMAC_Update(ctx, input2->buffer, input2->bufferSize); ++ HMAC_Final(ctx, output->buffer, (unsigned int *)(&output->bufferSize)); ++ ++ HMAC_CTX_free(ctx); ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC void opensslKdf(Buffer *password, Buffer *salt, int workfactor, Buffer *key, int kdfAlgo){ ++ if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA1 ){ ++ PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, ++ workfactor, EVP_sha1(), key->bufferSize, key->buffer); ++ }else if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA256 ){ ++ PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, ++ workfactor, EVP_sha256(), key->bufferSize, key->buffer); ++ }else if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA512 ){ ++ PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, ++ workfactor, EVP_sha512(), key->bufferSize, key->buffer); ++ } ++} ++ ++/************** End file hw_codec_openssl.c *********************************/ ++/************** Begin file hw_codec.c ***************************************/ ++ ++#include "securec.h" ++ ++typedef enum{ ++ OPERATE_CONTEXT_READ = 0, ++ OPERATE_CONTEXT_WRITE, ++ OPERATE_CONTEXT_BOTH ++}OperateContext; ++ ++CODEC_STATIC int sqlite3CodecIsHex(const unsigned char *buffer, int bufferSize){ ++ int i; ++ for(i = 0; i < bufferSize; i++){ ++ if((buffer[i] < '0' || buffer[i] > '9') && ++ (buffer[i] < 'a' || buffer[i] > 'f') && ++ (buffer[i] < 'A' || buffer[i] > 'F')){ ++ return 0; ++ } ++ } ++ return 1; ++} ++ ++CODEC_STATIC int sqlite3CodecHex2int(char input){ ++ if(input >= '0' && input <= '9'){ ++ return (int)(input - '0'); ++ }else if(input >= 'a' && input <= 'f'){ ++ return (int)(input - 'a') + 10; ++ }else if(input >= 'A' && input <= 'F'){ ++ return (int)(input - 'A') + 10; ++ }else{ ++ return 0; ++ } ++} ++ ++CODEC_STATIC void sqlite3CodecHex2Bin(unsigned char *inputBuffer, int inputBuffersize, unsigned char *outputBuffer){ ++ int i; ++ for(i = 0; i < inputBuffersize - 1; i += 2){ ++ outputBuffer[i / 2] = sqlite3CodecHex2int(inputBuffer[i]) << 4 | sqlite3CodecHex2int(inputBuffer[i + 1]); ++ } ++ return; ++} ++ ++CODEC_STATIC void sqlite3CodecBin2Hex(unsigned char *inputBuffer, int inputBuffersize, unsigned char *outputBuffer){ ++ char *buffer = NULL; ++ int i; ++ for(i = 0; i < inputBuffersize; i++){ ++ buffer = (char *)(outputBuffer + i * 2); ++ sqlite3_snprintf(3, buffer, "%02x ", inputBuffer[i]); ++ } ++ return; ++} ++ ++CODEC_STATIC int sqlite3CodecIfMemset(unsigned char *input, unsigned char val, int len){ ++ int i; ++ for(i = 0; i < len; i++){ ++ if(input[i] != val){ ++ return 0; ++ } ++ } ++ return 1; ++} ++ ++CODEC_STATIC int sqlite3CodecIsKeyInfoFormat(const unsigned char *input, int inputSize){ ++ return (input[0] == 'x') && (input[1] == '\'') && (input[inputSize - 1] == '\'') && (sqlite3CodecIsHex(input + 2, inputSize - 3)) ? 1 : 0; ++} ++ ++CODEC_STATIC void sqlite3CodecSetError(CodecContext *ctx, int error){ ++ if(ctx->pBt){ ++ ctx->pBt->pBt->pPager->errCode = error; ++ ctx->pBt->pBt->db->errCode = error; ++ } ++ return; ++} ++ ++CODEC_STATIC void sqlite3CodecClearDeriveKey(KeyContext *keyCtx){ ++ if(keyCtx->key != NULL){ ++ (void)memset_s(keyCtx->key, keyCtx->codecConst.keySize, 0, keyCtx->codecConst.keySize); ++ sqlite3_free(keyCtx->key); ++ keyCtx->key = NULL; ++ } ++ if(keyCtx->hmacKey != NULL){ ++ (void)memset_s(keyCtx->hmacKey, keyCtx->codecConst.keySize, 0, keyCtx->codecConst.keySize); ++ sqlite3_free(keyCtx->hmacKey); ++ keyCtx->hmacKey = NULL; ++ } ++ if(keyCtx->keyInfo != NULL){ ++ (void)memset_s(keyCtx->keyInfo, keyCtx->codecConst.keyInfoSize, 0, keyCtx->codecConst.keyInfoSize); ++ sqlite3_free(keyCtx->keyInfo); ++ keyCtx->keyInfo = NULL; ++ } ++ keyCtx->deriveFlag = 0; ++} ++ ++CODEC_STATIC void sqlite3CodecClearPassword(KeyContext *keyCtx){ ++ if(keyCtx->password != NULL){ ++ (void)memset_s(keyCtx->password, keyCtx->passwordSize, 0, keyCtx->passwordSize); ++ sqlite3_free(keyCtx->password); ++ keyCtx->password = NULL; ++ } ++ keyCtx->passwordSize = 0; ++} ++ ++CODEC_STATIC void sqlite3CodecInitDeriveKeyMemory(KeyContext *keyCtx){ ++ keyCtx->key = (unsigned char *)sqlite3Malloc(keyCtx->codecConst.keySize); ++ keyCtx->hmacKey = (unsigned char *)sqlite3Malloc(keyCtx->codecConst.keySize); ++ keyCtx->keyInfo = (unsigned char *)sqlite3Malloc(keyCtx->codecConst.keyInfoSize); ++ return; ++} ++ ++CODEC_STATIC int sqlite3CodecKeyCtxCmp(KeyContext *first, KeyContext *second){ ++ if(memcmp((unsigned char *)first, (unsigned char *)second, sizeof(CodecConstant))){ ++ return 0; ++ } ++ if(first->iter != second->iter){ ++ return 0; ++ } ++ if(first->passwordSize != second->passwordSize){ ++ return 0; ++ } ++ if((first->password != second->password) && memcmp(first->password, second->password, first->passwordSize)){ ++ return 0; ++ } ++ return 1; ++} ++ ++// This function will free all resources of key context, except it self. ++CODEC_STATIC void sqlite3CodecFreeKeyContext(KeyContext *keyCtx){ ++ sqlite3CodecClearDeriveKey(keyCtx); ++ sqlite3CodecClearPassword(keyCtx); ++ (void)opensslFreeCipher(keyCtx->codecConst.cipher); ++ (void)memset_s(keyCtx, sizeof(KeyContext), 0, KEY_CONTEXT_HEAD_SIZE); ++} ++ ++// You should clear key derive result of output before you call this function ++CODEC_STATIC int sqlite3CodecCopyDeriveKey(KeyContext *input, KeyContext *output){ ++ errno_t rc = EOK; ++ if(input->key != NULL && input->codecConst.keySize > 0){ ++ output->key = (unsigned char *)sqlite3Malloc(output->codecConst.keySize); ++ if(output->key == NULL){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_NOMEM; ++ } ++ rc = memcpy_s(output->key, output->codecConst.keySize, input->key, input->codecConst.keySize); ++ if(rc != EOK){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_ERROR; ++ } ++ } ++ if(input->hmacKey != NULL && input->codecConst.keySize > 0){ ++ output->hmacKey = (unsigned char *)sqlite3Malloc(output->codecConst.keySize); ++ if(output->hmacKey == NULL){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_NOMEM; ++ } ++ rc = memcpy_s(output->hmacKey, output->codecConst.keySize, input->hmacKey, input->codecConst.keySize); ++ if(rc != EOK){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_ERROR; ++ } ++ } ++ if(input->keyInfo != NULL && input->codecConst.keyInfoSize > 0){ ++ output->keyInfo = (unsigned char *)sqlite3Malloc(output->codecConst.keyInfoSize); ++ if(output->keyInfo == NULL){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_NOMEM; ++ } ++ rc = memcpy_s(output->keyInfo, output->codecConst.keyInfoSize, input->keyInfo, input->codecConst.keyInfoSize); ++ if(rc != EOK){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_ERROR; ++ } ++ } ++ return SQLITE_OK; ++} ++ ++// You should set all key infos including salt before you call this function ++CODEC_STATIC int sqlite3CodecDeriveKey(CodecContext *ctx, OperateContext whichKey){ ++ KeyContext *keyCtx = NULL; ++ KeyContext *secondKeyCtx = NULL; ++ switch(whichKey){ ++ case OPERATE_CONTEXT_READ: ++ keyCtx = ctx->readCtx; ++ secondKeyCtx = ctx->writeCtx; ++ break; ++ case OPERATE_CONTEXT_WRITE: ++ keyCtx = ctx->writeCtx; ++ secondKeyCtx = ctx->readCtx; ++ break; ++ default: ++ return SQLITE_ERROR; ++ } ++ if(keyCtx->password == NULL || keyCtx->passwordSize <= 0){ ++ return SQLITE_ERROR; ++ } ++ if(keyCtx->deriveFlag){ ++ return SQLITE_OK; ++ } ++ errno_t memcpyRc = EOK; ++ unsigned char salt[SALT_SIZE]; ++ if (ctx->pBt != NULL && sqlite3OsRead(ctx->pBt->pBt->pPager->fd, salt, SALT_SIZE, 0) == SQLITE_OK) { ++ assert(SALT_SIZE == FILE_HEADER_SIZE); ++ if (memcmp(SQLITE_FILE_HEADER, salt, SALT_SIZE) != 0 && memcmp(ctx->salt, salt, SALT_SIZE) != 0) { ++ memcpyRc = memcpy_s(ctx->salt, FILE_HEADER_SIZE, salt, SALT_SIZE); ++ if(memcpyRc != EOK){ ++ return SQLITE_ERROR; ++ } ++ } ++ } ++ sqlite3CodecInitDeriveKeyMemory(keyCtx); ++ if(keyCtx->key == NULL || keyCtx->hmacKey == NULL || keyCtx->keyInfo == NULL){ ++ sqlite3CodecClearDeriveKey(keyCtx); ++ return SQLITE_NOMEM; ++ } ++ if((keyCtx->passwordSize == keyCtx->codecConst.keyInfoSize) && ++ (sqlite3CodecIsKeyInfoFormat(keyCtx->password, keyCtx->passwordSize))){ ++ sqlite3CodecHex2Bin(keyCtx->password + 2, keyCtx->codecConst.keySize * 2, keyCtx->key); ++ sqlite3CodecHex2Bin(keyCtx->password + 2 + keyCtx->codecConst.keySize * 2, SALT_SIZE * 2, ctx->salt); ++ memcpyRc = memcpy_s(keyCtx->keyInfo, keyCtx->codecConst.keyInfoSize, keyCtx->password, keyCtx->passwordSize); ++ if(memcpyRc != EOK){ ++ return SQLITE_ERROR; ++ } ++ }else if((keyCtx->passwordSize == keyCtx->codecConst.keyInfoSize - SALT_SIZE * 2) && ++ (sqlite3CodecIsKeyInfoFormat(keyCtx->password, keyCtx->passwordSize))){ ++ sqlite3CodecHex2Bin(keyCtx->password + 2, keyCtx->codecConst.keySize * 2, keyCtx->key); ++ memcpyRc = memcpy_s(keyCtx->keyInfo, keyCtx->codecConst.keyInfoSize, keyCtx->password, keyCtx->passwordSize); ++ if(memcpyRc != EOK){ ++ return SQLITE_ERROR; ++ } ++ sqlite3CodecBin2Hex(ctx->salt, SALT_SIZE, keyCtx->keyInfo + 2 + keyCtx->codecConst.keySize * 2); ++ keyCtx->keyInfo[keyCtx->codecConst.keyInfoSize - 1] = '\''; ++ }else{ ++ Buffer password; ++ Buffer salt; ++ Buffer key; ++ password.buffer = keyCtx->password; ++ password.bufferSize = keyCtx->passwordSize; ++ salt.buffer = ctx->salt; ++ salt.bufferSize = SALT_SIZE; ++ key.buffer = keyCtx->key; ++ key.bufferSize = keyCtx->codecConst.keySize; ++ opensslKdf(&password, &salt, keyCtx->iter, &key, keyCtx->codecConst.kdfAlgo); ++ keyCtx->keyInfo[0] = 'x'; ++ keyCtx->keyInfo[1] = '\''; ++ sqlite3CodecBin2Hex(keyCtx->key, keyCtx->codecConst.keySize, keyCtx->keyInfo + 2); ++ sqlite3CodecBin2Hex(ctx->salt, SALT_SIZE, keyCtx->keyInfo + 2 + keyCtx->codecConst.keySize * 2); ++ keyCtx->keyInfo[keyCtx->codecConst.keyInfoSize - 1] = '\''; ++ } ++ int i; ++ for(i = 0; i < SALT_SIZE; i++){ ++ ctx->hmacSalt[i] = ctx->salt[i] ^ HMAC_SALT_MASK; ++ } ++ Buffer hmacPassword; ++ Buffer hmacSalt; ++ Buffer hmacKey; ++ hmacPassword.buffer = keyCtx->key; ++ hmacPassword.bufferSize = keyCtx->codecConst.keySize; ++ hmacSalt.buffer = ctx->hmacSalt; ++ hmacSalt.bufferSize = SALT_SIZE; ++ hmacKey.buffer = keyCtx->hmacKey; ++ hmacKey.bufferSize = keyCtx->codecConst.keySize; ++ opensslKdf(&hmacPassword, &hmacSalt, HMAC_ITER, &hmacKey, keyCtx->codecConst.kdfAlgo); ++ keyCtx->deriveFlag = 1; ++ if(sqlite3CodecKeyCtxCmp(keyCtx, secondKeyCtx)){ ++ sqlite3CodecClearDeriveKey(secondKeyCtx); ++ int rc = sqlite3CodecCopyDeriveKey(keyCtx, secondKeyCtx); ++ if(rc == SQLITE_OK){ ++ secondKeyCtx->deriveFlag = 1; ++ // clear password ++ if(!(ctx->savePassword)){ ++ sqlite3CodecClearPassword(secondKeyCtx); ++ } ++ } ++ } ++ // clear password ++ if(!(ctx->savePassword)){ ++ sqlite3CodecClearPassword(keyCtx); ++ } ++ return SQLITE_OK; ++} ++ ++// This function may clear key derive infos ++CODEC_STATIC int sqlite3CodecSetCodecConstant(KeyContext *keyCtx, const char *cipherName){ ++ if(keyCtx->codecConst.cipher){ ++ if(sqlite3StrICmp(cipherName, opensslGetCipherName(keyCtx->codecConst.cipher)) == 0){ ++ return SQLITE_OK; ++ } ++ } ++ sqlite3CodecClearDeriveKey(keyCtx); ++ void *cipher = opensslGetCipher(cipherName); ++ if(cipher != NULL){ ++ keyCtx->codecConst.cipher = cipher; ++ } else { ++ return SQLITE_ERROR; ++ } ++ keyCtx->codecConst.keySize = opensslGetKeySize(keyCtx->codecConst.cipher); ++ keyCtx->codecConst.keyInfoSize = (keyCtx->codecConst.keySize + SALT_SIZE) * 2 + 3; ++ keyCtx->codecConst.initVectorSize = opensslGetInitVectorSize(keyCtx->codecConst.cipher); ++ return SQLITE_OK; ++} ++ ++// You should clear key derive infos before you call this function ++CODEC_STATIC int sqlite3CodecSetIter(KeyContext *keyCtx, int iter){ ++ keyCtx->iter = iter; ++ return SQLITE_OK; ++} ++ ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++#define CIPHER_ID_AES_256_CBC 0 ++#define CIPHER_ID_AES_256_GCM 1 ++ ++#define CIPHER_TOTAL_NUM 2 ++ ++#define CIPHER_NAME_AES_256_CBC "aes-256-cbc" ++#define CIPHER_NAME_AES_256_GCM "aes-256-gcm" ++ ++struct CodecCipherNameId { ++ int cipherId; ++ const char *cipherName; ++}; ++ ++static const struct CodecCipherNameId g_cipherNameIdMap[CIPHER_TOTAL_NUM] = { ++ { CIPHER_ID_AES_256_CBC, CIPHER_NAME_AES_256_CBC }, ++ { CIPHER_ID_AES_256_GCM, CIPHER_NAME_AES_256_GCM } ++}; ++ ++SQLITE_PRIVATE void sqlite3CodecResetParameters(CodecParameter *p) ++{ ++ p->kdfIter = DEFAULT_ITER; ++ p->pageSize = DEFAULT_PAGE_SIZE; ++ p->cipher = CIPHER_ID_AES_256_GCM; ++ p->hmacAlgo = DEFAULT_HMAC_ALGORITHM; ++ p->kdfAlgo = DEFAULT_KDF_ALGORITHM; ++} ++ ++CODEC_STATIC void sqlite3CodecSetDefaultAttachCipher(CodecParameter *parm, const char *cipherName){ ++ int i; ++ for( i=0; icipher = g_cipherNameIdMap[i].cipherId; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return; ++ } ++ } ++ sqlite3_log(SQLITE_WARNING, "invalid attach cipher algorithm"); ++} ++ ++CODEC_STATIC const char *sqlite3CodecGetDefaultAttachCipher(CodecParameter *parm){ ++ const char *attachedCipher = CIPHER_NAME_AES_256_GCM; ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ if( (parm->cipher>=0) && (parm->ciphercipher].cipherName; ++ } ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return attachedCipher; ++} ++ ++CODEC_STATIC void sqlite3CodecSetDefaultAttachKdfIter(CodecParameter *parm, int iter){ ++ if( iter<=0 ){ ++ return; ++ } ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ parm->kdfIter = iter; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++} ++ ++CODEC_STATIC int sqlite3CodecGetDefaultAttachKdfIter(CodecParameter *parm){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ int iterNum = parm->kdfIter; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return iterNum; ++} ++ ++CODEC_STATIC void sqlite3CodecSetDefaultAttachHmacAlgo(CodecParameter *parm, int hmacAlgo){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ parm->hmacAlgo = hmacAlgo; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++} ++ ++CODEC_STATIC int sqlite3CodecGetDefaultAttachHmacAlgo(CodecParameter *parm){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ int hmacAlgo = parm->hmacAlgo; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return hmacAlgo; ++} ++ ++CODEC_STATIC void sqlite3CodecSetDefaultAttachKdfAlgo(CodecParameter *parm, int kdfAlgo){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ parm->kdfAlgo = kdfAlgo; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++} ++ ++CODEC_STATIC int sqlite3CodecGetDefaultAttachKdfAlgo(CodecParameter *parm){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ int kdfAlgo = parm->kdfAlgo; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return kdfAlgo; ++} ++ ++CODEC_STATIC void sqlite3CodecSetDefaultAttachPageSize(CodecParameter *parm, int pageSize){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ parm->pageSize = pageSize; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++} ++ ++CODEC_STATIC int sqlite3CodecGetDefaultAttachPageSize(CodecParameter *parm){ ++ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ int pageSize = parm->pageSize; ++ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); ++ return pageSize; ++} ++ ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ ++// You should clear key derive infos and password infos before you call this function ++CODEC_STATIC int sqlite3CodecSetPassword(KeyContext *keyCtx, const void *zKey, int nKey){ ++ keyCtx->passwordSize = nKey; ++ keyCtx->password = (unsigned char *)sqlite3Malloc(keyCtx->passwordSize); ++ if(keyCtx->password == NULL){ ++ return SQLITE_NOMEM; ++ } ++ errno_t rc = memcpy_s(keyCtx->password, keyCtx->passwordSize, zKey, nKey); ++ if(rc != EOK){ ++ sqlite3CodecClearPassword(keyCtx); ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++} ++ ++// You should clear key derive infos and password infos before you call this function ++CODEC_STATIC int sqlite3CodecSetHmacAlgorithm(KeyContext *keyCtx, int hmacAlgo){ ++ keyCtx->codecConst.hmacAlgo = hmacAlgo; ++ keyCtx->codecConst.hmacSize = opensslGetHmacSize(keyCtx); ++ int cipherBlockSize = opensslGetBlockSize(keyCtx->codecConst.cipher); ++ int blockSize = cipherBlockSize; ++ while(blockSize < MIN_BLOCK_SIZE){ ++ blockSize += cipherBlockSize; ++ } ++ int reserveSize = MAX_INIT_VECTOR_SIZE + keyCtx->codecConst.hmacSize; ++ if(reserveSize % blockSize == 0){ ++ keyCtx->codecConst.reserveSize = reserveSize; ++ }else{ ++ keyCtx->codecConst.reserveSize = (reserveSize / blockSize + 1) * blockSize; ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC int sqlite3CodecSetKdfAlgorithm(KeyContext *keyCtx, int kdfAlgo){ ++ keyCtx->codecConst.kdfAlgo = kdfAlgo; ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC int sqlite3CodecSetCipherPageSize(CodecContext *ctx, int size){ ++ if(!((size != 0) && ((size & (size - 1)) == 0)) || size < 512 || size > 65536) { ++ sqlite3_log(SQLITE_ERROR, "codec: cipher_page_size not a power of 2 and between 512 and 65536 inclusive(%d).", size); ++ return SQLITE_ERROR; ++ } ++ int cipherPageSize = ctx->readCtx->codecConst.cipherPageSize; ++ (void)memset_s(ctx->buffer, cipherPageSize, 0, cipherPageSize); ++ sqlite3_free(ctx->buffer); ++ ctx->readCtx->codecConst.cipherPageSize = size; ++ ctx->writeCtx->codecConst.cipherPageSize = size; ++ ++ ctx->buffer = (unsigned char *)sqlite3Malloc(size); ++ if (ctx->buffer == NULL) { ++ sqlite3_log(SQLITE_NOMEM, "codec: alloc mem failed when set cipher page size(%d).", size); ++ return SQLITE_NOMEM; ++ } ++ return SQLITE_OK; ++} ++ ++// You should clear output before you call this function ++CODEC_STATIC int sqlite3CodecCopyKeyContext(KeyContext *input, KeyContext *output){ ++ errno_t rc = memcpy_s(output, sizeof(KeyContext), input, KEY_CONTEXT_HEAD_SIZE); ++ if(rc != EOK){ ++ return SQLITE_ERROR; ++ } ++ if(input->password != NULL && input->passwordSize > 0){ ++ output->password = (unsigned char *)sqlite3Malloc(output->passwordSize); ++ if(output->password == NULL){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_NOMEM; ++ } ++ rc = memcpy_s(output->password, output->passwordSize, input->password, input->passwordSize); ++ if(rc != EOK){ ++ sqlite3CodecFreeKeyContext(output); ++ return SQLITE_ERROR; ++ } ++ } ++ return sqlite3CodecCopyDeriveKey(input, output); ++} ++ ++// You should clear key context before you call this function ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++CODEC_STATIC int sqlite3CodecInitKeyContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey, int attachFlag){ ++#else ++CODEC_STATIC int sqlite3CodecInitKeyContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey){ ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ int rc = SQLITE_OK; ++ KeyContext *keyCtx = ctx->readCtx; ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++ if( attachFlag!=0 ){ ++ CodecParameter *parm = &p->db->codecParm; ++ int hmacAlgo = sqlite3CodecGetDefaultAttachHmacAlgo(parm); ++ rc = sqlite3CodecSetCodecConstant(keyCtx, sqlite3CodecGetDefaultAttachCipher(parm)); ++ rc += sqlite3CodecSetIter(keyCtx, sqlite3CodecGetDefaultAttachKdfIter(parm)); ++ if( hmacAlgo!=0 ){ ++ rc += sqlite3CodecSetHmacAlgorithm(keyCtx, hmacAlgo); ++ } ++ int attachKdfAlgo = sqlite3CodecGetDefaultAttachKdfAlgo(parm); ++ if( attachKdfAlgo!=0 ){ ++ rc += sqlite3CodecSetKdfAlgorithm(keyCtx, attachKdfAlgo); ++ } ++ int cipherPageSize = sqlite3CodecGetDefaultAttachPageSize(parm); ++ if( cipherPageSize!=0 ){ ++ rc += sqlite3CodecSetCipherPageSize(ctx, cipherPageSize); ++ if ( rc != SQLITE_OK ) { ++ sqlite3CodecFreeKeyContext(keyCtx); ++ return SQLITE_ERROR; ++ } ++ rc += sqlite3BtreeSetPageSize(p, cipherPageSize, keyCtx->codecConst.reserveSize, 0); ++ } ++ }else{ ++ rc = sqlite3CodecSetCodecConstant(keyCtx, DEFAULT_CIPHER); ++ rc += sqlite3CodecSetIter(keyCtx, DEFAULT_ITER); ++ rc += sqlite3CodecSetHmacAlgorithm(keyCtx, DEFAULT_HMAC_ALGORITHM); ++ rc += sqlite3CodecSetKdfAlgorithm(keyCtx, DEFAULT_KDF_ALGORITHM); ++ } ++#else ++ rc = sqlite3CodecSetCodecConstant(keyCtx, DEFAULT_CIPHER); ++ rc += sqlite3CodecSetIter(keyCtx, DEFAULT_ITER); ++ rc += sqlite3CodecSetHmacAlgorithm(keyCtx, DEFAULT_HMAC_ALGORITHM); ++ rc += sqlite3CodecSetKdfAlgorithm(keyCtx, DEFAULT_KDF_ALGORITHM); ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ keyCtx->codecConst.rekeyHmacAlgo = DEFAULT_HMAC_ALGORITHM; ++ rc += sqlite3CodecSetPassword(keyCtx, zKey, nKey); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecFreeKeyContext(keyCtx); ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++} ++ ++// This function will free all resources of codec context, except it self. ++CODEC_STATIC void sqlite3CodecFreeContext(CodecContext *ctx){ ++ if(ctx->buffer){ ++ int cipherPageSize = ctx->readCtx->codecConst.cipherPageSize; ++ (void)memset_s(ctx->buffer, cipherPageSize, 0, cipherPageSize); ++ sqlite3_free(ctx->buffer); ++ ctx->buffer = NULL; ++ } ++ if(ctx->readCtx){ ++ sqlite3CodecFreeKeyContext(ctx->readCtx); ++ sqlite3_free(ctx->readCtx); ++ ctx->readCtx = NULL; ++ } ++ if(ctx->writeCtx){ ++ sqlite3CodecFreeKeyContext(ctx->writeCtx); ++ sqlite3_free(ctx->writeCtx); ++ ctx->writeCtx = NULL; ++ } ++ (void)memset_s(ctx, sizeof(CodecContext), 0, sizeof(CodecContext)); ++ return; ++} ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++CODEC_STATIC int sqlite3CodecInitContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey, int nDb){ ++ int attachFlag = (nDb > 1) ? 1 : 0; ++ int defaultPageSz = attachFlag ? p->db->codecParm.pageSize : DEFAULT_PAGE_SIZE; ++#else ++CODEC_STATIC int sqlite3CodecInitContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey){ ++ int defaultPageSz = DEFAULT_PAGE_SIZE; ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ sqlite3_file *fd = p->pBt->pPager->fd; ++ ctx->pBt = p; ++ ctx->savePassword = 0; ++ ctx->buffer = (unsigned char *)sqlite3Malloc(defaultPageSz); ++ ctx->readCtx = (KeyContext *)sqlite3Malloc(sizeof(KeyContext)); ++ ctx->writeCtx = (KeyContext *)sqlite3Malloc(sizeof(KeyContext)); ++ if(ctx->buffer == NULL || ctx->readCtx == NULL || ctx->writeCtx == NULL){ ++ sqlite3CodecFreeContext(ctx); ++ return SQLITE_NOMEM; ++ } ++ errno_t memsetRc = memset_s(ctx->buffer, defaultPageSz, 0, defaultPageSz); ++ memsetRc += memset_s(ctx->readCtx, sizeof(KeyContext), 0, sizeof(KeyContext)); ++ memsetRc += memset_s(ctx->writeCtx, sizeof(KeyContext), 0, sizeof(KeyContext)); ++ if(memsetRc != EOK){ ++ sqlite3CodecFreeContext(ctx); ++ return SQLITE_ERROR; ++ } ++ ctx->readCtx->codecConst.cipherPageSize = defaultPageSz; ++ ctx->writeCtx->codecConst.cipherPageSize = defaultPageSz; ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++ int rc = sqlite3CodecInitKeyContext(ctx, p, zKey, nKey, attachFlag); ++#else ++ int rc = sqlite3CodecInitKeyContext(ctx, p, zKey, nKey); ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ if(rc != SQLITE_OK){ ++ sqlite3CodecFreeContext(ctx); ++ return SQLITE_ERROR; ++ } ++ rc = sqlite3CodecCopyKeyContext(ctx->readCtx, ctx->writeCtx); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecFreeContext(ctx); ++ return SQLITE_ERROR; ++ } ++ if(fd == NULL || !(isOpen(fd)) || sqlite3OsRead(fd, ctx->salt, SALT_SIZE, 0) != SQLITE_OK){ ++ Buffer salt; ++ salt.buffer = ctx->salt; ++ salt.bufferSize = SALT_SIZE; ++ rc = opensslGetRandom(&salt); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecFreeContext(ctx); ++ return rc; ++ } ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC int sqlite3CodecGetDbIndex(sqlite3 *db, const char *zDb){ ++ int nDbIndex; ++ if(zDb == NULL){ ++ return 0; ++ } ++ for(nDbIndex = 0; nDbIndex < db->nDb; nDbIndex++){ ++ const char *zDbSName = db->aDb[nDbIndex].zDbSName; ++ if(strcmp(zDbSName, zDb) == 0){ ++ return nDbIndex; ++ } ++ } ++ return 0; ++} ++ ++CODEC_STATIC void sqlite3CodecTransPgno(Pgno input, unsigned char *output){ ++#ifdef CIPHER_BIG_ENDAIN ++ sqlite3Put4byte(output, input); ++#else ++ output[0] = (u8)input; ++ output[1] = (u8)(input>>8); ++ output[2] = (u8)(input>>16); ++ output[3] = (u8)(input>>24); ++#endif ++} ++ ++CODEC_STATIC int sqlite3CodecHmac(KeyContext *ctx, Pgno pgno, int bufferSize, unsigned char *input, unsigned char *output){ ++ Buffer key; ++ key.buffer = ctx->hmacKey; ++ key.bufferSize = ctx->codecConst.keySize; ++ Buffer input1; ++ input1.buffer = input; ++ input1.bufferSize = bufferSize; ++ Buffer input2; ++ unsigned char pgnoBuffer[sizeof(Pgno)]; ++ sqlite3CodecTransPgno(pgno, pgnoBuffer); ++ input2.buffer = pgnoBuffer; ++ input2.bufferSize = sizeof(Pgno); ++ Buffer outputBuffer; ++ outputBuffer.buffer = output; ++ outputBuffer.bufferSize = 0; ++ int rc = opensslHmac(&key, &input1, &input2, &outputBuffer, ctx->codecConst.hmacAlgo); ++ if(rc != SQLITE_OK || outputBuffer.bufferSize != ctx->codecConst.hmacSize){ ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC int sqlite3CodecCheckHmac(KeyContext *ctx, Pgno pgno, int bufferSize, unsigned char *input, unsigned char *expectResult){ ++ if (ctx->codecConst.hmacSize <= 0) { ++ return 1; ++ } ++ int rc = SQLITE_OK; ++ if (ctx->codecConst.hmacSize <= MAX_HMAC_SIZE) { ++ unsigned char buffer[MAX_HMAC_SIZE]; ++ rc = sqlite3CodecHmac(ctx, pgno, bufferSize, input, buffer); ++ if(rc != SQLITE_OK){ ++ return 1; ++ } ++ return memcmp(buffer, expectResult, ctx->codecConst.hmacSize); ++ } else { ++ unsigned char *output = (unsigned char *)malloc(ctx->codecConst.hmacSize); ++ if (output == NULL) { ++ return 1; ++ } ++ rc = sqlite3CodecHmac(ctx, pgno, bufferSize, input, output); ++ if(rc != SQLITE_OK){ ++ free(output); ++ return 1; ++ } ++ rc = memcmp(output, expectResult, ctx->codecConst.hmacSize); ++ free(output); ++ return rc; ++ } ++} ++ ++CODEC_STATIC int sqlite3CodecEncryptData(CodecContext *ctx, OperateContext whichKey, Pgno pgno, int bufferSize, unsigned char *input, unsigned char *output){ ++ KeyContext *keyCtx = NULL; ++ switch(whichKey){ ++ case OPERATE_CONTEXT_READ: ++ keyCtx = ctx->readCtx; ++ break; ++ case OPERATE_CONTEXT_WRITE: ++ keyCtx = ctx->writeCtx; ++ break; ++ default: ++ return SQLITE_ERROR; ++ } ++ int rc = SQLITE_OK; ++ if(!(keyCtx->deriveFlag)){ ++ rc = sqlite3CodecDeriveKey(ctx, whichKey); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ } ++ if(keyCtx->codecConst.keySize == 0){ ++ return SQLITE_ERROR; ++ } ++ Buffer inputBuffer; ++ inputBuffer.buffer = input; ++ inputBuffer.bufferSize = bufferSize - keyCtx->codecConst.reserveSize; ++ Buffer initVector; ++ initVector.buffer = output + inputBuffer.bufferSize; ++ initVector.bufferSize = keyCtx->codecConst.initVectorSize; ++ rc = opensslGetRandom(&initVector); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ void *cipherCtx = opensslGetCtx(keyCtx->codecConst.cipher, CODEC_OPERATION_ENCRYPT, keyCtx->key, initVector.buffer); ++ if(cipherCtx == NULL){ ++ return SQLITE_ERROR; ++ } ++ rc = opensslCipher(cipherCtx, &inputBuffer, output); ++ opensslFreeCtx(cipherCtx); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ rc = sqlite3CodecHmac(keyCtx, pgno, inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize, output, output + inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ return SQLITE_OK; ++} ++ ++CODEC_STATIC int sqlite3CodecDecryptData(CodecContext *ctx, OperateContext whichKey, Pgno pgno, int bufferSize, unsigned char *input, unsigned char *output){ ++ KeyContext *keyCtx = NULL; ++ switch(whichKey){ ++ case OPERATE_CONTEXT_READ: ++ keyCtx = ctx->readCtx; ++ break; ++ case OPERATE_CONTEXT_WRITE: ++ keyCtx = ctx->writeCtx; ++ break; ++ default: ++ return SQLITE_ERROR; ++ } ++ int rc = SQLITE_OK; ++ if(!(keyCtx->deriveFlag)){ ++ rc = sqlite3CodecDeriveKey(ctx, whichKey); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ } ++ if(keyCtx->codecConst.keySize == 0){ ++ return SQLITE_ERROR; ++ } ++ if(sqlite3CodecIfMemset(input, 0, bufferSize)){ ++ errno_t memsetRc = memset_s(output, bufferSize, 0, bufferSize); ++ if(memsetRc != EOK){ ++ return SQLITE_ERROR; ++ } ++ return SQLITE_OK; ++ }else{ ++ Buffer inputBuffer; ++ inputBuffer.buffer = input; ++ inputBuffer.bufferSize = bufferSize - keyCtx->codecConst.reserveSize; ++ if(sqlite3CodecCheckHmac(keyCtx, pgno, inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize, input, input + inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize)){ ++ sqlite3_log(SQLITE_ERROR, "codec: check hmac error at page %d, hmac %d, kdf %d, pageSize %d, iter %d.", ++ pgno, keyCtx->codecConst.hmacAlgo, keyCtx->codecConst.kdfAlgo, keyCtx->codecConst.cipherPageSize, keyCtx->iter); ++ return SQLITE_ERROR; ++ } ++ unsigned char *initVector = input + inputBuffer.bufferSize; ++ void *cipherCtx = opensslGetCtx(keyCtx->codecConst.cipher, CODEC_OPERATION_DECRYPT, keyCtx->key, initVector); ++ if(cipherCtx == NULL){ ++ return SQLITE_ERROR; ++ } ++ rc = opensslCipher(cipherCtx, &inputBuffer, output); ++ opensslFreeCtx(cipherCtx); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ } ++ return SQLITE_OK; ++} ++ ++void* sqlite3Codec(void *ctx, void *data, Pgno pgno, int mode){ ++ CodecContext *pCtx = (CodecContext *)ctx; ++ unsigned char *pData = (unsigned char *)data; ++ int offset = 0; ++ int rc = SQLITE_OK; ++ errno_t memcpyRc = EOK; ++ if(ctx == NULL || data == NULL){ ++ return pData; ++ } ++ if(pgno == 1){ ++ offset = FILE_HEADER_SIZE; ++ } ++ int cipherPageSize = pCtx->readCtx->codecConst.cipherPageSize; ++ switch(mode){ ++ case 0: ++ case 2: ++ case 3: ++ if(pgno == 1){ ++ memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, SQLITE_FILE_HEADER, FILE_HEADER_SIZE); ++ if(memcpyRc != EOK){ ++ sqlite3CodecSetError(pCtx, SQLITE_ERROR); ++ return pData; ++ } ++ } ++ rc = sqlite3CodecDecryptData(pCtx, OPERATE_CONTEXT_READ, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecSetError(pCtx, rc); ++ } ++ (void)memcpy_s(pData, cipherPageSize, pCtx->buffer, cipherPageSize); ++ return pData; ++ break; ++ case 6: ++ if(pgno == 1){ ++ memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, pCtx->salt, FILE_HEADER_SIZE); ++ if(memcpyRc != EOK){ ++ sqlite3CodecSetError(pCtx, SQLITE_ERROR); ++ return pData; ++ } ++ } ++ rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_WRITE, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecSetError(pCtx, rc); ++ return pData; ++ } ++ return pCtx->buffer; ++ break; ++ case 7: ++ if(pgno == 1){ ++ memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, pCtx->salt, FILE_HEADER_SIZE); ++ if(memcpyRc != EOK){ ++ sqlite3CodecSetError(pCtx, SQLITE_ERROR); ++ return pData; ++ } ++ } ++ rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_READ, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); ++ if(rc != SQLITE_OK){ ++ sqlite3CodecSetError(pCtx, rc); ++ return pData; ++ } ++ return pCtx->buffer; ++ break; ++ default: ++ return pData; ++ break; ++ } ++} ++ ++void sqlite3CodecDetach(void *ctx){ ++ if(ctx != NULL){ ++ sqlite3CodecFreeContext((CodecContext *)ctx); ++ sqlite3_free(ctx); ++ opensslDeactive(); ++ } ++ return; ++} ++ ++int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ ++ if(db == NULL){ ++ return SQLITE_ERROR; ++ } ++ Btree *p = db->aDb[nDb].pBt; ++ if(p == NULL || pKey == NULL || nKey <= 0){ ++ return SQLITE_OK; ++ } ++ opensslActive(); ++ CodecContext *ctx = (CodecContext *)sqlite3Malloc(sizeof(CodecContext)); ++ if(ctx == NULL){ ++ return SQLITE_NOMEM; ++ } ++ errno_t memsetRc = memset_s(ctx, sizeof(CodecContext), 0, sizeof(CodecContext)); ++ if(memsetRc != EOK){ ++ sqlite3_free(ctx); ++ return SQLITE_ERROR; ++ } ++ sqlite3_mutex_enter(db->mutex); ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++ int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); ++#else ++ int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey); ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ if(rc != SQLITE_OK){ ++ sqlite3_free(ctx); ++ return rc; ++ } ++ sqlite3PagerSetCodec(sqlite3BtreePager(p), sqlite3Codec, NULL, sqlite3CodecDetach, (void *)ctx); ++ ++ db->nextPagesize = ctx->readCtx->codecConst.cipherPageSize; ++ p->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; ++ sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); ++ sqlite3BtreeSecureDelete(p, 1); ++ if(isOpen(p->pBt->pPager->fd)){ ++ sqlite3BtreeSetAutoVacuum(p, SQLITE_DEFAULT_AUTOVACUUM); ++ } ++ ++ sqlite3_mutex_leave(db->mutex); ++ ++ return SQLITE_OK; ++} ++ ++void sqlite3CodecGetKey(sqlite3* db, int nDb, void **pKey, int *nKey) ++{ ++ Btree *p = db->aDb[nDb].pBt; ++ if(p == NULL){ ++ return; ++ } ++ CodecContext *ctx = (CodecContext *)(p->pBt->pPager->pCodec); ++ if(ctx){ ++ if(ctx->savePassword){ ++ *pKey = ctx->readCtx->password; ++ *nKey = ctx->readCtx->passwordSize; ++ }else{ ++ *pKey = ctx->readCtx->keyInfo; ++ *nKey = ctx->readCtx->codecConst.keyInfoSize; ++ } ++ }else{ ++ *pKey = NULL; ++ *nKey = 0; ++ } ++ return; ++} ++ ++int sqlite3_key(sqlite3 *db, const void *pKey, int nKey){ ++ return sqlite3_key_v2(db, "main", pKey, nKey); ++} ++ ++int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ ++ if(db == NULL || pKey == NULL || nKey <= 0){ ++ return SQLITE_ERROR; ++ } ++ int iDb = sqlite3CodecGetDbIndex(db, zDb); ++ return sqlite3CodecAttach(db, iDb, pKey, nKey); ++} ++ ++int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey){ ++ return sqlite3_rekey_v2(db, "main", pKey, nKey); ++} ++ ++int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ ++ if(db == NULL || pKey == NULL || nKey == 0){ ++ return SQLITE_ERROR; ++ } ++ int iDb = sqlite3CodecGetDbIndex(db, zDb); ++ Btree *p = db->aDb[iDb].pBt; ++ if(p == NULL){ ++ return SQLITE_OK; ++ } ++ int pageCount; ++ Pgno pgno; ++ PgHdr *page = NULL; ++ Pager *pPager = p->pBt->pPager; ++ CodecContext *ctx = (CodecContext *)(p->pBt->pPager->pCodec); ++ if(ctx == NULL){ ++ return SQLITE_OK; ++ } ++ sqlite3CodecClearDeriveKey(ctx->writeCtx); ++ sqlite3CodecClearPassword(ctx->writeCtx); ++ int rc = sqlite3CodecSetPassword(ctx->writeCtx, pKey, nKey); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ sqlite3_mutex_enter(db->mutex); ++ (void)sqlite3BtreeBeginTrans(p, 1, 0); ++ sqlite3PagerPagecount(pPager, &pageCount); ++ // support hmac algo changed by using rekey operation ++ int oldHmacAlgo = ctx->writeCtx->codecConst.hmacAlgo; ++ if( ctx->writeCtx->codecConst.rekeyHmacAlgo!=ctx->writeCtx->codecConst.hmacAlgo ){ ++ sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, ctx->writeCtx->codecConst.rekeyHmacAlgo); ++ sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, ctx->writeCtx->codecConst.rekeyHmacAlgo); ++ } ++ ++ for(pgno = 1; pgno <= (unsigned int)pageCount; pgno++){ ++ if(PAGER_SJ_PGNO(pPager) != pgno){ ++ rc = sqlite3PagerGet(pPager, pgno, &page, 0); ++ if(rc == SQLITE_OK){ ++ rc = sqlite3PagerWrite(page); ++ if(rc == SQLITE_OK){ ++ sqlite3PagerUnref(page); ++ }else{ ++ sqlite3_log(SQLITE_WARNING, "sqlite3_rekey_v2: error when writing page %d: errno = %d.", pgno, rc); ++ } ++ }else{ ++ sqlite3_log(SQLITE_WARNING, "sqlite3_rekey_v2: error when reading page %d: errno = %d.", pgno, rc); ++ } ++ } ++ } ++ if(rc == SQLITE_OK){ ++ (void)sqlite3BtreeCommit(p); ++ sqlite3CodecFreeKeyContext(ctx->readCtx); ++ (void)sqlite3CodecCopyKeyContext(ctx->writeCtx, ctx->readCtx); ++ }else{ ++ if( ctx->writeCtx->codecConst.rekeyHmacAlgo!=oldHmacAlgo ){ ++ sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, oldHmacAlgo); ++ sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, oldHmacAlgo); ++ } ++ (void)sqlite3BtreeRollback(p, SQLITE_ABORT_ROLLBACK, 0); ++ } ++ sqlite3_mutex_leave(db->mutex); ++ ++ return rc; ++} ++ ++void sqlite3_activate_see(const char* zPassPhrase){ ++ return; ++} ++ ++CODEC_STATIC void sqlite3CodecReturnPragmaResult(Parse *parse, const char *label, const char *value){ ++ Vdbe *v = sqlite3GetVdbe(parse); ++ sqlite3VdbeSetNumCols(v, 1); ++ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, label, SQLITE_STATIC); ++ sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, value, 0); ++ sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); ++ return; ++} ++ ++// Each configuration setting operation should be done before read/write DB file or there might be some error. ++int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, const char *zRight){ ++ Btree *p = db->aDb[iDb].pBt; ++ if(p == NULL){ ++ return 0; ++ } ++ CodecContext *ctx = (CodecContext *)(p->pBt->pPager->pCodec); ++#ifdef SQLITE_CODEC_ATTACH_CHANGED ++ CodecParameter *parm = &db->codecParm; ++ if(sqlite3StrICmp(zLeft, "cipher_default_attach_cipher") == 0 && zRight != NULL){ ++ (void)sqlite3CodecSetDefaultAttachCipher(parm, zRight); ++ return 1; ++ }else if(sqlite3StrICmp(zLeft, "cipher_default_attach_kdf_iter") == 0 && zRight != NULL){ ++ (void)sqlite3CodecSetDefaultAttachKdfIter(parm, atoi(zRight)); ++ return 1; ++ }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_hmac_algo")==0 && zRight!=NULL ){ ++ /* ++ ** Make sure to set the Kdf algorithm after setting the Hmac algorithm, or it will not take effect. ++ ** This behavior is to ensure backward compatible. ++ */ ++ if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA1)==0 ){ ++ sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA1); ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA1); ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ ++ sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA256); ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA256); ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ ++ sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA512); ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA512); ++ }else{ ++ return 0; ++ } ++ return 1; ++ }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_kdf_algo")==0 && zRight!=NULL ){ ++ if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA1)==0 ){ ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA1); ++ }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA256)==0 ){ ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA256); ++ }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA512)==0 ){ ++ sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA512); ++ }else{ ++ return 0; ++ } ++ return 1; ++ }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_page_size")==0 && zRight!=NULL ){ ++ (void)sqlite3CodecSetDefaultAttachPageSize(parm, atoi(zRight)); ++ return 1; ++ } ++#endif /* SQLITE_CODEC_ATTACH_CHANGED */ ++ if(ctx == NULL){ ++ return 0; ++ } ++ if(sqlite3StrICmp(zLeft, "codec_cipher") == 0){ ++ if(zRight){ ++ sqlite3_mutex_enter(db->mutex); ++ (void)sqlite3CodecSetCodecConstant(ctx->readCtx, zRight); ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, ctx->readCtx->codecConst.hmacAlgo); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, ctx->readCtx->codecConst.hmacAlgo); ++ sqlite3CodecFreeKeyContext(ctx->writeCtx); ++ (void)sqlite3CodecCopyKeyContext(ctx->readCtx, ctx->writeCtx); ++ sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); ++ sqlite3_mutex_leave(db->mutex); ++ }else{ ++ sqlite3CodecReturnPragmaResult(parse, "codec_cipher", opensslGetCipherName(ctx->writeCtx->codecConst.cipher)); ++ } ++ }else if(sqlite3StrICmp(zLeft, "codec_kdf_iter") == 0){ ++ if(zRight){ ++ (void)sqlite3CodecSetIter(ctx->readCtx, atoi(zRight)); ++ (void)sqlite3CodecSetIter(ctx->writeCtx, atoi(zRight)); ++ }else{ ++ char *iter = sqlite3_mprintf("%d", ctx->writeCtx->iter); ++ if(iter != NULL){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_kdf_iter", iter); ++ sqlite3_free(iter); ++ } ++ } ++ }else if( sqlite3StrICmp(zLeft, "codec_hmac_algo")==0 ){ ++ /* ++ ** Make sure to set the Kdf algorithm after setting the Hmac algorithm, or it will not take effect. ++ ** This behavior is to ensure backward compatible. ++ */ ++ if(zRight){ ++ sqlite3_mutex_enter(db->mutex); ++ if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA1)==0 ){ ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA1); ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA1); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA1); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA1); ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA256); ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA256); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA256); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA256); ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA512); ++ (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA512); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA512); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA512); ++ }else{ ++ sqlite3_mutex_leave(db->mutex); ++ return 0; ++ } ++ sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); ++ sqlite3_mutex_leave(db->mutex); ++ }else{ ++ if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA1); ++ }else if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA256); ++ }else if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA512); ++ } ++ } ++ }else if( sqlite3StrICmp(zLeft, "codec_kdf_algo")==0 ){ ++ if(zRight){ ++ sqlite3_mutex_enter(db->mutex); ++ if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA1)==0 ){ ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA1); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA1); ++ }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA256)==0 ){ ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA256); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA256); ++ }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA512)==0 ){ ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA512); ++ (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA512); ++ }else{ ++ sqlite3_mutex_leave(db->mutex); ++ return 0; ++ } ++ sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); ++ sqlite3_mutex_leave(db->mutex); ++ }else{ ++ if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA1 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA1); ++ }else if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA256 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA256); ++ }else if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA512 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA512); ++ } ++ } ++ }else if( sqlite3StrICmp(zLeft, "codec_page_size")==0 ){ ++ if(zRight){ ++ sqlite3_mutex_enter(db->mutex); ++ int rc = sqlite3CodecSetCipherPageSize(ctx, atoi(zRight)); ++ if (rc != SQLITE_OK){ ++ sqlite3_mutex_leave(db->mutex); ++ return 0; ++ } ++ sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); ++ sqlite3_mutex_leave(db->mutex); ++ } else { ++ char *pageSize = sqlite3_mprintf("%d", ctx->readCtx->codecConst.cipherPageSize); ++ if (pageSize != NULL) { ++ sqlite3CodecReturnPragmaResult(parse, "codec_page_size", pageSize); ++ sqlite3_free(pageSize); ++ } ++ } ++ }else if(sqlite3StrICmp(zLeft, "codec_rekey_hmac_algo") == 0){ ++ if(zRight){ ++ sqlite3_mutex_enter(db->mutex); ++ if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA1)==0 ){ ++ ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA1; ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ ++ ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA256; ++ }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ ++ ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA512; ++ }else{ ++ sqlite3_mutex_leave(db->mutex); ++ return 0; ++ } ++ sqlite3_mutex_leave(db->mutex); ++ }else{ ++ if( ctx->writeCtx->codecConst.rekeyHmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA1); ++ }else if( ctx->writeCtx->codecConst.rekeyHmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA256); ++ }else if( ctx->writeCtx->codecConst.rekeyHmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ ++ sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA512); ++ } ++ } ++ }else{ ++ return 0; ++ } ++ return 1; ++} ++ ++CODEC_STATIC int sqlite3CodecExportMetadata(sqlite3 *db, const char *dbName, const char *metaName){ ++ char *sql = sqlite3_mprintf("PRAGMA %s;", metaName); ++ if(sql == NULL){ ++ return SQLITE_NOMEM; ++ } ++ sqlite3_stmt *statement = NULL; ++ int rc = sqlite3_prepare_v2(db, sql, -1, &statement, NULL); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ rc = sqlite3_step(statement); ++ if(rc != SQLITE_ROW){ ++ sqlite3_finalize(statement); ++ return rc; ++ } ++ int metadata = sqlite3_column_int(statement, 0); ++ sqlite3_finalize(statement); ++ ++ sql = sqlite3_mprintf("PRAGMA %s.%s=%d;", dbName, metaName, metadata); ++ if(sql == NULL){ ++ return SQLITE_NOMEM; ++ } ++ rc = sqlite3_exec(db, sql, NULL, NULL, NULL); ++ sqlite3_free(sql); ++ return rc; ++} ++ ++CODEC_STATIC int sqlite3CodecBatchExportSql(sqlite3 *db, const char *sql, char **errMsg){ ++ sqlite3_stmt *statement = NULL; ++ int rc = sqlite3_prepare_v2(db, sql, -1, &statement, NULL); ++ if(rc != SQLITE_OK){ ++ return rc; ++ } ++ while(sqlite3_step(statement) == SQLITE_ROW){ ++ rc = sqlite3_exec(db, (char*)sqlite3_column_text(statement, 0), NULL, NULL, errMsg); ++ if(rc != SQLITE_OK){ ++ sqlite3_finalize(statement); ++ return rc; ++ } ++ } ++ sqlite3_finalize(statement); ++ return rc; ++} ++ ++void sqlite3CodecExportData(sqlite3_context *context, int argc, sqlite3_value **argv){ ++ sqlite3 *db = sqlite3_context_db_handle(context); ++ const char *dbName = (const char*) sqlite3_value_text(argv[0]); ++ ++ int rc = SQLITE_OK; ++ char *sql = NULL; ++ char *errMsg = NULL; ++ ++ u64 flagsBackup = db->flags; ++ u32 mDbFlagsBackup = db->mDbFlags; ++ int nChangeBackup = db->nChange; ++ int nTotalChangeBackup = db->nTotalChange; ++ int (*xTraceBackup)(u32,void*,void*,void*) = db->trace.xV2; ++ ++ db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; ++ db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); ++ db->mDbFlags |= DBFLAG_PreferBuiltin; ++ db->trace.xV2 = 0; ++ ++ rc = sqlite3CodecExportMetadata(db, dbName, "schema_version"); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ rc = sqlite3CodecExportMetadata(db, dbName, "user_version"); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ rc = sqlite3CodecExportMetadata(db, dbName, "application_id"); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'CREATE TABLE %s.' || substr(sql,14) FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence' AND rootpage>0;", dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'CREATE INDEX %s.' || substr(sql,14) FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %%';", dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'CREATE UNIQUE INDEX %s.' || substr(sql,21) FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %%';", dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'INSERT INTO %s.' || quote(name) || ' SELECT * FROM main.' || quote(name) || ';' FROM main.sqlite_master WHERE type = 'table' AND name!='sqlite_sequence' AND rootpage>0;", dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'DELETE FROM %s.' || quote(name) || ';' FROM %s.sqlite_master WHERE name='sqlite_sequence';", dbName, dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("SELECT 'INSERT INTO %s.' || quote(name) || ' SELECT * FROM main.' || quote(name) || ';' FROM %s.sqlite_master WHERE name=='sqlite_sequence';", dbName, dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3CodecBatchExportSql(db, sql, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++ sql = sqlite3_mprintf("INSERT INTO %s.sqlite_master SELECT type, name, tbl_name, rootpage, sql FROM main.sqlite_master WHERE type='view' OR type='trigger' OR (type='table' AND rootpage=0);", dbName, dbName); ++ if(sql == NULL){ ++ rc = SQLITE_NOMEM; ++ goto export_finish; ++ } ++ rc = sqlite3_exec(db, sql, NULL, NULL, &errMsg); ++ sqlite3_free(sql); ++ if(rc != SQLITE_OK){ ++ goto export_finish; ++ } ++export_finish: ++ db->flags = flagsBackup; ++ db->mDbFlags = mDbFlagsBackup; ++ db->nChange = nChangeBackup; ++ db->nTotalChange = nTotalChangeBackup; ++ db->trace.xV2 = xTraceBackup; ++ if(rc != SQLITE_OK){ ++ if(errMsg != NULL) { ++ sqlite3_result_error(context, errMsg, -1); ++ sqlite3DbFree(db, errMsg); ++ } else { ++ sqlite3_result_error(context, sqlite3ErrStr(rc), -1); ++ } ++ } ++ return; ++} ++/************** End file hw_codec.c *****************************************/ ++#endif /* SQLITE_HAS_CODEC */ ++ ++ ++#ifdef SQLITE_EXPORT_SYMBOLS ++/************** Begin hw export the symbols *****************************************/ ++#if defined(__GNUC__) ++# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) ++#elif defined(_MSC_VER) ++# define EXPORT_SYMBOLS __declspec(dllexport) ++#else ++# define EXPORT_SYMBOLS ++#endif ++ ++struct sqlite3_api_routines_hw { ++ int (*initialize)(); ++ int (*config)(int,...); ++ int (*key)(sqlite3*,const void*,int); ++ int (*key_v2)(sqlite3*,const char*,const void*,int); ++ int (*rekey)(sqlite3*,const void*,int); ++ int (*rekey_v2)(sqlite3*,const char*,const void*,int); ++}; ++ ++typedef struct sqlite3_api_routines_hw sqlite3_api_routines_hw; ++static const sqlite3_api_routines_hw sqlite3HwApis = { ++ sqlite3_initialize, ++ sqlite3_config, ++#ifdef SQLITE_HAS_CODEC ++ sqlite3_key, ++ sqlite3_key_v2, ++ sqlite3_rekey, ++ sqlite3_rekey_v2 ++#else ++ 0, ++ 0, ++ 0, ++ 0 ++#endif /* SQLITE_HAS_CODEC */ ++}; ++ ++EXPORT_SYMBOLS const sqlite3_api_routines *sqlite3_export_symbols = &sqlite3Apis; ++EXPORT_SYMBOLS const sqlite3_api_routines_hw *sqlite3_export_hw_symbols = &sqlite3HwApis; ++/************** End hw export the symbols *****************************************/ ++#endif /* SQLITE_EXPORT_SYMBOLS */ +-- +2.46.0.windows.1 + diff --git a/mock/sqlite/patch/0002-busy-debug.patch b/mock/sqlite/patch/0002-busy-debug.patch new file mode 100644 index 00000000..8fd1663b --- /dev/null +++ b/mock/sqlite/patch/0002-busy-debug.patch @@ -0,0 +1,813 @@ +From 4a29b0e98e03fea216795394ac340104e2e1bbfa Mon Sep 17 00:00:00 2001 +From: ryne3366 +Date: Tue, 25 Feb 2025 11:25:32 +0800 +Subject: [PATCH] suport busy debug + +Signed-off-by: ryne3366 +--- + src/sqlite3.c | 368 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 1 file changed, 355 insertions(+), 13 deletions(-) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index d2bfc23..1ec1ae9 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -38490,6 +38490,45 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){ + #endif + + /************** End of os_kv.c ***********************************************/ ++/************** Define dump function *****************************************/ ++#if SQLITE_OS_UNIX ++#define DB_LOCK_NUM 3 ++#define WAL_LOCK_NUM 9 ++// 8 wal lock, 1 SHM_DMS lock, 1 TRX lock ++#define MAX_LOCK_NUM (WAL_LOCK_NUM + 1) ++#define SHM_DMS_IDX (WAL_LOCK_NUM - 1) ++#define TRX_LOCK_IDX WAL_LOCK_NUM ++#define LOCK_BY_PROCESS 1 ++#define LOCK_BY_THREAD 0 ++#define DUMP_BUF_MAX_LEN 180 ++#define DUMP_MAX_STR_LEN 21 ++ ++typedef struct LocalLockStatus { ++ u8 busyLockIdx; ++ u8 busyLockType; ++ u8 lockByProcess; ++ u8 reserved; ++ u32 lockLen; ++ u32 busyLine; ++ int curTid; ++ u8 lockStatus[MAX_LOCK_NUM]; // last index is trx lock ++} LocalLockStatus; ++__thread LocalLockStatus g_lockStatus = {0}; ++ ++#define MARK_LAST_BUSY_LINE(rc) {if ((rc)==SQLITE_BUSY || (rc) == SQLITE_LOCKED) g_lockStatus.busyLine = __LINE__;} ++static void ResetLockStatus(void); ++static void TryRecordTid(int *tidBuf, int ofs, int lockLen); ++static void TryClearTid(int *tidBuf, int ofs, int lockLen); ++static void MarkLockBusy(u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess); ++static void MarkLockStatus(u32 lockIdx, u32 lockLen, u8 lockType); ++static void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess); ++#else ++#define MARK_LAST_BUSY_LINE(rc) ++#define ResetLockStatus(void) ++#define MarkLockBusy(A, B, C, D) ++#define MarkLockStatusByRc(A, B, C, D, E) ++#endif ++/************** End define dump function *************************************/ + /************** Begin file os_unix.c *****************************************/ + /* + ** 2004 May 22 +@@ -40313,6 +40352,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, + azFileLock(eFileLock))); ++ MarkLockStatus(TRX_LOCK_IDX, 1, eFileLock); + return SQLITE_OK; + } + +@@ -40337,6 +40377,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; ++ MarkLockBusy(TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_THREAD); + goto end_lock; + } + +@@ -40352,6 +40393,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; ++ MarkLockStatus(TRX_LOCK_IDX, 1, eFileLock); + goto end_lock; + } + +@@ -40373,6 +40415,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } ++ MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); + goto end_lock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; +@@ -40380,7 +40423,6 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + } + } + +- + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ +@@ -40396,7 +40438,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + } +- ++ MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); + /* Drop the temporary PENDING lock */ + lock.l_start = PENDING_BYTE; + lock.l_len = 1L; +@@ -40421,6 +40463,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; ++ MarkLockBusy(TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_THREAD); + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file +@@ -40445,6 +40488,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ + storeLastErrno(pFile, tErrno); + } + } ++ MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_PROCESS); + } + + +@@ -40638,7 +40682,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ + pFile->eFileLock = NO_LOCK; + } + } +- + /* Decrement the count of locks against this same file. When the + ** count reaches zero, close any other file descriptors whose close + ** was deferred because of outstanding locks. +@@ -42948,6 +42991,7 @@ struct unixShmNode { + sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK]; + #endif + int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ ++ int aLockTid[SQLITE_SHM_NLOCK]; + #ifdef SQLITE_DEBUG + u8 nextShmId; /* Next available unixShm.id value */ + #endif +@@ -43214,6 +43258,8 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pDbFd->iBusyTimeout = iSaveTimeout; + #endif ++ MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, EXCLUSIVE_LOCK, LOCK_BY_PROCESS); ++ MARK_LAST_BUSY_LINE(rc); + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to +@@ -43225,11 +43271,15 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; ++ MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); ++ MARK_LAST_BUSY_LINE(rc); + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); ++ MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); ++ MARK_LAST_BUSY_LINE(rc); + } + return rc; + } +@@ -43629,7 +43679,7 @@ static int unixShmLock( + return SQLITE_IOERR_SHMLOCK; + } + aLock = pShmNode->aLock; +- ++ int *aLockTid = pShmNode->aLockTid; + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); +@@ -43679,6 +43729,7 @@ static int unixShmLock( + assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) + || 0==(p->exclMask & mask) + ); ++ u8 useProcessLock = LOCK_BY_THREAD; + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) +@@ -43734,7 +43785,7 @@ static int unixShmLock( + p->sharedMask &= ~mask; + } + } +- ++ MarkLockStatusByRc(rc, ofst, n, NO_LOCK, useProcessLock); + if( bUnlock ){ + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ +@@ -43742,6 +43793,8 @@ static int unixShmLock( + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + } ++ useProcessLock = LOCK_BY_PROCESS; ++ TryClearTid(aLockTid, ofst, n); + } + }else if( flags & SQLITE_SHM_SHARED ){ + /* Case (b) - a shared lock. */ +@@ -43751,12 +43804,14 @@ static int unixShmLock( + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); ++ useProcessLock = LOCK_BY_PROCESS; + } +- ++ MarkLockStatusByRc(rc, ofst, n, SHARED_LOCK, useProcessLock); + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + aLock[ofst]++; ++ TryRecordTid(aLockTid, ofst, n); + } + }else{ + /* Case (c) - an exclusive lock. */ +@@ -43774,7 +43829,7 @@ static int unixShmLock( + break; + } + } +- ++ useProcessLock = LOCK_BY_PROCESS; + /* Get the exclusive locks at the system level. Then if successful + ** also update the in-memory values. */ + if( rc==SQLITE_OK ){ +@@ -43784,8 +43839,10 @@ static int unixShmLock( + for(ii=ofst; iinoSync ){ +@@ -62796,6 +62854,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } ++ MARK_LAST_BUSY_LINE(rc); + sqlite3EndBenignMalloc(); + }else{ + /* The journal file exists and no other connection has a reserved +@@ -62885,6 +62944,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); + + rc = pager_wait_on_lock(pPager, SHARED_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; +@@ -62921,6 +62981,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + if( rc!=SQLITE_OK ){ + goto failed; + } +@@ -63553,6 +63614,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory + */ + if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + if( rc!=SQLITE_OK ){ + return rc; + } +@@ -63565,6 +63627,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); ++ MARK_LAST_BUSY_LINE(rc); + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The +@@ -63572,8 +63635,10 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory + ** lock, but not when obtaining the RESERVED lock. + */ + rc = pagerLockDb(pPager, RESERVED_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + } + } + +@@ -65085,6 +65150,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); +@@ -65251,6 +65317,7 @@ static int pagerOpenWal(Pager *pPager){ + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); ++ MARK_LAST_BUSY_LINE(rc); + } + + /* Open the connection to the log file. If this operation fails, +@@ -65334,6 +65401,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ + if( !pPager->pWal ){ + int logexists = 0; + rc = pagerLockDb(pPager, SHARED_LOCK); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK ){ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists +@@ -65349,6 +65417,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ + */ + if( rc==SQLITE_OK && pPager->pWal ){ + rc = pagerExclusiveLock(pPager); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK ){ + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); +@@ -66883,6 +66952,7 @@ static int walIndexRecover(Wal *pWal){ + iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + if( rc ){ ++ MARK_LAST_BUSY_LINE(rc); + return rc; + } + +@@ -67720,6 +67790,7 @@ static int walCheckpoint( + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK ){ + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT; +@@ -67814,6 +67885,8 @@ static int walCheckpoint( + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); ++ } else { ++ MARK_LAST_BUSY_LINE(rc); + } + + if( rc==SQLITE_BUSY ){ +@@ -67833,11 +67906,13 @@ static int walCheckpoint( + SEH_INJECT_FAULT; + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; ++ MARK_LAST_BUSY_LINE(rc); + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as +@@ -68025,8 +68100,9 @@ SQLITE_PRIVATE int sqlite3WalClose( + walLimitSize(pWal, 0); + } + } ++ } else { ++ MARK_LAST_BUSY_LINE(rc); + } +- + walIndexClose(pWal, isDelete); + sqlite3OsClose(pWal->pWalFd); + if( isDelete ){ +@@ -68179,6 +68255,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; + } ++ MARK_LAST_BUSY_LINE(rc); + }else{ + int bWriteLock = pWal->writeLock; + if( bWriteLock +@@ -68202,6 +68279,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + } + } ++ MARK_LAST_BUSY_LINE(rc); + } + } + +@@ -68221,6 +68299,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; ++ MARK_LAST_BUSY_LINE(rc); + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } +@@ -68483,6 +68562,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ + ** so it takes care to hold an exclusive lock on the corresponding + ** WAL_READ_LOCK() while changing values. + */ ++static void DumpLocksByWal(Wal *pWal); + static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ + u32 mxReadMark; /* Largest aReadMark[] value */ +@@ -68525,6 +68605,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + return SQLITE_PROTOCOL; + } + if( *pCnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; ++#if SQLITE_OS_UNIX ++ if( cnt>=15 ) DumpLocksByWal(pWal); ++#endif /* SQLITE_OS_UNIX */ + #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor + ** to block for locks for approximately nDelay us. This affects three +@@ -68576,11 +68659,14 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + ** must be zeroed before the requested page is returned. + */ + rc = WAL_RETRY; ++ MARK_LAST_BUSY_LINE(SQLITE_BUSY); + }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ + walUnlockShared(pWal, WAL_RECOVER_LOCK); + rc = WAL_RETRY; ++ MARK_LAST_BUSY_LINE(SQLITE_BUSY); + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_BUSY_RECOVERY; ++ MARK_LAST_BUSY_LINE(SQLITE_BUSY); + } + } + if( rc!=SQLITE_OK ){ +@@ -68604,6 +68690,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); ++ MARK_LAST_BUSY_LINE(SQLITE_BUSY); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ +@@ -68621,6 +68708,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); ++ MARK_LAST_BUSY_LINE(SQLITE_BUSY); + return WAL_RETRY; + } + pWal->readLock = 0; +@@ -68665,6 +68753,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + }else if( rc!=SQLITE_BUSY ){ + return rc; + } ++ MARK_LAST_BUSY_LINE(rc); + } + } + if( mxI==0 ){ +@@ -68684,6 +68773,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + assert( rc!=SQLITE_BUSY_TIMEOUT ); + #endif + assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); ++ MARK_LAST_BUSY_LINE(rc); + return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the +@@ -68726,6 +68816,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); ++ MARK_LAST_BUSY_LINE(rc); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); +@@ -68863,7 +68954,7 @@ static int walBeginReadTransaction(Wal *pWal, int *pChanged){ + (void)walEnableBlocking(pWal); + rc = walLockShared(pWal, WAL_CKPT_LOCK); + walDisableBlocking(pWal); +- ++ MARK_LAST_BUSY_LINE(rc); + if( rc!=SQLITE_OK ){ + return rc; + } +@@ -69810,6 +69901,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( + ** it will not be invoked in this case. + */ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); ++ MARK_LAST_BUSY_LINE(rc); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ +@@ -69826,6 +69918,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); ++ MARK_LAST_BUSY_LINE(rc); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ +@@ -73685,6 +73778,7 @@ static void pageReinit(DbPage *pData){ + } + } + ++static void DumpLocksByPager(Pager *pPager); + /* + ** Invoke the busy handler for a btree. + */ +@@ -73692,7 +73786,13 @@ static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); +- return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); ++ int rc = sqlite3InvokeBusyHandler(&pBt->db->busyHandler); ++#if SQLITE_OS_UNIX ++ if (rc == 0) { ++ DumpLocksByPager(pBt->pPager); ++ } ++#endif /* SQLITE_OS_UNIX */ ++ return rc; + } + + /* +@@ -74123,7 +74223,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ + } + } + #endif +- ++ ResetLockStatus(); + /* Rollback any active transaction and free the handle structure. + ** The call to sqlite3BtreeRollback() drops any table-locks held by + ** this handle. +@@ -74788,7 +74888,7 @@ static SQLITE_NOINLINE int btreeBeginTrans( + + sqlite3BtreeEnter(p); + btreeIntegrity(p); +- ++ ResetLockStatus(); + /* If the btree is already in a write-transaction, or it + ** is already in a read-transaction and a read-transaction + ** is requested, this is a no-op. +@@ -82419,8 +82519,10 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int * + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); ++ ResetLockStatus(); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; ++ MARK_LAST_BUSY_LINE(rc); + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); + } +@@ -88466,6 +88568,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ + nTrans++; + } + rc = sqlite3PagerExclusiveLock(pPager); ++ MARK_LAST_BUSY_LINE(rc); + sqlite3BtreeLeave(pBt); + } + } +@@ -260441,9 +260544,248 @@ export_finish: + return; + } + /************** End file hw_codec.c *****************************************/ +-#endif /* SQLITE_HAS_CODEC */ ++#endif + ++#if SQLITE_OS_UNIX ++#include ++#include ++static inline int OsGetTid(void) ++{ ++#if defined(__linux__) ++ return (int)syscall(__NR_gettid); ++#elif defined(__APPLE__) ++ return (int)syscall(SYS_thread_selfid); ++#else ++ return 0; ++#endif ++} ++ ++static void ResetLockStatus(void) ++{ ++ (void)memset(&g_lockStatus, 0, sizeof(g_lockStatus)); ++ g_lockStatus.curTid = OsGetTid(); ++} ++/* ++** Record lock info, correspond wal aLock buf, 1 aLock: 1 ++*/ ++static inline void TryRecordTid(int *tidBuf, int ofs, int lockLen) ++{ ++ int lockOfs = ofs + lockLen; ++ for (int i = ofs; i < lockOfs; i++) { ++ if (tidBuf[i] == 0) { ++ tidBuf[i] = g_lockStatus.curTid; ++ } ++ } ++} ++/* ++** Clear locks info. ++*/ ++static inline void TryClearTid(int *tidBuf, int ofs, int lockLen) ++{ ++ int lockOfs = ofs + lockLen; ++ for (int i = ofs; i < lockOfs; i++) { ++ if (tidBuf[i] == g_lockStatus.curTid) { ++ tidBuf[i] = 0; ++ } ++ } ++} ++ ++static inline void MarkLockBusy(u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess) ++{ ++ g_lockStatus.busyLockIdx = lockIdx; ++ g_lockStatus.busyLockType = lockType; ++ g_lockStatus.lockByProcess = lockByProcess; ++ g_lockStatus.lockLen = lockLen; ++} ++ ++static void MarkLockStatus(u32 lockIdx, u32 lockLen, u8 lockType) ++{ ++ if (lockLen == 0 || (lockIdx + lockLen) > MAX_LOCK_NUM) { ++ sqlite3_log(SQLITE_ERROR, "Unexpect lock index %u lockLen %d!", lockIdx, lockLen); ++ return; ++ } ++ // only busy error code need record ++ if (g_lockStatus.lockLen != 0 && lockIdx == g_lockStatus.busyLockIdx) { ++ g_lockStatus.busyLockIdx = 0; ++ g_lockStatus.busyLockType = NO_LOCK; ++ g_lockStatus.lockLen = 0; ++ } ++ if (lockLen == 1) { ++ g_lockStatus.lockStatus[lockIdx] = lockType; ++ } else { ++ size_t len = sizeof(u8) * lockLen; ++ (void)memset(&g_lockStatus.lockStatus[lockIdx], lockType, len); ++ } ++} ++ ++static inline void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess) ++{ ++ if (rc == SQLITE_OK) { ++ MarkLockStatus(lockIdx, lockLen, lockType); ++ } else if (rc == SQLITE_BUSY) { ++ MarkLockBusy(lockIdx, lockLen, lockType, lockByProcess); ++ } ++} ++ ++static inline const char *TrxLockName(int eLock) ++{ ++ return eLock == NO_LOCK ? "NO_LOCK" : ++ eLock == RESERVED_LOCK ? "RESERVED" : ++ eLock == EXCLUSIVE_LOCK ? "EXCLUSIVE" : ++ eLock == SHARED_LOCK ? "SHARED" : ++ eLock == PENDING_LOCK ? "PENDING": ++ eLock == UNKNOWN_LOCK ? "UNKNOWN" : "UNKNOWN_LOCK"; ++} ++ ++static inline const char *IdxToLockName(u32 idx) ++{ ++ const char *lockName[MAX_LOCK_NUM] = {"write", "ckpt", "recover", "read0", ++ "read1", "read2", "read3", "read4", "wal_dms", "trxLock"}; ++ return (idx < MAX_LOCK_NUM) ? lockName[idx] : "errLock"; ++} ++ ++static void DumpHandleLock(char *dumpBuf, int dumpBufLen) ++{ ++ char *tmp = dumpBuf; ++ u8 *lockStatus = g_lockStatus.lockStatus; ++ int availLen = dumpBufLen - 1; ++ dumpBuf[availLen] = '\0'; ++ for (int i = 0; i < MAX_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { ++ if (lockStatus[i] != NO_LOCK) { ++ tmp[0] = '\0'; ++ sqlite3_snprintf(availLen, tmp, "<%s, %s>", IdxToLockName((u32)i), TrxLockName(lockStatus[i])); ++ int len = strlen(tmp); ++ tmp += len; ++ availLen -= len; ++ } ++ } ++ sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]BusyLine:%d, idx:%d, type:%d, fileLock:%d, len:%d, handleLocks:%s", ++ g_lockStatus.busyLine, g_lockStatus.busyLockIdx, g_lockStatus.busyLockType, g_lockStatus.lockByProcess, ++ g_lockStatus.lockLen, tmp != dumpBuf ? dumpBuf : "none"); ++} ++ ++static inline const char *FlockToName(int l_type) ++{ ++ return l_type == F_RDLCK ? "F_RDLCK" : ++ l_type == F_WRLCK ? "F_WRLCK" : ++ l_type == F_UNLCK ? "F_UNLCK" : "F_UNKNOWN"; ++} ++ ++static int DumpProcessLocks(int fd, struct flock *lock, const char *lockName, char *dumpBuf, int bufLen) ++{ ++ dumpBuf[0] = '\0'; ++ if (osFcntl(fd, F_GETLK, lock) != SQLITE_OK) { ++ sqlite3_log(SQLITE_ERROR, "[SQLite]Get wal file lock ofs %u failed, errno: %d", lock->l_start, errno); ++ return 0; ++ } ++ if (lock->l_type != F_UNLCK) { ++ sqlite3_snprintf(bufLen, dumpBuf, "<%s, pid:%u, %s>", lockName, lock->l_pid, FlockToName(lock->l_type)); ++ return strlen(dumpBuf); ++ } ++ return 0; ++} ++ ++static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) ++{ ++ unixInodeInfo *inode = file->pInode; ++ if (inode == NULL) { ++ sqlite3_log(SQLITE_ERROR, "[SQLite]Inode is null!"); ++ return; ++ } ++ sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]acqLock:%s, dbRef:%d, lockCnt:%d, curLock:%s, processLock:%d", ++ TrxLockName(file->eFileLock), inode->nRef, inode->nLock, TrxLockName(inode->eFileLock), inode->bProcessLock); ++ const char *lockName[DB_LOCK_NUM] = {"pending", "reserved", "shared_first"}; ++ char *tmp = dumpBuf; ++ int availLen = dumpBufLen - 1; ++ dumpBuf[availLen] = '\0'; ++ for (int i = 0; i < DB_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { ++ off_t ofs = i + PENDING_BYTE; ++ off_t lockLen = (ofs == SHARED_FIRST) ? SHARED_SIZE : 1; ++ struct flock lock = {.l_type = F_WRLCK, .l_start = ofs, .l_len = lockLen, .l_whence = SEEK_SET}; ++ int lockBufLen = DumpProcessLocks(file->h, &lock, lockName[i], tmp, availLen); ++ tmp += lockBufLen; ++ availLen -= lockBufLen; ++ } ++ if (tmp != dumpBuf) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]Trx locks: %s", dumpBuf); ++ } ++} ++ ++static void DumpWalLocks(unixFile *file, u8 walEnabled, char *dumpBuf, int dumpBufLen) ++{ ++ if (!walEnabled || file->pShm == NULL || file->pShm->pShmNode == NULL) { ++ sqlite3_log(SQLITE_ERROR, "[SQLite]Wal mode disabled!"); ++ return; ++ } ++ unixShmNode *pShmNode = file->pShm->pShmNode; ++ char *tmp = dumpBuf; ++ int availLen = dumpBufLen - 1; ++ dumpBuf[availLen] = '\0'; ++ for (int i = 0; i < WAL_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { ++ if (i < SQLITE_SHM_NLOCK && pShmNode->aLock[i]) { ++ tmp[0] = '\0'; ++ sqlite3_snprintf(availLen, tmp, "<%s, %d, tid:%d>", IdxToLockName((u32)i), pShmNode->aLock[i], ++ pShmNode->aLockTid[i]); ++ int strLen = strlen(tmp); ++ tmp += strLen; ++ availLen -= strLen; ++ } ++ off_t ofs = i + WALINDEX_LOCK_OFFSET; ++ struct flock lock = {.l_type = F_WRLCK, .l_start = ofs, .l_len = 1, .l_whence = SEEK_SET}; ++ int bufLen = DumpProcessLocks(pShmNode->hShm, &lock, IdxToLockName((u32)i), tmp, availLen); ++ tmp += bufLen; ++ availLen -= bufLen; ++ } ++ if (tmp != dumpBuf) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]Wal locks: %s", dumpBuf); ++ } ++} ++ ++static void DumpLocksInfo(unixFile *file, int walEnabled) ++{ ++ char *dumpBuf = sqlite3Malloc(DUMP_BUF_MAX_LEN); ++ if (dumpBuf == NULL) { ++ sqlite3_log(SQLITE_ERROR, "[SQLite]Can't alloc bufferSz %d for dump!", DUMP_BUF_MAX_LEN); ++ return; ++ } ++ DumpHandleLock(dumpBuf, DUMP_BUF_MAX_LEN); ++ DumpTrxProcessLocks(file, dumpBuf, DUMP_BUF_MAX_LEN); ++ DumpWalLocks(file, walEnabled, dumpBuf, DUMP_BUF_MAX_LEN); ++ sqlite3_free(dumpBuf); ++} ++ ++#ifndef SQLITE_OMIT_WAL ++static void DumpLocksByWal(Wal *pWal) ++{ ++ if (pWal == NULL) { ++ sqlite3_log(SQLITE_ERROR, "Wal ptr is NULL!"); ++ return; ++ } ++ if (pWal->pVfs == NULL || sqlite3_stricmp(pWal->pVfs->zName, "unix") != 0) { ++ return; ++ } ++ DumpLocksInfo((unixFile *)(pWal->pDbFd), 1); ++} ++#endif /* #ifndef SQLITE_OMIT_WAL */ ++ ++static void DumpLocksByPager(Pager *pPager) ++{ ++ if (pPager == NULL) { ++ sqlite3_log(SQLITE_ERROR, "Pager ptr is NULL!"); ++ return; ++ } ++ if (pPager->pVfs == NULL || sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0) { ++ return; ++ } ++#ifndef SQLITE_OMIT_WAL ++ DumpLocksInfo((unixFile *)(pPager->fd), pPager->pWal != NULL); ++#else /* #ifndef SQLITE_OMIT_WAL */ ++ DumpLocksInfo((unixFile *)(pPager->fd), 0); ++#endif /* #ifndef SQLITE_OMIT_WAL */ ++} ++#endif /* SQLITE_OS_UNIX */ + ++// hw export the symbols + #ifdef SQLITE_EXPORT_SYMBOLS + /************** Begin hw export the symbols *****************************************/ + #if defined(__GNUC__) +-- +2.34.1 + diff --git a/mock/sqlite/patch/0003-suport-meta-recovery.patch b/mock/sqlite/patch/0003-suport-meta-recovery.patch new file mode 100644 index 00000000..bbb574d8 --- /dev/null +++ b/mock/sqlite/patch/0003-suport-meta-recovery.patch @@ -0,0 +1,1346 @@ +From d35904a31c2fe03afdccba31b75a34deb3bbcdbb Mon Sep 17 00:00:00 2001 +From: ryne3366 +Date: Tue, 25 Feb 2025 11:28:45 +0800 +Subject: [PATCH] support meta recovery + +Signed-off-by: ryne3366 +--- + src/sqlite3.c | 1151 ++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 1140 insertions(+), 11 deletions(-) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 1ec1ae9..17d0d25 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -788,6 +788,7 @@ SQLITE_API int sqlite3_exec( + #define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ + #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ + #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ ++#define SQLITE_META_RECOVERED 66 /* meta page recovered*/ + /* end-of-error-codes */ + + /* +@@ -15626,7 +15627,6 @@ typedef int VList; + # define SQLITE_OS_UNIX 0 + #endif + +- + #endif /* SQLITE_OS_SETUP_H */ + + /************** End of os_setup.h ********************************************/ +@@ -45066,6 +45066,7 @@ static int unixOpen( + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; ++ sqlite3_log(SQLITE_WARNING, "Try open file readonly"); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; +@@ -58068,6 +58069,26 @@ struct PagerSavepoint { + #endif + }; + ++#if !defined(SQLITE_OS_UNIX) && defined(SQLITE_META_DWR) ++#undef SQLITE_META_DWR ++#endif ++ ++#ifdef SQLITE_META_DWR ++static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, const char *zRight); ++typedef struct MetaDwrHdr MetaDwrHdr; ++static int MetaDwrWriteHeader(Pager *pPager, MetaDwrHdr *hdr); ++static int MetaDwrUpdateMetaPages(Btree *pBt); ++static void MetaDwrPagerRelease(Pager *pPager); ++static int MetaDwrOpenFile(Pager *pPager, u8 openCreate); ++static void MetaDwrCheckVacuum(BtShared *pBt); ++static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion); ++static int MetaDwrOpenAndCheck(Btree *pBt); ++static void MetaDwrDisable(Btree *pBt); ++#define META_HEADER_CHANGED 1 ++#define META_SCHEMA_CHANGED 2 ++#define META_IN_RECOVERY 1 ++#define META_RECOVER_SUCCESS 2 ++#endif + /* + ** Bits of the Pager.doNotSpill flag. See further description below. + */ +@@ -58333,6 +58354,13 @@ struct Pager { + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ + char *zWal; /* File name for write-ahead log */ + #endif ++#ifdef SQLITE_META_DWR ++ u8 metaChanged; ++ sqlite3_file *metaFd; ++ MetaDwrHdr *metaHdr; ++ void *metaMapPage; ++ int (*xGetMethod)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ ++#endif + }; + + /* +@@ -58691,7 +58719,11 @@ static void setGetterMethod(Pager *pPager){ + pPager->xGet = getPageMMap; + #endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ ++#ifdef SQLITE_META_DWR ++ pPager->xGet = pPager->xGetMethod ? pPager->xGetMethod : getPageNormal; ++#else + pPager->xGet = getPageNormal; ++#endif + } + } + +@@ -59765,7 +59797,14 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } +- ++#ifdef SQLITE_META_DWR ++ if (bCommit && pPager->metaChanged != 0) { ++ sqlite3BeginBenignMalloc(); ++ (void)MetaDwrWriteHeader(pPager, pPager->metaHdr); ++ sqlite3EndBenignMalloc(); ++ pPager->metaChanged = 0; ++ } ++#endif + if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE +@@ -61860,6 +61899,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } ++#endif ++#ifdef SQLITE_META_DWR ++ MetaDwrPagerRelease(pPager); + #endif + pager_reset(pPager); + if( MEMDB ){ +@@ -62699,7 +62741,11 @@ act_like_temp_file: + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } +- ++#ifdef SQLITE_META_DWR ++ if( rc==SQLITE_OK && !memDb && !readOnly){ ++ (void)MetaDwrOpenFile(pPager, 0); ++ } ++#endif + /* If an error occurred above, free the Pager structure and close the file. + */ + if( rc!=SQLITE_OK ){ +@@ -63123,7 +63169,6 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); + } +- + failed: + if( rc!=SQLITE_OK ){ + assert( !MEMDB ); +@@ -64416,7 +64461,6 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ + pPager->eState = PAGER_READER; + return SQLITE_OK; + } +- + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + rc = pager_end_transaction(pPager, pPager->setSuper, 1); + return pager_error(pPager, rc); +@@ -70676,6 +70720,10 @@ struct BtShared { + #endif + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ + int nPreformatSize; /* Size of last cell written by TransferRow() */ ++#ifdef SQLITE_META_DWR ++ u32 maxMetaPage; ++ u32 metaRecoverStatus; ++#endif + }; + + /* +@@ -74077,8 +74125,8 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( + } + } + #endif +- *ppBtree = p; + ++ *ppBtree = p; + btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ +@@ -75048,7 +75096,12 @@ trans_begun: + rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } + } +- ++#ifdef SQLITE_META_DWR ++ if (rc == SQLITE_NOTADB || rc == SQLITE_CORRUPT) { ++ int rc1 = MetaDwrRecoverAndBeginTran(p, wrflag, pSchemaVersion); ++ rc = (rc1 == SQLITE_OK) ? SQLITE_OK : rc; ++ } ++#endif + btreeIntegrity(p); + sqlite3BtreeLeave(p); + return rc; +@@ -75437,6 +75490,9 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); ++#ifdef SQLITE_META_DWR ++ MetaDwrCheckVacuum(pBt); ++#endif + } + }else{ + rc = SQLITE_DONE; +@@ -75521,6 +75577,9 @@ static int autoVacuumCommit(Btree *p){ + put4byte(&pBt->pPage1->aData[28], nFin); + pBt->bDoTruncate = 1; + pBt->nPage = nFin; ++#ifdef SQLITE_META_DWR ++ MetaDwrCheckVacuum(pBt); ++#endif + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); +@@ -75577,6 +75636,9 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } ++#endif ++#ifdef SQLITE_META_DWR ++ (void)MetaDwrUpdateMetaPages(p); + #endif + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + sqlite3BtreeLeave(p); +@@ -81669,6 +81731,11 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; + } ++#endif ++#ifdef SQLITE_META_DWR ++ if (idx == 1 && pBt->pPager->metaFd) { ++ pBt->pPager->metaChanged = META_SCHEMA_CHANGED; ++ } + #endif + } + sqlite3BtreeLeave(p); +@@ -82082,7 +82149,6 @@ static int checkTreePage( + if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; + goto end_of_check; + } +- + /* Clear MemPage.isInit to make sure the corruption detection code in + ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; +@@ -83293,6 +83359,12 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + } + } + } ++ #ifdef SQLITE_META_DWR ++ if (rc == SQLITE_OK && p->pDest->pBt->pPager->metaFd) { ++ p->pDest->pBt->pPager->metaChanged = META_SCHEMA_CHANGED; ++ (void)MetaDwrUpdateMetaPages(p->pDest); ++ } ++ #endif + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } +@@ -83323,6 +83395,12 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + } + }else{ + sqlite3PagerTruncateImage(pDestPager, nDestTruncate); ++ #ifdef SQLITE_META_DWR ++ if (rc == SQLITE_OK && p->pDest->pBt->pPager->metaFd) { ++ p->pDest->pBt->pPager->metaChanged = META_SCHEMA_CHANGED; ++ (void)MetaDwrUpdateMetaPages(p->pDest); ++ } ++ #endif + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); + } + +@@ -140300,8 +140378,14 @@ SQLITE_PRIVATE void sqlite3Pragma( + /* sqlite3CodecPragma executes internal */ + goto pragma_out; + } +-#endif /* SQLITE_HAS_CODEC */ +- ++#endif ++/* END CODEC */ ++#ifdef SQLITE_META_DWR ++ if(PragmaMetaDoubleWrie(db, iDb, pParse, zLeft, zRight)) { ++ /* PragmaMetaDoubleWrie executes internal */ ++ goto pragma_out; ++ } ++#endif + /* Locate the pragma in the lookup table */ + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ){ +@@ -155771,7 +155855,6 @@ static void updateVirtualTable( + } + } + +- + if( eOnePass==ONEPASS_OFF ){ + /* End the virtual table scan */ + if( pSrc->nSrc==1 ){ +@@ -182655,6 +182738,10 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + zErr = "no more rows available"; + break; + } ++ case SQLITE_META_RECOVERED: { ++ zErr = "warning meta recover message"; ++ break; ++ } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rccheckFileId) { ++ unixFile *fd = (unixFile *)pPager->fd; ++ if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL) { ++ return SQLITE_INTERNAL; ++ } ++ if (fd->pInode->fileId.ino != hdr->dbFileInode) { ++ sqlite3_log(SQLITE_IOERR_DATA, "Ino mismatch file %llu dwr file %llu", ++ fd->pInode->fileId.ino, hdr->dbFileInode); ++ return SQLITE_IOERR_DATA; ++ } ++ } ++#endif ++ if (hdr->pageCnt > META_DWR_MAX_PAGES || hdr->version != META_DWR_VERSION || ++ hdr->magic != META_DWR_MAGIC || hdr->checkSum != META_DWR_MAGIC) { ++ sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file check wrong pageCnt %u, version %u, magic %u, checkSum %u", ++ hdr->pageCnt, hdr->version, hdr->magic, hdr->checkSum); ++ return SQLITE_IOERR_DATA; ++ } ++ if (hdr->pageSz != pPager->pageSize) { ++ sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file check wrong pageSz %u-%u", hdr->pageSz, pPager->pageSize); ++ return SQLITE_IOERR_DATA; ++ } ++ return SQLITE_OK; ++} ++ ++static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, const char *zRight) { ++ Btree *pBt = db->aDb[iDb].pBt; ++ if (pBt == NULL || zLeft == NULL || sqlite3StrICmp(zLeft, "meta_double_write") != 0) { ++ return 0; ++ } ++ Pager *pPager = pBt->pBt->pPager; ++ if (pPager == NULL) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "Invalid pager handle"); ++ return 1; ++ } ++ if (zRight == NULL) { ++ Vdbe *v = sqlite3GetVdbe(parse); ++ sqlite3VdbeSetNumCols(v, 1); ++ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "meta_double_write", SQLITE_STATIC); ++ sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pPager->metaFd ? "enabled" : "disabled", 0); ++ sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); ++ } else if (strncmp(zRight, "enabled", 7) == 0) { ++ sqlite3_mutex_enter(db->mutex); ++ // only support enabled meta double write ++ int rc = MetaDwrOpenAndCheck(pBt); ++ if (rc != SQLITE_OK) { ++ parse->nErr++; ++ parse->rc = rc; ++ } ++ sqlite3_mutex_leave(db->mutex); ++ } else if (strncmp(zRight, "disabled", 8) == 0) { ++ sqlite3_mutex_enter(db->mutex); ++ MetaDwrDisable(pBt); ++ sqlite3_mutex_leave(db->mutex); ++ } ++ return 1; ++} ++ ++static int GetBtreePageNo(MemPage *pPage, void *args) { ++ ScanPages *pInfo = (ScanPages *)args; ++ // realloc buffer to store pages ++ u32 pageCnt = pInfo->pageCnt; ++ if (pageCnt == pInfo->pageBufSize) { ++ u32 memSz = sizeof(Pgno) * ROUND8(pageCnt + 1); ++ Pgno *pages = sqlite3Malloc(memSz); ++ if (pages == NULL) { ++ sqlite3_log(SQLITE_NOMEM, "GetPages alloc buffer go wrong %u", memSz); ++ return SQLITE_NOMEM; ++ } ++ if (pageCnt != 0) { ++ memcpy(pages, pInfo->pages, pageCnt * sizeof(Pgno)); ++ } ++ sqlite3_free(pInfo->pages); ++ pInfo->pageBufSize = ROUND8(pageCnt + 1); ++ pInfo->pages = pages; ++ } ++ pInfo->pages[pageCnt] = pPage->pgno; ++ pInfo->pageCnt++; ++ if (pInfo->maxPageNo < pPage->pgno) { ++ pInfo->maxPageNo = pPage->pgno; ++ } ++ return 0; ++} ++ ++typedef int (*ScanFn)(MemPage *pPage, void *args); ++static SQLITE_NOINLINE int ScanOverflowPages( ++ MemPage *pPage, /* The page that contains the Cell */ ++ unsigned char *pCell, /* First byte of the Cell */ ++ CellInfo *pInfo, /* Size information about the cell */ ++ ScanFn fn, /* Scan pages function */ ++ void *args) { ++ BtShared *pBt; ++ Pgno ovflPgno; ++ int rc; ++ int nOvfl; ++ u32 ovflPageSize; ++ ++ if (pCell + pInfo->nSize > pPage->aDataEnd) { ++ /* Cell extends past end of page */ ++ return SQLITE_CORRUPT_BKPT; ++ } ++ ovflPgno = get4byte(pCell + pInfo->nSize - 4); ++ pBt = pPage->pBt; ++ assert(pBt->usableSize > 4); ++ ovflPageSize = pBt->usableSize - 4; ++ nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1) / ovflPageSize; ++ assert(nOvfl > 0 || ++ (CORRUPT_DB && (pInfo->nPayload + ovflPageSize) < ovflPageSize)); ++ while (nOvfl--) { ++ Pgno iNext = 0; ++ MemPage *pOvfl = 0; ++ if (ovflPgno < 2 || ovflPgno > btreePagecount(pBt)) { ++ return SQLITE_CORRUPT_BKPT; ++ } ++ rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); ++ if (rc) ++ return rc; ++ if (pOvfl) { ++ rc = fn(pOvfl, args); ++ if (rc) { ++ return rc; ++ } ++ sqlite3PagerUnref(pOvfl->pDbPage); ++ } ++ ovflPgno = iNext; ++ } ++ return SQLITE_OK; ++} ++ ++static int ScanBtreePage( ++ BtShared *pBt, /* The BTree that contains the table */ ++ Pgno pgno, /* Page number to clear */ ++ ScanFn fn, /* Scan pages function */ ++ void *args) { /* Scan pages args */ ++ MemPage *pPage; ++ int rc; ++ unsigned char *pCell; ++ int i; ++ int hdr; ++ CellInfo info; ++ ++ assert(sqlite3_mutex_held(pBt->mutex)); ++ if (pgno > btreePagecount(pBt)) { ++ return SQLITE_OK; ++ } ++ rc = getAndInitPage(pBt, pgno, &pPage, 0); ++ if (rc) { ++ return rc; ++ } ++ rc = fn(pPage, args); ++ if (rc) { ++ goto SCAN_PAGE_OUT; ++ } ++ hdr = pPage->hdrOffset; ++ for (i = pPage->nCell - 1; i >= 0; i--) { ++ pCell = findCell(pPage, i); ++ if (!pPage->leaf) { ++ rc = ScanBtreePage(pBt, get4byte(pCell), fn, args); ++ if (rc) { ++ goto SCAN_PAGE_OUT; ++ } ++ } ++ pPage->xParseCell(pPage, pCell, &info); ++ if (info.nLocal != info.nPayload) { ++ rc = ScanOverflowPages(pPage, pCell, &info, fn, args); ++ if (rc) { ++ goto SCAN_PAGE_OUT; ++ } ++ } ++ } ++ if (!pPage->leaf) { ++ rc = ScanBtreePage(pBt, get4byte(&pPage->aData[hdr + 8]), fn, args); ++ if (rc) { ++ goto SCAN_PAGE_OUT; ++ } ++ } ++SCAN_PAGE_OUT: ++ releasePage(pPage); ++ return rc; ++} ++ ++static inline int ScanMetaPages(Btree *pBt, ScanPages *pages) { ++ return ScanBtreePage(pBt->pBt, 1, GetBtreePageNo, pages); ++} ++ ++static int ReleaseMetaPages(ScanPages *pages) { ++ sqlite3_free(pages->pages); ++ pages->pages = NULL; ++ return SQLITE_OK; ++} ++ ++static void InitMetaHeader(MetaDwrHdr *hdr) { ++ (void)memset(hdr, 0, META_VERIFIED_HDR_LEN); ++ hdr->magic = META_DWR_MAGIC; ++ hdr->version = META_DWR_VERSION; ++ hdr->checkSum = META_DWR_MAGIC; ++} ++ ++static void MetaDwrReleaseHdr(MetaDwrHdr *hdr) { ++ if (!hdr) { ++ return; ++ } ++ sqlite3_free(hdr->zones); ++ sqlite3_free(hdr); ++} ++ ++static int ExpandMetaPageBuf(MetaDwrHdr *hdr, u32 minimalPageCnt, u32 bufHasData) { ++ if (minimalPageCnt < hdr->pageBufSize && hdr->zones != NULL) { ++ return SQLITE_OK; ++ } ++ int pageBufSz = ROUND8(MAX(hdr->pageCnt, minimalPageCnt)); ++ u8 *zones = (u8 *)sqlite3Malloc(pageBufSz * (sizeof(u8) + sizeof(Pgno))); ++ if (zones == NULL) { ++ return SQLITE_NOMEM_BKPT; ++ } ++ Pgno *pgnos = (Pgno *)(zones + pageBufSz); ++ if (hdr->zones != NULL) { ++ if (bufHasData && hdr->pageCnt > 0) { ++ (void)memcpy(zones, hdr->zones, hdr->pageCnt * sizeof(u8)); ++ (void)memcpy(pgnos, hdr->pages, hdr->pageCnt * sizeof(Pgno)); ++ } ++ sqlite3_free(hdr->zones); ++ } ++ hdr->pageBufSize = pageBufSz; ++ hdr->zones = zones; ++ hdr->pages = pgnos; ++ return SQLITE_OK; ++} ++ ++static MetaDwrHdr *AllocInitMetaHeaderDwr(Pager *pPager) { ++ MetaDwrHdr *hdr = sqlite3MallocZero(sizeof(MetaDwrHdr)); ++ if (hdr == NULL) { ++ return NULL; ++ } ++ InitMetaHeader(hdr); ++ int rc = ExpandMetaPageBuf(hdr, META_DWR_HEADER_DEFAULT_PAGE_CNT, 0); ++ if (rc != SQLITE_OK) { ++ MetaDwrReleaseHdr(hdr); ++ return NULL; ++ } ++ hdr->checkFileId = (pPager->pVfs != NULL && sqlite3_stricmp(pPager->pVfs->zName, "unix") == 0); ++ return hdr; ++} ++ ++static void MetaDwrCloseFile(Pager *pPager) { ++ if (!pPager->metaFd) { ++ return; ++ } ++#if SQLITE_OS_UNIX ++ if (pPager->metaMapPage) { ++ osMunmap(pPager->metaMapPage, META_DWR_HEADER_PAGE_SIZE); ++ pPager->metaMapPage = NULL; ++ } ++#endif ++ if (pPager->metaHdr && pPager->metaHdr->needSync > 0) { ++ (void)sqlite3OsSync(pPager->metaFd, SQLITE_SYNC_NORMAL); ++ } ++ sqlite3OsClose(pPager->metaFd); ++} ++ ++static void MetaDwrPagerRelease(Pager *pPager) { ++ MetaDwrCloseFile(pPager); ++ MetaDwrReleaseHdr(pPager->metaHdr); ++ pPager->metaHdr = NULL; ++ if (pPager->metaFd) { ++ sqlite3_free(pPager->metaFd); ++ pPager->metaFd = NULL; ++ } ++} ++ ++static inline int ReadFromHdrPage(Pager *pPager, void *data, int amt, i64 offset) { ++ if (pPager->metaMapPage) { ++ (void)memcpy(data, (u8 *)pPager->metaMapPage + offset, amt); ++ return SQLITE_OK; ++ } ++ return sqlite3OsRead(pPager->metaFd, data, amt, offset); ++} ++ ++static int MetaDwrReadHeader(Pager *pPager, MetaDwrHdr *hdr) { ++ i64 sz = 0; ++ int rc = sqlite3OsFileSize(pPager->metaFd, &sz); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "Meta dwr file size go wrong"); ++ return rc; ++ } ++ if (sz <= META_DWR_HEADER_PAGE_SIZE) { ++ rc = SQLITE_IOERR_DATA; ++ goto READ_META_OUT; ++ } ++ rc = ReadFromHdrPage(pPager, hdr, META_VERIFIED_HDR_LEN, 0); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "Meta dwr file header read wrong"); ++ goto READ_META_OUT; ++ } ++ rc = MetaDwrHeaderSimpleCheck(pPager, hdr); ++ if (rc != SQLITE_OK) { ++ goto READ_META_OUT; ++ } ++ // avoid realloc buffer if buf can't hold all pages ++ rc = ExpandMetaPageBuf(hdr, hdr->pageCnt, 0); ++ if (rc != SQLITE_OK) { ++ goto READ_META_OUT; ++ } ++ int zoneSize = hdr->pageCnt * sizeof(u8); ++ rc = ReadFromHdrPage(pPager, hdr->zones, zoneSize, META_VERIFIED_HDR_LEN); ++ if (rc != SQLITE_OK) { ++ goto READ_META_OUT; ++ } ++ rc = ReadFromHdrPage(pPager, hdr->pages, hdr->pageCnt * sizeof(Pgno), META_PAGE_NO_OFFSET); ++ if (rc != SQLITE_OK) { ++ goto READ_META_OUT; ++ } ++ for (u32 i = 0; i < hdr->pageCnt; i++) { ++ u8 zoneIdx = hdr->zones[i]; ++ if (zoneIdx != 0 && zoneIdx != 1 && zoneIdx != META_DWR_INVALID_ZONE) { ++ sqlite3_log(SQLITE_IOERR_DATA, "Invalid zoneIdx %d", zoneIdx); ++ rc = SQLITE_IOERR_DATA; ++ break; ++ } ++ } ++READ_META_OUT: ++ if (rc == SQLITE_IOERR_DATA) { ++ InitMetaHeader(hdr); ++ rc = SQLITE_OK; ++ } ++ return rc; ++} ++ ++static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) { ++ // 1 header page, idx correspond 2 zone pages ++ return META_DWR_HEADER_PAGE_SIZE + pageSz * (idx * 2 + zone); ++} ++ ++static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { ++ MetaDwrHdr *hdr = pBt->pPager->metaHdr; ++ // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie ++ const u8 *dbHdrInfo = &pBt->pPage1->aData[28]; ++ hdr->dbSize = sqlite3Get4byte(dbHdrInfo); ++#ifndef SQLITE_OMIT_WAL ++ if (pagerUseWal(pBt->pPager)) { ++ WalIndexHdr *pWalHdr = &pBt->pPager->pWal->hdr; ++ if (pWalHdr->isInit) { ++ hdr->mxFrameInWal = pWalHdr->mxFrame; ++ hdr->dbSize = pWalHdr->nPage; ++ } ++ } else { ++ hdr->mxFrameInWal = 0; ++ } ++#endif ++#if SQLITE_OS_UNIX ++ if (hdr->checkFileId) { ++ unixFile *fd = (unixFile *)pBt->pPager->fd; ++ if (fd == NULL || fd->pInode == NULL) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "update meta header invalid fd"); ++ hdr->hdrValid = 0; ++ return; ++ } ++ hdr->dbFileInode = fd->pInode->fileId.ino; ++ } ++#endif ++ hdr->freeListPageNo = sqlite3Get4byte(dbHdrInfo + 4); ++ hdr->freeListPageCnt = sqlite3Get4byte(dbHdrInfo + 8); ++ hdr->schemaCookie = sqlite3Get4byte(dbHdrInfo + 12); ++ hdr->hdrValid = 1; ++} ++ ++static inline int WriteToHdrPage(Pager *pPager, const void *data, int amt, i64 offset) { ++ if (pPager->metaMapPage) { ++ (void)memcpy((u8 *)pPager->metaMapPage + offset, data, amt); ++ return SQLITE_OK; ++ } ++ return sqlite3OsWrite(pPager->metaFd, data, amt, offset); ++} ++ ++static int MetaDwrWriteHeader(Pager *pPager, MetaDwrHdr *hdr) { ++ if (pPager->metaChanged == 0 || hdr == NULL || hdr->pageCnt == 0 || hdr->hdrValid == 0) { ++ return SQLITE_OK; ++ } ++ hdr->hdrValid = 0; ++ hdr->pageSz = pPager->pageSize; ++ hdr->dbSize = pPager->dbSize; ++ int rc = WriteToHdrPage(pPager, hdr, META_VERIFIED_HDR_LEN, 0); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "update meta header write hdr %u wrong", META_VERIFIED_HDR_LEN); ++ return rc; ++ } ++ if (hdr->zones) { ++ int zoneSize = hdr->pageCnt * sizeof(u8); ++ rc = WriteToHdrPage(pPager, hdr->zones, zoneSize, META_VERIFIED_HDR_LEN); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "update meta header write zonebuf %u wrong", zoneSize); ++ return rc; ++ } ++ } ++ if (hdr->pages) { ++ int pageBufSz = hdr->pageCnt * sizeof(Pgno); ++ rc = WriteToHdrPage(pPager, hdr->pages, pageBufSz, META_PAGE_NO_OFFSET); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "update meta header write pagebuf %u wrong", pageBufSz); ++ } ++ } ++ if (rc == SQLITE_OK) { ++ u64 size = CaculateMetaDwrWriteOffset((int)pPager->pageSize, hdr->pageCnt, 0); ++ rc = sqlite3OsTruncate(pPager->metaFd, size); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "update meta header truncate filesz %lu wrong", size); ++ } ++ i64 timeMs = 0; ++ sqlite3OsCurrentTimeInt64(pPager->pVfs, &timeMs); ++ if ((timeMs - hdr->lastSyncTime) > META_FILE_SYNC_TIMEOUT_MS || ++ hdr->needSync >= META_FILE_UPDATE_TIMES_PER_SYNC) { ++ rc = sqlite3OsSync(pPager->metaFd, SQLITE_SYNC_NORMAL); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "update meta header sync filesz %lu wrong", size); ++ } ++ hdr->lastSyncTime = timeMs; ++ hdr->needSync = 0; ++ } else { ++ hdr->needSync++; ++ } ++ } ++ return rc; ++} ++ ++static int MetaDwrFindPageIdx(MetaDwrHdr *hdr, u32 pgno, u32 *idx) { ++ for (u32 i = 0; i < hdr->pageCnt && i < META_DWR_MAX_PAGES; i++) { ++ if (pgno == hdr->pages[i]) { ++ *idx = i; ++ return 1; ++ } ++ } ++ return 0; ++} ++ ++static int MetaDwrWriteOnePage(Btree *pBt, PgHdr *pPage, MetaDwrHdr *hdr, u8 curZones, u32 idx) { ++ int rc = SQLITE_OK; ++ u8 pageExpand = 0; ++ if (hdr->pageCnt <= idx) { ++ rc = ExpandMetaPageBuf(hdr, idx + 1, 1); ++ if (rc != SQLITE_OK) { ++ return rc; ++ } ++ pageExpand = 1; ++ } ++ Pager *pPager = pBt->pBt->pPager; ++ // asume zone 0 or 1 ++ u8 zone = 1 - curZones; ++ int pageSz = sqlite3BtreeGetPageSize(pBt); ++ u64 ofs = CaculateMetaDwrWriteOffset(pageSz, idx, zone); ++ void *pData; ++#if defined(SQLITE_HAS_CODEC) ++ if ((pData = sqlite3PagerCodec(pPage)) == 0) ++ return SQLITE_NOMEM; ++#else ++ pData = pPage->pData; ++#endif ++ rc = sqlite3OsWrite(pPager->metaFd, pData, pageSz, ofs); ++ if (rc != SQLITE_OK) { ++ return rc; ++ } ++ hdr->zones[idx] = zone; ++ hdr->pages[idx] = pPage->pgno; ++ if (pageExpand) { ++ hdr->pageCnt++; ++ } ++ return SQLITE_OK; ++} ++ ++static int MetaDwrRestoreAllPages(Btree *pBt, const ScanPages *metaPages, MetaDwrHdr *hdr) { ++ u32 i = 0; ++ PgHdr *p = NULL; ++ int rc = SQLITE_OK; ++ for (i = 0; i < metaPages->pageCnt && i < META_DWR_MAX_PAGES; i++) { ++ Pgno pgno = metaPages->pages[i]; ++ if (pgno > btreePagecount(pBt->pBt)) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "pageno %d overlimit", pgno); ++ return SQLITE_CORRUPT_BKPT; ++ } ++ rc = sqlite3PagerGet(pBt->pBt->pPager, pgno, &p, 0); ++ if (rc) { ++ return rc; ++ } ++ rc = MetaDwrWriteOnePage(pBt, p, hdr, 1, i); ++ sqlite3PagerUnref(p); ++ if (rc) { ++ return rc; ++ } ++ } ++ MetaDwrUpdateHeaderDbInfo(pBt->pBt); ++ return rc; ++} ++ ++static inline const char *GetMetaFilePath(Pager *pPager) ++{ ++ return pPager->metaFd == NULL ? NULL : ((const char *)pPager->metaFd + ROUND8(pPager->pVfs->szOsFile)); ++} ++ ++static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { ++ if (pPager->metaFd || pPager->zFilename == NULL) { ++ return SQLITE_OK; ++ } ++ sqlite3BeginBenignMalloc(); ++ sqlite3_vfs *pVfs = pPager->pVfs; ++ int size = strlen(pPager->zFilename) + sizeof("-dwr"); ++ int szOsFile = ROUND8(pVfs->szOsFile); ++ sqlite3_file *metaFd = (sqlite3_file *)sqlite3MallocZero(szOsFile + size); ++ char *metaPath = (char *)metaFd + szOsFile; ++ if (metaFd == NULL) { ++ sqlite3EndBenignMalloc(); ++ sqlite3_log(SQLITE_NOMEM_BKPT, "sqlite alloc memsize %d go wrong", szOsFile + size); ++ return SQLITE_NOMEM_BKPT; ++ } ++ sqlite3_snprintf(size, metaPath, "%s-dwr", pPager->zFilename); ++ int exists = 0; ++ int rc = sqlite3OsAccess(pVfs, metaPath, SQLITE_ACCESS_EXISTS, &exists); ++ if (rc != SQLITE_OK) { ++ goto INIT_META_OUT; ++ } ++ if (!exists && !openCreate) { ++ sqlite3_free(metaFd); ++ goto INIT_META_OUT; ++ } ++ u32 flags = (SQLITE_OPEN_READWRITE | SQLITE_OPEN_SUPER_JOURNAL); ++ if (openCreate) { ++ flags |= SQLITE_OPEN_CREATE; ++ } ++ rc = sqlite3OsOpen(pVfs, metaPath, metaFd, (int)flags, 0); ++ if (rc != SQLITE_OK) { ++ goto INIT_META_OUT; ++ } ++#if SQLITE_OS_UNIX ++ if (pPager->metaMapPage == NULL) { ++ sqlite3_int64 sz = META_DWR_HEADER_PAGE_SIZE; ++ sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_CHUNK_SIZE, &sz); ++ sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz); ++ void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, ++ ((unixFile *)metaFd)->h, 0); ++ if (page != MAP_FAILED) { ++ pPager->metaMapPage = page; ++ } ++ } ++#endif ++ pPager->metaFd = metaFd; ++INIT_META_OUT: ++ sqlite3EndBenignMalloc(); ++ if (rc != SQLITE_OK && metaFd != NULL) { ++ sqlite3_free(metaFd); ++ } ++ return rc; ++} ++ ++void MetaDwrCheckVacuum(BtShared *pBt) { ++ if (!pBt || !pBt->pPager->metaFd) { ++ return; ++ } ++ if (pBt->nPage < pBt->maxMetaPage) { ++ pBt->pPager->metaChanged = META_SCHEMA_CHANGED; ++ } ++} ++ ++static inline u8 LocalMetaHdrValid(Pager *pPager) { ++ return pPager->metaMapPage != NULL && memcmp(pPager->metaMapPage, pPager->metaHdr, ++ META_VERIFIED_HDR_LEN) == 0; ++} ++ ++static int MetaDwrLoadHdr(Pager *pPager) { ++ if (!pPager->metaHdr) { ++ pPager->metaHdr = AllocInitMetaHeaderDwr(pPager); ++ if (pPager->metaHdr == NULL) { ++ return SQLITE_NOMEM_BKPT; ++ } ++ } ++ if (LocalMetaHdrValid(pPager)) { ++ return SQLITE_OK; ++ } ++ return MetaDwrReadHeader(pPager, pPager->metaHdr); ++} ++ ++static int MetaDwrLoadAndCheckMetaFile(BtShared *pBt, u8 reportErr) { ++ int rc = MetaDwrLoadHdr(pBt->pPager); ++ if (rc != SQLITE_OK) { ++ return rc; ++ } ++ MetaDwrHdr *hdr = pBt->pPager->metaHdr; ++ if (hdr->pageCnt == 0) { ++ return reportErr ? SQLITE_IOERR_DATA : SQLITE_OK; ++ } ++ // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie ++ u8 *dbHdrInfo = &pBt->pPage1->aData[28]; ++ if (hdr->dbSize != pBt->pPager->dbSize || hdr->dbSize != sqlite3Get4byte(dbHdrInfo) || ++ hdr->freeListPageNo != sqlite3Get4byte(dbHdrInfo + 4) || ++ hdr->freeListPageCnt != sqlite3Get4byte(dbHdrInfo + 8) || ++ hdr->schemaCookie != sqlite3Get4byte(dbHdrInfo + 12)) { ++ sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file expect %u-%u-%u-%u-%u but gotton %u-%u-%u-%u-%u", ++ pBt->pPager->dbSize, hdr->dbSize, sqlite3Get4byte(dbHdrInfo), sqlite3Get4byte(dbHdrInfo + 4), ++ sqlite3Get4byte(dbHdrInfo + 8), sqlite3Get4byte(dbHdrInfo + 12), hdr->dbSize, hdr->freeListPageNo, ++ hdr->freeListPageCnt, hdr->schemaCookie); ++ // reinit ++ InitMetaHeader(hdr); ++ if (reportErr) { ++ return SQLITE_IOERR_DATA; ++ } ++ } ++ return SQLITE_OK; ++} ++ ++static int MetaDwrReadOnePage(Pager *pPager, MetaDwrHdr *hdr, int idx, u8 *pData) { ++ u64 ofs = CaculateMetaDwrWriteOffset(pPager->pageSize, idx, hdr->zones[idx]); ++ int rc = sqlite3OsRead(pPager->metaFd, pData, pPager->pageSize, ofs); ++ CODEC1(pPager, pData, hdr->pages[idx], 3, rc = SQLITE_NOMEM_BKPT); ++ return rc; ++} ++ ++static int MetaDwrRecoverHeadPage( ++ Pager *pPager, /* The pager open on the database file */ ++ Pgno pgno, /* Page number to fetch */ ++ DbPage **pDbPage, ++ int flag) { ++ if (pPager->metaFd == NULL) { ++ return pgno == 1 ? SQLITE_NOTADB : SQLITE_CORRUPT; ++ } ++ sqlite3_pcache_page *pCachePage = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); ++ if (pCachePage == NULL) { ++ sqlite3_log(SQLITE_NOMEM_BKPT, "Get meta page wrong %d", pgno); ++ return SQLITE_NOMEM_BKPT; ++ } ++ DbPage *pPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pCachePage); ++ pPage->pPager = pPager; ++ assert(pCachePage != 0); ++ int rc = MetaDwrLoadHdr(pPager); ++ if (rc != SQLITE_OK) { ++ goto RELEASE_OUT; ++ } ++ MetaDwrHdr *hdr = pPager->metaHdr; ++ u8 walChecked = 0; ++#ifndef SQLITE_OMIT_WAL ++ if (pagerUseWal(pPager)) { ++ WalIndexHdr *pWalHdr = &pPager->pWal->hdr; ++ if (pWalHdr->isInit && pWalHdr->mxFrame != 0) { ++ if (hdr->mxFrameInWal != pWalHdr->mxFrame || hdr->dbSize != pWalHdr->nPage) { ++ rc = SQLITE_NOTADB; ++ sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr wal hdr expect %u-%u but gotten %u-%u", ++ hdr->mxFrameInWal, hdr->dbSize, pWalHdr->mxFrame, pWalHdr->nPage); ++ goto RELEASE_OUT; ++ } else { ++ walChecked = 1; ++ } ++ } ++ } ++ if (walChecked == 0) { ++ i64 size = 0; ++ rc = sqlite3OsFileSize(pPager->fd, &size); ++ if (rc != SQLITE_OK) { ++ rc = SQLITE_NOTADB; ++ sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr get db file size go wrong"); ++ goto RELEASE_OUT; ++ } ++ i64 expectSz = (i64)hdr->dbSize * (i64)hdr->pageSz; ++ if (size != expectSz) { ++ rc = SQLITE_NOTADB; ++ sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr expect file size %lu but gotten size %llu", ++ expectSz, size); ++ goto RELEASE_OUT; ++ } ++ } ++#endif ++ rc = SQLITE_NOTADB; ++ for (u32 i = 0; i < hdr->pageCnt; i++) { ++ if (hdr->pages[i] != pgno) { ++ continue; ++ } ++ rc = MetaDwrReadOnePage(pPager, hdr, i, sqlite3PagerGetData(pPage)); ++ if (rc == SQLITE_OK) { ++ *pDbPage = pPage; ++ if (pPage->pgno == 1) { ++ memcpy(&pPager->dbFileVers, &((u8 *)pPage->pData)[24], sizeof(pPager->dbFileVers)); ++ } ++ pager_set_pagehash(pPage); ++ } ++ break; ++ } ++RELEASE_OUT: ++ if (rc != SQLITE_OK && pPage != NULL) { ++ sqlite3PcacheDrop(pPage); ++ } ++ return rc; ++} ++ ++static int MetaDwrRestoreChangedPages(Btree *pBt) { ++ Pager *pPager = pBt->pBt->pPager; ++ MetaDwrHdr *hdr = pPager->metaHdr; ++ u8 *zones = sqlite3MallocZero(hdr->pageBufSize * sizeof(u8)); ++ if (zones == NULL) { ++ sqlite3_log(SQLITE_NOMEM_BKPT, "Alloc zones buffer size %u go wrong", hdr->pageBufSize * sizeof(u8)); ++ return SQLITE_NOMEM_BKPT; ++ } ++ if (hdr->pageCnt > 0) { ++ memcpy(zones, hdr->zones, hdr->pageCnt * sizeof(u8)); ++ } ++ u32 idx = 0; ++ PgHdr *p = 0; ++ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); ++ int rc = SQLITE_OK; ++ for (p = pList; p; p = p->pDirty) { ++ if (MetaDwrFindPageIdx(hdr, p->pgno, &idx) == 0) { ++ continue; ++ } ++ rc = MetaDwrWriteOnePage(pBt, p, hdr, zones[idx], idx); ++ if (rc != SQLITE_OK) { ++ break; ++ } ++ } ++ if (rc == SQLITE_OK) { ++ MetaDwrUpdateHeaderDbInfo(pBt->pBt); ++ } ++ sqlite3_free(zones); ++ return rc; ++} ++ ++static int MetaDwrUpdateMetaPages(Btree *pBt) { ++ Pager *pPager = pBt->pBt->pPager; ++ if (!pPager || !pPager->metaFd || pPager->memDb || pPager->readOnly || pBt->pBt->pPage1 == NULL) { ++ return SQLITE_OK; ++ } ++ if (pPager->metaChanged == 0) { ++ if ((pBt->pBt->pPage1->pDbPage->flags & PGHDR_DIRTY) == 0) { ++ return SQLITE_OK; ++ } ++ pPager->metaChanged = META_HEADER_CHANGED; ++ } ++ sqlite3BeginBenignMalloc(); ++ int rc = MetaDwrLoadHdr(pPager); ++ if (rc != SQLITE_OK) { ++ goto UPDATE_OUT; ++ } ++ // only update header page ++ if (pPager->metaChanged == META_HEADER_CHANGED) { ++ rc = MetaDwrRestoreChangedPages(pBt); ++ goto UPDATE_OUT; ++ } ++ // update schema pages ++ ScanPages metaPages = {0}; ++ rc = ScanMetaPages(pBt, &metaPages); ++ if (rc != SQLITE_OK) { ++ goto UPDATE_OUT; ++ } ++ MetaDwrHdr *hdr = pPager->metaHdr; ++ // rewrite ++ if (hdr->pageCnt == 0 || hdr->pageCnt != metaPages.pageCnt || pBt->pBt->nPage > pBt->pBt->maxMetaPage || ++ memcmp(hdr->pages, metaPages.pages, hdr->pageCnt * sizeof(Pgno))) { ++ // if page numbers unorderred, restore all pages ++ rc = MetaDwrRestoreAllPages(pBt, &metaPages, hdr); ++ } else { ++ rc = MetaDwrRestoreChangedPages(pBt); ++ } ++ if (rc == SQLITE_OK) { ++ pBt->pBt->maxMetaPage = metaPages.maxPageNo; ++ } ++ ReleaseMetaPages(&metaPages); ++UPDATE_OUT: ++ sqlite3EndBenignMalloc(); ++ return rc; ++} ++ ++static int MetaDwrRecoverSinglePage(Btree *pBt, int pgno, u8 *pData) { ++ if (pgno < 1 || pBt == NULL) { ++ return SQLITE_CORRUPT_BKPT; ++ } ++ Pager *pPager = sqlite3BtreePager(pBt); ++ DbPage *pDbPage = NULL; ++ int rc = sqlite3PagerGet(pPager, pgno, &pDbPage, 0); ++ if (rc) { ++ return rc; ++ } ++ if ((rc = sqlite3PagerWrite(pDbPage)) == SQLITE_OK) { ++ memcpy(sqlite3PagerGetData(pDbPage), pData, pPager->pageSize); ++ } else { ++ sqlite3_log(rc, "Dwr recoverwrite meta page %d failed", pgno); ++ } ++ sqlite3PagerUnref(pDbPage); ++ return rc; ++} ++ ++static int MetaDwrCheckMeta(Btree *pBt) { ++ int nErr = 0; ++ Pgno aRoot[2] = {0, 1}; // quick check and only check root btree ++ sqlite3_value aMem[2] = {0}; ++ char *errStr = NULL; ++ int rc = sqlite3BtreeIntegrityCheck(pBt->db, pBt, &aRoot[0], &aMem[0], 2, SQLITE_INTEGRITY_CHECK_ERROR_MAX, ++ &nErr, &errStr); ++ if (nErr == 0) { ++ assert(errStr == 0); ++ return SQLITE_OK; ++ } ++ if (rc != SQLITE_OK) { ++ sqlite3_free(errStr); ++ return rc; ++ } ++ sqlite3_log(SQLITE_WARNING_DUMP, "Integrity check %s", errStr); ++ sqlite3_free(errStr); ++ return SQLITE_CORRUPT; ++} ++ ++static int MetaDwrBeginTrans(Btree *pBt, int wrflag) { ++ pBt->pBt->btsFlags &= ~BTS_READ_ONLY; ++ Pager *pPager = pBt->pBt->pPager; ++ void *xGetMethod = pPager->xGet; ++ pPager->xGetMethod = MetaDwrRecoverHeadPage; ++ pPager->xGet = MetaDwrRecoverHeadPage; ++ int rc = sqlite3BtreeBeginTrans(pBt, wrflag, 0); ++ pPager->xGet = xGetMethod; ++ pPager->xGetMethod = 0; ++ if (rc == SQLITE_OK) { ++ sqlite3PagerWrite(pBt->pBt->pPage1->pDbPage); ++ sqlite3_log(rc, "sqlite fix meta header"); ++ } ++ return rc; ++} ++ ++static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion) ++{ ++ Pager *pPager = pBt->pBt->pPager; ++ assert(sqlite3_mutex_held(pBt->pBt->mutex)); ++ if (!pPager->metaFd || pBt->pBt->metaRecoverStatus || pPager->readOnly || pPager->memDb) { ++ return SQLITE_NOTADB; ++ } ++ int rc = MetaDwrLoadHdr(pPager); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "MetaDwr load header failed"); ++ return rc; ++ } ++ pBt->pBt->metaRecoverStatus = META_IN_RECOVERY; ++ rc = MetaDwrBeginTrans(pBt, 2); ++ if (rc != SQLITE_OK) { ++ return rc; ++ } ++ void *pData = NULL; ++ pPager->metaChanged = META_HEADER_CHANGED; ++ MetaDwrHdr *hdr = pPager->metaHdr; ++ sqlite3_log(SQLITE_WARNING_DUMP, "sqlite meta recover %u frames", hdr->pageCnt); ++ int szPage = sqlite3BtreeGetPageSize(pBt); ++ pData = sqlite3Malloc(szPage); ++ if (pData == NULL) { ++ rc = SQLITE_NOMEM; ++ sqlite3_log(rc, "Dwr malloc mem size %d failed", szPage); ++ goto DWR_RECOVER_OUT; ++ } ++ for (u32 i = 0; i < hdr->pageCnt; i++) { ++ rc = MetaDwrReadOnePage(pPager, hdr, i, pData); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "Dwr read %d meta page failed ", i); ++ break; ++ } ++ rc = MetaDwrRecoverSinglePage(pBt, hdr->pages[i], pData); ++ if (rc != SQLITE_OK) { ++ sqlite3_log(rc, "Dwr recover %d meta page failed ", i); ++ break; ++ } ++ } ++DWR_RECOVER_OUT: ++ /* Close the transaction, if one was opened. */ ++ if (rc == SQLITE_OK) { ++ sqlite3BtreeCommit(pBt); ++ } else { ++ (void)sqlite3BtreeRollback(pBt, SQLITE_ABORT_ROLLBACK, 0); ++ } ++ if (rc == SQLITE_OK) { ++ rc = sqlite3BtreeBeginTrans(pBt, wrflag, pSchemaVersion); ++ } ++ if (rc == SQLITE_OK) { ++ pBt->pBt->metaRecoverStatus = META_RECOVER_SUCCESS; ++ } ++ if (pData) { ++ sqlite3_free(pData); ++ } ++ return rc; ++} ++ ++static int Sqlite3MetaDwrCheckRestore(Btree *pBt) { ++ Pager *pPager = pBt->pBt->pPager; ++ int rc = MetaDwrOpenFile(pPager, 1); ++ if (rc != SQLITE_OK) { ++ return rc; ++ } ++ ScanPages metaPages = {0}; ++ rc = ScanMetaPages(pBt, &metaPages); ++ if (rc != SQLITE_OK || metaPages.pageCnt == 0) { ++ goto CHK_RESTORE_OUT; ++ } ++ rc = MetaDwrLoadAndCheckMetaFile(pBt->pBt, 0); ++ if (rc != SQLITE_OK) { ++ goto CHK_RESTORE_OUT; ++ } ++ MetaDwrHdr *hdr = pPager->metaHdr; ++ if (hdr->pageCnt == 0 || hdr->pageCnt != metaPages.pageCnt || ++ memcmp(hdr->pages, metaPages.pages, hdr->pageCnt * sizeof(Pgno))) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "sqlite meta restore all"); ++ rc = MetaDwrRestoreAllPages(pBt, &metaPages, hdr); ++ if (rc == SQLITE_OK) { ++ pPager->metaChanged = META_SCHEMA_CHANGED; ++ rc = MetaDwrWriteHeader(pPager, hdr); ++ pPager->metaChanged = 0; ++ } ++ } ++ if (rc == SQLITE_OK) { ++ pBt->pBt->maxMetaPage = metaPages.maxPageNo; ++ } ++CHK_RESTORE_OUT: ++ ReleaseMetaPages(&metaPages); ++ return rc; ++} ++ ++static inline u8 IsConnectionValidForCheck(Pager *pPager) ++{ ++#if SQLITE_OS_UNIX ++ unixFile *fd = (unixFile *)pPager->fd; ++ // unix and only one connection exist ++ if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL || ++ sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0 || fd->pInode->nRef != 1) { ++ return 0; ++ } ++ return 1; ++#else ++ return 0; ++#endif ++} ++ ++static int MetaDwrOpenAndCheck(Btree *pBt) ++{ ++ Pager *pPager = pBt->pBt->pPager; ++ if (pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { ++ return SQLITE_OK; ++ } ++#ifdef SQLITE_HAS_CODEC ++ // not support codec right now ++ if (pPager->xCodec) { ++ return SQLITE_OK; ++ } ++#endif ++ sqlite3BtreeEnter(pBt); ++ int rc = SQLITE_OK; ++ int openedTransaction = 0; ++ int tnxState = sqlite3BtreeTxnState(pBt); ++ if (tnxState == SQLITE_TXN_NONE) { ++ rc = sqlite3BtreeBeginTrans(pBt, 0, 0); ++ if (rc != SQLITE_OK) { ++ goto DWR_OPEN_OUT; ++ } ++ openedTransaction = 1; ++ } ++ rc = MetaDwrCheckMeta(pBt); ++ if (rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB) { ++ // keep txn status after recover ++ rc = MetaDwrRecoverAndBeginTran(pBt, tnxState == SQLITE_TXN_WRITE ? 1 : 0, 0); ++ goto DWR_OPEN_OUT; ++ } ++ rc = Sqlite3MetaDwrCheckRestore(pBt); ++DWR_OPEN_OUT: ++ if (rc == SQLITE_OK && pBt->pBt->metaRecoverStatus == META_RECOVER_SUCCESS) { ++ rc = MetaDwrCheckMeta(pBt); ++ if (rc == SQLITE_OK) { ++ rc = SQLITE_META_RECOVERED; ++ } ++ } ++ /* Close the transaction, if one was opened. */ ++ if (openedTransaction) { ++ sqlite3BtreeCommit(pBt); ++ } ++ sqlite3BtreeLeave(pBt); ++ return rc; ++} + ++static void MetaDwrDisable(Btree *pBt) ++{ ++ Pager *pPager = pBt->pBt->pPager; ++ if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { ++ return; ++ } ++#ifdef SQLITE_HAS_CODEC ++ // not support codec right now ++ if (pPager->xCodec) { ++ return; ++ } ++#endif ++ sqlite3BtreeEnter(pBt); ++ MetaDwrCloseFile(pPager); ++ MetaDwrReleaseHdr(pPager->metaHdr); ++ pPager->metaHdr = NULL; ++ const char *metaPath = GetMetaFilePath(pPager); ++ if (metaPath != NULL) { ++ (void)osUnlink(metaPath); ++ } ++ if (pPager->metaFd) { ++ sqlite3_free(pPager->metaFd); ++ pPager->metaFd = NULL; ++ } ++ sqlite3BtreeLeave(pBt); ++} ++#endif + #if SQLITE_OS_UNIX + #include + #include +-- +2.34.1 + diff --git a/mock/sqlite/patch/0004-Enable-and-optimize-ICU.patch b/mock/sqlite/patch/0004-Enable-and-optimize-ICU.patch new file mode 100644 index 00000000..41747c37 --- /dev/null +++ b/mock/sqlite/patch/0004-Enable-and-optimize-ICU.patch @@ -0,0 +1,2119 @@ +From 5526b966d88575fd479dfe69af0c70c4ac43de14 Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Tue, 25 Feb 2025 17:02:54 +0800 +Subject: [PATCH] Enable and optimize ICU + +Signed-off-by: MartinChoo <214582617@qq.com> +--- + src/sqlite3.c | 1065 ++++------------------------------------------ + src/sqlite3icu.c | 925 ++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 1009 insertions(+), 981 deletions(-) + create mode 100644 src/sqlite3icu.c + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 17d0d25..f348f3c 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -2506,6 +2506,7 @@ struct sqlite3_mem_methods { + #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ + #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ + #define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ ++#define SQLITE_CONFIG_ENABLE_ICU 31 /* boolean */ + + /* + ** CAPI3REF: Database Connection Configuration Options +@@ -3283,6 +3284,17 @@ SQLITE_API int sqlite3_get_table( + ); + SQLITE_API void sqlite3_free_table(char **result); + ++// hw export the symbols ++#ifdef SQLITE_EXPORT_SYMBOLS ++#if defined(__GNUC__) ++# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) ++#elif defined(_MSC_VER) ++# define EXPORT_SYMBOLS __declspec(dllexport) ++#else ++# define EXPORT_SYMBOLS ++#endif ++#endif // SQLITE_EXPORT_SYMBOLS ++ + /* + ** CAPI3REF: Formatted String Printing Functions + ** +@@ -181005,6 +181017,7 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + /************** End of rtree.h ***********************************************/ + /************** Continuing where we left off in main.c ***********************/ + #endif ++#include "sqlite3tokenizer.h" + #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + /************** Include sqliteicu.h in the middle of main.c ******************/ + /************** Begin file sqliteicu.h ***************************************/ +@@ -181029,13 +181042,54 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + extern "C" { + #endif /* __cplusplus */ + +-SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); ++SQLITE_PRIVATE int sqlite3IcuInitInner(sqlite3 *db); + + #if 0 + } /* extern "C" */ + #endif /* __cplusplus */ + + /************** End of sqliteicu.h *******************************************/ ++#ifndef _WIN32 ++#include ++#endif ++ ++typedef void (*sqlite3Fts3IcuTokenizerModule_ptr)(sqlite3_tokenizer_module const** ppModule); ++typedef int (*sqlite3IcuInit_ptr)(sqlite3 *db); ++static sqlite3Fts3IcuTokenizerModule_ptr tokenModulePtr = NULL; ++static sqlite3IcuInit_ptr icuInitPtr = NULL; ++static u32 icuEnable = 0u; ++static u32 icuInit = 0u; ++static void *g_library = NULL; ++ ++int sqlite3IcuModuleInit(){ ++ int rc = SQLITE_OK; ++ if( icuInit ){ ++ return rc; ++ } ++#ifndef _WIN32 ++ g_library = dlopen("libsqliteicu.z.so", RTLD_LAZY); ++ if( g_library==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load icu so failed"); ++ return SQLITE_ERROR; ++ } ++ tokenModulePtr = (sqlite3Fts3IcuTokenizerModule_ptr)dlsym(g_library, "sqlite3Fts3IcuTokenizerModule"); ++ icuInitPtr = (sqlite3IcuInit_ptr)dlsym(g_library, "sqlite3IcuInit"); ++ if( tokenModulePtr==NULL || icuInitPtr==NULL ){ ++ sqlite3_log(SQLITE_ERROR, "load icu init function failed"); ++ return SQLITE_ERROR; ++ } ++ icuInit = 1u; ++#endif ++ return rc; ++} ++ ++SQLITE_PRIVATE int sqlite3IcuInitInner(sqlite3 *db) ++{ ++ if( !icuEnable ){ ++ return SQLITE_OK; ++ } ++ return icuInitPtr(db); ++} + /************** Continuing where we left off in main.c ***********************/ + #endif + +@@ -181075,7 +181129,7 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { + sqlite3Fts5Init, + #endif + #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +- sqlite3IcuInit, ++ sqlite3IcuInitInner, + #endif + #ifdef SQLITE_ENABLE_RTREE + sqlite3RtreeInit, +@@ -181467,6 +181521,19 @@ SQLITE_API int sqlite3_shutdown(void){ + SQLITE_API int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; ++ va_start(ap, op); ++ ++#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) ++ if( op==SQLITE_CONFIG_ENABLE_ICU ){ ++ int iVal = va_arg(ap, int); ++ if( iVal==0 ){ ++ icuEnable = 0u; ++ }else{ ++ icuEnable = 1u; ++ } ++ return rc; ++ } ++#endif /* SQLITE_ENABLE_ICU */ + + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes +@@ -181484,7 +181551,6 @@ SQLITE_API int sqlite3_config(int op, ...){ + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } + +- va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe +@@ -184609,6 +184675,12 @@ static int openDatabase( + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + ++#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) ++ if( icuEnable ){ ++ rc = sqlite3IcuModuleInit(); ++ if( rc!=SQLITE_OK ) return rc; ++ } ++#endif + + /* Load compiled-in extensions */ + for(i=0; rc==SQLITE_OK && i arg1 arg2) +- ** +- ** then argc is set to 2, and the argv[] array contains pointers +- ** to the strings "arg1" and "arg2". +- ** +- ** This method should return either SQLITE_OK (0), or an SQLite error +- ** code. If SQLITE_OK is returned, then *ppTokenizer should be set +- ** to point at the newly created tokenizer structure. The generic +- ** sqlite3_tokenizer.pModule variable should not be initialized by +- ** this callback. The caller will do so. +- */ +- int (*xCreate)( +- int argc, /* Size of argv array */ +- const char *const*argv, /* Tokenizer argument strings */ +- sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +- ); +- +- /* +- ** Destroy an existing tokenizer. The fts3 module calls this method +- ** exactly once for each successful call to xCreate(). +- */ +- int (*xDestroy)(sqlite3_tokenizer *pTokenizer); +- +- /* +- ** Create a tokenizer cursor to tokenize an input buffer. The caller +- ** is responsible for ensuring that the input buffer remains valid +- ** until the cursor is closed (using the xClose() method). +- */ +- int (*xOpen)( +- sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ +- const char *pInput, int nBytes, /* Input buffer */ +- sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ +- ); +- +- /* +- ** Destroy an existing tokenizer cursor. The fts3 module calls this +- ** method exactly once for each successful call to xOpen(). +- */ +- int (*xClose)(sqlite3_tokenizer_cursor *pCursor); +- +- /* +- ** Retrieve the next token from the tokenizer cursor pCursor. This +- ** method should either return SQLITE_OK and set the values of the +- ** "OUT" variables identified below, or SQLITE_DONE to indicate that +- ** the end of the buffer has been reached, or an SQLite error code. +- ** +- ** *ppToken should be set to point at a buffer containing the +- ** normalized version of the token (i.e. after any case-folding and/or +- ** stemming has been performed). *pnBytes should be set to the length +- ** of this buffer in bytes. The input text that generated the token is +- ** identified by the byte offsets returned in *piStartOffset and +- ** *piEndOffset. *piStartOffset should be set to the index of the first +- ** byte of the token in the input buffer. *piEndOffset should be set +- ** to the index of the first byte just past the end of the token in +- ** the input buffer. +- ** +- ** The buffer *ppToken is set to point at is managed by the tokenizer +- ** implementation. It is only required to be valid until the next call +- ** to xNext() or xClose(). +- */ +- /* TODO(shess) current implementation requires pInput to be +- ** nul-terminated. This should either be fixed, or pInput/nBytes +- ** should be converted to zInput. +- */ +- int (*xNext)( +- sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ +- const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ +- int *piStartOffset, /* OUT: Byte offset of token in input buffer */ +- int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ +- int *piPosition /* OUT: Number of tokens returned before this one */ +- ); +- +- /*********************************************************************** +- ** Methods below this point are only available if iVersion>=1. +- */ +- +- /* +- ** Configure the language id of a tokenizer cursor. +- */ +- int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +-}; +- +-struct sqlite3_tokenizer { +- const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ +- /* Tokenizer implementations will typically add additional fields */ +-}; +- +-struct sqlite3_tokenizer_cursor { +- sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ +- /* Tokenizer implementations will typically add additional fields */ +-}; + + int fts3_global_term_cnt(int iTerm, int iCol); + int fts3_term_cnt(int iTerm, int iCol); +@@ -191576,9 +191540,6 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module co + #ifndef SQLITE_DISABLE_FTS3_UNICODE + SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); + #endif +-#ifdef SQLITE_ENABLE_ICU +-SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +-#endif + + /* + ** Initialize the fts3 extension. If this extension is built as part +@@ -191597,7 +191558,14 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + + #ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; +- sqlite3Fts3IcuTokenizerModule(&pIcu); ++ if( icuEnable ){ ++ if( tokenModulePtr!=NULL ){ ++ tokenModulePtr(&pIcu); ++ }else{ ++ sqlite3_log(SQLITE_ERROR, "icu module ptr is null"); ++ return SQLITE_ERROR; ++ } ++ } + #endif + + #ifndef SQLITE_DISABLE_FTS3_UNICODE +@@ -191633,7 +191601,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + || sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode) + #endif + #ifdef SQLITE_ENABLE_ICU +- || (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) ++ || (icuEnable && pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) + #endif + ){ + rc = SQLITE_NOMEM; +@@ -217955,862 +217923,6 @@ SQLITE_API int sqlite3_rtree_init( + #endif + + /************** End of rtree.c ***********************************************/ +-/************** Begin file icu.c *********************************************/ +-/* +-** 2007 May 6 +-** +-** The author disclaims copyright to this source code. In place of +-** a legal notice, here is a blessing: +-** +-** May you do good and not evil. +-** May you find forgiveness for yourself and forgive others. +-** May you share freely, never taking more than you give. +-** +-************************************************************************* +-** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +-** +-** This file implements an integration between the ICU library +-** ("International Components for Unicode", an open-source library +-** for handling unicode data) and SQLite. The integration uses +-** ICU to provide the following to SQLite: +-** +-** * An implementation of the SQL regexp() function (and hence REGEXP +-** operator) using the ICU uregex_XX() APIs. +-** +-** * Implementations of the SQL scalar upper() and lower() functions +-** for case mapping. +-** +-** * Integration of ICU and SQLite collation sequences. +-** +-** * An implementation of the LIKE operator that uses ICU to +-** provide case-independent matching. +-*/ +- +-#if !defined(SQLITE_CORE) \ +- || defined(SQLITE_ENABLE_ICU) \ +- || defined(SQLITE_ENABLE_ICU_COLLATIONS) +- +-/* Include ICU headers */ +-#include +-#include +-#include +-#include +- +-/* #include */ +- +-#ifndef SQLITE_CORE +-/* #include "sqlite3ext.h" */ +- SQLITE_EXTENSION_INIT1 +-#else +-/* #include "sqlite3.h" */ +-#endif +- +-/* +-** This function is called when an ICU function called from within +-** the implementation of an SQL scalar function returns an error. +-** +-** The scalar function context passed as the first argument is +-** loaded with an error message based on the following two args. +-*/ +-static void icuFunctionError( +- sqlite3_context *pCtx, /* SQLite scalar function context */ +- const char *zName, /* Name of ICU function that failed */ +- UErrorCode e /* Error code returned by ICU function */ +-){ +- char zBuf[128]; +- sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); +- zBuf[127] = '\0'; +- sqlite3_result_error(pCtx, zBuf, -1); +-} +- +-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) +- +-/* +-** Maximum length (in bytes) of the pattern in a LIKE or GLOB +-** operator. +-*/ +-#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +-# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +-#endif +- +-/* +-** Version of sqlite3_free() that is always a function, never a macro. +-*/ +-static void xFree(void *p){ +- sqlite3_free(p); +-} +- +-/* +-** This lookup table is used to help decode the first byte of +-** a multi-byte UTF8 character. It is copied here from SQLite source +-** code file utf8.c. +-*/ +-static const unsigned char icuUtf8Trans1[] = { +- 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, +- 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +- 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, +- 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, +- 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, +- 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +- 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, +- 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +-}; +- +-#define SQLITE_ICU_READ_UTF8(zIn, c) \ +- c = *(zIn++); \ +- if( c>=0xc0 ){ \ +- c = icuUtf8Trans1[c-0xc0]; \ +- while( (*zIn & 0xc0)==0x80 ){ \ +- c = (c<<6) + (0x3f & *(zIn++)); \ +- } \ +- } +- +-#define SQLITE_ICU_SKIP_UTF8(zIn) \ +- assert( *zIn ); \ +- if( *(zIn++)>=0xc0 ){ \ +- while( (*zIn & 0xc0)==0x80 ){zIn++;} \ +- } +- +- +-/* +-** Compare two UTF-8 strings for equality where the first string is +-** a "LIKE" expression. Return true (1) if they are the same and +-** false (0) if they are different. +-*/ +-static int icuLikeCompare( +- const uint8_t *zPattern, /* LIKE pattern */ +- const uint8_t *zString, /* The UTF-8 string to compare against */ +- const UChar32 uEsc /* The escape character */ +-){ +- static const uint32_t MATCH_ONE = (uint32_t)'_'; +- static const uint32_t MATCH_ALL = (uint32_t)'%'; +- +- int prevEscape = 0; /* True if the previous character was uEsc */ +- +- while( 1 ){ +- +- /* Read (and consume) the next character from the input pattern. */ +- uint32_t uPattern; +- SQLITE_ICU_READ_UTF8(zPattern, uPattern); +- if( uPattern==0 ) break; +- +- /* There are now 4 possibilities: +- ** +- ** 1. uPattern is an unescaped match-all character "%", +- ** 2. uPattern is an unescaped match-one character "_", +- ** 3. uPattern is an unescaped escape character, or +- ** 4. uPattern is to be handled as an ordinary character +- */ +- if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ +- /* Case 1. */ +- uint8_t c; +- +- /* Skip any MATCH_ALL or MATCH_ONE characters that follow a +- ** MATCH_ALL. For each MATCH_ONE, skip one character in the +- ** test string. +- */ +- while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ +- if( c==MATCH_ONE ){ +- if( *zString==0 ) return 0; +- SQLITE_ICU_SKIP_UTF8(zString); +- } +- zPattern++; +- } +- +- if( *zPattern==0 ) return 1; +- +- while( *zString ){ +- if( icuLikeCompare(zPattern, zString, uEsc) ){ +- return 1; +- } +- SQLITE_ICU_SKIP_UTF8(zString); +- } +- return 0; +- +- }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ +- /* Case 2. */ +- if( *zString==0 ) return 0; +- SQLITE_ICU_SKIP_UTF8(zString); +- +- }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ +- /* Case 3. */ +- prevEscape = 1; +- +- }else{ +- /* Case 4. */ +- uint32_t uString; +- SQLITE_ICU_READ_UTF8(zString, uString); +- uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); +- uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); +- if( uString!=uPattern ){ +- return 0; +- } +- prevEscape = 0; +- } +- } +- +- return *zString==0; +-} +- +-/* +-** Implementation of the like() SQL function. This function implements +-** the build-in LIKE operator. The first argument to the function is the +-** pattern and the second argument is the string. So, the SQL statements: +-** +-** A LIKE B +-** +-** is implemented as like(B, A). If there is an escape character E, +-** +-** A LIKE B ESCAPE E +-** +-** is mapped to like(B, A, E). +-*/ +-static void icuLikeFunc( +- sqlite3_context *context, +- int argc, +- sqlite3_value **argv +-){ +- const unsigned char *zA = sqlite3_value_text(argv[0]); +- const unsigned char *zB = sqlite3_value_text(argv[1]); +- UChar32 uEsc = 0; +- +- /* Limit the length of the LIKE or GLOB pattern to avoid problems +- ** of deep recursion and N*N behavior in patternCompare(). +- */ +- if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ +- sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); +- return; +- } +- +- +- if( argc==3 ){ +- /* The escape character string must consist of a single UTF-8 character. +- ** Otherwise, return an error. +- */ +- int nE= sqlite3_value_bytes(argv[2]); +- const unsigned char *zE = sqlite3_value_text(argv[2]); +- int i = 0; +- if( zE==0 ) return; +- U8_NEXT(zE, i, nE, uEsc); +- if( i!=nE){ +- sqlite3_result_error(context, +- "ESCAPE expression must be a single character", -1); +- return; +- } +- } +- +- if( zA && zB ){ +- sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); +- } +-} +- +-/* +-** Function to delete compiled regexp objects. Registered as +-** a destructor function with sqlite3_set_auxdata(). +-*/ +-static void icuRegexpDelete(void *p){ +- URegularExpression *pExpr = (URegularExpression *)p; +- uregex_close(pExpr); +-} +- +-/* +-** Implementation of SQLite REGEXP operator. This scalar function takes +-** two arguments. The first is a regular expression pattern to compile +-** the second is a string to match against that pattern. If either +-** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +-** is 1 if the string matches the pattern, or 0 otherwise. +-** +-** SQLite maps the regexp() function to the regexp() operator such +-** that the following two are equivalent: +-** +-** zString REGEXP zPattern +-** regexp(zPattern, zString) +-** +-** Uses the following ICU regexp APIs: +-** +-** uregex_open() +-** uregex_matches() +-** uregex_close() +-*/ +-static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ +- UErrorCode status = U_ZERO_ERROR; +- URegularExpression *pExpr; +- UBool res; +- const UChar *zString = sqlite3_value_text16(apArg[1]); +- +- (void)nArg; /* Unused parameter */ +- +- /* If the left hand side of the regexp operator is NULL, +- ** then the result is also NULL. +- */ +- if( !zString ){ +- return; +- } +- +- pExpr = sqlite3_get_auxdata(p, 0); +- if( !pExpr ){ +- const UChar *zPattern = sqlite3_value_text16(apArg[0]); +- if( !zPattern ){ +- return; +- } +- pExpr = uregex_open(zPattern, -1, 0, 0, &status); +- +- if( U_SUCCESS(status) ){ +- sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); +- pExpr = sqlite3_get_auxdata(p, 0); +- } +- if( !pExpr ){ +- icuFunctionError(p, "uregex_open", status); +- return; +- } +- } +- +- /* Configure the text that the regular expression operates on. */ +- uregex_setText(pExpr, zString, -1, &status); +- if( !U_SUCCESS(status) ){ +- icuFunctionError(p, "uregex_setText", status); +- return; +- } +- +- /* Attempt the match */ +- res = uregex_matches(pExpr, 0, &status); +- if( !U_SUCCESS(status) ){ +- icuFunctionError(p, "uregex_matches", status); +- return; +- } +- +- /* Set the text that the regular expression operates on to a NULL +- ** pointer. This is not really necessary, but it is tidier than +- ** leaving the regular expression object configured with an invalid +- ** pointer after this function returns. +- */ +- uregex_setText(pExpr, 0, 0, &status); +- +- /* Return 1 or 0. */ +- sqlite3_result_int(p, res ? 1 : 0); +-} +- +-/* +-** Implementations of scalar functions for case mapping - upper() and +-** lower(). Function upper() converts its input to upper-case (ABC). +-** Function lower() converts to lower-case (abc). +-** +-** ICU provides two types of case mapping, "general" case mapping and +-** "language specific". Refer to ICU documentation for the differences +-** between the two. +-** +-** To utilise "general" case mapping, the upper() or lower() scalar +-** functions are invoked with one argument: +-** +-** upper('ABC') -> 'abc' +-** lower('abc') -> 'ABC' +-** +-** To access ICU "language specific" case mapping, upper() or lower() +-** should be invoked with two arguments. The second argument is the name +-** of the locale to use. Passing an empty string ("") or SQL NULL value +-** as the second argument is the same as invoking the 1 argument version +-** of upper() or lower(). +-** +-** lower('I', 'en_us') -> 'i' +-** lower('I', 'tr_tr') -> '\u131' (small dotless i) +-** +-** http://www.icu-project.org/userguide/posix.html#case_mappings +-*/ +-static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ +- const UChar *zInput; /* Pointer to input string */ +- UChar *zOutput = 0; /* Pointer to output buffer */ +- int nInput; /* Size of utf-16 input string in bytes */ +- int nOut; /* Size of output buffer in bytes */ +- int cnt; +- int bToUpper; /* True for toupper(), false for tolower() */ +- UErrorCode status; +- const char *zLocale = 0; +- +- assert(nArg==1 || nArg==2); +- bToUpper = (sqlite3_user_data(p)!=0); +- if( nArg==2 ){ +- zLocale = (const char *)sqlite3_value_text(apArg[1]); +- } +- +- zInput = sqlite3_value_text16(apArg[0]); +- if( !zInput ){ +- return; +- } +- nOut = nInput = sqlite3_value_bytes16(apArg[0]); +- if( nOut==0 ){ +- sqlite3_result_text16(p, "", 0, SQLITE_STATIC); +- return; +- } +- +- for(cnt=0; cnt<2; cnt++){ +- UChar *zNew = sqlite3_realloc(zOutput, nOut); +- if( zNew==0 ){ +- sqlite3_free(zOutput); +- sqlite3_result_error_nomem(p); +- return; +- } +- zOutput = zNew; +- status = U_ZERO_ERROR; +- if( bToUpper ){ +- nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); +- }else{ +- nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); +- } +- +- if( U_SUCCESS(status) ){ +- sqlite3_result_text16(p, zOutput, nOut, xFree); +- }else if( status==U_BUFFER_OVERFLOW_ERROR ){ +- assert( cnt==0 ); +- continue; +- }else{ +- icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); +- } +- return; +- } +- assert( 0 ); /* Unreachable */ +-} +- +-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ +- +-/* +-** Collation sequence destructor function. The pCtx argument points to +-** a UCollator structure previously allocated using ucol_open(). +-*/ +-static void icuCollationDel(void *pCtx){ +- UCollator *p = (UCollator *)pCtx; +- ucol_close(p); +-} +- +-/* +-** Collation sequence comparison function. The pCtx argument points to +-** a UCollator structure previously allocated using ucol_open(). +-*/ +-static int icuCollationColl( +- void *pCtx, +- int nLeft, +- const void *zLeft, +- int nRight, +- const void *zRight +-){ +- UCollationResult res; +- UCollator *p = (UCollator *)pCtx; +- res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); +- switch( res ){ +- case UCOL_LESS: return -1; +- case UCOL_GREATER: return +1; +- case UCOL_EQUAL: return 0; +- } +- assert(!"Unexpected return value from ucol_strcoll()"); +- return 0; +-} +- +-/* +-** Implementation of the scalar function icu_load_collation(). +-** +-** This scalar function is used to add ICU collation based collation +-** types to an SQLite database connection. It is intended to be called +-** as follows: +-** +-** SELECT icu_load_collation(, ); +-** +-** Where is a string containing an ICU locale identifier (i.e. +-** "en_AU", "tr_TR" etc.) and is the name of the +-** collation sequence to create. +-*/ +-static void icuLoadCollation( +- sqlite3_context *p, +- int nArg, +- sqlite3_value **apArg +-){ +- sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); +- UErrorCode status = U_ZERO_ERROR; +- const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ +- const char *zName; /* SQL Collation sequence name (eg. "japanese") */ +- UCollator *pUCollator; /* ICU library collation object */ +- int rc; /* Return code from sqlite3_create_collation_x() */ +- +- assert(nArg==2 || nArg==3); +- (void)nArg; /* Unused parameter */ +- zLocale = (const char *)sqlite3_value_text(apArg[0]); +- zName = (const char *)sqlite3_value_text(apArg[1]); +- +- if( !zLocale || !zName ){ +- return; +- } +- +- pUCollator = ucol_open(zLocale, &status); +- if( !U_SUCCESS(status) ){ +- icuFunctionError(p, "ucol_open", status); +- return; +- } +- assert(p); +- if(nArg==3){ +- const char *zOption = (const char*)sqlite3_value_text(apArg[2]); +- static const struct { +- const char *zName; +- UColAttributeValue val; +- } aStrength[] = { +- { "PRIMARY", UCOL_PRIMARY }, +- { "SECONDARY", UCOL_SECONDARY }, +- { "TERTIARY", UCOL_TERTIARY }, +- { "DEFAULT", UCOL_DEFAULT_STRENGTH }, +- { "QUARTERNARY", UCOL_QUATERNARY }, +- { "IDENTICAL", UCOL_IDENTICAL }, +- }; +- unsigned int i; +- for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ +- sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); +- sqlite3_str_appendf(pStr, +- "unknown collation strength \"%s\" - should be one of:", +- zOption); +- for(i=0; izName, p->nArg, p->enc, +- p->iContext ? (void*)db : (void*)0, +- p->xFunc, 0, 0 +- ); +- } +- +- return rc; +-} +- +-#if !SQLITE_CORE +-#ifdef _WIN32 +-__declspec(dllexport) +-#endif +-SQLITE_API int sqlite3_icu_init( +- sqlite3 *db, +- char **pzErrMsg, +- const sqlite3_api_routines *pApi +-){ +- SQLITE_EXTENSION_INIT2(pApi) +- return sqlite3IcuInit(db); +-} +-#endif +- +-#endif +- +-/************** End of icu.c *************************************************/ +-/************** Begin file fts3_icu.c ****************************************/ +-/* +-** 2007 June 22 +-** +-** The author disclaims copyright to this source code. In place of +-** a legal notice, here is a blessing: +-** +-** May you do good and not evil. +-** May you find forgiveness for yourself and forgive others. +-** May you share freely, never taking more than you give. +-** +-************************************************************************* +-** This file implements a tokenizer for fts3 based on the ICU library. +-*/ +-/* #include "fts3Int.h" */ +-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +-#ifdef SQLITE_ENABLE_ICU +- +-/* #include */ +-/* #include */ +-/* #include "fts3_tokenizer.h" */ +- +-#include +-/* #include */ +-/* #include */ +-#include +- +-typedef struct IcuTokenizer IcuTokenizer; +-typedef struct IcuCursor IcuCursor; +- +-struct IcuTokenizer { +- sqlite3_tokenizer base; +- char *zLocale; +-}; +- +-struct IcuCursor { +- sqlite3_tokenizer_cursor base; +- +- UBreakIterator *pIter; /* ICU break-iterator object */ +- int nChar; /* Number of UChar elements in pInput */ +- UChar *aChar; /* Copy of input using utf-16 encoding */ +- int *aOffset; /* Offsets of each character in utf-8 input */ +- +- int nBuffer; +- char *zBuffer; +- +- int iToken; +-}; +- +-/* +-** Create a new tokenizer instance. +-*/ +-static int icuCreate( +- int argc, /* Number of entries in argv[] */ +- const char * const *argv, /* Tokenizer creation arguments */ +- sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +-){ +- IcuTokenizer *p; +- int n = 0; +- +- if( argc>0 ){ +- n = strlen(argv[0])+1; +- } +- p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); +- if( !p ){ +- return SQLITE_NOMEM; +- } +- memset(p, 0, sizeof(IcuTokenizer)); +- +- if( n ){ +- p->zLocale = (char *)&p[1]; +- memcpy(p->zLocale, argv[0], n); +- } +- +- *ppTokenizer = (sqlite3_tokenizer *)p; +- +- return SQLITE_OK; +-} +- +-/* +-** Destroy a tokenizer +-*/ +-static int icuDestroy(sqlite3_tokenizer *pTokenizer){ +- IcuTokenizer *p = (IcuTokenizer *)pTokenizer; +- sqlite3_free(p); +- return SQLITE_OK; +-} +- +-/* +-** Prepare to begin tokenizing a particular string. The input +-** string to be tokenized is pInput[0..nBytes-1]. A cursor +-** used to incrementally tokenize this string is returned in +-** *ppCursor. +-*/ +-static int icuOpen( +- sqlite3_tokenizer *pTokenizer, /* The tokenizer */ +- const char *zInput, /* Input string */ +- int nInput, /* Length of zInput in bytes */ +- sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +-){ +- IcuTokenizer *p = (IcuTokenizer *)pTokenizer; +- IcuCursor *pCsr; +- +- const int32_t opt = U_FOLD_CASE_DEFAULT; +- UErrorCode status = U_ZERO_ERROR; +- int nChar; +- +- UChar32 c; +- int iInput = 0; +- int iOut = 0; +- +- *ppCursor = 0; +- +- if( zInput==0 ){ +- nInput = 0; +- zInput = ""; +- }else if( nInput<0 ){ +- nInput = strlen(zInput); +- } +- nChar = nInput+1; +- pCsr = (IcuCursor *)sqlite3_malloc64( +- sizeof(IcuCursor) + /* IcuCursor */ +- ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ +- (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ +- ); +- if( !pCsr ){ +- return SQLITE_NOMEM; +- } +- memset(pCsr, 0, sizeof(IcuCursor)); +- pCsr->aChar = (UChar *)&pCsr[1]; +- pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; +- +- pCsr->aOffset[iOut] = iInput; +- U8_NEXT(zInput, iInput, nInput, c); +- while( c>0 ){ +- int isError = 0; +- c = u_foldCase(c, opt); +- U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); +- if( isError ){ +- sqlite3_free(pCsr); +- return SQLITE_ERROR; +- } +- pCsr->aOffset[iOut] = iInput; +- +- if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); +- if( !U_SUCCESS(status) ){ +- sqlite3_free(pCsr); +- return SQLITE_ERROR; +- } +- pCsr->nChar = iOut; +- +- ubrk_first(pCsr->pIter); +- *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; +- return SQLITE_OK; +-} +- +-/* +-** Close a tokenization cursor previously opened by a call to icuOpen(). +-*/ +-static int icuClose(sqlite3_tokenizer_cursor *pCursor){ +- IcuCursor *pCsr = (IcuCursor *)pCursor; +- ubrk_close(pCsr->pIter); +- sqlite3_free(pCsr->zBuffer); +- sqlite3_free(pCsr); +- return SQLITE_OK; +-} +- +-/* +-** Extract the next token from a tokenization cursor. +-*/ +-static int icuNext( +- sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ +- const char **ppToken, /* OUT: *ppToken is the token text */ +- int *pnBytes, /* OUT: Number of bytes in token */ +- int *piStartOffset, /* OUT: Starting offset of token */ +- int *piEndOffset, /* OUT: Ending offset of token */ +- int *piPosition /* OUT: Position integer of token */ +-){ +- IcuCursor *pCsr = (IcuCursor *)pCursor; +- +- int iStart = 0; +- int iEnd = 0; +- int nByte = 0; +- +- while( iStart==iEnd ){ +- UChar32 c; +- +- iStart = ubrk_current(pCsr->pIter); +- iEnd = ubrk_next(pCsr->pIter); +- if( iEnd==UBRK_DONE ){ +- return SQLITE_DONE; +- } +- +- while( iStartaChar, iWhite, pCsr->nChar, c); +- if( u_isspace(c) ){ +- iStart = iWhite; +- }else{ +- break; +- } +- } +- assert(iStart<=iEnd); +- } +- +- do { +- UErrorCode status = U_ZERO_ERROR; +- if( nByte ){ +- char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); +- if( !zNew ){ +- return SQLITE_NOMEM; +- } +- pCsr->zBuffer = zNew; +- pCsr->nBuffer = nByte; +- } +- +- u_strToUTF8( +- pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ +- &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ +- &status /* Output success/failure */ +- ); +- } while( nByte>pCsr->nBuffer ); +- +- *ppToken = pCsr->zBuffer; +- *pnBytes = nByte; +- *piStartOffset = pCsr->aOffset[iStart]; +- *piEndOffset = pCsr->aOffset[iEnd]; +- *piPosition = pCsr->iToken++; +- +- return SQLITE_OK; +-} +- +-/* +-** The set of routines that implement the simple tokenizer +-*/ +-static const sqlite3_tokenizer_module icuTokenizerModule = { +- 0, /* iVersion */ +- icuCreate, /* xCreate */ +- icuDestroy, /* xCreate */ +- icuOpen, /* xOpen */ +- icuClose, /* xClose */ +- icuNext, /* xNext */ +- 0, /* xLanguageid */ +-}; +- +-/* +-** Set *ppModule to point at the implementation of the ICU tokenizer. +-*/ +-SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( +- sqlite3_tokenizer_module const**ppModule +-){ +- *ppModule = &icuTokenizerModule; +-} +- +-#endif /* defined(SQLITE_ENABLE_ICU) */ +-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +- +-/************** End of fts3_icu.c ********************************************/ + /************** Begin file sqlite3rbu.c **************************************/ + /* + ** 2014 August 30 +@@ -261916,15 +261028,6 @@ static void DumpLocksByPager(Pager *pPager) + + // hw export the symbols + #ifdef SQLITE_EXPORT_SYMBOLS +-/************** Begin hw export the symbols *****************************************/ +-#if defined(__GNUC__) +-# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) +-#elif defined(_MSC_VER) +-# define EXPORT_SYMBOLS __declspec(dllexport) +-#else +-# define EXPORT_SYMBOLS +-#endif +- + struct sqlite3_api_routines_hw { + int (*initialize)(); + int (*config)(int,...); +diff --git a/src/sqlite3icu.c b/src/sqlite3icu.c +new file mode 100644 +index 0000000..75aa78c +--- /dev/null ++++ b/src/sqlite3icu.c +@@ -0,0 +1,925 @@ ++/****************************************************************************** ++** This file is an amalgamation of many separate C source files from SQLite ++** version 3.40.1. By combining all the individual C code files into this ++** single large file, the entire code can be compiled as a single translation ++** unit. This allows many compilers to do optimizations that would not be ++** possible if the files were compiled separately. Performance improvements ++** of 5% or more are commonly seen when SQLite is compiled as a single ++** translation unit. ++** ++** This file is all you need to compile SQLite. To use SQLite in other ++** programs, you need this file and the "sqlite3.h" header file that defines ++** the programming interface to the SQLite library. (If you do not have ++** the "sqlite3.h" header file at hand, you will find a copy embedded within ++** the text of this file. Search for "Begin file sqlite3.h" to find the start ++** of the embedded sqlite3.h header file.) Additional code files may be needed ++** if you want a wrapper to interface SQLite with your choice of programming ++** language. The code for the "sqlite3" command-line shell is also in a ++** separate file. This file contains only code for the core SQLite library. ++*/ ++/* ++** 2019.09.02-Complete codec logic for encryption and decryption. ++** Huawei Technologies Co, Ltd. ++*/ ++/************** Begin file icu.c *********************************************/ ++/* ++** 2007 May 6 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++************************************************************************* ++** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ ++** ++** This file implements an integration between the ICU library ++** ("International Components for Unicode", an open-source library ++** for handling unicode data) and SQLite. The integration uses ++** ICU to provide the following to SQLite: ++** ++** * An implementation of the SQL regexp() function (and hence REGEXP ++** operator) using the ICU uregex_XX() APIs. ++** ++** * Implementations of the SQL scalar upper() and lower() functions ++** for case mapping. ++** ++** * Integration of ICU and SQLite collation sequences. ++** ++** * An implementation of the LIKE operator that uses ICU to ++** provide case-independent matching. ++*/ ++#include ++#include ++#include ++#include ++#include ++ ++#include "sqlite3icu.h" ++#include "sqlite3.h" ++ ++#ifdef HARMONY_OS ++#include "common/unicode/putil.h" ++#endif ++ ++#if !defined(SQLITE_CORE) \ ++ || defined(SQLITE_ENABLE_ICU) \ ++ || defined(SQLITE_ENABLE_ICU_COLLATIONS) ++ ++/* Include ICU headers */ ++#include ++#include ++#include ++#include ++ ++#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) ++ /* This case when the file really is being compiled as a loadable ++ ** extension */ ++# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; ++# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; ++# define SQLITE_EXTENSION_INIT3 \ ++ extern const sqlite3_api_routines *sqlite3_api; ++#else ++ /* This case when the file is being statically linked into the ++ ** application */ ++# define SQLITE_EXTENSION_INIT1 /*no-op*/ ++# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ ++# define SQLITE_EXTENSION_INIT3 /*no-op*/ ++#endif ++ ++/* #include */ ++ ++#ifndef SQLITE_CORE ++/* #include "sqlite3ext.h" */ ++ SQLITE_EXTENSION_INIT1 ++#else ++/* #include "sqlite3.h" */ ++#endif ++ ++// hw export the symbols ++#ifdef SQLITE_EXPORT_SYMBOLS ++#if defined(__GNUC__) ++# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) ++#elif defined(_MSC_VER) ++# define EXPORT_SYMBOLS __declspec(dllexport) ++#else ++# define EXPORT_SYMBOLS ++#endif ++#endif ++ ++EXPORT_SYMBOLS SQLITE_API int sqlite3IcuInit(sqlite3 *db); ++#ifdef SQLITE_ENABLE_ICU ++EXPORT_SYMBOLS SQLITE_API void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); ++#endif ++/* ++** This function is called when an ICU function called from within ++** the implementation of an SQL scalar function returns an error. ++** ++** The scalar function context passed as the first argument is ++** loaded with an error message based on the following two args. ++*/ ++static void icuFunctionError( ++ sqlite3_context *pCtx, /* SQLite scalar function context */ ++ const char *zName, /* Name of ICU function that failed */ ++ UErrorCode e /* Error code returned by ICU function */ ++){ ++ char zBuf[128]; ++ sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); ++ zBuf[127] = '\0'; ++ sqlite3_result_error(pCtx, zBuf, -1); ++} ++ ++#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) ++ ++/* ++** Maximum length (in bytes) of the pattern in a LIKE or GLOB ++** operator. ++*/ ++#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH ++# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 ++#endif ++ ++/* ++** Version of sqlite3_free() that is always a function, never a macro. ++*/ ++static void xFree(void *p){ ++ sqlite3_free(p); ++} ++ ++/* ++** This lookup table is used to help decode the first byte of ++** a multi-byte UTF8 character. It is copied here from SQLite source ++** code file utf8.c. ++*/ ++static const unsigned char icuUtf8Trans1[] = { ++ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, ++ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, ++ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ++ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, ++ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ++ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, ++}; ++ ++#define SQLITE_ICU_READ_UTF8(zIn, c) \ ++ c = *(zIn++); \ ++ if( c>=0xc0 ){ \ ++ c = icuUtf8Trans1[c-0xc0]; \ ++ while( (*zIn & 0xc0)==0x80 ){ \ ++ c = (c<<6) + (0x3f & *(zIn++)); \ ++ } \ ++ } ++ ++#define SQLITE_ICU_SKIP_UTF8(zIn) \ ++ assert( *zIn ); \ ++ if( *(zIn++)>=0xc0 ){ \ ++ while( (*zIn & 0xc0)==0x80 ){zIn++;} \ ++ } ++ ++ ++/* ++** Compare two UTF-8 strings for equality where the first string is ++** a "LIKE" expression. Return true (1) if they are the same and ++** false (0) if they are different. ++*/ ++static int icuLikeCompare( ++ const uint8_t *zPattern, /* LIKE pattern */ ++ const uint8_t *zString, /* The UTF-8 string to compare against */ ++ const UChar32 uEsc /* The escape character */ ++){ ++ static const uint32_t MATCH_ONE = (uint32_t)'_'; ++ static const uint32_t MATCH_ALL = (uint32_t)'%'; ++ ++ int prevEscape = 0; /* True if the previous character was uEsc */ ++ ++ while( 1 ){ ++ ++ /* Read (and consume) the next character from the input pattern. */ ++ uint32_t uPattern; ++ SQLITE_ICU_READ_UTF8(zPattern, uPattern); ++ if( uPattern==0 ) break; ++ ++ /* There are now 4 possibilities: ++ ** ++ ** 1. uPattern is an unescaped match-all character "%", ++ ** 2. uPattern is an unescaped match-one character "_", ++ ** 3. uPattern is an unescaped escape character, or ++ ** 4. uPattern is to be handled as an ordinary character ++ */ ++ if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ ++ /* Case 1. */ ++ uint8_t c; ++ ++ /* Skip any MATCH_ALL or MATCH_ONE characters that follow a ++ ** MATCH_ALL. For each MATCH_ONE, skip one character in the ++ ** test string. ++ */ ++ while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ ++ if( c==MATCH_ONE ){ ++ if( *zString==0 ) return 0; ++ SQLITE_ICU_SKIP_UTF8(zString); ++ } ++ zPattern++; ++ } ++ ++ if( *zPattern==0 ) return 1; ++ ++ while( *zString ){ ++ if( icuLikeCompare(zPattern, zString, uEsc) ){ ++ return 1; ++ } ++ SQLITE_ICU_SKIP_UTF8(zString); ++ } ++ return 0; ++ ++ }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ ++ /* Case 2. */ ++ if( *zString==0 ) return 0; ++ SQLITE_ICU_SKIP_UTF8(zString); ++ ++ }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ ++ /* Case 3. */ ++ prevEscape = 1; ++ ++ }else{ ++ /* Case 4. */ ++ uint32_t uString; ++ SQLITE_ICU_READ_UTF8(zString, uString); ++ uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); ++ uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); ++ if( uString!=uPattern ){ ++ return 0; ++ } ++ prevEscape = 0; ++ } ++ } ++ ++ return *zString==0; ++} ++ ++/* ++** Implementation of the like() SQL function. This function implements ++** the build-in LIKE operator. The first argument to the function is the ++** pattern and the second argument is the string. So, the SQL statements: ++** ++** A LIKE B ++** ++** is implemented as like(B, A). If there is an escape character E, ++** ++** A LIKE B ESCAPE E ++** ++** is mapped to like(B, A, E). ++*/ ++static void icuLikeFunc( ++ sqlite3_context *context, ++ int argc, ++ sqlite3_value **argv ++){ ++ const unsigned char *zA = sqlite3_value_text(argv[0]); ++ const unsigned char *zB = sqlite3_value_text(argv[1]); ++ UChar32 uEsc = 0; ++ ++ /* Limit the length of the LIKE or GLOB pattern to avoid problems ++ ** of deep recursion and N*N behavior in patternCompare(). ++ */ ++ if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ ++ sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); ++ return; ++ } ++ ++ ++ if( argc==3 ){ ++ /* The escape character string must consist of a single UTF-8 character. ++ ** Otherwise, return an error. ++ */ ++ int nE= sqlite3_value_bytes(argv[2]); ++ const unsigned char *zE = sqlite3_value_text(argv[2]); ++ int i = 0; ++ if( zE==0 ) return; ++ U8_NEXT(zE, i, nE, uEsc); ++ if( i!=nE){ ++ sqlite3_result_error(context, ++ "ESCAPE expression must be a single character", -1); ++ return; ++ } ++ } ++ ++ if( zA && zB ){ ++ sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); ++ } ++} ++ ++/* ++** Function to delete compiled regexp objects. Registered as ++** a destructor function with sqlite3_set_auxdata(). ++*/ ++static void icuRegexpDelete(void *p){ ++ URegularExpression *pExpr = (URegularExpression *)p; ++ uregex_close(pExpr); ++} ++ ++/* ++** Implementation of SQLite REGEXP operator. This scalar function takes ++** two arguments. The first is a regular expression pattern to compile ++** the second is a string to match against that pattern. If either ++** argument is an SQL NULL, then NULL Is returned. Otherwise, the result ++** is 1 if the string matches the pattern, or 0 otherwise. ++** ++** SQLite maps the regexp() function to the regexp() operator such ++** that the following two are equivalent: ++** ++** zString REGEXP zPattern ++** regexp(zPattern, zString) ++** ++** Uses the following ICU regexp APIs: ++** ++** uregex_open() ++** uregex_matches() ++** uregex_close() ++*/ ++static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ++ UErrorCode status = U_ZERO_ERROR; ++ URegularExpression *pExpr; ++ UBool res; ++ const UChar *zString = sqlite3_value_text16(apArg[1]); ++ ++ (void)nArg; /* Unused parameter */ ++ ++ /* If the left hand side of the regexp operator is NULL, ++ ** then the result is also NULL. ++ */ ++ if( !zString ){ ++ return; ++ } ++ ++ pExpr = sqlite3_get_auxdata(p, 0); ++ if( !pExpr ){ ++ const UChar *zPattern = sqlite3_value_text16(apArg[0]); ++ if( !zPattern ){ ++ return; ++ } ++ pExpr = uregex_open(zPattern, -1, 0, 0, &status); ++ ++ if( U_SUCCESS(status) ){ ++ sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); ++ pExpr = sqlite3_get_auxdata(p, 0); ++ } ++ if( !pExpr ){ ++ icuFunctionError(p, "uregex_open", status); ++ return; ++ } ++ } ++ ++ /* Configure the text that the regular expression operates on. */ ++ uregex_setText(pExpr, zString, -1, &status); ++ if( !U_SUCCESS(status) ){ ++ icuFunctionError(p, "uregex_setText", status); ++ return; ++ } ++ ++ /* Attempt the match */ ++ res = uregex_matches(pExpr, 0, &status); ++ if( !U_SUCCESS(status) ){ ++ icuFunctionError(p, "uregex_matches", status); ++ return; ++ } ++ ++ /* Set the text that the regular expression operates on to a NULL ++ ** pointer. This is not really necessary, but it is tidier than ++ ** leaving the regular expression object configured with an invalid ++ ** pointer after this function returns. ++ */ ++ uregex_setText(pExpr, 0, 0, &status); ++ ++ /* Return 1 or 0. */ ++ sqlite3_result_int(p, res ? 1 : 0); ++} ++ ++/* ++** Implementations of scalar functions for case mapping - upper() and ++** lower(). Function upper() converts its input to upper-case (ABC). ++** Function lower() converts to lower-case (abc). ++** ++** ICU provides two types of case mapping, "general" case mapping and ++** "language specific". Refer to ICU documentation for the differences ++** between the two. ++** ++** To utilise "general" case mapping, the upper() or lower() scalar ++** functions are invoked with one argument: ++** ++** upper('ABC') -> 'abc' ++** lower('abc') -> 'ABC' ++** ++** To access ICU "language specific" case mapping, upper() or lower() ++** should be invoked with two arguments. The second argument is the name ++** of the locale to use. Passing an empty string ("") or SQL NULL value ++** as the second argument is the same as invoking the 1 argument version ++** of upper() or lower(). ++** ++** lower('I', 'en_us') -> 'i' ++** lower('I', 'tr_tr') -> '\u131' (small dotless i) ++** ++** http://www.icu-project.org/userguide/posix.html#case_mappings ++*/ ++static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ++ const UChar *zInput; /* Pointer to input string */ ++ UChar *zOutput = 0; /* Pointer to output buffer */ ++ int nInput; /* Size of utf-16 input string in bytes */ ++ int nOut; /* Size of output buffer in bytes */ ++ int cnt; ++ int bToUpper; /* True for toupper(), false for tolower() */ ++ UErrorCode status; ++ const char *zLocale = 0; ++ ++ assert(nArg==1 || nArg==2); ++ bToUpper = (sqlite3_user_data(p)!=0); ++ if( nArg==2 ){ ++ zLocale = (const char *)sqlite3_value_text(apArg[1]); ++ } ++ ++ zInput = sqlite3_value_text16(apArg[0]); ++ if( !zInput ){ ++ return; ++ } ++ nOut = nInput = sqlite3_value_bytes16(apArg[0]); ++ if( nOut==0 ){ ++ sqlite3_result_text16(p, "", 0, SQLITE_STATIC); ++ return; ++ } ++ ++ for(cnt=0; cnt<2; cnt++){ ++ UChar *zNew = sqlite3_realloc(zOutput, nOut); ++ if( zNew==0 ){ ++ sqlite3_free(zOutput); ++ sqlite3_result_error_nomem(p); ++ return; ++ } ++ zOutput = zNew; ++ status = U_ZERO_ERROR; ++ if( bToUpper ){ ++ nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); ++ }else{ ++ nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); ++ } ++ ++ if( U_SUCCESS(status) ){ ++ sqlite3_result_text16(p, zOutput, nOut, xFree); ++ }else if( status==U_BUFFER_OVERFLOW_ERROR ){ ++ assert( cnt==0 ); ++ continue; ++ }else{ ++ icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); ++ } ++ return; ++ } ++ assert( 0 ); /* Unreachable */ ++} ++ ++#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ ++ ++/* ++** Collation sequence destructor function. The pCtx argument points to ++** a UCollator structure previously allocated using ucol_open(). ++*/ ++static void icuCollationDel(void *pCtx){ ++ UCollator *p = (UCollator *)pCtx; ++ ucol_close(p); ++} ++ ++/* ++** Collation sequence comparison function. The pCtx argument points to ++** a UCollator structure previously allocated using ucol_open(). ++*/ ++static int icuCollationColl( ++ void *pCtx, ++ int nLeft, ++ const void *zLeft, ++ int nRight, ++ const void *zRight ++){ ++ UCollationResult res; ++ UCollator *p = (UCollator *)pCtx; ++ res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); ++ switch( res ){ ++ case UCOL_LESS: return -1; ++ case UCOL_GREATER: return +1; ++ case UCOL_EQUAL: return 0; ++ } ++ assert(!"Unexpected return value from ucol_strcoll()"); ++ return 0; ++} ++ ++/* ++** Implementation of the scalar function icu_load_collation(). ++** ++** This scalar function is used to add ICU collation based collation ++** types to an SQLite database connection. It is intended to be called ++** as follows: ++** ++** SELECT icu_load_collation(, ); ++** ++** Where is a string containing an ICU locale identifier (i.e. ++** "en_AU", "tr_TR" etc.) and is the name of the ++** collation sequence to create. ++*/ ++static void icuLoadCollation( ++ sqlite3_context *p, ++ int nArg, ++ sqlite3_value **apArg ++){ ++ sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); ++ UErrorCode status = U_ZERO_ERROR; ++ const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ ++ const char *zName; /* SQL Collation sequence name (eg. "japanese") */ ++ UCollator *pUCollator; /* ICU library collation object */ ++ int rc; /* Return code from sqlite3_create_collation_x() */ ++ ++ assert(nArg==2 || nArg==3); ++ (void)nArg; /* Unused parameter */ ++ zLocale = (const char *)sqlite3_value_text(apArg[0]); ++ zName = (const char *)sqlite3_value_text(apArg[1]); ++ ++ if( !zLocale || !zName ){ ++ return; ++ } ++ ++ pUCollator = ucol_open(zLocale, &status); ++ if( !U_SUCCESS(status) ){ ++ icuFunctionError(p, "ucol_open", status); ++ return; ++ } ++ assert(p); ++ if(nArg==3){ ++ const char *zOption = (const char*)sqlite3_value_text(apArg[2]); ++ static const struct { ++ const char *zName; ++ UColAttributeValue val; ++ } aStrength[] = { ++ { "PRIMARY", UCOL_PRIMARY }, ++ { "SECONDARY", UCOL_SECONDARY }, ++ { "TERTIARY", UCOL_TERTIARY }, ++ { "DEFAULT", UCOL_DEFAULT_STRENGTH }, ++ { "QUARTERNARY", UCOL_QUATERNARY }, ++ { "IDENTICAL", UCOL_IDENTICAL }, ++ }; ++ unsigned int i; ++ for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ ++ sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); ++ sqlite3_str_appendf(pStr, ++ "unknown collation strength \"%s\" - should be one of:", ++ zOption); ++ for(i=0; izName, p->nArg, p->enc, ++ p->iContext ? (void*)db : (void*)0, ++ p->xFunc, 0, 0 ++ ); ++ } ++ ++ return rc; ++} ++ ++#if !SQLITE_CORE ++#ifdef _WIN32 ++__declspec(dllexport) ++#endif ++SQLITE_API int sqlite3_icu_init( ++ sqlite3 *db, ++ char **pzErrMsg, ++ const sqlite3_api_routines *pApi ++){ ++ SQLITE_EXTENSION_INIT2(pApi) ++ return sqlite3IcuInit(db); ++} ++#endif ++ ++#endif ++ ++/************** End of icu.c *************************************************/ ++/************** Begin file fts3_icu.c ****************************************/ ++/* ++** 2007 June 22 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++************************************************************************* ++** This file implements a tokenizer for fts3 based on the ICU library. ++*/ ++/* #include "fts3Int.h" */ ++#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) ++#ifdef SQLITE_ENABLE_ICU ++ ++/* #include */ ++/* #include */ ++/* #include "fts3_tokenizer.h" */ ++ ++#include ++/* #include */ ++/* #include */ ++#include ++ ++typedef struct IcuTokenizer IcuTokenizer; ++typedef struct IcuCursor IcuCursor; ++ ++struct IcuTokenizer { ++ sqlite3_tokenizer base; ++ char *zLocale; ++}; ++ ++struct IcuCursor { ++ sqlite3_tokenizer_cursor base; ++ ++ UBreakIterator *pIter; /* ICU break-iterator object */ ++ int nChar; /* Number of UChar elements in pInput */ ++ UChar *aChar; /* Copy of input using utf-16 encoding */ ++ int *aOffset; /* Offsets of each character in utf-8 input */ ++ ++ int nBuffer; ++ char *zBuffer; ++ ++ int iToken; ++}; ++ ++/* ++** Create a new tokenizer instance. ++*/ ++static int icuCreate( ++ int argc, /* Number of entries in argv[] */ ++ const char * const *argv, /* Tokenizer creation arguments */ ++ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ ++){ ++ IcuTokenizer *p; ++ int n = 0; ++ ++ if( argc>0 ){ ++ n = strlen(argv[0])+1; ++ } ++ p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); ++ if( !p ){ ++ return SQLITE_NOMEM; ++ } ++ memset(p, 0, sizeof(IcuTokenizer)); ++ ++ if( n ){ ++ p->zLocale = (char *)&p[1]; ++ memcpy(p->zLocale, argv[0], n); ++ } ++ ++ *ppTokenizer = (sqlite3_tokenizer *)p; ++ ++ return SQLITE_OK; ++} ++ ++/* ++** Destroy a tokenizer ++*/ ++static int icuDestroy(sqlite3_tokenizer *pTokenizer){ ++ IcuTokenizer *p = (IcuTokenizer *)pTokenizer; ++ sqlite3_free(p); ++ return SQLITE_OK; ++} ++ ++/* ++** Prepare to begin tokenizing a particular string. The input ++** string to be tokenized is pInput[0..nBytes-1]. A cursor ++** used to incrementally tokenize this string is returned in ++** *ppCursor. ++*/ ++static int icuOpen( ++ sqlite3_tokenizer *pTokenizer, /* The tokenizer */ ++ const char *zInput, /* Input string */ ++ int nInput, /* Length of zInput in bytes */ ++ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ ++){ ++ IcuTokenizer *p = (IcuTokenizer *)pTokenizer; ++ IcuCursor *pCsr; ++ ++ const int32_t opt = U_FOLD_CASE_DEFAULT; ++ UErrorCode status = U_ZERO_ERROR; ++ int nChar; ++ ++ UChar32 c; ++ int iInput = 0; ++ int iOut = 0; ++ ++ *ppCursor = 0; ++ ++ if( zInput==0 ){ ++ nInput = 0; ++ zInput = ""; ++ }else if( nInput<0 ){ ++ nInput = strlen(zInput); ++ } ++ nChar = nInput+1; ++ pCsr = (IcuCursor *)sqlite3_malloc64( ++ sizeof(IcuCursor) + /* IcuCursor */ ++ ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ ++ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ ++ ); ++ if( !pCsr ){ ++ return SQLITE_NOMEM; ++ } ++ memset(pCsr, 0, sizeof(IcuCursor)); ++ pCsr->aChar = (UChar *)&pCsr[1]; ++ pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; ++ ++ pCsr->aOffset[iOut] = iInput; ++ U8_NEXT(zInput, iInput, nInput, c); ++ while( c>0 ){ ++ int isError = 0; ++ c = u_foldCase(c, opt); ++ U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); ++ if( isError ){ ++ sqlite3_free(pCsr); ++ return SQLITE_ERROR; ++ } ++ pCsr->aOffset[iOut] = iInput; ++ ++ if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); ++ if( !U_SUCCESS(status) ){ ++ sqlite3_free(pCsr); ++ return SQLITE_ERROR; ++ } ++ pCsr->nChar = iOut; ++ ++ ubrk_first(pCsr->pIter); ++ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; ++ return SQLITE_OK; ++} ++ ++/* ++** Close a tokenization cursor previously opened by a call to icuOpen(). ++*/ ++static int icuClose(sqlite3_tokenizer_cursor *pCursor){ ++ IcuCursor *pCsr = (IcuCursor *)pCursor; ++ ubrk_close(pCsr->pIter); ++ sqlite3_free(pCsr->zBuffer); ++ sqlite3_free(pCsr); ++ return SQLITE_OK; ++} ++ ++/* ++** Extract the next token from a tokenization cursor. ++*/ ++static int icuNext( ++ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ ++ const char **ppToken, /* OUT: *ppToken is the token text */ ++ int *pnBytes, /* OUT: Number of bytes in token */ ++ int *piStartOffset, /* OUT: Starting offset of token */ ++ int *piEndOffset, /* OUT: Ending offset of token */ ++ int *piPosition /* OUT: Position integer of token */ ++){ ++ IcuCursor *pCsr = (IcuCursor *)pCursor; ++ ++ int iStart = 0; ++ int iEnd = 0; ++ int nByte = 0; ++ ++ while( iStart==iEnd ){ ++ UChar32 c; ++ ++ iStart = ubrk_current(pCsr->pIter); ++ iEnd = ubrk_next(pCsr->pIter); ++ if( iEnd==UBRK_DONE ){ ++ return SQLITE_DONE; ++ } ++ ++ while( iStartaChar, iWhite, pCsr->nChar, c); ++ if( u_isspace(c) ){ ++ iStart = iWhite; ++ }else{ ++ break; ++ } ++ } ++ assert(iStart<=iEnd); ++ } ++ ++ do { ++ UErrorCode status = U_ZERO_ERROR; ++ if( nByte ){ ++ char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); ++ if( !zNew ){ ++ return SQLITE_NOMEM; ++ } ++ pCsr->zBuffer = zNew; ++ pCsr->nBuffer = nByte; ++ } ++ ++ u_strToUTF8( ++ pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ ++ &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ ++ &status /* Output success/failure */ ++ ); ++ } while( nByte>pCsr->nBuffer ); ++ ++ *ppToken = pCsr->zBuffer; ++ *pnBytes = nByte; ++ *piStartOffset = pCsr->aOffset[iStart]; ++ *piEndOffset = pCsr->aOffset[iEnd]; ++ *piPosition = pCsr->iToken++; ++ ++ return SQLITE_OK; ++} ++ ++/* ++** The set of routines that implement the simple tokenizer ++*/ ++static const sqlite3_tokenizer_module icuTokenizerModule = { ++ 0, /* iVersion */ ++ icuCreate, /* xCreate */ ++ icuDestroy, /* xCreate */ ++ icuOpen, /* xOpen */ ++ icuClose, /* xClose */ ++ icuNext, /* xNext */ ++ 0, /* xLanguageid */ ++}; ++ ++/* ++** Set *ppModule to point at the implementation of the ICU tokenizer. ++*/ ++EXPORT_SYMBOLS SQLITE_API void sqlite3Fts3IcuTokenizerModule( ++ sqlite3_tokenizer_module const**ppModule ++){ ++ *ppModule = &icuTokenizerModule; ++} ++ ++#endif /* defined(SQLITE_ENABLE_ICU) */ ++#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ ++ ++/************** End of fts3_icu.c ********************************************/ +-- +2.47.0.windows.2 + diff --git a/mock/sqlite/patch/0005-Report-corruption-when-runtime-decteted.patch b/mock/sqlite/patch/0005-Report-corruption-when-runtime-decteted.patch new file mode 100644 index 00000000..2af605a1 --- /dev/null +++ b/mock/sqlite/patch/0005-Report-corruption-when-runtime-decteted.patch @@ -0,0 +1,1446 @@ +From 0fdc896d7434be7ad9a3947aaca13d5ce760547d Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Tue, 25 Feb 2025 22:16:42 +0800 +Subject: [PATCH] Report corruption when runtime detected + +Signed-off-by: MartinChoo <214582617@qq.com> +--- + src/sqlite3.c | 717 ++++++++++++++++++++++++++++++++++++++++++-------- + 1 file changed, 605 insertions(+), 112 deletions(-) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index f348f3c..0666938 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -2474,6 +2474,21 @@ struct sqlite3_mem_methods { + ** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and + ** recommended case) then the integer is always filled with zero, regardless + ** if its initial value. ++** ++** [[SQLITE_CONFIG_CORRUPTION]]

    SQLITE_CONFIG_CORRUPTION
    ++**
    The SQLITE_CONFIG_CORRUPTION option is used to configure the SQLite ++** global [ corruption error]. ++** (^The SQLITE_CONFIG_CORRUPTION option takes two arguments: a pointer to a ++** function with a call signature of void(*)(void*,const void*), ++** and a pointer to void. ^If the function pointer is not NULL, it is ++** invoked to process each data corruption event. ^If the ++** function pointer is NULL, no=op will do when corruption detect. ++** ^The void pointer that is the second argument to SQLITE_CONFIG_CORRUPTION is ++** passed through as the first parameter to the application-defined corruption ++** function whenever that function is invoked. ^The second parameter to ++** the corruption function is a corruption message after formatting via [sqlite3_snprintf()]. ++** In a multi-threaded application, the application-defined corruption ++** function must be threadsafe.
    + ** + */ + #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +@@ -2506,7 +2521,8 @@ struct sqlite3_mem_methods { + #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ + #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ + #define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ +-#define SQLITE_CONFIG_ENABLE_ICU 31 /* boolean */ ++#define SQLITE_CONFIG_CORRUPTION 31 /* xCorruption */ ++#define SQLITE_CONFIG_ENABLE_ICU 32 /* boolean */ + + /* + ** CAPI3REF: Database Connection Configuration Options +@@ -20360,6 +20376,8 @@ struct Sqlite3Config { + int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ + unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ ++ void (*xCorruption)(void *, const void *); ++ void *pCorruptionArg; + /* vvvv--- must be last ---vvv */ + #ifdef SQLITE_DEBUG + sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */ +@@ -20627,6 +20645,57 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprLis + } \ + } + ++#define SQLITE_PRINT_CORRUPT_SIZE (SQLITE_PRINT_BUF_SIZE * 2) ++ ++#define SQLITE_CORRUPT_CONTEXT(N,P,T,O,S,M,R) \ ++ { .nPage=(N), .pgno=(P), .type=(T), \ ++ .zoneRange={(O),(S)}, .zMsg=(M), .reservedArgs=(R) } ++ ++typedef struct { ++ int offset; ++ size_t size; ++}sqlite3CorruptRange; ++ ++typedef enum { ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ CORRUPT_TYPE_PAGE_BTREE_INTERIOR, ++ CORRUPT_TYPE_PAGE_INDEX_LEAF, ++ CORRUPT_TYPE_PAGE_INDEX_INTERIOR, ++ CORRUPT_TYPE_PAGE_OVERFLOW, ++ CORRUPT_TYPE_PAGE_PTR_MAP, ++ CORRUPT_TYPE_PAGE_FREE_LIST, ++ CORRUPT_TYPE_FRAME_WAL, ++ CORRUPT_TYPE_ENTRY_JOURNAL, ++ CORRUPT_TYPE_VDBE, ++ CORRUPT_TYPE_FILE_HEADER, ++ CORRUPT_TYPE_UNKOWN, ++} CorruptType; ++ ++typedef struct { ++ size_t nPage; /* Number of pages */ ++ unsigned int pgno; /* Page number for corrupted page */ ++ CorruptType type; ++ sqlite3CorruptRange zoneRange; ++ const char *zMsg; ++ void *reservedArgs; ++}sqlite3CorruptContext; ++ ++// Encode buffer with base16, return size after encode ++static size_t sqlite3base16Encode(const unsigned char *buffer, size_t bufSize, char *encodeBuf, size_t encodeBufSize) ++{ ++ if (buffer == NULL || bufSize == 0 || encodeBuf == NULL || encodeBufSize == 0) { ++ return 0; ++ } ++ static const char base16Code[] = "0123456789ABCDEF"; ++ size_t i = 0; ++ for (; i < bufSize && (i * 2 < encodeBufSize - 1); i++) { ++ *encodeBuf++ = base16Code[(buffer[i] >> 4) & 0x0F]; ++ *encodeBuf++ = base16Code[buffer[i] & 0x0F]; ++ } ++ *encodeBuf = '\0'; ++ return i * 2; ++} ++ + /* + ** The SQLITE_*_BKPT macros are substitutes for the error codes with + ** the same name but without the _BKPT suffix. These macros invoke +@@ -20635,10 +20704,11 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprLis + ** to set a debugger breakpoint. + */ + SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType); +-SQLITE_PRIVATE int sqlite3CorruptError(int); ++SQLITE_PRIVATE int sqlite3CorruptError(int lineno, sqlite3CorruptContext *context); + SQLITE_PRIVATE int sqlite3MisuseError(int); + SQLITE_PRIVATE int sqlite3CantopenError(int); +-#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) ++#define SQLITE_CORRUPT_REPORT(context) sqlite3CorruptError(__LINE__,(context)) ++#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__,NULL) + #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) + #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) + #ifdef SQLITE_DEBUG +@@ -20650,12 +20720,13 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); + # define SQLITE_NOMEM_BKPT SQLITE_NOMEM + # define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM + #endif +-#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) ++#if (defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO)) + SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); +-# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) ++# define SQLITE_CORRUPT_PGNO(P,context) sqlite3CorruptPgnoError(__LINE__,(P)) + #else +-# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) ++# define SQLITE_CORRUPT_PGNO(P,context) sqlite3CorruptError(__LINE__,(context)) + #endif ++# define SQLITE_CORRUPT_REPORT_PGNO(context) SQLITE_CORRUPT_PGNO((context)->pgno,(context)) + + /* + ** FTS3 and FTS4 both require virtual table support +@@ -23111,6 +23182,8 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ ++ 0, /* xCorruption */ ++ 0, /* pCorruptionArg */ + #ifdef SQLITE_DEBUG + {0,0,0,0,0,0}, /* aTune */ + #endif +@@ -60758,7 +60831,7 @@ static int readDbPage(PgHdr *pPg){ + assert( isOpen(pPager->fd) ); + + if( pagerUseWal(pPager) ){ +- rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); ++ rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); // find in wal-index + if( rc ) return rc; + } + if( iFrame ){ +@@ -63281,7 +63354,12 @@ static int getPageNormal( + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + +- if( pgno==0 ) return SQLITE_CORRUPT_BKPT; ++ if( pgno==0 ) { ++ const char *zMsg = "pgno should not be 0"; ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, 0, CORRUPT_TYPE_UNKOWN, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); ++ } + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + pPg = 0; +@@ -63313,7 +63391,11 @@ static int getPageNormal( + ** (2) Never try to fetch the locking page + */ + if( pgno==PAGER_SJ_PGNO(pPager) ){ +- rc = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "try fetching the locking page(%u)", pgno); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, pgno, CORRUPT_TYPE_UNKOWN, ++ -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT(&context); + goto pager_acquire_err; + } + +@@ -63399,7 +63481,10 @@ static int getPageMMap( + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ +- return SQLITE_CORRUPT_BKPT; ++ const char *zMsg = "pgno should not be 0"; ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, 0, CORRUPT_TYPE_UNKOWN, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); +@@ -65010,7 +65095,11 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i + if( pPgOld ){ + if( NEVER(pPgOld->nRef>1) ){ + sqlite3PagerUnrefNotNull(pPgOld); +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "page(%u) should be no references, ref cnt:%d", pgno, (int)pPgOld->nRef); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, pgno, CORRUPT_TYPE_UNKOWN, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + if( pPager->tempFile ){ +@@ -66940,7 +67029,13 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ + /* Write the aPgno[] array entry and the hash-table slot. */ + nCollide = idx; + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ +- if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; ++ if( (nCollide--)==0 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "no place for page(%u) to map into WAL, idx:%d", iPage, idx); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, iPage, CORRUPT_TYPE_FRAME_WAL, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); ++ } + } + sLoc.aPgno[idx-1] = iPage; + AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); +@@ -67889,7 +67984,13 @@ static int walCheckpoint( + ** database plus the amount of data in the wal file, plus the + ** maximum size of the pending-byte page (65536 bytes), then + ** must be corruption somewhere. */ +- rc = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "final db size unexpected,nSize=%lld,mxFrame=%u,pageSize=%d,nReq=%lld", ++ nSize, pWal->hdr.mxFrame, szPage, nReq); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, 0, CORRUPT_TYPE_FRAME_WAL, ++ -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT(&context); + }else{ + sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); + } +@@ -69201,7 +69302,11 @@ static int walFindFrame( + } + if( (nCollide--)==0 ){ + *piRead = 0; +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "mis-match page(%u) to map into WAL", pgno); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, pgno, CORRUPT_TYPE_FRAME_WAL, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + iKey = walNextHash(iKey); + } +@@ -70005,7 +70110,12 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ +- rc = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "mis-match between pageSize=%d and bufferSize=%d, mxFrame=%u", ++ walPagesize(pWal),nBuf,pWal->hdr.mxFrame); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, 0, CORRUPT_TYPE_FRAME_WAL, ++ -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT(&context); + }else{ + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); + } +@@ -71469,7 +71579,7 @@ static int checkDbHeaderValid(sqlite3 *db, int iDbpage, u8 *zBuf){ + ** with the page number and filename associated with the (MemPage*). + */ + #ifdef SQLITE_DEBUG +-int corruptPageError(int lineno, MemPage *p){ ++int corruptPageError(int lineno, MemPage *p, sqlite3CorruptContext *context){ + char *zMsg; + sqlite3BeginBenignMalloc(); + zMsg = sqlite3_mprintf("database corruption page %u of %s", +@@ -71480,11 +71590,11 @@ int corruptPageError(int lineno, MemPage *p){ + sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); + } + sqlite3_free(zMsg); +- return SQLITE_CORRUPT_BKPT; ++ return SQLITE_CORRUPT_REPORT(context); + } +-# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) ++# define SQLITE_CORRUPT_PAGE(context,pMemPage) corruptPageError(__LINE__, (pMemPage),(context)) + #else +-# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) ++# define SQLITE_CORRUPT_PAGE(context,pMemPage) SQLITE_CORRUPT_PGNO((pMemPage)->pgno,(context)) + #endif + + /* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +@@ -72210,7 +72320,12 @@ static int btreeMoveto( + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ +- rc = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected fields in total:%u, should != 0 and < %u", ++ pIdxKey->nField,pKeyInfo->nAllField); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pCur->pBt->nPage, 0, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT(&context); + }else{ + rc = sqlite3BtreeIndexMoveto(pCur, pIdxKey, pRes); + } +@@ -72407,7 +72522,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + + assert( pBt->autoVacuum ); +- if( key==0 ){ ++ if( key==0 ){ // The pgno of each entry on ptrmap page starts from 3, an unexpected pgno indicates data corrupted + *pRC = SQLITE_CORRUPT_BKPT; + return; + } +@@ -72421,12 +72536,24 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ +- *pRC = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode((unsigned char *)sqlite3PagerGetExtra(pDbPage), 8, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "page(%u) been initialized before as a btree page, base16:%s", ++ iPtrmap, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, ++ -1, 0, zMsg, NULL); ++ *pRC = SQLITE_CORRUPT_REPORT(&context); + goto ptrmap_exit; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ +- *pRC = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect offset in ptrmap page(%u), target:%u, page usableSize=%u", ++ iPtrmap, key, pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, ++ -1, 0, zMsg, NULL); ++ *pRC = SQLITE_CORRUPT_REPORT(&context); + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); +@@ -72471,7 +72598,12 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect offset in ptrmap page(%d), target:%u, page usableSize=%u", ++ iPtrmap, key, pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + assert( offset <= (int)pBt->usableSize-5 ); + assert( pEType!=0 ); +@@ -72479,7 +72611,15 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + + sqlite3PagerUnref(pDbPage); +- if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap); ++ if( *pEType<1 || *pEType>5 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // 5 bytes for each entry on ptrmap page ++ (void)sqlite3base16Encode(pPtrmap, 5, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect entry type:%d, base16:%s", (int)*pEType, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, ++ offset, 5, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT_PGNO(&context); ++ } + return SQLITE_OK; + } + +@@ -72919,7 +73059,14 @@ static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ + Pgno ovfl; + if( SQLITE_OVERFLOW(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); +- *pRC = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // Output cell header as much as possible, 4 bytes for overflow pgno ++ (void)sqlite3base16Encode(pCell, info.nSize - info.nLocal - 4, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cell overflow, offset=%d, rest=%d, length=%u, base16:%s", ++ (int)(pCell - pPage->aData), (int)(pSrc->aDataEnd - pCell), info.nSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ pCell - pPage->aData, info.nSize, zMsg, NULL); ++ *pRC = SQLITE_CORRUPT_REPORT(&context); + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); +@@ -72977,10 +73124,29 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); +- if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iFree>usableSize-4 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Output first 8 bytes as it's page header ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset=%d overflow, usableSize=%d, base16:%s", ++ iFree, usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ 0, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); +- if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iFree2>usableSize-4 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data, 4, xBuffer, sizeof(xBuffer)); // Output first freeblock's header 4 bytes ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "1st freeblock's next pointer overflow, point:%d, usableSize=%d, base16:%s", ++ iFree2, usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iFree, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; +@@ -72988,16 +73154,51 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( top>=iFree ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print first 8 bytes which is page header ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "1st freeblock's offset:%d should > CellContentArea's offset:%d, base16:%s", ++ iFree, top, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iFree, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( iFree2 ){ +- if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iFree+sz>iFree2 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print first 8 bytes which is page header ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "the 1st 2 freeblocks mis-order, 1st block offset:%d, size:%d, 2nd block offset:%d, base16:%s", ++ iFree, sz, iFree2, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iFree, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + sz2 = get2byte(&data[iFree2+2]); +- if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iFree2+sz2 > usableSize ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data + iFree2, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to 2nd block ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "the 2nd freeblock overflow, offset:%d, size:%d, usableSize:%d, base16:%s", ++ iFree2, sz2, usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iFree2, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + }else if( iFree+sz>usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data + iFree, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to 1st block ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "the 1st freeblock overflow, offset:%d, size:%d, usableSize:%d, base16:%s", iFree, sz, usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iFree, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + + cbrk = top+sz; +@@ -73030,13 +73231,24 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ + ** if PRAGMA cell_size_check=ON. + */ + if( pc>iCellLast ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // Print 4 bytes belong to 1st block ++ (void)sqlite3base16Encode(data + cellOffset + i*2, 2, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "%d-th cell pointer:%d out of range[%d, %d], base16:%s", ++ i, pc, iCellStart, iCellLast, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ cellOffset + i*2, 2, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( pc>=0 && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "move %d-th cell from %d using unexpected size:%d", i, pc, size); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); +@@ -73050,7 +73262,13 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ + defragment_out: + assert( pPage->nFree>=0 ); + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "after defragment, free bytes should not change, fragment bytes:%d, free space:%d, total:%d", ++ (int)data[hdr+7], cbrk-iCellFirst, pPage->nFree); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( cbrk>=iCellFirst ); + put2byte(&data[hdr+5], cbrk); +@@ -73075,7 +73293,7 @@ defragment_out: + ** will be ignored if adding the extra space to the fragmentation count + ** causes the fragmentation count to exceed 60. + */ +-static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ++static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ // search on B-tree page + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ +@@ -73107,7 +73325,15 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + return &aData[pc]; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ +- *pRc = SQLITE_CORRUPT_PAGE(pPg); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(aData + pc, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to free block ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "freeblock rest bytes:%d begin at %d which cost %d, still exceed usableSize:%u, base16:%s", ++ x, pc, nByte, pPg->pBt->usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ pc, 4, zMsg, NULL); ++ *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account +@@ -73122,14 +73348,25 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + if( pc<=iAddr ){ + if( pc ){ + /* The next slot in the chain comes before the current slot */ +- *pRc = SQLITE_CORRUPT_PAGE(pPg); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pTmp, 2, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to free block ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "the next slot:%d in chain comes before current slot:%d, base16:%s", pc, iAddr, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ iAddr, 2, zMsg, NULL); ++ *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); + } + return 0; + } + } + if( pc>maxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ +- *pRc = SQLITE_CORRUPT_PAGE(pPg); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "free slot:%d overflow, end:%d", pc, maxPC+nByte-4); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); + } + return 0; + } +@@ -73177,10 +73414,16 @@ static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; + }else{ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print 8 bytes belong to page header ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected cellContentArea offset:%d, base16:%s", top, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + }else if( top>(int)pPage->pBt->usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + + /* If there is enough space between gap and top for one more cell pointer, +@@ -73197,7 +73440,11 @@ static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + *pIdx = g2 = (int)(pSpace-data); + if( g2<=gap ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cellpointers(end:%d) overlap with freeblock(%d)", gap, g2); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + }else{ + return SQLITE_OK; + } +@@ -73276,12 +73523,23 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + while( (iFreeBlk = get2byte(&data[iPtr]))pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + iPtr = iFreeBlk; + } + if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data + iPtr, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to freeblock ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d overflow, base16:%s", (int)iFreeBlk, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, iPtr, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); + +@@ -73293,10 +73551,24 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; +- if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iEnd>iFreeBlk ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d overlaps with pre block's end:%u", ++ (int)iFreeBlk, iEnd); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(data + iFreeBlk, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to freeblock ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d, end:%u overflow, usableSize:%u, base16:%s", ++ (int)iFreeBlk, iEnd, pPage->pBt->usableSize, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, iFreeBlk, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); +@@ -73309,13 +73581,27 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ +- if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( iPtrEnd>iStart ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "check pre freeblock end:%d overlaps with the pending free block:%d", ++ iPtrEnd, (int)iStart); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, iPtr, iPtrEnd - iPtr, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; + } + } +- if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); ++ if( nFrag>data[hdr+7] ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "fragment free bytes:%d increase unexpectly, should be %d", ++ (int)nFrag, (int)data[hdr+7]); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + data[hdr+7] -= nFrag; + } + pTmp = &data[hdr+5]; +@@ -73329,8 +73615,21 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ +- if( iStart= the beginning of the CellContentArea:%d", (int)x, (int)iStart); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } ++ if( iPtr!=hdr+1 ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "1st freeblock's pos incorrect, hdr:%d, iPtr:%d", (int)hdr, (int)iPtr); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); ++ } + put2byte(&data[hdr+1], iFreeBlk); + put2byte(&data[hdr+5], iEnd); + }else{ +@@ -73391,7 +73690,13 @@ static int decodeFlags(MemPage *pPage, int flagByte){ + pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, + pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); + #endif /* LOG_DUMP */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pPage->aData, 8, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unrecognized flag:%d, base16:%s", flagByte, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + }else{ + pPage->childPtrSize = 4; +@@ -73422,7 +73727,13 @@ static int decodeFlags(MemPage *pPage, int flagByte){ + pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, + pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); + #endif /* LOG_DUMP */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pPage->aData, 8, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unrecognized flag:%d, base16:%s", flagByte, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + } + return SQLITE_OK; +@@ -73473,12 +73784,20 @@ static int btreeComputeFreeSpace(MemPage *pPage){ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "the 1st freeblock:%d before all cells:%d", pc, top); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + while( 1 ){ + if( pc>iCellLast ){ + /* Freeblock off the end of the page */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock end:%d out of page range:%d", pc, iCellLast); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); +@@ -73488,11 +73807,19 @@ static int btreeComputeFreeSpace(MemPage *pPage){ + } + if( next>0 ){ + /* Freeblock not in ascending order */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "all freeblocks should order by asc, pre:%d, cur:%u", pc, next); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( pc+size>(unsigned int)usableSize ){ + /* Last freeblock extends past page end */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "last freeblock overflow, offset:%d, size:%u", pc, size); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + } + +@@ -73504,7 +73831,13 @@ static int btreeComputeFreeSpace(MemPage *pPage){ + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize || nFreepBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + pPage->nFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +@@ -73535,12 +73868,20 @@ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cell pointer:%d indicate out of range:[%d, %d]", pc, iCellFirst, iCellLast); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected cell size:%d,offset:%d, out of range:%d", sz, pc, usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + } + return SQLITE_OK; +@@ -73572,7 +73913,7 @@ static int btreeInitPage(MemPage *pPage){ + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); +@@ -73586,7 +73927,12 @@ static int btreeInitPage(MemPage *pPage){ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "too many cells(%d) for the page:%u, offset:%d, out of range:%u", ++ (int)pPage->nCell, pPage->pgno, (int)pPage->hdrOffset + 3, MX_CELL(pBt)); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only +@@ -73729,7 +74075,11 @@ static int getAndInitPage( + + if( pgno>btreePagecount(pBt) ){ + *ppPage = 0; +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "page number(%u) > db file size(%u)", pgno, btreePagecount(pBt)); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ +@@ -75199,7 +75549,12 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + if( eType==PTRMAP_OVERFLOW2 ){ + /* The pointer is always the first 4 bytes of the page in this case. */ + if( get4byte(pPage->aData)!=iFrom ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "1st 4 bytes of ovrflow page(%u) point to next(%u), should be %u", ++ pPage->pgno, get4byte(pPage->aData), iFrom); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_PTR_MAP, ++ 0, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + put4byte(pPage->aData, iTo); + }else{ +@@ -75218,7 +75573,13 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + pPage->xParseCell(pPage, pCell, &info); + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "btree cell contain ovrflow pointer overflow, offset:%d, size:%u, usableSize:%u", ++ (int)(pCell - pPage->aData), info.nSize, pPage->pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_PTR_MAP, pCell - pPage->aData, info.nSize, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); +@@ -75227,7 +75588,13 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + } + }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "btree cell contain child pointer overflow, offset:%d, size:4, usableSize:%u", ++ (int)(pCell - pPage->aData), pPage->pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_PTR_MAP, pCell - pPage->aData, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( get4byte(pCell)==iFrom ){ + put4byte(pCell, iTo); +@@ -75239,7 +75606,11 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + if( i==nCell ){ + if( eType!=PTRMAP_BTREE || + get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "missing pointer point to overflow page on btree page(%u)", pPage->pgno); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); + } +@@ -75372,7 +75743,11 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ + return rc; + } + if( eType==PTRMAP_ROOTPAGE ){ +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "try vacuum root page(%u), should not happened", nFin); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, PTRMAP_PAGENO(pBt, iLastPg), ++ CORRUPT_TYPE_PAGE_PTR_MAP, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + + if( eType==PTRMAP_FREEPAGE ){ +@@ -76448,7 +76823,12 @@ static int accessPayload( + assert( eOp==0 || eOp==1 ); + assert( pCur->eState==CURSOR_VALID ); + if( pCur->ix>=pPage->nCell ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cell index:%u exceed limit:%u on the page:%u", ++ pCur->ix, pPage->nCell, pPage->pgno); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( cursorHoldsMutex(pCur) ); + +@@ -76463,7 +76843,12 @@ static int accessPayload( + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "base on payload size:%u, the max offset(%d) should > %d", ++ pCur->info.nLocal, (int)(aPayload - pPage->aData), (int)(pBt->usableSize - pCur->info.nLocal)); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ 0, 8, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + + /* Check if data must be read/written to/from the btree page itself. */ +@@ -76534,7 +76919,16 @@ static int accessPayload( + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ + /* If required, populate the overflow page-list cache. */ +- if( nextPage > pBt->nPage ) return SQLITE_CORRUPT_BKPT; ++ if( nextPage > pBt->nPage ){ ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(aPayload + pCur->info.nLocal, 4, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow page:%u should not exceed the size of database file, base16:%s", ++ nextPage, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ aPayload - pPage->aData + pCur->info.nLocal, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); ++ } + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); +@@ -76618,7 +77012,11 @@ static int accessPayload( + + if( rc==SQLITE_OK && amt>0 ){ + /* Overflow chain ends prematurely */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow chain ends prematurely, rest:%d", amt); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + return rc; + } +@@ -76770,7 +77168,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); +- rc = SQLITE_CORRUPT_PGNO(newPgno); ++ rc = SQLITE_CORRUPT_PGNO(newPgno, NULL); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; +@@ -76905,7 +77303,12 @@ static int moveToRoot(BtCursor *pCur){ + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ +- return SQLITE_CORRUPT_PAGE(pCur->pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "the page(%u) state illegal, isInit:%u, pKeyInfo%s0, intKey:%u", ++ pRoot->pgno, pRoot->isInit, ((pCur->pKeyInfo==0)?"==":"!="), pRoot->intKey); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pCur->pBt->nPage, pRoot->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pCur->pPage); + } + + skip_init: +@@ -77166,7 +77569,11 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( + if( pPage->intKeyLeaf ){ + while( 0x80 <= *(pCell++) ){ + if( pCell>=pPage->aDataEnd ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cell idx(%d) point to a cell should not out of page", idx); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + } + } +@@ -77449,7 +77856,12 @@ bypass_moveto_root: + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ +- rc = SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "nCell:%d illegal, usableSize:%u, nPage:%u", ++ nCell, pCur->pBt->usableSize, pCur->pBt->nPage); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_PAGE(&context, pPage); + goto moveto_index_finish; + } + pCellKey = sqlite3Malloc( nCell+nOverrun ); +@@ -77523,7 +77935,7 @@ bypass_moveto_root: + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); +- rc = SQLITE_CORRUPT_PGNO(chldPg); ++ rc = SQLITE_CORRUPT_PGNO(chldPg, NULL); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; +@@ -77807,7 +78219,13 @@ static int allocateBtreePage( + n = get4byte(&pPage1->aData[36]); + testcase( n==mxPage-1 ); + if( n>=mxPage ){ +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE*3] = {0}; ++ (void)sqlite3base16Encode(pPage1->aData, 100, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "total pages(%u) in freelist should not over the total size of db file(%u), base16:%s", n, mxPage, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, 1, CORRUPT_TYPE_FILE_HEADER, 36, 4, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + if( n>0 ){ + /* There are pages on the freelist. Reuse one of those pages. */ +@@ -77863,7 +78281,12 @@ static int allocateBtreePage( + } + testcase( iTrunk==mxPage ); + if( iTrunk>mxPage || nSearch++ > n ){ +- rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "freelist trunk page(%u) should <= the size of db(%u)", ++ iTrunk, mxPage); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, (pPrevTrunk ? pPrevTrunk->pgno : 1), ++ CORRUPT_TYPE_PAGE_FREE_LIST, -1, 0, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT_PGNO(&context); + }else{ + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); + } +@@ -77892,7 +78315,14 @@ static int allocateBtreePage( + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); + }else if( k>(u32)(pBt->usableSize/4 - 2) ){ + /* Value of k is out of range. Database corruption */ +- rc = SQLITE_CORRUPT_PGNO(iTrunk); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pTrunk->aData, 8, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "total leaf pages(%u) on trunk page over limit(%u), base16:%s", ++ k, (u32)(pBt->usableSize/4 - 2), xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, ++ 0, 8, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT_PGNO(&context); + goto end_allocate_page; + #ifndef SQLITE_OMIT_AUTOVACUUM + }else if( searchList +@@ -77926,7 +78356,14 @@ static int allocateBtreePage( + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + if( iNewTrunk>mxPage ){ +- rc = SQLITE_CORRUPT_PGNO(iTrunk); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pTrunk->aData, 12, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "leaf page's pgno(%u) on trunk page exceed db file size(%u), base16:%s", iNewTrunk, mxPage, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, ++ 8, 4, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT_PGNO(&context); + goto end_allocate_page; + } + testcase( iNewTrunk==mxPage ); +@@ -77991,7 +78428,14 @@ static int allocateBtreePage( + iPage = get4byte(&aData[8+closest*4]); + testcase( iPage==mxPage ); + if( iPage>mxPage || iPage<2 ){ +- rc = SQLITE_CORRUPT_PGNO(iTrunk); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(aData+8+closest*4, 4, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "leaf page's pgno(%u) out of range:[3, %d], base16:%s", ++ iPage, mxPage, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, ++ 8+closest*4, 4, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT_PGNO(&context); + goto end_allocate_page; + } + testcase( iPage==mxPage ); +@@ -78176,7 +78620,14 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ + nLeaf = get4byte(&pTrunk->aData[4]); + assert( pBt->usableSize>32 ); + if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ +- rc = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pTrunk->aData, 4, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "the number of leaf page(%u) on trunk page(%d) exceed limit, base16:%s", ++ nLeaf, iTrunk, xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, ++ 0, 4, zMsg, NULL); ++ rc = SQLITE_CORRUPT_REPORT(&context); + goto freepage_out; + } + if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ +@@ -78265,7 +78716,12 @@ static SQLITE_NOINLINE int clearCellOverflow( + testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd ); + if( pCell + pInfo->nSize > pPage->aDataEnd ){ + /* Cell extends past end of page */ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow end of page, pgno:%u, offset:%d, size:%u, usableSize:%u", ++ pPage->pgno, (int)(pCell - pPage->aData), pInfo->nSize, pPage->pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(btreePagecount(pPage->pBt), pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; +@@ -78282,7 +78738,14 @@ static SQLITE_NOINLINE int clearCellOverflow( + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the + ** file the database must be corrupt. */ +- return SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; ++ (void)sqlite3base16Encode(pCell + pInfo->nSize - 4, 4, xBuffer, sizeof(xBuffer)); ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow page's pgno(%u) illegal, out of range:[2, %u], base16:%s", ++ ovflPgno, btreePagecount(pBt), xBuffer); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(btreePagecount(pBt), pPage->pgno, ++ CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_REPORT(&context); + } + if( nOvfl ){ + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); +@@ -78558,7 +79021,12 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ + testcase( pc==(u32)get2byte(&data[hdr+5]) ); + testcase( pc+sz==pPage->pBt->usableSize ); + if( pc+sz > pPage->pBt->usableSize ){ +- *pRC = SQLITE_CORRUPT_BKPT; ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "cell offset:%u size:%d, idx:%d overflow, usableSize:%u", ++ pc, sz, idx, pPage->pBt->usableSize); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ *pRC = SQLITE_CORRUPT_REPORT(&context); + return; + } + rc = freeSpace(pPage, pc, sz); +@@ -79700,7 +80168,7 @@ static int balance_nonroot( + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ +- rc = SQLITE_CORRUPT_PAGE(pOld); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pOld); + goto balance_cleanup; + } + +@@ -79724,7 +80192,7 @@ static int balance_nonroot( + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); + if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ +- rc = SQLITE_CORRUPT_PAGE(pOld); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pOld); + goto balance_cleanup; + } + limit = pOld->aiOvfl[0]; +@@ -80367,7 +80835,7 @@ static int anotherValidCursor(BtCursor *pCur){ + && pOther->eState==CURSOR_VALID + && pOther->pPage==pCur->pPage + ){ +- return SQLITE_CORRUPT_PAGE(pCur->pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pCur->pPage); + } + } + return SQLITE_OK; +@@ -80427,7 +80895,7 @@ static int balance(BtCursor *pCur){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ +- rc = SQLITE_CORRUPT_PAGE(pPage); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pPage); + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; +@@ -80591,7 +81059,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( + rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); + if( rc ) return rc; + if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ +- rc = SQLITE_CORRUPT_PAGE(pPage); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pPage); + }else{ + if( iOffset+ovflPageSize<(u32)nTotal ){ + ovflPgno = get4byte(pPage->aData); +@@ -80619,7 +81087,14 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; ++ sqlite3_snprintf(sizeof(zMsg), zMsg, ++ "cell payload cursor point to(%d), size:%u overlaps with non-cell content area:[%u, %d]", ++ (int)(pCur->info.pPayload - pPage->aData), pCur->info.nLocal, pPage->cellOffset, ++ (int)(pPage->aDataEnd - pPage->aData)); ++ sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, ++ -1, 0, zMsg, NULL); ++ return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( pCur->info.nLocal==nTotal ){ + /* The entire cell is local */ +@@ -80700,7 +81175,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( + ** Which can only happen if the SQLITE_NoSchemaError flag was set when + ** the schema was loaded. This cannot be asserted though, as a user might + ** set the flag, load the schema, and then unset the flag. */ +- return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); ++ return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot, NULL); + } + } + +@@ -80823,7 +81298,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( + if( pPage->nFree<0 ){ + if( NEVER(pCur->eState>CURSOR_INVALID) ){ + /* ^^^^^--- due to the moveToRoot() call above */ +- rc = SQLITE_CORRUPT_PAGE(pPage); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pPage); + }else{ + rc = btreeComputeFreeSpace(pPage); + } +@@ -80865,7 +81340,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( + CellInfo info; + assert( idx>=0 ); + if( idx>=pPage->nCell ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ +@@ -80892,10 +81367,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + if( oldCell+szNew > pPage->aDataEnd ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; +@@ -80997,7 +81472,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 + nIn = pSrc->info.nLocal; + aIn = pSrc->info.pPayload; + if( aIn+nIn>pSrc->pPage->aDataEnd ){ +- return SQLITE_CORRUPT_PAGE(pSrc->pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pSrc->pPage); + } + nRem = pSrc->info.nPayload; + if( nIn==nRem && nInpPage->maxLocal ){ +@@ -81022,7 +81497,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 + + if( nRem>nIn ){ + if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ +- return SQLITE_CORRUPT_PAGE(pSrc->pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pSrc->pPage); + } + ovflIn = get4byte(&pSrc->info.pPayload[nIn]); + } +@@ -81118,7 +81593,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ + assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); + if( rc || pCur->eState!=CURSOR_VALID ) return rc; + }else{ +- return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); ++ return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot, NULL); + } + } + assert( pCur->eState==CURSOR_VALID ); +@@ -81127,14 +81602,14 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ + iCellIdx = pCur->ix; + pPage = pCur->pPage; + if( pPage->nCell<=iCellIdx ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ +- return SQLITE_CORRUPT_PAGE(pPage); ++ return SQLITE_CORRUPT_PAGE(NULL, pPage); + } + + /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must +@@ -81225,7 +81700,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ + n = pCur->pPage->pgno; + } + pCell = findCell(pLeaf, pLeaf->nCell-1); +- if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); ++ if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(NULL, pLeaf); + nCell = pLeaf->xCellSize(pLeaf, pCell); + assert( MX_CELL_SIZE(pBt) >= nCell ); + pTmp = pBt->pTmpSpace; +@@ -81341,7 +81816,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ + */ + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + if( pgnoRoot>btreePagecount(pBt) ){ +- return SQLITE_CORRUPT_PGNO(pgnoRoot); ++ return SQLITE_CORRUPT_PGNO(pgnoRoot, NULL); + } + pgnoRoot++; + +@@ -81389,7 +81864,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ + } + rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); + if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ +- rc = SQLITE_CORRUPT_PGNO(pgnoRoot); ++ rc = SQLITE_CORRUPT_PGNO(pgnoRoot, NULL); + } + if( rc!=SQLITE_OK ){ + releasePage(pRoot); +@@ -81479,14 +81954,14 @@ static int clearDatabasePage( + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>btreePagecount(pBt) ){ +- return SQLITE_CORRUPT_PGNO(pgno); ++ return SQLITE_CORRUPT_PGNO(pgno, NULL); + } + rc = getAndInitPage(pBt, pgno, &pPage, 0); + if( rc ) return rc; + if( (pBt->openFlags & BTREE_SINGLE)==0 + && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) + ){ +- rc = SQLITE_CORRUPT_PAGE(pPage); ++ rc = SQLITE_CORRUPT_PAGE(NULL, pPage); + goto cleardatabasepage_out; + } + hdr = pPage->hdrOffset; +@@ -81590,7 +82065,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ + assert( p->inTrans==TRANS_WRITE ); + assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ +- return SQLITE_CORRUPT_PGNO(iTable); ++ return SQLITE_CORRUPT_PGNO(iTable, NULL); + } + + rc = sqlite3BtreeClearTable(p, iTable, 0); +@@ -85920,7 +86395,7 @@ static int growOpArray(Vdbe *v, int nOp){ + ** + ** Other useful labels for breakpoints include: + ** test_trace_breakpoint(pc,pOp) +-** sqlite3CorruptError(lineno) ++** sqlite3CorruptError(lineno,context) + ** sqlite3MisuseError(lineno) + ** sqlite3CantopenError(lineno) + */ +@@ -94157,7 +94632,7 @@ SQLITE_API int sqlite3_found_count = 0; + ** + ** Other useful labels for breakpoints include: + ** test_addop_breakpoint(pc,pOp) +-** sqlite3CorruptError(lineno) ++** sqlite3CorruptError(lineno,context) + ** sqlite3MisuseError(lineno) + ** sqlite3CantopenError(lineno) + */ +@@ -181855,6 +182330,12 @@ SQLITE_API int sqlite3_config(int op, ...){ + #endif + break; + } ++ case SQLITE_CONFIG_CORRUPTION: { ++ typedef void(*CORRUPTION_FUNC_t)(void*, const void*); ++ sqlite3GlobalConfig.xCorruption = va_arg(ap, CORRUPTION_FUNC_t); ++ sqlite3GlobalConfig.pCorruptionArg = va_arg(ap, void*); ++ break; ++ } + + default: { + rc = SQLITE_ERROR; +@@ -185037,9 +185518,21 @@ SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ + zType, lineno, 20+sqlite3_sourceid()); + return iErr; + } +-SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ ++SQLITE_PRIVATE int sqlite3CorruptError(int lineno, sqlite3CorruptContext *context){ + testcase( sqlite3GlobalConfig.xLog!=0 ); +- return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); ++ if (context!=NULL && sqlite3GlobalConfig.xCorruption != 0) { ++ char zMsg[SQLITE_PRINT_BUF_SIZE] = {0}; /* Complete corruption log message */ ++ sqlite3_snprintf(sizeof(zMsg), zMsg, "pgno:%u,type:%d,range:{%d,%d},line:%d", ++ context->pgno, (int)context->type, (int)context->zoneRange.offset, (int)context->zoneRange.size, lineno); ++ sqlite3GlobalConfig.xCorruption(sqlite3GlobalConfig.pCorruptionArg, zMsg); ++ } ++ char zCorruptMsg[SQLITE_PRINT_BUF_SIZE * 10] = {0}; ++ if (context!=NULL && context->zMsg != NULL){ ++ sqlite3_snprintf(sizeof(zCorruptMsg), zCorruptMsg, "database corruption, %s", context->zMsg); ++ } else { ++ sqlite3_snprintf(sizeof(zCorruptMsg), zCorruptMsg, "database corruption"); ++ } ++ return sqlite3ReportError(SQLITE_CORRUPT, lineno, zCorruptMsg); + } + SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); +-- +2.47.0.windows.2 + diff --git a/mock/sqlite/patch/0006-Add-extention-cksumvfs-and-check-page.patch b/mock/sqlite/patch/0006-Add-extention-cksumvfs-and-check-page.patch new file mode 100644 index 00000000..c8d12eeb --- /dev/null +++ b/mock/sqlite/patch/0006-Add-extention-cksumvfs-and-check-page.patch @@ -0,0 +1,1079 @@ +From 506b1d3f9ea38fd7abc841ebc41649723247c1ee Mon Sep 17 00:00:00 2001 +From: MartinChoo <214582617@qq.com> +Date: Wed, 26 Feb 2025 10:16:41 +0800 +Subject: [PATCH] Add extention:cksumvfs and check page + +--- + ext/misc/cksumvfs.c | 916 ++++++++++++++++++++++++++++++++++++++++++++ + src/sqlite3.c | 99 ++++- + 2 files changed, 1008 insertions(+), 7 deletions(-) + create mode 100644 ext/misc/cksumvfs.c + +diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c +new file mode 100644 +index 0000000..6f4c55c +--- /dev/null ++++ b/ext/misc/cksumvfs.c +@@ -0,0 +1,916 @@ ++/* ++** 2020-04-20 ++** ++** The author disclaims copyright to this source code. In place of ++** a legal notice, here is a blessing: ++** ++** May you do good and not evil. ++** May you find forgiveness for yourself and forgive others. ++** May you share freely, never taking more than you give. ++** ++****************************************************************************** ++** ++** This file implements a VFS shim that writes a checksum on each page ++** of an SQLite database file. When reading pages, the checksum is verified ++** and an error is raised if the checksum is incorrect. ++** ++** COMPILING ++** ++** This extension requires SQLite 3.32.0 or later. It uses the ++** sqlite3_database_file_object() interface which was added in ++** version 3.32.0, so it will not link with an earlier version of ++** SQLite. ++** ++** To build this extension as a separately loaded shared library or ++** DLL, use compiler command-lines similar to the following: ++** ++** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so ++** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib ++** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll ++** ++** You may want to add additional compiler options, of course, ++** according to the needs of your project. ++** ++** If you want to statically link this extension with your product, ++** then compile it like any other C-language module but add the ++** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that ++** it is being statically linked rather than dynamically linked ++** ++** LOADING ++** ++** To load this extension as a shared library, you first have to ++** bring up a dummy SQLite database connection to use as the argument ++** to the sqlite3_load_extension() API call. Then you invoke the ++** sqlite3_load_extension() API and shutdown the dummy database ++** connection. All subsequent database connections that are opened ++** will include this extension. For example: ++** ++** sqlite3 *db; ++** sqlite3_open(":memory:", &db); ++** sqlite3_load_extension(db, "./cksumvfs"); ++** sqlite3_close(db); ++** ++** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and ++** statically linked against the application, initialize it using ++** a single API call as follows: ++** ++** sqlite3_register_cksumvfs(); ++** ++** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new ++** default VFS and it uses the prior default VFS as the next VFS ++** down in the stack. This is normally what you want. However, in ++** complex situations where multiple VFS shims are being loaded, ++** it might be important to ensure that cksumvfs is loaded in the ++** correct order so that it sequences itself into the default VFS ++** Shim stack in the right order. ++** ++** USING ++** ++** Open database connections using the sqlite3_open() or ++** sqlite3_open_v2() interfaces, as normal. Ordinary database files ++** (without a checksum) will operate normally. Databases with ++** checksums will return an SQLITE_IOERR_DATA error if a page is ++** encountered that contains an invalid checksum. ++** ++** Checksumming only works on databases that have a reserve-bytes ++** value of exactly 8. The default value for reserve-bytes is 0. ++** Hence, newly created database files will omit the checksum by ++** default. To create a database that includes a checksum, change ++** the reserve-bytes value to 8 by runing: ++** ++** int n = 8; ++** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); ++** ++** If you do this immediately after creating a new database file, ++** before anything else has been written into the file, then that ++** might be all that you need to do. Otherwise, the API call ++** above should be followed by: ++** ++** sqlite3_exec(db, "VACUUM", 0, 0, 0); ++** ++** It never hurts to run the VACUUM, even if you don't need it. ++** If the database is in WAL mode, you should shutdown and ++** reopen all database connections before continuing. ++** ++** From the CLI, use the ".filectrl reserve_bytes 8" command, ++** followed by "VACUUM;". ++** ++** Note that SQLite allows the number of reserve-bytes to be ++** increased but not decreased. So if a database file already ++** has a reserve-bytes value greater than 8, there is no way to ++** activate checksumming on that database, other than to dump ++** and restore the database file. Note also that other extensions ++** might also make use of the reserve-bytes. Checksumming will ++** be incompatible with those other extensions. ++** ++** VERIFICATION OF CHECKSUMS ++** ++** If any checksum is incorrect, the "PRAGMA quick_check" command ++** will find it. To verify that checksums are actually enabled ++** and running, use the following query: ++** ++** SELECT count(*), verify_checksum(data) ++** FROM sqlite_dbpage ++** GROUP BY 2; ++** ++** There are three possible outputs form the verify_checksum() ++** function: 1, 0, and NULL. 1 is returned if the checksum is ++** correct. 0 is returned if the checksum is incorrect. NULL ++** is returned if the page is unreadable. If checksumming is ++** enabled, the read will fail if the checksum is wrong, so the ++** usual result from verify_checksum() on a bad checksum is NULL. ++** ++** If everything is OK, the query above should return a single ++** row where the second column is 1. Any other result indicates ++** either that there is a checksum error, or checksum validation ++** is disabled. ++** ++** CONTROLLING CHECKSUM VERIFICATION ++** ++** The cksumvfs extension implements a new PRAGMA statement that can ++** be used to disable, re-enable, or query the status of checksum ++** verification: ++** ++** PRAGMA checksum_verification; -- query status ++** PRAGMA checksum_verification=OFF; -- disable verification ++** PRAGMA checksum_verification=ON; -- re-enable verification ++** ++** The "checksum_verification" pragma will return "1" (true) or "0" ++** (false) if checksum verification is enabled or disabled, respectively. ++** "Verification" in this context means the feature that causes ++** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while ++** reading. Checksums are always kept up-to-date as long as the ++** reserve-bytes value of the database is 8, regardless of the setting ++** of this pragma. Checksum verification can be disabled (for example) ++** to do forensic analysis of a database that has previously reported ++** a checksum error. ++** ++** The "checksum_verification" pragma will always respond with "0" if ++** the database file does not have a reserve-bytes value of 8. The ++** pragma will return no rows at all if the cksumvfs extension is ++** not loaded. ++** ++** IMPLEMENTATION NOTES ++** ++** The checksum is stored in the last 8 bytes of each page. This ++** module only operates if the "bytes of reserved space on each page" ++** value at offset 20 the SQLite database header is exactly 8. If ++** the reserved-space value is not 8, this module is a no-op. ++*/ ++#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_CKSUMVFS_STATIC) ++# define SQLITE_CKSUMVFS_STATIC ++#endif ++#ifdef SQLITE_CKSUMVFS_STATIC ++# include "sqlite3.h" ++#else ++# include "sqlite3ext.h" ++ SQLITE_EXTENSION_INIT1 ++#endif ++#include ++#include ++ ++// hw export the symbols ++#ifdef SQLITE_EXPORT_SYMBOLS ++#if defined(__GNUC__) ++# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) ++#elif defined(_MSC_VER) ++# define EXPORT_SYMBOLS __declspec(dllexport) ++#else ++# define EXPORT_SYMBOLS ++#endif ++#endif ++ ++/* ++** Forward declaration of objects used by this utility ++*/ ++typedef struct sqlite3_vfs CksmVfs; ++typedef struct CksmFile CksmFile; ++ ++/* ++** Useful datatype abbreviations ++*/ ++#if !defined(SQLITE_AMALGAMATION) ++ typedef unsigned char u8; ++ typedef unsigned int u32; ++#endif ++ ++/* Access to a lower-level VFS that (might) implement dynamic loading, ++** access to randomness, etc. ++*/ ++#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) ++#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) ++ ++/* An open file */ ++struct CksmFile { ++ sqlite3_file base; /* IO methods */ ++ const char *zFName; /* Original name of the file */ ++ char computeCksm; /* True to compute checksums. ++ ** Always true if reserve size is 8. */ ++ char verifyCksm; /* True to verify checksums */ ++ char isWal; /* True if processing a WAL file */ ++ char inCkpt; /* Currently doing a checkpoint */ ++ CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ ++}; ++ ++/* ++** Methods for CksmFile ++*/ ++static int cksmClose(sqlite3_file*); ++static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); ++static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); ++static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); ++static int cksmSync(sqlite3_file*, int flags); ++static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); ++static int cksmLock(sqlite3_file*, int); ++static int cksmUnlock(sqlite3_file*, int); ++static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); ++static int cksmFileControl(sqlite3_file*, int op, void *pArg); ++static int cksmSectorSize(sqlite3_file*); ++static int cksmDeviceCharacteristics(sqlite3_file*); ++static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); ++static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); ++static void cksmShmBarrier(sqlite3_file*); ++static int cksmShmUnmap(sqlite3_file*, int deleteFlag); ++static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); ++static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); ++ ++/* ++** Methods for CksmVfs ++*/ ++static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); ++static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); ++static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); ++static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); ++static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); ++static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); ++static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); ++static void cksmDlClose(sqlite3_vfs*, void*); ++static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); ++static int cksmSleep(sqlite3_vfs*, int microseconds); ++static int cksmCurrentTime(sqlite3_vfs*, double*); ++static int cksmGetLastError(sqlite3_vfs*, int, char *); ++static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); ++static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); ++static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); ++static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); ++ ++static sqlite3_vfs cksm_vfs = { ++ 3, /* iVersion (set when registered) */ ++ 0, /* szOsFile (set when registered) */ ++ 1024, /* mxPathname */ ++ 0, /* pNext */ ++ "cksmvfs", /* zName */ ++ 0, /* pAppData (set when registered) */ ++ cksmOpen, /* xOpen */ ++ cksmDelete, /* xDelete */ ++ cksmAccess, /* xAccess */ ++ cksmFullPathname, /* xFullPathname */ ++ cksmDlOpen, /* xDlOpen */ ++ cksmDlError, /* xDlError */ ++ cksmDlSym, /* xDlSym */ ++ cksmDlClose, /* xDlClose */ ++ cksmRandomness, /* xRandomness */ ++ cksmSleep, /* xSleep */ ++ cksmCurrentTime, /* xCurrentTime */ ++ cksmGetLastError, /* xGetLastError */ ++ cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ ++ cksmSetSystemCall, /* xSetSystemCall */ ++ cksmGetSystemCall, /* xGetSystemCall */ ++ cksmNextSystemCall /* xNextSystemCall */ ++}; ++ ++static const sqlite3_io_methods cksm_io_methods = { ++ 3, /* iVersion */ ++ cksmClose, /* xClose */ ++ cksmRead, /* xRead */ ++ cksmWrite, /* xWrite */ ++ cksmTruncate, /* xTruncate */ ++ cksmSync, /* xSync */ ++ cksmFileSize, /* xFileSize */ ++ cksmLock, /* xLock */ ++ cksmUnlock, /* xUnlock */ ++ cksmCheckReservedLock, /* xCheckReservedLock */ ++ cksmFileControl, /* xFileControl */ ++ cksmSectorSize, /* xSectorSize */ ++ cksmDeviceCharacteristics, /* xDeviceCharacteristics */ ++ cksmShmMap, /* xShmMap */ ++ cksmShmLock, /* xShmLock */ ++ cksmShmBarrier, /* xShmBarrier */ ++ cksmShmUnmap, /* xShmUnmap */ ++ cksmFetch, /* xFetch */ ++ cksmUnfetch /* xUnfetch */ ++}; ++ ++/* Do byte swapping on a unsigned 32-bit integer */ ++#define BYTESWAP32(x) ( \ ++ (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ ++ + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ ++) ++ ++/* Compute a checksum on a buffer */ ++static void cksmCompute( ++ u8 *a, /* Content to be checksummed */ ++ int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ ++ u8 *aOut /* OUT: Final 8-byte checksum value output */ ++){ ++ u32 s1 = 0, s2 = 0; ++ u32 *aData = (u32*)a; ++ u32 *aEnd = (u32*)&a[nByte]; ++ u32 x = 1; ++ ++ assert( nByte>=8 ); ++ assert( (nByte&0x00000007)==0 ); ++ assert( nByte<=65536 ); ++ ++ if( 1 == *(u8*)&x ){ ++ /* Little-endian */ ++ do { ++ s1 += *aData++ + s2; ++ s2 += *aData++ + s1; ++ }while( aData65536 || (nByte & (nByte-1))!=0 ) return; ++ cksmCompute(data, nByte-8, cksum); ++ sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); ++} ++ ++#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME ++/* ++** SQL function: initialize_cksumvfs(SCHEMANAME) ++** ++** This SQL functions (whose name is actually determined at compile-time ++** by the value of the SQLITE_CKSUMVFS_INIT_FUNCNAME macro) invokes: ++** ++** sqlite3_file_control(db, SCHEMANAME, SQLITE_FCNTL_RESERVE_BYTE, &n); ++** ++** In order to set the reserve bytes value to 8, so that cksumvfs will ++** operation. This feature is provided (if and only if the ++** SQLITE_CKSUMVFS_INIT_FUNCNAME compile-time option is set to a string ++** which is the name of the SQL function) so as to provide the ability ++** to invoke the file-control in programming languages that lack ++** direct access to the sqlite3_file_control() interface (ex: Java). ++** ++** This interface is undocumented, apart from this comment. Usage ++** example: ++** ++** 1. Compile with -DSQLITE_CKSUMVFS_INIT_FUNCNAME="ckvfs_init" ++** 2. Run: "SELECT cksum_init('main'); VACUUM;" ++*/ ++static void cksmInitFunc( ++ sqlite3_context *context, ++ int argc, ++ sqlite3_value **argv ++){ ++ int nByte = 8; ++ const char *zSchemaName = (const char*)sqlite3_value_text(argv[0]); ++ sqlite3 *db = sqlite3_context_db_handle(context); ++ sqlite3_file_control(db, zSchemaName, SQLITE_FCNTL_RESERVE_BYTES, &nByte); ++ /* Return NULL */ ++} ++#endif /* SQLITE_CKSUMBFS_INIT_FUNCNAME */ ++ ++/* ++** Close a cksm-file. ++*/ ++static int cksmClose(sqlite3_file *pFile){ ++ CksmFile *p = (CksmFile *)pFile; ++ if( p->pPartner ){ ++ assert( p->pPartner->pPartner==p ); ++ p->pPartner->pPartner = 0; ++ p->pPartner = 0; ++ } ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xClose(pFile); ++} ++ ++/* ++** Set the computeCkSm and verifyCksm flags, if they need to be ++** changed. ++*/ ++static void cksmSetFlags(CksmFile *p, int hasCorrectReserveSize){ ++ if( hasCorrectReserveSize!=p->computeCksm ){ ++ p->computeCksm = p->verifyCksm = hasCorrectReserveSize; ++ if( p->pPartner ){ ++ p->pPartner->verifyCksm = hasCorrectReserveSize; ++ p->pPartner->computeCksm = hasCorrectReserveSize; ++ } ++ } ++} ++ ++static void EncodeReservedBytesIntoBase16(const u8 *reserved, int len, char *encodeStr, int maxLen){ ++ static const char baseCode[] = "0123456789ABCDEF"; ++ for(int i=0; i> 4) & 0x0F]; ++ *encodeStr++ = baseCode[reserved[i] & 0x0F]; ++ } ++ *encodeStr = '0'; ++} ++ ++/* ++** Read data from a cksm-file. ++*/ ++static int cksmRead( ++ sqlite3_file *pFile, ++ void *zBuf, ++ int iAmt, ++ sqlite_int64 iOfst ++){ ++ int rc; ++ CksmFile *p = (CksmFile *)pFile; ++ pFile = ORIGFILE(pFile); ++ rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); ++ if( rc==SQLITE_OK ){ ++ if( iOfst==0 && iAmt>=100 && ( ++ memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 ++ )){ ++ u8 *d = (u8*)zBuf; ++ char hasCorrectReserveSize = (d[20]==8); ++ cksmSetFlags(p, hasCorrectReserveSize); ++ } ++ /* Verify the checksum if ++ ** (1) the size indicates that we are dealing with a complete ++ ** database page, only support pageSize:4K ++ ** (2) checksum verification is enabled ++ ** (3) we are not in the middle of checkpoint ++ */ ++ if( iAmt==4096 /* (1) */ ++ && p->verifyCksm /* (2) */ ++ && !p->inCkpt /* (3) */ ++ ){ ++ u8 cksum[8]; ++ cksmCompute((u8*)zBuf, iAmt-8, cksum); ++ if( memcmp((u8*)zBuf+iAmt-8, cksum, 8)!=0 ){ ++ char expect[18] = {0}; ++ char actual[18] = {0}; ++ EncodeReservedBytesIntoBase16((u8 *)zBuf+iAmt-8, 8, expect, 18); ++ EncodeReservedBytesIntoBase16(cksum, 8, actual, 18); ++ sqlite3_log(SQLITE_IOERR_DATA, "checksum fault offset %lld of \"%s\", amt:%d, expect:%s, actual:%s", ++ iOfst, p->zFName, iAmt, expect, actual); ++ rc = SQLITE_IOERR_DATA; ++ } ++ } ++ } ++ return rc; ++} ++ ++/* ++** Write data to a cksm-file. ++*/ ++static int cksmWrite( ++ sqlite3_file *pFile, ++ const void *zBuf, ++ int iAmt, ++ sqlite_int64 iOfst ++){ ++ CksmFile *p = (CksmFile *)pFile; ++ pFile = ORIGFILE(pFile); ++ if( iOfst==0 && iAmt>=100 && ( ++ memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 ++ )){ ++ u8 *d = (u8*)zBuf; ++ char hasCorrectReserveSize = (d[20]==8); ++ cksmSetFlags(p, hasCorrectReserveSize); ++ } ++ /* If the write size is appropriate for a database page and if ++ ** checksums where ever enabled, then it will be safe to compute ++ ** the checksums. The reserve byte size might have increased, but ++ ** it will never decrease. And because it cannot decrease, the ++ ** checksum will not overwrite anything. ++ */ ++ if( iAmt==4096 ++ && p->computeCksm ++ && !p->inCkpt ++ ){ ++ cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); ++ } ++ return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); ++} ++ ++/* ++** Truncate a cksm-file. ++*/ ++static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xTruncate(pFile, size); ++} ++ ++/* ++** Sync a cksm-file. ++*/ ++static int cksmSync(sqlite3_file *pFile, int flags){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xSync(pFile, flags); ++} ++ ++/* ++** Return the current file-size of a cksm-file. ++*/ ++static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ ++ CksmFile *p = (CksmFile *)pFile; ++ pFile = ORIGFILE(p); ++ return pFile->pMethods->xFileSize(pFile, pSize); ++} ++ ++/* ++** Lock a cksm-file. ++*/ ++static int cksmLock(sqlite3_file *pFile, int eLock){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xLock(pFile, eLock); ++} ++ ++/* ++** Unlock a cksm-file. ++*/ ++static int cksmUnlock(sqlite3_file *pFile, int eLock){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xUnlock(pFile, eLock); ++} ++ ++/* ++** Check if another file-handle holds a RESERVED lock on a cksm-file. ++*/ ++static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xCheckReservedLock(pFile, pResOut); ++} ++ ++/* ++** File control method. For custom operations on a cksm-file. ++*/ ++static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ ++ int rc; ++ CksmFile *p = (CksmFile*)pFile; ++ pFile = ORIGFILE(pFile); ++ if( op==SQLITE_FCNTL_PRAGMA ){ ++ char **azArg = (char**)pArg; ++ assert( azArg[1]!=0 ); ++ if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ ++ char *zArg = azArg[2]; ++ if( zArg!=0 ){ ++ if( (zArg[0]>='1' && zArg[0]<='9') ++ || sqlite3_strlike("enable%",zArg,0)==0 ++ || sqlite3_stricmp("yes",zArg)==0 ++ || sqlite3_stricmp("on",zArg)==0 ++ ){ ++ p->verifyCksm = p->computeCksm; ++ }else{ ++ p->verifyCksm = 0; ++ } ++ if( p->pPartner ) p->pPartner->verifyCksm = p->verifyCksm; ++ } ++ azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); ++ return SQLITE_OK; ++ }else if( p->computeCksm && azArg[2]!=0 ++ && sqlite3_stricmp(azArg[1], "page_size")==0 ){ ++ /* Do not allow page size changes on a checksum database */ ++ return SQLITE_OK; ++ } ++ }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ ++ p->inCkpt = op==SQLITE_FCNTL_CKPT_START; ++ if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; ++ }else if( op==SQLITE_FCNTL_CKSM_FILE ){ ++ /* This VFS needs to obtain a pointer to the corresponding database ++ ** file handle from within xOpen() calls to open wal files. To do this, ++ ** it uses the sqlite3_database_file_object() API to obtain a pointer ++ ** to the file-handle used by SQLite to access the db file. This is ++ ** fine if cksmvfs happens to be the top-level VFS, but not if there ++ ** are one or more wrapper VFS. To handle this case, this file-control ++ ** is used to extract the cksmvfs file-handle from any wrapper file ++ ** handle. */ ++ sqlite3_file **ppFile = (sqlite3_file**)pArg; ++ *ppFile = (sqlite3_file*)p; ++ return SQLITE_OK; ++ } ++ rc = pFile->pMethods->xFileControl(pFile, op, pArg); ++ if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ ++ *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); ++ } ++ return rc; ++} ++ ++/* ++** Return the sector-size in bytes for a cksm-file. ++*/ ++static int cksmSectorSize(sqlite3_file *pFile){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xSectorSize(pFile); ++} ++ ++/* ++** Return the device characteristic flags supported by a cksm-file. ++*/ ++static int cksmDeviceCharacteristics(sqlite3_file *pFile){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xDeviceCharacteristics(pFile); ++} ++ ++/* Create a shared memory file mapping */ ++static int cksmShmMap( ++ sqlite3_file *pFile, ++ int iPg, ++ int pgsz, ++ int bExtend, ++ void volatile **pp ++){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); ++} ++ ++/* Perform locking on a shared-memory segment */ ++static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xShmLock(pFile,offset,n,flags); ++} ++ ++/* Memory barrier operation on shared memory */ ++static void cksmShmBarrier(sqlite3_file *pFile){ ++ pFile = ORIGFILE(pFile); ++ pFile->pMethods->xShmBarrier(pFile); ++} ++ ++/* Unmap a shared memory segment */ ++static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ ++ pFile = ORIGFILE(pFile); ++ return pFile->pMethods->xShmUnmap(pFile,deleteFlag); ++} ++ ++/* Fetch a page of a memory-mapped file */ ++static int cksmFetch( ++ sqlite3_file *pFile, ++ sqlite3_int64 iOfst, ++ int iAmt, ++ void **pp ++){ ++ CksmFile *p = (CksmFile *)pFile; ++ if( p->computeCksm ){ ++ *pp = 0; ++ return SQLITE_OK; ++ } ++ pFile = ORIGFILE(pFile); ++ if( pFile->pMethods->iVersion>2 && pFile->pMethods->xFetch ){ ++ return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); ++ } ++ *pp = 0; ++ return SQLITE_OK; ++} ++ ++/* Release a memory-mapped page */ ++static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ ++ pFile = ORIGFILE(pFile); ++ if( pFile->pMethods->iVersion>2 && pFile->pMethods->xUnfetch ){ ++ return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); ++ } ++ return SQLITE_OK; ++} ++ ++/* ++** Open a cksm file handle. ++*/ ++static int cksmOpen( ++ sqlite3_vfs *pVfs, ++ const char *zName, ++ sqlite3_file *pFile, ++ int flags, ++ int *pOutFlags ++){ ++ CksmFile *p; ++ sqlite3_file *pSubFile; ++ sqlite3_vfs *pSubVfs; ++ int rc; ++ pSubVfs = ORIGVFS(pVfs); ++ if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ ++ return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); ++ } ++ p = (CksmFile*)pFile; ++ memset(p, 0, sizeof(*p)); ++ pSubFile = ORIGFILE(pFile); ++ pFile->pMethods = &cksm_io_methods; ++ rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); ++ if( rc ) goto cksm_open_done; ++ if( flags & SQLITE_OPEN_WAL ){ ++ sqlite3_file *pDb = sqlite3_database_file_object(zName); ++ rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); ++ assert( rc==SQLITE_OK ); ++ p->pPartner = (CksmFile*)pDb; ++ assert( p->pPartner->pPartner==0 ); ++ p->pPartner->pPartner = p; ++ p->isWal = 1; ++ p->computeCksm = p->pPartner->computeCksm; ++ }else{ ++ p->isWal = 0; ++ p->computeCksm = 0; ++ } ++ p->zFName = zName; ++cksm_open_done: ++ if( rc ) pFile->pMethods = 0; ++ return rc; ++} ++ ++/* ++** All other VFS methods are pass-thrus. ++*/ ++static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ ++ return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); ++} ++static int cksmAccess( ++ sqlite3_vfs *pVfs, ++ const char *zPath, ++ int flags, ++ int *pResOut ++){ ++ return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); ++} ++static int cksmFullPathname( ++ sqlite3_vfs *pVfs, ++ const char *zPath, ++ int nOut, ++ char *zOut ++){ ++ return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); ++} ++static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ ++ return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); ++} ++static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ ++ ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); ++} ++static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ ++ return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); ++} ++static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ ++ ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); ++} ++static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ ++ return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); ++} ++static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ ++ return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); ++} ++static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ ++ return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); ++} ++static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ ++ return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); ++} ++static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ ++ sqlite3_vfs *pOrig = ORIGVFS(pVfs); ++ int rc; ++ assert( pOrig->iVersion>=2 ); ++ if( pOrig->xCurrentTimeInt64 ){ ++ rc = pOrig->xCurrentTimeInt64(pOrig, p); ++ }else{ ++ double r; ++ rc = pOrig->xCurrentTime(pOrig, &r); ++ *p = (sqlite3_int64)(r*86400000.0); ++ } ++ return rc; ++} ++static int cksmSetSystemCall( ++ sqlite3_vfs *pVfs, ++ const char *zName, ++ sqlite3_syscall_ptr pCall ++){ ++ return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); ++} ++static sqlite3_syscall_ptr cksmGetSystemCall( ++ sqlite3_vfs *pVfs, ++ const char *zName ++){ ++ return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); ++} ++static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ ++ return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); ++} ++ ++/* Register the verify_checksum() SQL function. ++*/ ++static int cksmRegisterFunc( ++ sqlite3 *db, ++ char **pzErrMsg, ++ const sqlite3_api_routines *pApi ++){ ++ int rc; ++ if( db==0 ) return SQLITE_OK; ++ rc = sqlite3_create_function(db, "verify_checksum", 1, ++ SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, ++ 0, cksmVerifyFunc, 0, 0); ++#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME ++ (void)sqlite3_create_function(db, SQLITE_CKSUMVFS_INIT_FUNCNAME, 1, ++ SQLITE_UTF8|SQLITE_DIRECTONLY, ++ 0, cksmInitFunc, 0, 0); ++#endif ++ return rc; ++} ++ ++/* ++** Register the cksum VFS as the default VFS for the system. ++** Also make arrangements to automatically register the "verify_checksum()" ++** SQL function on each new database connection. ++*/ ++static int cksmRegisterVfs(void){ ++ int rc = SQLITE_OK; ++ sqlite3_vfs *pOrig; ++ if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; ++ pOrig = sqlite3_vfs_find(0); ++ if( pOrig==0 ) return SQLITE_ERROR; ++ cksm_vfs.iVersion = pOrig->iVersion; ++ cksm_vfs.pAppData = pOrig; ++ cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); ++ rc = sqlite3_vfs_register(&cksm_vfs, 1); ++ if( rc==SQLITE_OK ){ ++ rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); ++ } ++ return rc; ++} ++ ++#if defined(SQLITE_CKSUMVFS_STATIC) ++/* This variant of the initializer runs when the extension is ++** statically linked. ++*/ ++int sqlite3_register_cksumvfs(const char *NotUsed){ ++ (void)NotUsed; ++ return cksmRegisterVfs(); ++} ++int sqlite3_unregister_cksumvfs(void){ ++ if( sqlite3_vfs_find("cksmvfs") ){ ++ sqlite3_vfs_unregister(&cksm_vfs); ++ sqlite3_cancel_auto_extension((void(*)(void))cksmRegisterFunc); ++ } ++ return SQLITE_OK; ++} ++#endif /* defined(SQLITE_CKSUMVFS_STATIC */ ++ ++#if !defined(SQLITE_CKSUMVFS_STATIC) ++/* This variant of the initializer function is used when the ++** extension is shared library to be loaded at run-time. ++*/ ++#ifdef _WIN32 ++__declspec(dllexport) ++#endif ++/* ++** This routine is called by sqlite3_load_extension() when the ++** extension is first loaded. ++***/ ++int sqlite3_cksumvfs_init( ++ sqlite3 *db, ++ char **pzErrMsg, ++ const sqlite3_api_routines *pApi ++){ ++ int rc; ++ SQLITE_EXTENSION_INIT2(pApi); ++ (void)pzErrMsg; /* not used */ ++ rc = cksmRegisterFunc(db, 0, 0); ++ if( rc==SQLITE_OK ){ ++ rc = cksmRegisterVfs(); ++ } ++ if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; ++ return rc; ++} ++#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ ++ ++#ifdef SQLITE_CKSUMVFS_STATIC ++struct sqlite3_api_routines_cksumvfs { ++ int (*register_cksumvfs)(const char *); ++ int (*unregister_cksumvfs)(); ++}; ++typedef struct sqlite3_api_routines_cksumvfs sqlite3_api_routines_cksumvfs; ++static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { ++ sqlite3_register_cksumvfs, ++ sqlite3_unregister_cksumvfs ++}; ++ ++EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; ++#endif +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 0666938..7e9dcbf 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -58992,17 +58992,70 @@ static int jrnlBufferSize(Pager *pPager){ + ** and debugging only. + */ + #ifdef SQLITE_CHECK_PAGES +-/* +-** Return a 32-bit hash of the page data for pPage. +-*/ +-static u32 pager_datahash(int nByte, unsigned char *pData){ ++#if defined (__arm__) || defined (__aarch64__) ++#include ++u32 deep_fast_hash_arm(void *src, int srcLen){ ++ uint16_t chunkSize = srcLen/4; ++ uint8_t *u8p_src = (uint8_t *)src; ++ uint16x8_t m_prime = vdupq_n_u16(44497); ++ uint16x8_t m_res0 = vdupq_n_u16(0); ++ uint16x8_t m_res1 = vdupq_n_u16(0); ++ uint16x8_t m_res2 = vdupq_n_u16(0); ++ uint16x8_t m_res3 = vdupq_n_u16(0); ++ uint16x8_t m_res4 = vdupq_n_u16(0); ++ uint16x8_t m_res5 = vdupq_n_u16(0); ++ uint16x8_t m_res6 = vdupq_n_u16(0); ++ uint16x8_t m_res7 = vdupq_n_u16(0); ++ ++ for(int i=0; ipPager->pageSize, (unsigned char *)pPage->pData); + } +@@ -59018,8 +59071,15 @@ static void pager_set_pagehash(PgHdr *pPage){ + #define CHECK_PAGE(x) checkPage(x) + static void checkPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; +- assert( pPager->eState!=PAGER_ERROR ); +- assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); ++ if( pPager->eState==PAGER_ERROR ){ ++ return; ++ } ++ if( pPg->flags&PGHDR_DIRTY ) { ++ return; ++ } ++ if( pPg->pageHash!=pager_pagehash(pPg) ){ ++ sqlite3_log(SQLITE_CORRUPT, "cache corruption occurs through checking page(%u)", pPg->pgno); ++ } + } + + #else +@@ -185754,12 +185814,16 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ ++#ifndef SQLITE_CKSUMVFS_STATIC ++ rc = SQLITE_OK; ++#else + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=255 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; ++#endif + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; +@@ -261521,6 +261585,27 @@ static void DumpLocksByPager(Pager *pPager) + + // hw export the symbols + #ifdef SQLITE_EXPORT_SYMBOLS ++#ifndef SQLITE_CKSUMVFS_STATIC ++int sqlite3_register_cksumvfs(const char *NotUsed){ ++ return SQLITE_MISUSE; ++} ++int sqlite3_unregister_cksumvfs(void){ ++ return SQLITE_MISUSE; ++} ++ ++struct sqlite3_api_routines_cksumvfs { ++ int (*register_cksumvfs)(const char *); ++ int (*unregister_cksumvfs)(); ++}; ++typedef struct sqlite3_api_routines_cksumvfs sqlite3_api_routines_cksumvfs; ++static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { ++ sqlite3_register_cksumvfs, ++ sqlite3_unregister_cksumvfs ++}; ++ ++EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; ++#endif ++ + struct sqlite3_api_routines_hw { + int (*initialize)(); + int (*config)(int,...); +-- +2.47.0.windows.2 + diff --git a/mock/sqlite/patch/0007-BugFix-CurrVersion.patch b/mock/sqlite/patch/0007-BugFix-CurrVersion.patch new file mode 100644 index 00000000..bbd61dce --- /dev/null +++ b/mock/sqlite/patch/0007-BugFix-CurrVersion.patch @@ -0,0 +1,72 @@ +From 5dd4cd181892660bbf665994c42d2fd18e8846a5 Mon Sep 17 00:00:00 2001 +From: linzhuobin1 +Date: Sat, 1 Mar 2025 16:29:16 +0800 +Subject: [PATCH] IssueNo:#IBPWV3 Description: fix a potential uaf in fts3. + Sig: SIG_DataManagement Feature or Bugfix:Bugfix Binary Source:No/Yes + TDD:Pass XTS:Pass Pretest:Pass + +Signed-off-by: linzhuobin1 +--- + src/sqlite3.c | 39 +++++++++++++++++++++++++++++++++++++++ + 1 file changed, 39 insertions(+) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 7e9dcbf..2084441 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -205751,6 +205751,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ + return rc; + } + ++/* ++** Expression node pExpr is an MSR phrase. This function restarts pExpr ++** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned ++** if successful, or an SQLite error code otherwise. ++*/ ++int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){ ++ int rc = SQLITE_OK; ++ if( pExpr->bEof==0 ){ ++ i64 iDocid = pExpr->iDocid; ++ fts3EvalRestart(pCsr, pExpr, &rc); ++ while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){ ++ fts3EvalNextRow(pCsr, pExpr, &rc); ++ if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB; ++ } ++ } ++ return rc; ++} ++ ++/* ++** If expression pExpr is a phrase expression that uses an MSR query, ++** restart it as a regular, non-incremental query. Return SQLITE_OK ++** if successful, or an SQLite error code otherwise. ++*/ ++static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ ++ TermOffsetCtx *p = (TermOffsetCtx*)ctx; ++ int rc = SQLITE_OK; ++ if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){ ++ rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr); ++ pExpr->pPhrase->bIncr = 0; ++ } ++ return rc; ++} ++ + /* + ** Implementation of offsets() function. + */ +@@ -205787,6 +205820,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + ++ /* If a query restart will be required, do it here, rather than later of ++ ** after pointers to poslist buffers that may be invalidated by a restart ++ ** have been saved. */ ++ rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx); ++ if( rc!=SQLITE_OK ) goto offsets_out; ++ + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ +-- +2.46.0.windows.1 + diff --git a/mock/sqlite/patch/BUILD.gn b/mock/sqlite/patch/BUILD.gn new file mode 100644 index 00000000..2abaadd9 --- /dev/null +++ b/mock/sqlite/patch/BUILD.gn @@ -0,0 +1,41 @@ +# Copyright (C) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build/ohos.gni") + +sqlite_src_dir = "//third_party/sqlite" +sqlite_dst_dir = root_out_dir + "/patched_sqlite" +sqlite_patch_dir = sqlite_src_dir + "/patch" + +action("apply_patch") { + visibility = [ "*" ] + script = "${sqlite_patch_dir}/apply_patch.sh" + inputs = [ "$sqlite_src_dir" ] + outputs = [ + "$sqlite_dst_dir/ext/misc/cksumvfs.c", + "$sqlite_dst_dir/src/shell.c", + "$sqlite_dst_dir/src/sqlite3.c", + "$sqlite_dst_dir/src/sqlite3icu.c", + "$sqlite_dst_dir/include/sqlite3ext.h", + "$sqlite_dst_dir/include/sqlite3.h", + "$sqlite_dst_dir/include/sqlite3sym.h", + "$sqlite_dst_dir/include/sqlite3icu.h", + "$sqlite_dst_dir/include/sqlite3tokenizer.h", + ] + + args = [ + rebase_path(sqlite_src_dir, root_build_dir), + rebase_path(sqlite_dst_dir, root_build_dir), + rebase_path(sqlite_patch_dir, root_build_dir), + ] +} diff --git a/mock/sqlite/patch/apply_patch.sh b/mock/sqlite/patch/apply_patch.sh new file mode 100644 index 00000000..22af1f81 --- /dev/null +++ b/mock/sqlite/patch/apply_patch.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (C) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +cur_dir=$(pwd) +src_dir=$1 +dst_dir=$2 +patch_dir=$3 + +echo "cur_dir: $cur_dir" +echo "src_dir: $src_dir" +echo "dst_dir: $dst_dir" +echo "patch_dir: $patch_dir" + +if [ "$src_dir" == "" ] || [ "$dst_dir" == "" ] || [ "$patch_dir" == "" ]; then + echo "invalid params for apply_patch." +exit 1 +fi + +echo "check and clear patched_sqlite_dir: $dst_dir" +if [ -d "$dst_dir" ]; then + echo "remove $dst_dir begin" + rm -rf "$dst_dir" +fi + +echo "create patched_sqlite_dir: $dst_dir" +mkdir -p $dst_dir + +echo "copy $src_dir/* to $dst_dir/" +cp -fa $src_dir/* $dst_dir + +ls -l $patch_dir/*.patch +if [ $? -ne 0 ]; then + echo "WARNING: no patch." + exit 0 +fi + +PATCH_FILES=$(realpath $(ls $patch_dir/*.patch)) +echo "found the following patchs:" +echo "$PATCH_FILES" + +cd $dst_dir +echo "pwd: $(pwd)" + +for patch_file in $PATCH_FILES; do + echo "applying patch: $patch_file" + patch -p1 -i "$patch_file" + if [ $? -ne 0 ]; then + echo "failed to apply patch: $patch_file" + exit 1 + fi +done + +cd $cur_dir +exit 0 diff --git a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/BUILD.gn b/mock/sqlite/sqlite.gni similarity index 64% rename from data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/BUILD.gn rename to mock/sqlite/sqlite.gni index 8c49df4a..0f9c7918 100644 --- a/data_object/frameworks/jskitsimpl/collaboration_edit/test/unittest/src/BUILD.gn +++ b/mock/sqlite/sqlite.gni @@ -11,19 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import("//build/test.gni") - -module_output_path = "data_object/collaboration_edit" - -ohos_js_unittest("CollaborationEditJsTest") { - module_out_path = module_output_path - - hap_profile = "../config.json" - - certificate_profile = "../openharmony_sx.p7b" -} - -group("unittest") { - testonly = true - deps = [ ":CollaborationEditJsTest" ] +declare_args() { + sqlite_support_check_pages = false } diff --git a/mock/sqlite/src/shell.c b/mock/sqlite/src/shell.c index dc66c400..b08671e1 100644 --- a/mock/sqlite/src/shell.c +++ b/mock/sqlite/src/shell.c @@ -34,17 +34,19 @@ /* This needs to come before any includes for MSVC compiler */ #define _CRT_SECURE_NO_WARNINGS #endif +typedef unsigned int u32; +typedef unsigned short int u16; /* ** Optionally #include a user-defined header, whereby compilation options -** may be set prior to where they take effect, but after platform setup. +** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include ** file. Note that this macro has a like effect on sqlite3.c compilation. */ +# define SHELL_STRINGIFY_(f) #f +# define SHELL_STRINGIFY(f) SHELL_STRINGIFY_(f) #ifdef SQLITE_CUSTOM_INCLUDE -# define INC_STRINGIFY_(f) #f -# define INC_STRINGIFY(f) INC_STRINGIFY_(f) -# include INC_STRINGIFY(SQLITE_CUSTOM_INCLUDE) +# include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) #endif /* @@ -55,6 +57,15 @@ # define SQLITE_OS_WINRT 0 #endif +/* +** If SQLITE_SHELL_FIDDLE is defined then the shell is modified +** somewhat for use as a WASM module in a web browser. This flag +** should only be used when building the "fiddle" web application, as +** the browser-mode build has much different user input requirements +** and this build mode rewires the user input subsystem to account for +** that. +*/ + /* ** Warning pragmas copied from msvc.h in the core. */ @@ -94,10 +105,19 @@ # define _LARGEFILE_SOURCE 1 #endif +#if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE) +/* +** emcc requires _POSIX_SOURCE (or one of several similar defines) +** to expose strdup(). +*/ +# define _POSIX_SOURCE +#endif + #include #include #include #include +#include #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -110,7 +130,7 @@ typedef unsigned char u8; #if !defined(_WIN32) && !defined(WIN32) # include -# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) # include # endif #endif @@ -165,6 +185,14 @@ typedef unsigned char u8; # define SHELL_USE_LOCAL_GETLINE 1 #endif +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif #if defined(_WIN32) || defined(WIN32) # if SQLITE_OS_WINRT @@ -191,7 +219,7 @@ typedef unsigned char u8; /* Make sure isatty() has a prototype. */ extern int isatty(int); -# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) /* popen and pclose are not C89 functions and so are ** sometimes omitted from the header */ extern FILE *popen(const char*,const char*); @@ -218,1936 +246,6909 @@ typedef unsigned char u8; #if SQLITE_OS_WINRT #include #endif +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN #include /* string conversion routines only needed on Win32 */ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); -extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); -extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif -/* On Windows, we normally run with output mode of TEXT so that \n characters -** are automatically translated into \r\n. However, this behavior needs -** to be disabled in some cases (ex: when generating CSV output and when -** rendering quoted strings that contain \n characters). The following -** routines take care of that. -*/ -#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT -static void setBinaryMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_BINARY); -} -static void setTextMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_TEXT); -} +/* Use console I/O package as a direct INCLUDE. */ +#define SQLITE_INTERNAL_LINKAGE static + +#ifdef SQLITE_SHELL_FIDDLE +/* Deselect most features from the console I/O package for Fiddle. */ +# define SQLITE_CIO_NO_REDIRECT +# define SQLITE_CIO_NO_CLASSIFY +# define SQLITE_CIO_NO_TRANSLATE +# define SQLITE_CIO_NO_SETMODE +#endif +/************************* Begin ../ext/consio/console_io.h ******************/ +/* +** 2023 November 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +******************************************************************************** +** This file exposes various interfaces used for console and other I/O +** by the SQLite project command-line tools. These interfaces are used +** at either source conglomeration time, compilation time, or run time. +** This source provides for either inclusion into conglomerated, +** "single-source" forms or separate compilation then linking. +** +** Platform dependencies are "hidden" here by various stratagems so +** that, provided certain conditions are met, the programs using this +** source or object code compiled from it need no explicit conditional +** compilation in their source for their console and stream I/O. +** +** The symbols and functionality exposed here are not a public API. +** This code may change in tandem with other project code as needed. +** +** When this .h file and its companion .c are directly incorporated into +** a source conglomeration (such as shell.c), the preprocessor symbol +** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O +** translation for Windows is effected for the build. +*/ +#define HAVE_CONSOLE_IO_H 1 +#ifndef SQLITE_INTERNAL_LINKAGE +# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */ +# include #else -# define setBinaryMode(X,Y) -# define setTextMode(X,Y) +# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */ #endif - -/* True if the timer is enabled */ -static int enableTimer = 0; - -/* Return the current wall-clock time */ -static sqlite3_int64 timeOfDay(void){ - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs==0 ) return 0; /* Never actually happens */ - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); - }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); - } - return t; -} - -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include -#include - -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ -}; -#define getrusage(A,B) memset(B,0,sizeof(*B)) +#ifndef SQLITE3_H +/* # include "sqlite3.h" */ #endif -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ - -/* -** Begin timing an operation -*/ -static void beginTimer(void){ - if( enableTimer ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); - } -} +#ifndef SQLITE_CIO_NO_CLASSIFY -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); -} +/* Define enum for use with following function. */ +typedef enum StreamsAreConsole { + SAC_NoConsole = 0, + SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4, + SAC_AnyConsole = 0x7 +} StreamsAreConsole; /* -** Print the timing results. +** Classify the three standard I/O streams according to whether +** they are connected to a console attached to the process. +** +** Returns the bit-wise OR of SAC_{In,Out,Err}Console values, +** or SAC_NoConsole if none of the streams reaches a console. +** +** This function should be called before any I/O is done with +** the given streams. As a side-effect, the given inputs are +** recorded so that later I/O operations on them may be done +** differently than the C library FILE* I/O would be done, +** iff the stream is used for the I/O functions that follow, +** and to support the ones that use an implicit stream. +** +** On some platforms, stream or console mode alteration (aka +** "Setup") may be made which is undone by consoleRestore(). */ -static void endTimer(void){ - if( enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); - } -} - -#define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() -#define HAS_TIMER 1 - -#elif (defined(_WIN32) || defined(WIN32)) - -/* Saved resource information for the beginning of an operation */ -static HANDLE hProcess; -static FILETIME ftKernelBegin; -static FILETIME ftUserBegin; -static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, - LPFILETIME, LPFILETIME); -static GETPROCTIMES getProcessTimesAddr = NULL; +SQLITE_INTERNAL_LINKAGE StreamsAreConsole +consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ); +/* A usual call for convenience: */ +#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr) /* -** Check to see if we have timer support. Return 1 if necessary -** support found (or found previously). +** After an initial call to consoleClassifySetup(...), renew +** the same setup it effected. (A call not after is an error.) +** This will restore state altered by consoleRestore(); +** +** Applications which run an inferior (child) process which +** inherits the same I/O streams may call this function after +** such a process exits to guard against console mode changes. */ -static int hasTimer(void){ - if( getProcessTimesAddr ){ - return 1; - } else { -#if !SQLITE_OS_WINRT - /* GetProcessTimes() isn't supported in WIN95 and some other Windows - ** versions. See if the version we are running on has it, and if it - ** does, save off a pointer to it and the current process handle. - */ - hProcess = GetCurrentProcess(); - if( hProcess ){ - HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); - if( NULL != hinstLib ){ - getProcessTimesAddr = - (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); - if( NULL != getProcessTimesAddr ){ - return 1; - } - FreeLibrary(hinstLib); - } - } -#endif - } - return 0; -} +SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void); /* -** Begin timing an operation +** Undo any side-effects left by consoleClassifySetup(...). +** +** This should be called after consoleClassifySetup() and +** before the process terminates normally. It is suitable +** for use with the atexit() C library procedure. After +** this call, no console I/O should be done until one of +** console{Classify or Renew}Setup(...) is called again. +** +** Applications which run an inferior (child) process that +** inherits the same I/O streams might call this procedure +** before so that said process will have a console setup +** however users have configured it or come to expect. */ -static void beginTimer(void){ - if( enableTimer && getProcessTimesAddr ){ - FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess,&ftCreation,&ftExit, - &ftKernelBegin,&ftUserBegin); - ftWallBegin = timeOfDay(); - } -} +SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ); -/* Return the difference of two FILETIME structs in seconds */ -static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite_int64 i64Start = *((sqlite_int64 *) pStart); - sqlite_int64 i64End = *((sqlite_int64 *) pEnd); - return (double) ((i64End - i64Start) / 10000000.0); -} +#else /* defined(SQLITE_CIO_NO_CLASSIFY) */ +# define consoleClassifySetup(i,o,e) +# define consoleRenewSetup() +# define consoleRestore() +#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */ +#ifndef SQLITE_CIO_NO_REDIRECT /* -** Print the timing results. +** Set stream to be used for the functions below which write +** to "the designated X stream", where X is Output or Error. +** Returns the previous value. +** +** Alternatively, pass the special value, invalidFileStream, +** to get the designated stream value without setting it. +** +** Before the designated streams are set, they default to +** those passed to consoleClassifySetup(...), and before +** that is called they default to stdout and stderr. +** +** It is error to close a stream so designated, then, without +** designating another, use the corresponding {o,e}Emit(...). */ -static void endTimer(void){ - if( enableTimer && getProcessTimesAddr){ - FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; - sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); - } -} - -#define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() -#define HAS_TIMER hasTimer() - +SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream; +SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf); +# ifdef CONSIO_SET_ERROR_STREAM +SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf); +# endif #else -#define BEGIN_TIMER -#define END_TIMER -#define HAS_TIMER 0 -#endif +# define setOutputStream(pf) +# define setErrorStream(pf) +#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */ +#ifndef SQLITE_CIO_NO_TRANSLATE /* -** Used to prevent warnings about unused parameters +** Emit output like fprintf(). If the output is going to the +** console and translation from UTF-8 is necessary, perform +** the needed translation. Otherwise, write formatted output +** to the provided stream almost as-is, possibly with newline +** translation as specified by set{Binary,Text}Mode(). */ -#define UNUSED_PARAMETER(x) (void)(x) +SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...); +/* Like fPrintfUtf8 except stream is always the designated output. */ +SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...); +/* Like fPrintfUtf8 except stream is always the designated error. */ +SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...); /* -** Number of elements in an array +** Emit output like fputs(). If the output is going to the +** console and translation from UTF-8 is necessary, perform +** the needed translation. Otherwise, write given text to the +** provided stream almost as-is, possibly with newline +** translation as specified by set{Binary,Text}Mode(). */ -#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) +SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO); +/* Like fPutsUtf8 except stream is always the designated output. */ +SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z); +/* Like fPutsUtf8 except stream is always the designated error. */ +SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z); /* -** If the following flag is set, then command execution stops -** at an error if we are not interactive. +** Emit output like fPutsUtf8(), except that the length of the +** accepted char or character sequence is limited by nAccept. +** +** Returns the number of accepted char values. */ -static int bail_on_error = 0; +#ifdef CONSIO_SPUTB +SQLITE_INTERNAL_LINKAGE int +fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept); +/* Like fPutbUtf8 except stream is always the designated output. */ +#endif +SQLITE_INTERNAL_LINKAGE int +oPutbUtf8(const char *cBuf, int nAccept); +/* Like fPutbUtf8 except stream is always the designated error. */ +#ifdef CONSIO_EPUTB +SQLITE_INTERNAL_LINKAGE int +ePutbUtf8(const char *cBuf, int nAccept); +#endif /* -** Threat stdin as an interactive input if the following variable -** is true. Otherwise, assume stdin is connected to a file or pipe. +** Collect input like fgets(...) with special provisions for input +** from the console on platforms that require same. Defers to the +** C library fgets() when input is not from the console. Newline +** translation may be done as set by set{Binary,Text}Mode(). As a +** convenience, pfIn==NULL is treated as stdin. */ -static int stdin_is_interactive = 1; +SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn); +/* Like fGetsUtf8 except stream is always the designated input. */ +/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */ -/* -** On Windows systems we have to know if standard output is a console -** in order to translate UTF-8 into MBCS. The following variable is -** true if translation is required. -*/ -static int stdout_is_console = 1; +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ +#ifndef SQLITE_CIO_NO_SETMODE /* -** The following is the open SQLite database. We make a pointer -** to this database a static variable so that it can be accessed -** by the SIGINT handler to interrupt database processing. +** Set given stream for binary mode, where newline translation is +** not done, or for text mode where, for some platforms, newlines +** are translated to the platform's conventional char sequence. +** If bFlush true, flush the stream. +** +** An additional side-effect is that if the stream is one passed +** to consoleClassifySetup() as an output, it is flushed first. +** +** Note that binary/text mode has no effect on console I/O +** translation. On all platforms, newline to the console starts +** a new line and CR,LF chars from the console become a newline. */ -static sqlite3 *globalDb = 0; +SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush); +SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush); +#endif -/* -** True if an interrupt (Control-C) has been received. -*/ -static volatile int seenInterrupt = 0; +#ifdef SQLITE_CIO_PROMPTED_IN +typedef struct Prompts { + int numPrompts; + const char **azPrompts; +} Prompts; -#ifdef SQLITE_DEBUG /* -** Out-of-memory simulator variables +** Macros for use of a line editor. +** +** The following macros define operations involving use of a +** line-editing library or simple console interaction. +** A "T" argument is a text (char *) buffer or filename. +** A "N" argument is an integer. +** +** SHELL_ADD_HISTORY(T) // Record text as line(s) of history. +** SHELL_READ_HISTORY(T) // Read history from file named by T. +** SHELL_WRITE_HISTORY(T) // Write history to file named by T. +** SHELL_STIFLE_HISTORY(N) // Limit history to N entries. +** +** A console program which does interactive console input is +** expected to call: +** SHELL_READ_HISTORY(T) before collecting such input; +** SHELL_ADD_HISTORY(T) as record-worthy input is taken; +** SHELL_STIFLE_HISTORY(N) after console input ceases; then +** SHELL_WRITE_HISTORY(T) before the program exits. */ -static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ -static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ -static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ -#endif /* SQLITE_DEBUG */ /* -** This is the name of our program. It is set in main(), used -** in a number of other places, mostly for error messages. +** Retrieve a single line of input text from an input stream. +** +** If pfIn is the input stream passed to consoleClassifySetup(), +** and azPrompt is not NULL, then a prompt is issued before the +** line is collected, as selected by the isContinuation flag. +** Array azPrompt[{0,1}] holds the {main,continuation} prompt. +** +** If zBufPrior is not NULL then it is a buffer from a prior +** call to this routine that can be reused, or will be freed. +** +** The result is stored in space obtained from malloc() and +** must either be freed by the caller or else passed back to +** this function as zBufPrior for reuse. +** +** This function may call upon services of a line-editing +** library to interactively collect line edited input. */ -static char *Argv0; - +SQLITE_INTERNAL_LINKAGE char * +shellGetLine(FILE *pfIn, char *zBufPrior, int nLen, + short isContinuation, Prompts azPrompt); +#endif /* defined(SQLITE_CIO_PROMPTED_IN) */ /* -** Prompt strings. Initialized in main. Settable with -** .prompt main continue +** TBD: Define an interface for application(s) to generate +** completion candidates for use by the line-editor. +** +** This may be premature; the CLI is the only application +** that does this. Yet, getting line-editing melded into +** console I/O is desirable because a line-editing library +** may have to establish console operating mode, possibly +** in a way that interferes with the above functionality. */ -static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ -static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ -/* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, translate the -** output from UTF-8 into MBCS. -*/ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) ){ - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); -} -#elif !defined(utf8_printf) -# define utf8_printf fprintf +#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE)) +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +SQLITE_INTERNAL_LINKAGE const char* +zSkipValidUtf8(const char *z, int nAccept, long ccm); + #endif +/************************* End ../ext/consio/console_io.h ********************/ +/************************* Begin ../ext/consio/console_io.c ******************/ /* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). +** 2023 November 4 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +******************************************************************************** +** This file implements various interfaces used for console and stream I/O +** by the SQLite project command-line tools, as explained in console_io.h . +** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there. */ -#if !defined(raw_printf) -# define raw_printf fprintf + +#ifndef SQLITE_CDECL +# define SQLITE_CDECL #endif -/* Indicate out-of-memory and exit. */ -static void shell_out_of_memory(void){ - raw_printf(stderr,"Error: out of memory\n"); - exit(1); -} +#ifndef SHELL_NO_SYSINC +# include +# include +# include +# include +# include +/* # include "sqlite3.h" */ +#endif +#ifndef HAVE_CONSOLE_IO_H +# include "console_io.h" +#endif +#if defined(_MSC_VER) +# pragma warning(disable : 4204) +#endif -#ifdef SQLITE_DEBUG -/* This routine is called when a simulated OOM occurs. It is broken -** out as a separate routine to make it easy to set a breakpoint on -** the OOM -*/ -void shellOomFault(void){ - if( oomRepeat>0 ){ - oomRepeat--; - }else{ - oomCounter--; - } -} -#endif /* SQLITE_DEBUG */ +#ifndef SQLITE_CIO_NO_TRANSLATE +# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT +# ifndef SHELL_NO_SYSINC +# include +# include +# undef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# include +# endif +# define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */ +# else +# ifndef SHELL_NO_SYSINC +# include +# endif +# define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */ +# endif +#else +# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */ +#endif -#ifdef SQLITE_DEBUG -/* This routine is a replacement malloc() that is used to simulate -** Out-Of-Memory (OOM) errors for testing purposes. -*/ -static void *oomMalloc(int nByte){ - if( oomCounter ){ - if( oomCounter==1 ){ - shellOomFault(); - return 0; - }else{ - oomCounter--; - } - } - return defaultMalloc(nByte); +#if CIO_WIN_WC_XLATE +static HANDLE handleOfFile(FILE *pf){ + int fileDesc = _fileno(pf); + union { intptr_t osfh; HANDLE fh; } fid = { + (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE + }; + return fid.fh; } -#endif /* SQLITE_DEBUG */ +#endif -#ifdef SQLITE_DEBUG -/* Register the OOM simulator. This must occur before any memory -** allocations */ -static void registerOomSimulator(void){ - sqlite3_mem_methods mem; - sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem); - defaultMalloc = mem.xMalloc; - mem.xMalloc = oomMalloc; - sqlite3_config(SQLITE_CONFIG_MALLOC, &mem); +#ifndef SQLITE_CIO_NO_TRANSLATE +typedef struct PerStreamTags { +# if CIO_WIN_WC_XLATE + HANDLE hx; + DWORD consMode; + char acIncomplete[4]; +# else + short reachesConsole; +# endif + FILE *pf; +} PerStreamTags; + +/* Define NULL-like value for things which can validly be 0. */ +# define SHELL_INVALID_FILE_PTR ((FILE *)~0) +# if CIO_WIN_WC_XLATE +# define SHELL_INVALID_CONS_MODE 0xFFFF0000 +# endif + +# if CIO_WIN_WC_XLATE +# define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \ + {0,0,0,0}, SHELL_INVALID_FILE_PTR } +# else +# define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR } +# endif + +/* Quickly say whether a known output is going to the console. */ +# if CIO_WIN_WC_XLATE +static short pstReachesConsole(PerStreamTags *ppst){ + return (ppst->hx != INVALID_HANDLE_VALUE); } -#endif +# else +# define pstReachesConsole(ppst) 0 +# endif -/* -** Write I/O traces to the following stream. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif +# if CIO_WIN_WC_XLATE +static void restoreConsoleArb(PerStreamTags *ppst){ + if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode); +} +# else +# define restoreConsoleArb(ppst) +# endif -/* -** This routine works like printf in that its first argument is a -** format string and subsequent arguments are values to be substituted -** in place of % fields. The result of formatting this string -** is written to iotrace. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ - va_list ap; - char *z; - if( iotrace==0 ) return; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - utf8_printf(iotrace, "%s", z); - sqlite3_free(z); +/* Say whether FILE* appears to be a console, collect associated info. */ +static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ +# if CIO_WIN_WC_XLATE + short rv = 0; + DWORD dwCM = SHELL_INVALID_CONS_MODE; + HANDLE fh = handleOfFile(pf); + ppst->pf = pf; + if( INVALID_HANDLE_VALUE != fh ){ + rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM)); + } + ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE; + ppst->consMode = dwCM; + return rv; +# else + ppst->pf = pf; + ppst->reachesConsole = ( (short)isatty(fileno(pf)) ); + return ppst->reachesConsole; +# endif } -#endif -/* -** Output string zUtf to stream pOut as w characters. If w is negative, -** then right-justify the text. W is the width in UTF-8 characters, not -** in bytes. This is different from the %*.*s specification in printf -** since with %*.*s the width is measured in bytes, not characters. -*/ -static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ - int i; - int n; - int aw = w<0 ? -w : w; - for(i=n=0; zUtf[i]; i++){ - if( (zUtf[i]&0xc0)!=0x80 ){ - n++; - if( n==aw ){ - do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); - break; - } - } - } - if( n>=aw ){ - utf8_printf(pOut, "%.*s", i, zUtf); - }else if( w<0 ){ - utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); - }else{ - utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); - } +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) +# endif + +# if CIO_WIN_WC_XLATE +/* Define console modes for use with the Windows Console API. */ +# define SHELL_CONI_MODE \ + (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \ + | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT) +# define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \ + | ENABLE_VIRTUAL_TERMINAL_PROCESSING) +# endif + +typedef struct ConsoleInfo { + PerStreamTags pstSetup[3]; + PerStreamTags pstDesignated[3]; + StreamsAreConsole sacSetup; +} ConsoleInfo; + +static short isValidStreamInfo(PerStreamTags *ppst){ + return (ppst->pf != SHELL_INVALID_FILE_PTR); } +static ConsoleInfo consoleInfo = { + { /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER }, + { /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER }, + SAC_NoConsole /* sacSetup */ +}; -/* -** Determines if a string is a number of not. -*/ -static int isNumber(const char *z, int *realnum){ - if( *z=='-' || *z=='+' ) z++; - if( !IsDigit(*z) ){ - return 0; +SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0; + +# if CIO_WIN_WC_XLATE +static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){ + if( pstReachesConsole(ppst) ){ + DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE; + SetConsoleMode(ppst->hx, cm); } - z++; - if( realnum ) *realnum = 0; - while( IsDigit(*z) ){ z++; } - if( *z=='.' ){ - z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; +} +# else +# define maybeSetupAsConsole(ppst,odir) +# endif + +SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){ +# if CIO_WIN_WC_XLATE + int ix = 0; + while( ix < 6 ){ + PerStreamTags *ppst = (ix<3)? + &consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3]; + maybeSetupAsConsole(ppst, (ix % 3)>0); + ++ix; } - if( *z=='e' || *z=='E' ){ - z++; - if( *z=='+' || *z=='-' ) z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; +# endif +} + +SQLITE_INTERNAL_LINKAGE StreamsAreConsole +consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){ + StreamsAreConsole rv = SAC_NoConsole; + FILE* apf[3] = { pfIn, pfOut, pfErr }; + int ix; + for( ix = 2; ix >= 0; --ix ){ + PerStreamTags *ppst = &consoleInfo.pstSetup[ix]; + if( streamOfConsole(apf[ix], ppst) ){ + rv |= (SAC_InConsole< 0 ) fflush(apf[ix]); } - return *z==0; + consoleInfo.sacSetup = rv; + consoleRenewSetup(); + return rv; } -/* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -*/ -static int strlen30(const char *z){ - const char *z2 = z; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); +SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){ +# if CIO_WIN_WC_XLATE + static ConsoleInfo *pci = &consoleInfo; + if( pci->sacSetup ){ + int ix; + for( ix=0; ix<3; ++ix ){ + if( pci->sacSetup & (SAC_InConsole<pstSetup[ix]; + SetConsoleMode(ppst->hx, ppst->consMode); + } + } + } +# endif } +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ -/* -** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +#ifdef SQLITE_CIO_INPUT_REDIR +/* Say whether given FILE* is among those known, via either +** consoleClassifySetup() or set{Output,Error}Stream, as +** readable, and return an associated PerStreamTags pointer +** if so. Otherwise, return 0. */ -static int strlenChar(const char *z){ - int n = 0; - while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; - } - return n; +static PerStreamTags * isKnownReadable(FILE *pf){ + static PerStreamTags *apst[] = { + &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0 + }; + int ix = 0; + do { + if( apst[ix]->pf == pf ) break; + } while( apst[++ix] != 0 ); + return apst[ix]; } +#endif -/* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. +#ifndef SQLITE_CIO_NO_TRANSLATE +/* Say whether given FILE* is among those known, via either +** consoleClassifySetup() or set{Output,Error}Stream, as +** writable, and return an associated PerStreamTags pointer +** if so. Otherwise, return 0. */ -static FILE * openChrSource(const char *zFile){ -#ifdef _WIN32 - struct _stat x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; +static PerStreamTags * isKnownWritable(FILE *pf){ + static PerStreamTags *apst[] = { + &consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2], + &consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0 + }; + int ix = 0; + do { + if( apst[ix]->pf == pf ) break; + } while( apst[++ix] != 0 ); + return apst[ix]; +} + +static FILE *designateEmitStream(FILE *pf, unsigned chix){ + FILE *rv = consoleInfo.pstDesignated[chix].pf; + if( pf == invalidFileStream ) return rv; + else{ + /* Setting a possibly new output stream. */ + PerStreamTags *ppst = isKnownWritable(pf); + if( ppst != 0 ){ + PerStreamTags pst = *ppst; + consoleInfo.pstDesignated[chix] = pst; + }else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]); } return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return fopen(zFile, "rb"); - }else{ - return 0; - } -#endif -#undef STAT_CHR_SRC } -/* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails. -** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. -*/ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; - int n = 0; +SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){ + return designateEmitStream(pf, 1); +} +# ifdef CONSIO_SET_ERROR_STREAM +SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){ + return designateEmitStream(pf, 2); +} +# endif +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ - while( 1 ){ - if( n+100>nLine ){ - nLine = nLine*2 + 100; - zLine = realloc(zLine, nLine); - if( zLine==0 ) shell_out_of_memory(); - } - if( fgets(&zLine[n], nLine - n, in)==0 ){ - if( n==0 ){ - free(zLine); - return 0; - } - zLine[n] = 0; - break; - } - while( zLine[n] ) n++; - if( n>0 && zLine[n-1]=='\n' ){ - n--; - if( n>0 && zLine[n-1]=='\r' ) n--; - zLine[n] = 0; - break; - } - } -#if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the - ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive && in==stdin ){ - char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); - if( zTrans ){ - int nTrans = strlen30(zTrans)+1; - if( nTrans>nLine ){ - zLine = realloc(zLine, nTrans); - if( zLine==0 ) shell_out_of_memory(); +#ifndef SQLITE_CIO_NO_SETMODE +# if CIO_WIN_WC_XLATE +static void setModeFlushQ(FILE *pf, short bFlush, int mode){ + if( bFlush ) fflush(pf); + _setmode(_fileno(pf), mode); +} +# else +# define setModeFlushQ(f, b, m) if(b) fflush(f) +# endif + +SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){ + setModeFlushQ(pf, bFlush, _O_BINARY); +} +SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){ + setModeFlushQ(pf, bFlush, _O_TEXT); +} +# undef setModeFlushQ + +#else /* defined(SQLITE_CIO_NO_SETMODE) */ +# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0) +# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0) +#endif /* defined(SQLITE_CIO_NO_SETMODE) */ + +#ifndef SQLITE_CIO_NO_TRANSLATE +# if CIO_WIN_WC_XLATE +/* Write buffer cBuf as output to stream known to reach console, +** limited to ncTake char's. Return ncTake on success, else 0. */ +static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){ + int rv = 0; + if( z!=NULL ){ + int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0); + if( nwc > 0 ){ + WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR)); + if( zw!=NULL ){ + nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc); + if( nwc > 0 ){ + /* Translation from UTF-8 to UTF-16, then WCHARs out. */ + if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){ + rv = ncTake; + } + } + sqlite3_free(zw); } - memcpy(zLine, zTrans, nTrans); - sqlite3_free(zTrans); } } -#endif /* defined(_WIN32) || defined(WIN32) */ - return zLine; + return rv; } -/* -** Retrieve a single line of input text. -** -** If in==0 then read from standard input and prompt before each line. -** If isContinuation is true, then a continuation prompt is appropriate. -** If isContinuation is zero, then the main prompt should be used. -** -** If zPrior is not NULL then it is a buffer from a prior call to this -** routine that can be reused. -** -** The result is stored in space obtained from malloc() and must either -** be freed by the caller or else passed back into this routine via the -** zPrior argument for reuse. -*/ -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ - char *zPrompt; - char *zResult; - if( in!=0 ){ - zResult = local_getline(zPrior, in); +/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */ +static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){ + char *z = sqlite3_vmprintf(zFormat, ap); + if( z ){ + int rv = conZstrEmit(ppst, z, (int)strlen(z)); + sqlite3_free(z); + return rv; + }else return 0; +} +# endif /* CIO_WIN_WC_XLATE */ + +# ifdef CONSIO_GET_EMIT_STREAM +static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix, + PerStreamTags *ppst){ + PerStreamTags *rv = isKnownWritable(pf); + short isValid = (rv!=0)? isValidStreamInfo(rv) : 0; + if( rv != 0 && isValid ) return rv; + streamOfConsole(pf, ppst); + return ppst; +} +# endif + +/* Get stream info, either for designated output or error stream when +** chix equals 1 or 2, or for an arbitrary stream when chix == 0. +** In either case, ppst references a caller-owned PerStreamTags +** struct which may be filled in if none of the known writable +** streams is being held by consoleInfo. The ppf parameter is a +** byref output when chix!=0 and a byref input when chix==0. + */ +static PerStreamTags * +getEmitStreamInfo(unsigned chix, PerStreamTags *ppst, + /* in/out */ FILE **ppf){ + PerStreamTags *ppstTry; + FILE *pfEmit; + if( chix > 0 ){ + ppstTry = &consoleInfo.pstDesignated[chix]; + if( !isValidStreamInfo(ppstTry) ){ + ppstTry = &consoleInfo.pstSetup[chix]; + pfEmit = ppst->pf; + }else pfEmit = ppstTry->pf; + if( !isValidStreamInfo(ppstTry) ){ + pfEmit = (chix > 1)? stderr : stdout; + ppstTry = ppst; + streamOfConsole(pfEmit, ppstTry); + } + *ppf = pfEmit; }else{ - zPrompt = isContinuation ? continuePrompt : mainPrompt; -#if SHELL_USE_LOCAL_GETLINE - printf("%s", zPrompt); - fflush(stdout); - zResult = local_getline(zPrior, stdin); -#else - free(zPrior); - zResult = shell_readline(zPrompt); - if( zResult && *zResult ) shell_add_history(zResult); -#endif + ppstTry = isKnownWritable(*ppf); + if( ppstTry != 0 ) return ppstTry; + streamOfConsole(*ppf, ppst); + return ppst; } - return zResult; + return ppstTry; } - -/* -** Return the value of a hexadecimal digit. Return -1 if the input -** is not a hex digit. -*/ -static int hexDigitValue(char c){ - if( c>='0' && c<='9' ) return c - '0'; - if( c>='a' && c<='f' ) return c - 'a' + 10; - if( c>='A' && c<='F' ) return c - 'A' + 10; - return -1; +SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){ + va_list ap; + int rv; + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif + assert(zFormat!=0); + va_start(ap, zFormat); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + rv = conioVmPrintf(ppst, zFormat, ap); + }else{ +# endif + rv = vfprintf(pfOut, zFormat, ap); +# if CIO_WIN_WC_XLATE + } +# endif + va_end(ap); + return rv; } -/* -** Interpret zArg as an integer value, possibly with suffixes. -*/ -static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { - { "KiB", 1024 }, - { "MiB", 1024*1024 }, - { "GiB", 1024*1024*1024 }, - { "KB", 1000 }, - { "MB", 1000000 }, - { "GB", 1000000000 }, - { "K", 1000 }, - { "M", 1000000 }, - { "G", 1000000000 }, - }; - int i; - int isNeg = 0; - if( zArg[0]=='-' ){ - isNeg = 1; - zArg++; - }else if( zArg[0]=='+' ){ - zArg++; - } - if( zArg[0]=='0' && zArg[1]=='x' ){ - int x; - zArg += 2; - while( (x = hexDigitValue(zArg[0]))>=0 ){ - v = (v<<4) + x; - zArg++; - } +SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){ + va_list ap; + int rv; + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# else + getEmitStreamInfo(2, &pst, &pfErr); +# endif + assert(zFormat!=0); + va_start(ap, zFormat); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + rv = conioVmPrintf(ppst, zFormat, ap); }else{ - while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; - zArg++; - } +# endif + rv = vfprintf(pfErr, zFormat, ap); +# if CIO_WIN_WC_XLATE } - for(i=0; ipf) ) restoreConsoleArb(ppst); + }else{ +# endif + rv = vfprintf(pfO, zFormat, ap); +# if CIO_WIN_WC_XLATE } - return isNeg? -v : v; +# endif + va_end(ap); + return rv; } -/* -** A variable length string to which one can append text. -*/ -typedef struct ShellText ShellText; -struct ShellText { - char *z; - int n; - int nAlloc; -}; +SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){ + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); +# else + getEmitStreamInfo(0, &pst, &pfO); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + int rv; + maybeSetupAsConsole(ppst, 1); + rv = conZstrEmit(ppst, z, (int)strlen(z)); + if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); + return rv; + }else { +# endif + return (fputs(z, pfO)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif +} -/* -** Initialize and destroy a ShellText object -*/ -static void initText(ShellText *p){ - memset(p, 0, sizeof(*p)); +SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){ + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# else + getEmitStreamInfo(2, &pst, &pfErr); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z)); + else { +# endif + return (fputs(z, pfErr)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif } -static void freeText(ShellText *p){ - free(p->z); - initText(p); + +SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){ + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif + assert(z!=0); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z)); + else { +# endif + return (fputs(z, pfOut)<0)? 0 : (int)strlen(z); +# if CIO_WIN_WC_XLATE + } +# endif } -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. -** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. -*/ -static void appendText(ShellText *p, char const *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ - len = nAppend+p->n+1; - if( quote ){ - len += 2; - for(i=0; i=0 => char count, nAccept<0 => character + */ +SQLITE_INTERNAL_LINKAGE const char* +zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; } } + return z; +} +#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/ + +#ifndef SQLITE_CIO_NO_TRANSLATE +# ifdef CONSIO_SPUTB +SQLITE_INTERNAL_LINKAGE int +fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){ + assert(pfO!=0); +# if CIO_WIN_WC_XLATE + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ + PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); + if( pstReachesConsole(ppst) ){ + int rv; + maybeSetupAsConsole(ppst, 1); + rv = conZstrEmit(ppst, cBuf, nAccept); + if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); + return rv; + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfO); +# if CIO_WIN_WC_XLATE + } +# endif +} +# endif - if( p->z==0 || p->n+len>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - if( p->z==0 ) shell_out_of_memory(); +SQLITE_INTERNAL_LINKAGE int +oPutbUtf8(const char *cBuf, int nAccept){ + FILE *pfOut; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ +# if CIO_WIN_WC_XLATE + PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut); +# else + getEmitStreamInfo(1, &pst, &pfOut); +# endif +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + return conZstrEmit(ppst, cBuf, nAccept); + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfOut); +# if CIO_WIN_WC_XLATE } +# endif +} - if( quote ){ - char *zCsr = p->z+p->n; - *zCsr++ = quote; - for(i=0; in = (int)(zCsr - p->z); - *zCsr = '\0'; +# ifdef CONSIO_EPUTB +SQLITE_INTERNAL_LINKAGE int +ePutbUtf8(const char *cBuf, int nAccept){ + FILE *pfErr; + PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ + PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); +# if CIO_WIN_WC_XLATE + if( pstReachesConsole(ppst) ){ + return conZstrEmit(ppst, cBuf, nAccept); + }else { +# endif + return (int)fwrite(cBuf, 1, nAccept, pfErr); +# if CIO_WIN_WC_XLATE + } +# endif +} +# endif /* defined(CONSIO_EPUTB) */ + +SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ + if( pfIn==0 ) pfIn = stdin; +# if CIO_WIN_WC_XLATE + if( pfIn == consoleInfo.pstSetup[0].pf + && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){ +# if CIO_WIN_WC_XLATE==1 +# define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */ + WCHAR wcBuf[SHELL_GULP+1]; + int lend = 0, noc = 0; + if( ncMax > 0 ) cBuf[0] = 0; + while( noc < ncMax-8-1 && !lend ){ + /* There is room for at least 2 more characters and a 0-terminator. */ + int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4; +# undef SHELL_GULP + DWORD nbr = 0; + BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0); + if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){ + /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */ + DWORD nbrx; + bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0); + if( bRC ) nbr += nbrx; + } + if( !bRC || (noc==0 && nbr==0) ) return 0; + if( nbr > 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0); + if( nmb != 0 && noc+nmb <= ncMax ){ + int iseg = noc; + nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0); + noc += nmb; + /* Fixup line-ends as coded by Windows for CR (or "Enter".) + ** This is done without regard for any setMode{Text,Binary}() + ** call that might have been done on the interactive input. + */ + if( noc > 0 ){ + if( cBuf[noc-1]=='\n' ){ + lend = 1; + if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n'; + } + } + /* Check for ^Z (anywhere in line) too, to act as EOF. */ + while( iseg < noc ){ + if( cBuf[iseg]=='\x1a' ){ + noc = iseg; /* Chop ^Z and anything following. */ + lend = 1; /* Counts as end of line too. */ + break; + } + ++iseg; + } + }else break; /* Drop apparent garbage in. (Could assert.) */ + }else break; + } + /* If got nothing, (after ^Z chop), must be at end-of-file. */ + if( noc > 0 ){ + cBuf[noc] = 0; + return cBuf; + }else return 0; +# endif }else{ - memcpy(p->z+p->n, zAppend, nAppend); - p->n += nAppend; - p->z[p->n] = '\0'; +# endif + return fgets(cBuf, ncMax, pfIn); +# if CIO_WIN_WC_XLATE } +# endif } +#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ -/* -** Attempt to determine if identifier zName needs to be quoted, either -** because it contains non-alphanumeric characters, or because it is an -** SQLite keyword. Be conservative in this estimate: When in doubt assume -** that quoting is required. -** -** Return '"' if quoting is required. Return 0 if no quoting is required. -*/ -static char quoteChar(const char *zName){ - int i; - if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; - for(i=0; zName[i]; i++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; +#if defined(_MSC_VER) +# pragma warning(default : 4204) +#endif + +#undef SHELL_INVALID_FILE_PTR + +/************************* End ../ext/consio/console_io.c ********************/ + +#ifndef SQLITE_SHELL_FIDDLE + +/* From here onward, fgets() is redirected to the console_io library. */ +# define fgets(b,n,f) fGetsUtf8(b,n,f) +/* + * Define macros for emitting output text in various ways: + * sputz(s, z) => emit 0-terminated string z to given stream s + * sputf(s, f, ...) => emit varargs per format f to given stream s + * oputz(z) => emit 0-terminated string z to default stream + * oputf(f, ...) => emit varargs per format f to default stream + * eputz(z) => emit 0-terminated string z to error stream + * eputf(f, ...) => emit varargs per format f to error stream + * oputb(b, n) => emit char buffer b[0..n-1] to default stream + * + * Note that the default stream is whatever has been last set via: + * setOutputStream(FILE *pf) + * This is normally the stream that CLI normal output goes to. + * For the stand-alone CLI, it is stdout with no .output redirect. + * + * The ?putz(z) forms are required for the Fiddle builds for string literal + * output, in aid of enforcing format string to argument correspondence. + */ +# define sputz(s,z) fPutsUtf8(z,s) +# define sputf fPrintfUtf8 +# define oputz(z) oPutsUtf8(z) +# define oputf oPrintfUtf8 +# define eputz(z) ePutsUtf8(z) +# define eputf ePrintfUtf8 +# define oputb(buf,na) oPutbUtf8(buf,na) + +#else +/* For Fiddle, all console handling and emit redirection is omitted. */ +/* These next 3 macros are for emitting formatted output. When complaints + * from the WASM build are issued for non-formatted output, (when a mere + * string literal is to be emitted, the ?putz(z) forms should be used. + * (This permits compile-time checking of format string / argument mismatch.) + */ +# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) +# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +# define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +/* These next 3 macros are for emitting simple string literals. */ +# define oputz(z) fputs(z,stdout) +# define eputz(z) fputs(z,stderr) +# define sputz(fp,z) fputs(z,fp) +# define oputb(buf,na) fwrite(buf,1,na,stdout) +#endif + +/* True if the timer is enabled */ +static int enableTimer = 0; + +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); +} +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); +} + +/* Return the current wall-clock time */ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); } - return sqlite3_keyword_check(zName, i) ? '"' : 0; + return t; } +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include +#include + +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + /* -** Construct a fake object name and column list to describe the structure -** of the view, virtual table, or table valued function zSchema.zName. +** Begin timing an operation */ -static char *shellFakeSchema( - sqlite3 *db, /* The database connection containing the vtab */ - const char *zSchema, /* Schema of the database holding the vtab */ - const char *zName /* The name of the virtual table */ -){ - sqlite3_stmt *pStmt = 0; - char *zSql; - ShellText s; - char cQuote; - char *zDiv = "("; - int nRow = 0; - - zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", - zSchema ? zSchema : "main", zName); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - initText(&s); - if( zSchema ){ - cQuote = quoteChar(zSchema); - if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; - appendText(&s, zSchema, cQuote); - appendText(&s, ".", 0); - } - cQuote = quoteChar(zName); - appendText(&s, zName, cQuote); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); - nRow++; - appendText(&s, zDiv, 0); - zDiv = ","; - cQuote = quoteChar(zCol); - appendText(&s, zCol, cQuote); - } - appendText(&s, ")", 0); - sqlite3_finalize(pStmt); - if( nRow==0 ){ - freeText(&s); - s.z = 0; +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); } - return s.z; } -/* -** SQL function: shell_module_schema(X) -** -** Return a fake schema for the table-valued function or eponymous virtual -** table X. +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. */ -static void shellModuleSchema( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - const char *zName = (const char*)sqlite3_value_text(apVal[0]); - char *zFake = shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName); - UNUSED_PARAMETER(nVal); - if( zFake ){ - sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), - -1, sqlite3_free); - free(zFake); +static void endTimer(void){ + if( enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + sputf(stdout, "Run Time: real %.3f user %f sys %f\n", + (iEnd - iBegin)*0.001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } } +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + /* -** SQL function: shell_add_schema(S,X) -** -** Add the schema name X to the CREATE statement in S and return the result. -** Examples: -** -** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); -** -** Also works on -** -** CREATE INDEX -** CREATE UNIQUE INDEX -** CREATE VIEW -** CREATE TRIGGER -** CREATE VIRTUAL TABLE -** -** This UDF is used by the .schema command to insert the schema name of -** attached databases into the middle of the sqlite_schema.sql field. +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). */ -static void shellAddSchemaName( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - static const char *aPrefix[] = { - "TABLE", - "INDEX", - "UNIQUE INDEX", - "VIEW", - "TRIGGER", - "VIRTUAL TABLE" - }; - int i = 0; - const char *zIn = (const char*)sqlite3_value_text(apVal[0]); - const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); - const char *zName = (const char*)sqlite3_value_text(apVal[2]); - sqlite3 *db = sqlite3_context_db_handle(pCtx); - UNUSED_PARAMETER(nVal); - if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){ - for(i=0; i -#include -#include -#include -#include -#include -#include +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) /* -** We may need several defines that should have been in "sys/stat.h". +** If the following flag is set, then command execution stops +** at an error if we are not interactive. */ - -#ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) -#endif - -#ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) -#endif - -#ifndef S_ISLNK -#define S_ISLNK(mode) (0) -#endif +static int bail_on_error = 0; /* -** We may need to provide the "mode_t" type. +** Treat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. */ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; -#endif +static int stdin_is_interactive = 1; /* -** We may need to provide the "ino_t" type. +** On Windows systems we need to know if standard output is a console +** in order to show that UTF-16 translation is done in the sign-on +** banner. The following variable is true if it is the console. */ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif +static int stdout_is_console = 1; /* -** We need to define "NAME_MAX" if it was not present in "limits.h". +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. */ +static sqlite3 *globalDb = 0; -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -#endif +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; /* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. */ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif +static char *Argv0; /* -** We need to provide the necessary structures and related types. +** Prompt strings. Initialized in main. Settable with +** .prompt main continue */ +#define PROMPT_LEN_MAX 20 +/* First line prompt. default: "sqlite> " */ +static char mainPrompt[PROMPT_LEN_MAX]; +/* Continuation prompt. default: " ...> " */ +static char continuePrompt[PROMPT_LEN_MAX]; + +/* This is variant of the standard-library strncpy() routine with the +** one change that the destination string is always zero-terminated, even +** if there is no zero-terminator in the first n-1 characters of the source +** string. +*/ +static char *shell_strncpy(char *dest, const char *src, size_t n){ + size_t i; + for(i=0; iinParenLevel += ni; + if( ni==0 ) p->inParenLevel = 0; + p->zScannerAwaits = 0; +} + +/* Record that a lexeme is opened, or closed with args==0. */ +static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ + if( s!=0 || c==0 ){ + p->zScannerAwaits = s; + p->acAwait[0] = 0; + }else{ + p->acAwait[0] = c; + p->zScannerAwaits = p->acAwait; + } +} -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ -}; -#endif - -#ifndef DIR_DEFINED -#define DIR_DEFINED -typedef struct DIR DIR; -typedef DIR *LPDIR; -struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ -}; -#endif - -/* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir() or -** readdir_r() functions. -*/ +/* Upon demand, derive the continuation prompt to display. */ +static char *dynamicContinuePrompt(void){ + if( continuePrompt[0]==0 + || (dynPrompt.zScannerAwaits==0 && dynPrompt.inParenLevel == 0) ){ + return continuePrompt; + }else{ + if( dynPrompt.zScannerAwaits ){ + size_t ncp = strlen(continuePrompt); + size_t ndp = strlen(dynPrompt.zScannerAwaits); + if( ndp > ncp-3 ) return continuePrompt; + strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + }else{ + if( dynPrompt.inParenLevel>9 ){ + shell_strncpy(dynPrompt.dynamicPrompt, "(..", 4); + }else if( dynPrompt.inParenLevel<0 ){ + shell_strncpy(dynPrompt.dynamicPrompt, ")x!", 4); + }else{ + shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); + dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); + } + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); + } + } + return dynPrompt.dynamicPrompt; +} +#endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif +/* Indicate out-of-memory and exit. */ +static void shell_out_of_memory(void){ + eputz("Error: out of memory\n"); + exit(1); +} -/* -** Provide the function prototype for the POSIX compatiable getenv() -** function. This function is not thread-safe. +/* Check a pointer to see if it is NULL. If it is NULL, exit with an +** out-of-memory error. */ - -extern const char *windirent_getenv(const char *name); +static void shell_check_oom(const void *p){ + if( p==0 ) shell_out_of_memory(); +} /* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), readdir_r(), and closedir() POSIX functions. +** Write I/O traces to the following stream. */ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.h ********************/ -/************************* Begin test_windirent.c ******************/ /* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. */ - -#if defined(_WIN32) && defined(_MSC_VER) -/* #include "test_windirent.h" */ +#ifdef SQLITE_ENABLE_IOTRACE +static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sputf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif /* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. +** Output string zUtf to Out stream as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. */ -const char *windirent_getenv( - const char *name -){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ - return 0; +static void utf8_width_print(int w, const char *zUtf){ + int i; + int n; + int aw = w<0 ? -w : w; + if( zUtf==0 ) zUtf = ""; + for(i=n=0; zUtf[i]; i++){ + if( (zUtf[i]&0xc0)!=0x80 ){ + n++; + if( n==aw ){ + do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + break; + } + } + } + if( n>=aw ){ + oputf("%.*s", i, zUtf); + }else if( w<0 ){ + oputf("%*s%s", aw-n, "", zUtf); }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; + oputf("%s%*s", zUtf, aw-n, ""); } } + /* -** Implementation of the POSIX opendir() function using the MSVCRT. +** Determines if a string is a number of not. */ -LPDIR opendir( - const char *dirname -){ - struct _finddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); - - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); - - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !IsDigit(*z) ){ + return 0; } - - memset(&data, 0, sizeof(struct _finddata_t)); - _snprintf(data.name, namesize, "%s\\*", dirname); - dirp->d_handle = _findfirst(data.name, &data); - - if( dirp->d_handle==BAD_INTPTR_T ){ - closedir(dirp); - return NULL; + z++; + if( realnum ) *realnum = 0; + while( IsDigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: - - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); - return NULL; - } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; } - - dirp->d_first.d_attributes = data.attrib; - strncpy(dirp->d_first.d_name, data.name, NAME_MAX); - dirp->d_first.d_name[NAME_MAX] = '\0'; - - return dirp; + return *z==0; } /* -** Implementation of the POSIX readdir() function using the MSVCRT. +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. */ -LPDIRENT readdir( - LPDIR dirp -){ - struct _finddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} - return &dirp->d_first; +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0xc0&*(z++))!=0x80 ) n++; } - -next: - - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; - strncpy(dirp->d_next.d_name, data.name, NAME_MAX); - dirp->d_next.d_name[NAME_MAX] = '\0'; - - return &dirp->d_next; + return n; } /* -** Implementation of the POSIX readdir_r() function using the MSVCRT. +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. */ -INT readdir_r( - LPDIR dirp, - LPDIRENT entry, - LPDIRENT *result -){ - struct _finddata_t data; - - if( dirp==NULL ) return EBADF; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - entry->d_ino = dirp->d_first.d_ino; - entry->d_attributes = dirp->d_first.d_attributes; - strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; - - *result = entry; - return 0; +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; } - -next: - - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ){ - *result = NULL; - return ENOENT; + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return fopen(zFile, "rb"); + }else{ + return 0; } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - entry->d_ino = (ino_t)-1; /* not available */ - entry->d_attributes = data.attrib; - strncpy(entry->d_name, data.name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; - - *result = entry; - return 0; +#endif +#undef STAT_CHR_SRC } /* -** Implementation of the POSIX closedir() function using the MSVCRT. +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. */ -INT closedir( - LPDIR dirp -){ - INT result = 0; - - if( dirp==NULL ) return EINVAL; +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); + while( 1 ){ + if( n+100>nLine ){ + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + shell_check_oom(zLine); + } + if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; + zLine[n] = 0; + break; + } } - - sqlite3_free(dirp); - return result; + return zLine; } -#endif /* defined(WIN32) && defined(_MSC_VER) */ - -/************************* End test_windirent.c ********************/ -#define dirent DIRENT -#endif -/************************* Begin ../ext/misc/shathree.c ******************/ /* -** 2017-03-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This SQLite extension implements functions that compute SHA3 hashes. -** Two SQL functions are implemented: -** -** sha3(X,SIZE) -** sha3_query(Y,SIZE) +** Retrieve a single line of input text. ** -** The sha3(X) function computes the SHA3 hash of the input X, or NULL if -** X is NULL. +** If in==0 then read from standard input and prompt before each line. +** If isContinuation is true, then a continuation prompt is appropriate. +** If isContinuation is zero, then the main prompt should be used. ** -** The sha3_query(Y) function evalutes all queries in the SQL statements of Y -** and returns a hash of their results. +** If zPrior is not NULL then it is a buffer from a prior call to this +** routine that can be reused. ** -** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm -** is used. If SIZE is included it must be one of the integers 224, 256, -** 384, or 512, to determine SHA3 hash variant that is computed. +** The result is stored in space obtained from malloc() and must either +** be freed by the caller or else passed back into this routine via the +** zPrior argument for reuse. */ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include -#include - -#ifndef SQLITE_AMALGAMATION -/* typedef sqlite3_uint64 u64; */ -#endif /* SQLITE_AMALGAMATION */ +#ifndef SQLITE_SHELL_FIDDLE +static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ + char *zPrompt; + char *zResult; + if( in!=0 ){ + zResult = local_getline(zPrior, in); + }else{ + zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; +#if SHELL_USE_LOCAL_GETLINE + sputz(stdout, zPrompt); + fflush(stdout); + do{ + zResult = local_getline(zPrior, stdin); + zPrior = 0; + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); +#else + free(zPrior); + zResult = shell_readline(zPrompt); + while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); + } + if( zResult && *zResult ) shell_add_history(zResult); +#endif + } + return zResult; +} +#endif /* !SQLITE_SHELL_FIDDLE */ -/****************************************************************************** -** The Hash Engine -*/ /* -** Macros to determine whether the machine is big or little endian, -** and whether or not that determination is run-time or compile-time. -** -** For best performance, an attempt is made to guess at the byte-order -** using C-preprocessor macros. If that is unsuccessful, or if -** -DSHA3_BYTEORDER=0 is set, then byte-order is determined -** at run-time. +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. */ -#ifndef SHA3_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SHA3_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) -# define SHA3_BYTEORDER 4321 -# else -# define SHA3_BYTEORDER 0 -# endif -#endif +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( IsDigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; iu.s[0]) -# define a01 (p->u.s[1]) -# define a02 (p->u.s[2]) -# define a03 (p->u.s[3]) -# define a04 (p->u.s[4]) -# define a10 (p->u.s[5]) -# define a11 (p->u.s[6]) -# define a12 (p->u.s[7]) -# define a13 (p->u.s[8]) -# define a14 (p->u.s[9]) -# define a20 (p->u.s[10]) -# define a21 (p->u.s[11]) -# define a22 (p->u.s[12]) -# define a23 (p->u.s[13]) -# define a24 (p->u.s[14]) -# define a30 (p->u.s[15]) -# define a31 (p->u.s[16]) -# define a32 (p->u.s[17]) -# define a33 (p->u.s[18]) -# define a34 (p->u.s[19]) -# define a40 (p->u.s[20]) -# define a41 (p->u.s[21]) -# define a42 (p->u.s[22]) -# define a43 (p->u.s[23]) -# define a44 (p->u.s[24]) -# define ROL64(a,x) ((a<>(64-x))) - - for(i=0; i<24; i+=4){ - c0 = a00^a10^a20^a30^a40; - c1 = a01^a11^a21^a31^a41; - c2 = a02^a12^a22^a32^a42; - c3 = a03^a13^a23^a33^a43; - c4 = a04^a14^a24^a34^a44; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + free(p->z); + initText(p); +} - b0 = (a00^d0); - b1 = ROL64((a11^d1), 44); - b2 = ROL64((a22^d2), 43); - b3 = ROL64((a33^d3), 21); - b4 = ROL64((a44^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i]; - a11 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(ShellText *p, const char *zAppend, char quote){ + i64 len; + i64 i; + i64 nAppend = strlen30(zAppend); - b2 = ROL64((a20^d0), 3); - b3 = ROL64((a31^d1), 45); - b4 = ROL64((a42^d2), 61); - b0 = ROL64((a03^d3), 28); - b1 = ROL64((a14^d4), 20); - a20 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; iz==0 || p->n+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + shell_check_oom(p->z); + } - b1 = ROL64((a10^d0), 36); - b2 = ROL64((a21^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a43^d3), 56); - b0 = ROL64((a04^d4), 27); - a10 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} - b3 = ROL64((a30^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a02^d2), 62); - b1 = ROL64((a13^d3), 55); - b2 = ROL64((a24^d4), 39); - a30 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( zName==0 ) return '"'; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} - c0 = a00^a20^a40^a10^a30; - c1 = a11^a31^a01^a21^a41; - c2 = a22^a42^a12^a32^a02; - c3 = a33^a03^a23^a43^a13; - c4 = a44^a14^a34^a04^a24; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +/* +** Construct a fake object name and column list to describe the structure +** of the view, virtual table, or table valued function zSchema.zName. +*/ +static char *shellFakeSchema( + sqlite3 *db, /* The database connection containing the vtab */ + const char *zSchema, /* Schema of the database holding the vtab */ + const char *zName /* The name of the virtual table */ +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + ShellText s; + char cQuote; + char *zDiv = "("; + int nRow = 0; - b0 = (a00^d0); - b1 = ROL64((a31^d1), 44); - b2 = ROL64((a12^d2), 43); - b3 = ROL64((a43^d3), 21); - b4 = ROL64((a24^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+1]; - a31 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); + zSql = sqlite3_mprintf("PRAGMA \"%w\".table_info=%Q;", + zSchema ? zSchema : "main", zName); + shell_check_oom(zSql); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + initText(&s); + if( zSchema ){ + cQuote = quoteChar(zSchema); + if( cQuote && sqlite3_stricmp(zSchema,"temp")==0 ) cQuote = 0; + appendText(&s, zSchema, cQuote); + appendText(&s, ".", 0); + } + cQuote = quoteChar(zName); + appendText(&s, zName, cQuote); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zCol = (const char*)sqlite3_column_text(pStmt, 1); + nRow++; + appendText(&s, zDiv, 0); + zDiv = ","; + if( zCol==0 ) zCol = ""; + cQuote = quoteChar(zCol); + appendText(&s, zCol, cQuote); + } + appendText(&s, ")", 0); + sqlite3_finalize(pStmt); + if( nRow==0 ){ + freeText(&s); + s.z = 0; + } + return s.z; +} - b2 = ROL64((a40^d0), 3); - b3 = ROL64((a21^d1), 45); - b4 = ROL64((a02^d2), 61); - b0 = ROL64((a33^d3), 28); - b1 = ROL64((a14^d4), 20); - a40 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); +/* +** SQL function: strtod(X) +** +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. +*/ +static void shellStrtod( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + char *z = (char*)sqlite3_value_text(apVal[0]); + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} - b4 = ROL64((a30^d0), 18); - b0 = ROL64((a11^d1), 1); - b1 = ROL64((a42^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a04^d4), 8); - a30 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); +/* +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. +*/ +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +} - b1 = ROL64((a20^d0), 36); - b2 = ROL64((a01^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a13^d3), 56); - b0 = ROL64((a44^d4), 27); - a20 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); - b3 = ROL64((a10^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a22^d2), 62); - b1 = ROL64((a03^d3), 55); - b2 = ROL64((a34^d4), 39); - a10 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); +/* +** SQL function: shell_module_schema(X) +** +** Return a fake schema for the table-valued function or eponymous virtual +** table X. +*/ +static void shellModuleSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zName; + char *zFake; + UNUSED_PARAMETER(nVal); + zName = (const char*)sqlite3_value_text(apVal[0]); + zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} - c0 = a00^a40^a30^a20^a10; - c1 = a31^a21^a11^a01^a41; - c2 = a12^a02^a42^a32^a22; - c3 = a43^a33^a23^a13^a03; - c4 = a24^a14^a04^a44^a34; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); +/* +** SQL function: shell_add_schema(S,X) +** +** Add the schema name X to the CREATE statement in S and return the result. +** Examples: +** +** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); +** +** Also works on +** +** CREATE INDEX +** CREATE UNIQUE INDEX +** CREATE VIEW +** CREATE TRIGGER +** CREATE VIRTUAL TABLE +** +** This UDF is used by the .schema command to insert the schema name of +** attached databases into the middle of the sqlite_schema.sql field. +*/ +static void shellAddSchemaName( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + static const char *aPrefix[] = { + "TABLE", + "INDEX", + "UNIQUE INDEX", + "VIEW", + "TRIGGER", + "VIRTUAL TABLE" + }; + int i = 0; + const char *zIn = (const char*)sqlite3_value_text(apVal[0]); + const char *zSchema = (const char*)sqlite3_value_text(apVal[1]); + const char *zName = (const char*)sqlite3_value_text(apVal[2]); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + UNUSED_PARAMETER(nVal); + if( zIn!=0 && cli_strncmp(zIn, "CREATE ", 7)==0 ){ + for(i=0; i +#include +#include +#include +#include +#include +#include - b3 = ROL64((a40^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a42^d2), 62); - b1 = ROL64((a43^d3), 55); - b2 = ROL64((a44^d4), 39); - a40 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); +/* +** We may need several defines that should have been in "sys/stat.h". +*/ + +#ifndef S_ISREG +#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif + +#ifndef S_ISDIR +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif + +#ifndef S_ISLNK +#define S_ISLNK(mode) (0) +#endif + +/* +** We may need to provide the "mode_t" type. +*/ + +#ifndef MODE_T_DEFINED + #define MODE_T_DEFINED + typedef unsigned short mode_t; +#endif + +/* +** We may need to provide the "ino_t" type. +*/ + +#ifndef INO_T_DEFINED + #define INO_T_DEFINED + typedef unsigned short ino_t; +#endif + +/* +** We need to define "NAME_MAX" if it was not present in "limits.h". +*/ + +#ifndef NAME_MAX +# ifdef FILENAME_MAX +# define NAME_MAX (FILENAME_MAX) +# else +# define NAME_MAX (260) +# endif +#endif + +/* +** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". +*/ + +#ifndef NULL_INTPTR_T +# define NULL_INTPTR_T ((intptr_t)(0)) +#endif + +#ifndef BAD_INTPTR_T +# define BAD_INTPTR_T ((intptr_t)(-1)) +#endif + +/* +** We need to provide the necessary structures and related types. +*/ + +#ifndef DIRENT_DEFINED +#define DIRENT_DEFINED +typedef struct DIRENT DIRENT; +typedef DIRENT *LPDIRENT; +struct DIRENT { + ino_t d_ino; /* Sequence number, do not use. */ + unsigned d_attributes; /* Win32 file attributes. */ + char d_name[NAME_MAX + 1]; /* Name within the directory. */ +}; +#endif + +#ifndef DIR_DEFINED +#define DIR_DEFINED +typedef struct DIR DIR; +typedef DIR *LPDIR; +struct DIR { + intptr_t d_handle; /* Value returned by "_findfirst". */ + DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ + DIRENT d_next; /* DIRENT constructed based on "_findnext". */ +}; +#endif + +/* +** Provide a macro, for use by the implementation, to determine if a +** particular directory entry should be skipped over when searching for +** the next directory entry that should be returned by the readdir() or +** readdir_r() functions. +*/ + +#ifndef is_filtered +# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) +#endif + +/* +** Provide the function prototype for the POSIX compatible getenv() +** function. This function is not thread-safe. +*/ + +extern const char *windirent_getenv(const char *name); + +/* +** Finally, we can provide the function prototypes for the opendir(), +** readdir(), readdir_r(), and closedir() POSIX functions. +*/ + +extern LPDIR opendir(const char *dirname); +extern LPDIRENT readdir(LPDIR dirp); +extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); +extern INT closedir(LPDIR dirp); + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.h ********************/ +/************************* Begin test_windirent.c ******************/ +/* +** 2015 November 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement most of the opendir() family of +** POSIX functions on Win32 using the MSVCRT. +*/ + +#if defined(_WIN32) && defined(_MSC_VER) +/* #include "test_windirent.h" */ + +/* +** Implementation of the POSIX getenv() function using the Win32 API. +** This function is not thread-safe. +*/ +const char *windirent_getenv( + const char *name +){ + static char value[32768]; /* Maximum length, per MSDN */ + DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ + DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ + + memset(value, 0, sizeof(value)); + dwRet = GetEnvironmentVariableA(name, value, dwSize); + if( dwRet==0 || dwRet>dwSize ){ + /* + ** The function call to GetEnvironmentVariableA() failed -OR- + ** the buffer is not large enough. Either way, return NULL. + */ + return 0; + }else{ + /* + ** The function call to GetEnvironmentVariableA() succeeded + ** -AND- the buffer contains the entire value. + */ + return value; } } /* -** Initialize a new hash. iSize determines the size of the hash -** in bits and should be one of 224, 256, 384, or 512. Or iSize -** can be zero to use the default hash size of 256 bits. +** Implementation of the POSIX opendir() function using the MSVCRT. */ -static void SHA3Init(SHA3Context *p, int iSize){ - memset(p, 0, sizeof(*p)); - if( iSize>=128 && iSize<=512 ){ - p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; - }else{ - p->nRate = (1600 - 2*256)/8; +LPDIR opendir( + const char *dirname +){ + struct _finddata_t data; + LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); + SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); + + if( dirp==NULL ) return NULL; + memset(dirp, 0, sizeof(DIR)); + + /* TODO: Remove this if Unix-style root paths are not used. */ + if( sqlite3_stricmp(dirname, "/")==0 ){ + dirname = windirent_getenv("SystemDrive"); } -#if SHA3_BYTEORDER==1234 - /* Known to be little-endian at compile-time. No-op */ -#elif SHA3_BYTEORDER==4321 - p->ixMask = 7; /* Big-endian */ -#else - { - static unsigned int one = 1; - if( 1==*(unsigned char*)&one ){ - /* Little endian. No byte swapping. */ - p->ixMask = 0; - }else{ - /* Big endian. Byte swap. */ - p->ixMask = 7; + + memset(&data, 0, sizeof(struct _finddata_t)); + _snprintf(data.name, namesize, "%s\\*", dirname); + dirp->d_handle = _findfirst(data.name, &data); + + if( dirp->d_handle==BAD_INTPTR_T ){ + closedir(dirp); + return NULL; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ){ +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + closedir(dirp); + return NULL; } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + } + + dirp->d_first.d_attributes = data.attrib; + strncpy(dirp->d_first.d_name, data.name, NAME_MAX); + dirp->d_first.d_name[NAME_MAX] = '\0'; + + return dirp; +} + +/* +** Implementation of the POSIX readdir() function using the MSVCRT. +*/ +LPDIRENT readdir( + LPDIR dirp +){ + struct _finddata_t data; + + if( dirp==NULL ) return NULL; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + return &dirp->d_first; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + dirp->d_next.d_ino++; + dirp->d_next.d_attributes = data.attrib; + strncpy(dirp->d_next.d_name, data.name, NAME_MAX); + dirp->d_next.d_name[NAME_MAX] = '\0'; + + return &dirp->d_next; +} + +/* +** Implementation of the POSIX readdir_r() function using the MSVCRT. +*/ +INT readdir_r( + LPDIR dirp, + LPDIRENT entry, + LPDIRENT *result +){ + struct _finddata_t data; + + if( dirp==NULL ) return EBADF; + + if( dirp->d_first.d_ino==0 ){ + dirp->d_first.d_ino++; + dirp->d_next.d_ino++; + + entry->d_ino = dirp->d_first.d_ino; + entry->d_attributes = dirp->d_first.d_attributes; + strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; + } + +next: + + memset(&data, 0, sizeof(struct _finddata_t)); + if( _findnext(dirp->d_handle, &data)==-1 ){ + *result = NULL; + return ENOENT; + } + + /* TODO: Remove this block to allow hidden and/or system files. */ + if( is_filtered(data) ) goto next; + + entry->d_ino = (ino_t)-1; /* not available */ + entry->d_attributes = data.attrib; + strncpy(entry->d_name, data.name, NAME_MAX); + entry->d_name[NAME_MAX] = '\0'; + + *result = entry; + return 0; +} + +/* +** Implementation of the POSIX closedir() function using the MSVCRT. +*/ +INT closedir( + LPDIR dirp +){ + INT result = 0; + + if( dirp==NULL ) return EINVAL; + + if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ + result = _findclose(dirp->d_handle); + } + + sqlite3_free(dirp); + return result; +} + +#endif /* defined(WIN32) && defined(_MSC_VER) */ + +/************************* End test_windirent.c ********************/ +#define dirent DIRENT +#endif +/************************* Begin ../ext/misc/memtrace.c ******************/ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/memtrace.c ********************/ +/************************* Begin ../ext/misc/pcachetrace.c ******************/ +/* +** 2023-06-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 +** mechanism to add a tracing layer on top of pluggable page cache of +** SQLite. If this extension is registered prior to sqlite3_initialize(), +** it will cause all page cache activities to be logged on standard output, +** or to some other FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --pcachetrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/pcachetrace.c ********************/ +/************************* Begin ../ext/misc/shathree.c ******************/ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_query(Y,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. +** +** The sha3_query(Y) function evaluates all queries in the SQL statements of Y +** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_AMALGAMATION +/* typedef sqlite3_uint64 u64; */ +#endif /* SQLITE_AMALGAMATION */ + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + c0 = a00^a20^a40^a10^a30; + c1 = a11^a31^a01^a21^a41; + c2 = a22^a42^a12^a32^a02; + c3 = a33^a03^a23^a43^a13; + c4 = a44^a14^a34^a04^a24; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a31^d1), 44); + b2 = ROL64((a12^d2), 43); + b3 = ROL64((a43^d3), 21); + b4 = ROL64((a24^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+1]; + a31 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a40^d0), 3); + b3 = ROL64((a21^d1), 45); + b4 = ROL64((a02^d2), 61); + b0 = ROL64((a33^d3), 28); + b1 = ROL64((a14^d4), 20); + a40 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a30^d0), 18); + b0 = ROL64((a11^d1), 1); + b1 = ROL64((a42^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a04^d4), 8); + a30 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a20^d0), 36); + b2 = ROL64((a01^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a13^d3), 56); + b0 = ROL64((a44^d4), 27); + a20 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a10^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a22^d2), 62); + b1 = ROL64((a03^d3), 55); + b2 = ROL64((a34^d4), 39); + a10 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + c0 = a00^a40^a30^a20^a10; + c1 = a31^a21^a11^a01^a41; + c2 = a12^a02^a42^a32^a22; + c3 = a43^a33^a23^a13^a03; + c4 = a24^a14^a04^a44^a34; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a21^d1), 44); + b2 = ROL64((a42^d2), 43); + b3 = ROL64((a13^d3), 21); + b4 = ROL64((a34^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+2]; + a21 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a30^d0), 3); + b3 = ROL64((a01^d1), 45); + b4 = ROL64((a22^d2), 61); + b0 = ROL64((a43^d3), 28); + b1 = ROL64((a14^d4), 20); + a30 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a10^d0), 18); + b0 = ROL64((a31^d1), 1); + b1 = ROL64((a02^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a44^d4), 8); + a10 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a40^d0), 36); + b2 = ROL64((a11^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a03^d3), 56); + b0 = ROL64((a24^d4), 27); + a40 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a20^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a12^d2), 62); + b1 = ROL64((a33^d3), 55); + b2 = ROL64((a04^d4), 39); + a20 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + c0 = a00^a30^a10^a40^a20; + c1 = a21^a01^a31^a11^a41; + c2 = a42^a22^a02^a32^a12; + c3 = a13^a43^a23^a03^a33; + c4 = a34^a14^a44^a24^a04; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a01^d1), 44); + b2 = ROL64((a02^d2), 43); + b3 = ROL64((a03^d3), 21); + b4 = ROL64((a04^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+3]; + a01 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a10^d0), 3); + b3 = ROL64((a11^d1), 45); + b4 = ROL64((a12^d2), 61); + b0 = ROL64((a13^d3), 28); + b1 = ROL64((a14^d4), 20); + a10 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a20^d0), 18); + b0 = ROL64((a21^d1), 1); + b1 = ROL64((a22^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a24^d4), 8); + a20 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a30^d0), 36); + b2 = ROL64((a31^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a33^d3), 56); + b0 = ROL64((a34^d4), 27); + a30 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a40^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a42^d2), 62); + b1 = ROL64((a43^d3), 55); + b2 = ROL64((a44^d4), 39); + a40 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; + if( aData==0 ) return; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void sha3_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z ){ + n = (int)strlen(z); + sha3_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + } + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + sha3_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + sha3_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, + SQLITE_UTF8 | SQLITE_DIRECTONLY, + 0, sha3QueryFunc, 0, 0); + } + return rc; +} + +/************************* End ../ext/misc/shathree.c ********************/ +/************************* Begin ../ext/misc/uint.c ******************/ +/* +** 2020-04-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the UINT collating sequence. +** +** UINT works like BINARY for text, except that embedded strings +** of digits compare in numeric order. +** +** * Leading zeros are handled properly, in the sense that +** they do not mess of the maginitude comparison of embedded +** strings of digits. "x00123y" is equal to "x123y". +** +** * Only unsigned integers are recognized. Plus and minus +** signs are ignored. Decimal points and exponential notation +** are ignored. +** +** * Embedded integers can be of arbitrary length. Comparison +** is *not* limited integers that can be expressed as a +** 64-bit machine integer. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* +** Compare text in lexicographic order, except strings of digits +** compare in numeric order. +*/ +static int uintCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + int i=0, j=0, x; + (void)notUsed; + while( i +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. +*/ +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; + int iExp = 0; + + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_from_text_failed; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_from_text_failed; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. +*/ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc( nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 1); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 1); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. A := A + B. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pB ){ + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + } + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + decimalMul(pA, pB); + if( pA->oom ){ + goto mul_end; + } + decimal_result(context, pA); + +mul_end: + decimal_free(pA); + decimal_free(pB); +} + +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + +#ifdef _WIN32 + +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + int iArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "decimal_sum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, + decimalSumStep, decimalSumFinalize, + decimalSumValue, decimalSumInverse, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_collation(db, "decimal", SQLITE_UTF8, + 0, decimalCollFunc); + } + return rc; +} + +/************************* End ../ext/misc/decimal.c ********************/ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +/************************* Begin ../ext/misc/base64.c ******************/ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters and to +** terminate any encoded blob having non-zero length. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes matching that of USASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +/* #include "sqlite3ext.h" */ + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +/************************* End ../ext/misc/base64.c ********************/ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#define OMIT_BASE85_CHECKER +/************************* Begin ../ext/misc/base85.c ******************/ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +/* # include "sqlite3ext.h" */ + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +/* typedef unsigned char u8; */ +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (((unsigned long)pIn[0])<<24) | + (pIn[1]<<16) | (pIn[2]<<8) | pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + case 3: + *pOut++ = (qv >> 16)&0xff; + case 2: + *pOut++ = (qv >> 8)&0xff; + case 1: + *pOut++ = qv&0xff; + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +# endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 + +#endif +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif + +/************************* End ../ext/misc/base85.c ********************/ +/************************* Begin ../ext/misc/ieee754.c ******************/ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(2,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + if( 1-e >= 64 ){ + m = 0; + }else{ + m >>= 1-e; + } + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + + +#ifdef _WIN32 + +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_inc", 2, 0, ieee754inc }, + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. +** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** The virtual table also has a rowid, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return that member of a generate_series(...) sequence whose 0-based +** index is ix. The 0th member is given by smBase. The sequence members +** progress per ix increment by smStep. +*/ +static sqlite3_int64 genSeqMember( + sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix +){ + static const sqlite3_uint64 mxI64 = + ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; + if( ix>=mxI64 ){ + /* Get ix into signed i64 range. */ + ix -= mxI64; + /* With 2's complement ALU, this next can be 1 step, but is split into + * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ + smBase += (mxI64/2) * smStep; + smBase += (mxI64 - mxI64/2) * smStep; + } + /* Under UBSAN (or on 1's complement machines), must do this last term + * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ + if( ix>=2 ){ + sqlite3_int64 ix2 = (sqlite3_int64)ix/2; + smBase += ix2*smStep; + ix -= ix2; + } + return smBase + ((sqlite3_int64)ix)*smStep; +} + +/* typedef unsigned char u8; */ + +typedef struct SequenceSpec { + sqlite3_int64 iBase; /* Starting value ("start") */ + sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ + sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ + sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ + sqlite3_int64 iValueNow; /* Current value during generation */ + u8 isNotEOF; /* Sequence generation not exhausted */ + u8 isReversing; /* Sequence is being reverse generated */ +} SequenceSpec; + +/* +** Prepare a SequenceSpec for use in generating an integer series +** given initialized iBase, iTerm and iStep values. Sequence is +** initialized per given isReversing. Other members are computed. +*/ +static void setupSequence( SequenceSpec *pss ){ + int bSameSigns; + pss->uSeqIndexMax = 0; + pss->isNotEOF = 0; + bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); + if( pss->iTerm < pss->iBase ){ + sqlite3_uint64 nuspan = 0; + if( bSameSigns ){ + nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iBase>=0 and iTerm<0 . */ + nuspan = 1; + nuspan += pss->iBase; + nuspan += -(pss->iTerm+1); + } + if( pss->iStep<0 ){ + pss->isNotEOF = 1; + if( nuspan==ULONG_MAX ){ + pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; + }else if( pss->iStep>LLONG_MIN ){ + pss->uSeqIndexMax = nuspan/-pss->iStep; + } + } + }else if( pss->iTerm > pss->iBase ){ + sqlite3_uint64 puspan = 0; + if( bSameSigns ){ + puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iTerm>=0 and iBase<0 . */ + puspan = 1; + puspan += pss->iTerm; + puspan += -(pss->iBase+1); + } + if( pss->iStep>0 ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = puspan/pss->iStep; + } + }else if( pss->iTerm == pss->iBase ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = 0; + } + pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; + pss->iValueNow = (pss->isReversing) + ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) + : pss->iBase; +} + +/* +** Progress sequence generator to yield next value, if any. +** Leave its state to either yield next value or be at EOF. +** Return whether there is a next value, or 0 at EOF. +*/ +static int progressSequence( SequenceSpec *pss ){ + if( !pss->isNotEOF ) return 0; + if( pss->isReversing ){ + if( pss->uSeqIndexNow > 0 ){ + pss->uSeqIndexNow--; + pss->iValueNow -= pss->iStep; + }else{ + pss->isNotEOF = 0; + } + }else{ + if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ + pss->uSeqIndexNow++; + pss->iValueNow += pss->iStep; + }else{ + pss->isNotEOF = 0; + } + } + return pss->isNotEOF; +} + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + SequenceSpec ss; /* (this) Derived class data */ +}; + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pUnused, + int argcUnused, const char *const*argvUnused, + sqlite3_vtab **ppVtab, + char **pzErrUnused +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + (void)pUnused; + (void)argcUnused; + (void)argvUnused; + (void)pzErrUnused; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + (void)pUnused; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + progressSequence( & pCur->ss ); + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->ss.iBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + default: x = pCur->ss.iValueNow; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +#ifndef LARGEST_UINT64 +#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) +#endif + +/* +** Return the rowid for the current row, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_uint64 n = pCur->ss.uSeqIndexNow; + *pRowid = (sqlite3_int64)((nss.isNotEOF; +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 0x01: start=VALUE +** 0x02: stop=VALUE +** 0x04: step=VALUE +** 0x08: descending order +** 0x10: ascending order +** 0x20: LIMIT VALUE +** 0x40: OFFSET VALUE +** +** This routine should initialize the cursor and position it so that it +** is pointing at the first row, or pointing off the end of the table +** (so that seriesEof() will return true) if the table is empty. +*/ +static int seriesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStrUnused, + int argc, sqlite3_value **argv +){ + series_cursor *pCur = (series_cursor *)pVtabCursor; + int i = 0; + (void)idxStrUnused; + if( idxNum & 0x01 ){ + pCur->ss.iBase = sqlite3_value_int64(argv[i++]); + }else{ + pCur->ss.iBase = 0; + } + if( idxNum & 0x02 ){ + pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); + }else{ + pCur->ss.iTerm = 0xffffffff; + } + if( idxNum & 0x04 ){ + pCur->ss.iStep = sqlite3_value_int64(argv[i++]); + if( pCur->ss.iStep==0 ){ + pCur->ss.iStep = 1; + }else if( pCur->ss.iStep<0 ){ + if( (idxNum & 0x10)==0 ) idxNum |= 0x08; + } + }else{ + pCur->ss.iStep = 1; + } + if( idxNum & 0x20 ){ + sqlite3_int64 iLimit = sqlite3_value_int64(argv[i++]); + sqlite3_int64 iTerm; + if( idxNum & 0x40 ){ + sqlite3_int64 iOffset = sqlite3_value_int64(argv[i++]); + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } + } + if( iLimit>=0 ){ + iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; + if( pCur->ss.iStep<0 ){ + if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + }else{ + if( iTermss.iTerm ) pCur->ss.iTerm = iTerm; + } + } + } + for(i=0; iss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; + break; + } + } + if( idxNum & 0x08 ){ + pCur->ss.isReversing = pCur->ss.iStep > 0; + }else{ + pCur->ss.isReversing = pCur->ss.iStep < 0; + } + setupSequence( &pCur->ss ); + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** 0x01 start = $value -- constraint exists +** 0x02 stop = $value -- constraint exists +** 0x04 step = $value -- constraint exists +** 0x08 output is in descending order +** 0x10 output is in ascending order +** 0x20 LIMIT $value -- constraint exists +** 0x40 OFFSET $value -- constraint exists +*/ +static int seriesBestIndex( + sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo +){ + int i, j; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif + int unusableMask = 0; /* Mask of unusable constraints */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + int aIdx[5]; /* Constraints on start, stop, step, LIMIT, OFFSET */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that the start, stop, and step columns + ** are the last three columns in the virtual table. */ + assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); + assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); + + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; /* 0 for start, 1 for stop, 2 for step */ + int iMask; /* bitmask for those column */ + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } + if( pConstraint->iColumniColumn - SERIES_COLUMN_START; + assert( iCol>=0 && iCol<=2 ); + iMask = 1 << iCol; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + continue; + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idxNum |= iMask; + aIdx[iCol] = i; + } + } + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<5; i++){ + if( (j = aIdx[i])>=0 ){ + pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; + } + } + /* The current generate_column() implementation requires at least one + ** argument (the START value). Legacy versions assumed START=0 if the + ** first argument was omitted. Compile with -DZERO_ARGUMENT_GENERATE_SERIES + ** to obtain the legacy behavior */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( !bStartSeen ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf( + "first argument to \"generate_series()\" missing or unusable"); + return SQLITE_ERROR; + } +#endif + if( (unusableMask & ~idxNum)!=0 ){ + /* The start, stop, and step columns are inputs. Therefore if there + ** are unusable constraints on any of start, stop, or step then + ** this plan is unusable */ + return SQLITE_CONSTRAINT; + } + if( (idxNum & 0x03)==0x03 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ + if( pIdxInfo->aOrderBy[0].desc ){ + idxNum |= 0x08; + }else{ + idxNum |= 0x10; + } + pIdxInfo->orderByConsumed = 1; + } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; + }else{ + /* If either boundary is missing, we have to generate a huge span + ** of numbers. Make this case very expensive so that the query + ** planner will work hard to avoid it. */ + pIdxInfo->estimatedRows = 2147483647; + } + pIdxInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 + +#endif +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3_libversion_number()<3008012 && pzErrMsg!=0 ){ + *pzErrMsg = sqlite3_mprintf( + "generate_series() requires SQLite 3.8.12 or later"); + return SQLITE_ERROR; + } + rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); +#endif + return rc; +} + +/************************* End ../ext/misc/series.c ********************/ +/************************* Begin ../ext/misc/regexp.c ******************/ +/* +** 2012-11-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** The code in this file implements a compact but reasonably +** efficient regular-expression matcher for posix extended regular +** expressions against UTF8 text. +** +** This file is an SQLite extension. It registers a single function +** named "regexp(A,B)" where A is the regular expression and B is the +** string to be matched. By registering this function, SQLite will also +** then implement the "B regexp A" operator. Note that with the function +** the regular expression comes first, but with the operator it comes +** second. +** +** The following regular expression syntax is supported: +** +** X* zero or more occurrences of X +** X+ one or more occurrences of X +** X? zero or one occurrences of X +** X{p,q} between p and q occurrences of X +** (X) match X +** X|Y X or Y +** ^X X occurring at the beginning of the string +** X$ X occurring at the end of the string +** . Match any single character +** \c Character c where c is one of \{}()[]|*+?. +** \c C-language escapes for c in afnrtv. ex: \t or \n +** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX +** \xXX Where XX is exactly 2 hex digits, unicode value XX +** [abc] Any single character from the set abc +** [^abc] Any single character not in the set abc +** [a-z] Any single character in the range a-z +** [^a-z] Any single character not in the range a-z +** \b Word boundary +** \w Word character. [A-Za-z0-9_] +** \W Non-word character +** \d Digit +** \D Non-digit +** \s Whitespace character +** \S Non-whitespace character +** +** A nondeterministic finite automaton (NFA) is used for matching, so the +** performance is bounded by O(N*M) where N is the size of the regular +** expression and M is the size of the input string. The matcher never +** exhibits exponential behavior. Note that the X{p,q} operator expands +** to p copies of X following by q-p copies of X? and that the size of the +** regular expression in the O(N*M) performance bound is computed after +** this expansion. +*/ +#include +#include +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 + +/* +** The following #defines change the names of some functions implemented in +** this file to prevent name collisions with C-library functions of the +** same name. +*/ +#define re_match sqlite3re_match +#define re_compile sqlite3re_compile +#define re_free sqlite3re_free + +/* The end-of-input character */ +#define RE_EOF 0 /* End of input */ +#define RE_START 0xfffffff /* Start of input - larger than an UTF-8 */ + +/* The NFA is implemented as sequence of opcodes taken from the following +** set. Each opcode has a single integer argument. +*/ +#define RE_OP_MATCH 1 /* Match the one character in the argument */ +#define RE_OP_ANY 2 /* Match any one character. (Implements ".") */ +#define RE_OP_ANYSTAR 3 /* Special optimized version of .* */ +#define RE_OP_FORK 4 /* Continue to both next and opcode at iArg */ +#define RE_OP_GOTO 5 /* Jump to opcode at iArg */ +#define RE_OP_ACCEPT 6 /* Halt and indicate a successful match */ +#define RE_OP_CC_INC 7 /* Beginning of a [...] character class */ +#define RE_OP_CC_EXC 8 /* Beginning of a [^...] character class */ +#define RE_OP_CC_VALUE 9 /* Single value in a character class */ +#define RE_OP_CC_RANGE 10 /* Range of values in a character class */ +#define RE_OP_WORD 11 /* Perl word character [A-Za-z0-9_] */ +#define RE_OP_NOTWORD 12 /* Not a perl word character */ +#define RE_OP_DIGIT 13 /* digit: [0-9] */ +#define RE_OP_NOTDIGIT 14 /* Not a digit */ +#define RE_OP_SPACE 15 /* space: [ \t\n\r\v\f] */ +#define RE_OP_NOTSPACE 16 /* Not a digit */ +#define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ +#define RE_OP_ATSTART 18 /* Currently at the start of the string */ + +#if defined(SQLITE_DEBUG) +/* Opcode names used for symbolic debugging */ +static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", + "ATSTART", +}; +#endif /* SQLITE_DEBUG */ + + +/* Each opcode is a "state" in the NFA */ +typedef unsigned short ReStateNumber; + +/* Because this is an NFA and not a DFA, multiple states can be active at +** once. An instance of the following object records all active states in +** the NFA. The implementation is optimized for the common case where the +** number of actives states is small. +*/ +typedef struct ReStateSet { + unsigned nState; /* Number of current states */ + ReStateNumber *aState; /* Current states */ +} ReStateSet; + +/* An input string read one character at a time. +*/ +typedef struct ReInput ReInput; +struct ReInput { + const unsigned char *z; /* All text */ + int i; /* Next byte to read */ + int mx; /* EOF when i>=mx */ +}; + +/* A compiled NFA (or an NFA that is in the process of being compiled) is +** an instance of the following object. +*/ +typedef struct ReCompiled ReCompiled; +struct ReCompiled { + ReInput sIn; /* Regular expression text */ + const char *zErr; /* Error message to return */ + char *aOp; /* Operators for the virtual machine */ + int *aArg; /* Arguments to each operator */ + unsigned (*xNextChar)(ReInput*); /* Next character function */ + unsigned char zInit[12]; /* Initial text to match */ + int nInit; /* Number of bytes in zInit */ + unsigned nState; /* Number of entries in aOp[] and aArg[] */ + unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ +}; + +/* Add a state to the given state set if it is not already there */ +static void re_add_state(ReStateSet *pSet, int newState){ + unsigned i; + for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; + pSet->aState[pSet->nState++] = (ReStateNumber)newState; +} + +/* Extract the next unicode character from *pzIn and return it. Advance +** *pzIn to the first byte past the end of the character returned. To +** be clear: this routine converts utf8 to unicode. This routine is +** optimized for the common case where the next character is a single byte. +*/ +static unsigned re_next_char(ReInput *p){ + unsigned c; + if( p->i>=p->mx ) return 0; + c = p->z[p->i++]; + if( c>=0x80 ){ + if( (c&0xe0)==0xc0 && p->imx && (p->z[p->i]&0xc0)==0x80 ){ + c = (c&0x1f)<<6 | (p->z[p->i++]&0x3f); + if( c<0x80 ) c = 0xfffd; + }else if( (c&0xf0)==0xe0 && p->i+1mx && (p->z[p->i]&0xc0)==0x80 + && (p->z[p->i+1]&0xc0)==0x80 ){ + c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f); + p->i += 2; + if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd; + }else if( (c&0xf8)==0xf0 && p->i+2mx && (p->z[p->i]&0xc0)==0x80 + && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){ + c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6) + | (p->z[p->i+2]&0x3f); + p->i += 3; + if( c<=0xffff || c>0x10ffff ) c = 0xfffd; + }else{ + c = 0xfffd; + } + } + return c; +} +static unsigned re_next_char_nocase(ReInput *p){ + unsigned c = re_next_char(p); + if( c>='A' && c<='Z' ) c += 'a' - 'A'; + return c; +} + +/* Return true if c is a perl "word" character: [A-Za-z0-9_] */ +static int re_word_char(int c){ + return (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +/* Return true if c is a "digit" character: [0-9] */ +static int re_digit_char(int c){ + return (c>='0' && c<='9'); +} + +/* Return true if c is a perl "space" character: [ \t\r\n\v\f] */ +static int re_space_char(int c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* Run a compiled regular expression on the zero-terminated input +** string zIn[]. Return true on a match and false if there is no match. +*/ +static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ + ReStateSet aStateSet[2], *pThis, *pNext; + ReStateNumber aSpace[100]; + ReStateNumber *pToFree; + unsigned int i = 0; + unsigned int iSwap = 0; + int c = RE_START; + int cPrev = 0; + int rc = 0; + ReInput in; + + in.z = zIn; + in.i = 0; + in.mx = nIn>=0 ? nIn : (int)strlen((char const*)zIn); + + /* Look for the initial prefix match, if there is one. */ + if( pRe->nInit ){ + unsigned char x = pRe->zInit[0]; + while( in.i+pRe->nInit<=in.mx + && (zIn[in.i]!=x || + strncmp((const char*)zIn+in.i, (const char*)pRe->zInit, pRe->nInit)!=0) + ){ + in.i++; + } + if( in.i+pRe->nInit>in.mx ) return 0; + c = RE_START-1; + } + + if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ + pToFree = 0; + aStateSet[0].aState = aSpace; + }else{ + pToFree = sqlite3_malloc64( sizeof(ReStateNumber)*2*pRe->nState ); + if( pToFree==0 ) return -1; + aStateSet[0].aState = pToFree; + } + aStateSet[1].aState = &aStateSet[0].aState[pRe->nState]; + pNext = &aStateSet[1]; + pNext->nState = 0; + re_add_state(pNext, 0); + while( c!=RE_EOF && pNext->nState>0 ){ + cPrev = c; + c = pRe->xNextChar(&in); + pThis = pNext; + pNext = &aStateSet[iSwap]; + iSwap = 1 - iSwap; + pNext->nState = 0; + for(i=0; inState; i++){ + int x = pThis->aState[i]; + switch( pRe->aOp[x] ){ + case RE_OP_MATCH: { + if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); + break; + } + case RE_OP_ATSTART: { + if( cPrev==RE_START ) re_add_state(pThis, x+1); + break; + } + case RE_OP_ANY: { + if( c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_WORD: { + if( re_word_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTWORD: { + if( !re_word_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_DIGIT: { + if( re_digit_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTDIGIT: { + if( !re_digit_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_SPACE: { + if( re_space_char(c) ) re_add_state(pNext, x+1); + break; + } + case RE_OP_NOTSPACE: { + if( !re_space_char(c) && c!=0 ) re_add_state(pNext, x+1); + break; + } + case RE_OP_BOUNDARY: { + if( re_word_char(c)!=re_word_char(cPrev) ) re_add_state(pThis, x+1); + break; + } + case RE_OP_ANYSTAR: { + re_add_state(pNext, x); + re_add_state(pThis, x+1); + break; + } + case RE_OP_FORK: { + re_add_state(pThis, x+pRe->aArg[x]); + re_add_state(pThis, x+1); + break; + } + case RE_OP_GOTO: { + re_add_state(pThis, x+pRe->aArg[x]); + break; + } + case RE_OP_ACCEPT: { + rc = 1; + goto re_match_end; + } + case RE_OP_CC_EXC: { + if( c==0 ) break; + /* fall-through */ goto re_op_cc_inc; + } + case RE_OP_CC_INC: re_op_cc_inc: { + int j = 1; + int n = pRe->aArg[x]; + int hit = 0; + for(j=1; j>0 && jaOp[x+j]==RE_OP_CC_VALUE ){ + if( pRe->aArg[x+j]==c ){ + hit = 1; + j = -1; + } + }else{ + if( pRe->aArg[x+j]<=c && pRe->aArg[x+j+1]>=c ){ + hit = 1; + j = -1; + }else{ + j++; + } + } + } + if( pRe->aOp[x]==RE_OP_CC_EXC ) hit = !hit; + if( hit ) re_add_state(pNext, x+n); + break; + } + } + } + } + for(i=0; inState; i++){ + int x = pNext->aState[i]; + while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x]; + if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; } + } +re_match_end: + sqlite3_free(pToFree); + return rc; +} + +/* Resize the opcode and argument arrays for an RE under construction. +*/ +static int re_resize(ReCompiled *p, int N){ + char *aOp; + int *aArg; + aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); + if( aOp==0 ) return 1; + p->aOp = aOp; + aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); + if( aArg==0 ) return 1; + p->aArg = aArg; + p->nAlloc = N; + return 0; +} + +/* Insert a new opcode and argument into an RE under construction. The +** insertion point is just prior to existing opcode iBefore. +*/ +static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ + int i; + if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0; + for(i=p->nState; i>iBefore; i--){ + p->aOp[i] = p->aOp[i-1]; + p->aArg[i] = p->aArg[i-1]; + } + p->nState++; + p->aOp[iBefore] = (char)op; + p->aArg[iBefore] = arg; + return iBefore; +} + +/* Append a new opcode and argument to the end of the RE under construction. +*/ +static int re_append(ReCompiled *p, int op, int arg){ + return re_insert(p, p->nState, op, arg); +} + +/* Make a copy of N opcodes starting at iStart onto the end of the RE +** under construction. +*/ +static void re_copy(ReCompiled *p, int iStart, int N){ + if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; + memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); + memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); + p->nState += N; +} + +/* Return true if c is a hexadecimal digit character: [0-9a-fA-F] +** If c is a hex digit, also set *pV = (*pV)*16 + valueof(c). If +** c is not a hex digit *pV is unchanged. +*/ +static int re_hex(int c, int *pV){ + if( c>='0' && c<='9' ){ + c -= '0'; + }else if( c>='a' && c<='f' ){ + c -= 'a' - 10; + }else if( c>='A' && c<='F' ){ + c -= 'A' - 10; + }else{ + return 0; } -#endif + *pV = (*pV)*16 + (c & 0xff); + return 1; } -/* -** Make consecutive calls to the SHA3Update function to add new content -** to the hash +/* A backslash character has been seen, read the next character and +** return its interpretation. */ -static void SHA3Update( - SHA3Context *p, - const unsigned char *aData, - unsigned int nData -){ - unsigned int i = 0; -#if SHA3_BYTEORDER==1234 - if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ - for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; - p->nLoaded += 8; - if( p->nLoaded>=p->nRate ){ - KeccakF1600Step(p); - p->nLoaded = 0; - } +static unsigned re_esc_char(ReCompiled *p){ + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; + static const char zTrans[] = "\a\f\n\r\t\v"; + int i, v = 0; + char c; + if( p->sIn.i>=p->sIn.mx ) return 0; + c = p->sIn.z[p->sIn.i]; + if( c=='u' && p->sIn.i+4sIn.mx ){ + const unsigned char *zIn = p->sIn.z + p->sIn.i; + if( re_hex(zIn[1],&v) + && re_hex(zIn[2],&v) + && re_hex(zIn[3],&v) + && re_hex(zIn[4],&v) + ){ + p->sIn.i += 5; + return v; } } -#endif - for(; iu.x[p->nLoaded] ^= aData[i]; -#elif SHA3_BYTEORDER==4321 - p->u.x[p->nLoaded^0x07] ^= aData[i]; -#else - p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; -#endif - p->nLoaded++; - if( p->nLoaded==p->nRate ){ - KeccakF1600Step(p); - p->nLoaded = 0; + if( c=='x' && p->sIn.i+2sIn.mx ){ + const unsigned char *zIn = p->sIn.z + p->sIn.i; + if( re_hex(zIn[1],&v) + && re_hex(zIn[2],&v) + ){ + p->sIn.i += 3; + return v; } } + for(i=0; zEsc[i] && zEsc[i]!=c; i++){} + if( zEsc[i] ){ + if( i<6 ) c = zTrans[i]; + p->sIn.i++; + }else{ + p->zErr = "unknown \\ escape"; + } + return c; } -/* -** After all content has been added, invoke SHA3Final() to compute -** the final hash. The function returns a pointer to the binary -** hash value. +/* Forward declaration */ +static const char *re_subcompile_string(ReCompiled*); + +/* Peek at the next byte of input */ +static unsigned char rePeek(ReCompiled *p){ + return p->sIn.isIn.mx ? p->sIn.z[p->sIn.i] : 0; +} + +/* Compile RE text into a sequence of opcodes. Continue up to the +** first unmatched ")" character, then return. If an error is found, +** return a pointer to the error message string. */ -static unsigned char *SHA3Final(SHA3Context *p){ - unsigned int i; - if( p->nLoaded==p->nRate-1 ){ - const unsigned char c1 = 0x86; - SHA3Update(p, &c1, 1); - }else{ - const unsigned char c2 = 0x06; - const unsigned char c3 = 0x80; - SHA3Update(p, &c2, 1); - p->nLoaded = p->nRate - 1; - SHA3Update(p, &c3, 1); +static const char *re_subcompile_re(ReCompiled *p){ + const char *zErr; + int iStart, iEnd, iGoto; + iStart = p->nState; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + while( rePeek(p)=='|' ){ + iEnd = p->nState; + re_insert(p, iStart, RE_OP_FORK, iEnd + 2 - iStart); + iGoto = re_append(p, RE_OP_GOTO, 0); + p->sIn.i++; + zErr = re_subcompile_string(p); + if( zErr ) return zErr; + p->aArg[iGoto] = p->nState - iGoto; + } + return 0; +} + +/* Compile an element of regular expression text (anything that can be +** an operand to the "|" operator). Return NULL on success or a pointer +** to the error message if there is a problem. +*/ +static const char *re_subcompile_string(ReCompiled *p){ + int iPrev = -1; + int iStart; + unsigned c; + const char *zErr; + while( (c = p->xNextChar(&p->sIn))!=0 ){ + iStart = p->nState; + switch( c ){ + case '|': + case ')': { + p->sIn.i--; + return 0; + } + case '(': { + zErr = re_subcompile_re(p); + if( zErr ) return zErr; + if( rePeek(p)!=')' ) return "unmatched '('"; + p->sIn.i++; + break; + } + case '.': { + if( rePeek(p)=='*' ){ + re_append(p, RE_OP_ANYSTAR, 0); + p->sIn.i++; + }else{ + re_append(p, RE_OP_ANY, 0); + } + break; + } + case '*': { + if( iPrev<0 ) return "'*' without operand"; + re_insert(p, iPrev, RE_OP_GOTO, p->nState - iPrev + 1); + re_append(p, RE_OP_FORK, iPrev - p->nState + 1); + break; + } + case '+': { + if( iPrev<0 ) return "'+' without operand"; + re_append(p, RE_OP_FORK, iPrev - p->nState); + break; + } + case '?': { + if( iPrev<0 ) return "'?' without operand"; + re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); + break; + } + case '$': { + re_append(p, RE_OP_MATCH, RE_EOF); + break; + } + case '^': { + re_append(p, RE_OP_ATSTART, 0); + break; + } + case '{': { + int m = 0, n = 0; + int sz, j; + if( iPrev<0 ) return "'{m,n}' without operand"; + while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } + n = m; + if( c==',' ){ + p->sIn.i++; + n = 0; + while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } + } + if( c!='}' ) return "unmatched '{'"; + if( n>0 && nsIn.i++; + sz = p->nState - iPrev; + if( m==0 ){ + if( n==0 ) return "both m and n are zero in '{m,n}'"; + re_insert(p, iPrev, RE_OP_FORK, sz+1); + iPrev++; + n--; + }else{ + for(j=1; j0 ){ + re_append(p, RE_OP_FORK, -sz); + } + break; + } + case '[': { + unsigned int iFirst = p->nState; + if( rePeek(p)=='^' ){ + re_append(p, RE_OP_CC_EXC, 0); + p->sIn.i++; + }else{ + re_append(p, RE_OP_CC_INC, 0); + } + while( (c = p->xNextChar(&p->sIn))!=0 ){ + if( c=='[' && rePeek(p)==':' ){ + return "POSIX character classes not supported"; + } + if( c=='\\' ) c = re_esc_char(p); + if( rePeek(p)=='-' ){ + re_append(p, RE_OP_CC_RANGE, c); + p->sIn.i++; + c = p->xNextChar(&p->sIn); + if( c=='\\' ) c = re_esc_char(p); + re_append(p, RE_OP_CC_RANGE, c); + }else{ + re_append(p, RE_OP_CC_VALUE, c); + } + if( rePeek(p)==']' ){ p->sIn.i++; break; } + } + if( c==0 ) return "unclosed '['"; + if( p->nState>iFirst ) p->aArg[iFirst] = p->nState - iFirst; + break; + } + case '\\': { + int specialOp = 0; + switch( rePeek(p) ){ + case 'b': specialOp = RE_OP_BOUNDARY; break; + case 'd': specialOp = RE_OP_DIGIT; break; + case 'D': specialOp = RE_OP_NOTDIGIT; break; + case 's': specialOp = RE_OP_SPACE; break; + case 'S': specialOp = RE_OP_NOTSPACE; break; + case 'w': specialOp = RE_OP_WORD; break; + case 'W': specialOp = RE_OP_NOTWORD; break; + } + if( specialOp ){ + p->sIn.i++; + re_append(p, specialOp, 0); + }else{ + c = re_esc_char(p); + re_append(p, RE_OP_MATCH, c); + } + break; + } + default: { + re_append(p, RE_OP_MATCH, c); + break; + } + } + iPrev = iStart; } - for(i=0; inRate; i++){ - p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + return 0; +} + +/* Free and reclaim all the memory used by a previously compiled +** regular expression. Applications should invoke this routine once +** for every call to re_compile() to avoid memory leaks. +*/ +static void re_free(ReCompiled *pRe){ + if( pRe ){ + sqlite3_free(pRe->aOp); + sqlite3_free(pRe->aArg); + sqlite3_free(pRe); } - return &p->u.x[p->nRate]; } -/* End of the hashing logic -*****************************************************************************/ /* -** Implementation of the sha3(X,SIZE) function. -** -** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default -** size is 256. If X is a BLOB, it is hashed as is. -** For all other non-NULL types of input, X is converted into a UTF-8 string -** and the string is hashed without the trailing 0x00 terminator. The hash -** of a NULL value is NULL. +** Compile a textual regular expression in zIn[] into a compiled regular +** expression suitable for us by re_match() and return a pointer to the +** compiled regular expression in *ppRe. Return NULL on success or an +** error message if something goes wrong. */ -static void sha3Func( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - SHA3Context cx; - int eType = sqlite3_value_type(argv[0]); - int nByte = sqlite3_value_bytes(argv[0]); - int iSize; - if( argc==1 ){ - iSize = 256; +static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ + ReCompiled *pRe; + const char *zErr; + int i, j; + + *ppRe = 0; + pRe = sqlite3_malloc( sizeof(*pRe) ); + if( pRe==0 ){ + return "out of memory"; + } + memset(pRe, 0, sizeof(*pRe)); + pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; + if( re_resize(pRe, 30) ){ + re_free(pRe); + return "out of memory"; + } + if( zIn[0]=='^' ){ + zIn++; }else{ - iSize = sqlite3_value_int(argv[1]); - if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ - sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " - "384 512", -1); - return; - } + re_append(pRe, RE_OP_ANYSTAR, 0); } - if( eType==SQLITE_NULL ) return; - SHA3Init(&cx, iSize); - if( eType==SQLITE_BLOB ){ - SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + pRe->sIn.z = (unsigned char*)zIn; + pRe->sIn.i = 0; + pRe->sIn.mx = (int)strlen(zIn); + zErr = re_subcompile_re(pRe); + if( zErr ){ + re_free(pRe); + return zErr; + } + if( pRe->sIn.i>=pRe->sIn.mx ){ + re_append(pRe, RE_OP_ACCEPT, 0); + *ppRe = pRe; }else{ - SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + re_free(pRe); + return "unrecognized character"; } - sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); -} -/* Compute a string using sqlite3_vsnprintf() with a maximum length -** of 50 bytes and add it to the hash. -*/ -static void hash_step_vformat( - SHA3Context *p, /* Add content to this context */ - const char *zFormat, - ... -){ - va_list ap; - int n; - char zBuf[50]; - va_start(ap, zFormat); - sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); - va_end(ap); - n = (int)strlen(zBuf); - SHA3Update(p, (unsigned char*)zBuf, n); + /* The following is a performance optimization. If the regex begins with + ** ".*" (if the input regex lacks an initial "^") and afterwards there are + ** one or more matching characters, enter those matching characters into + ** zInit[]. The re_match() routine can then search ahead in the input + ** string looking for the initial match without having to run the whole + ** regex engine over the string. Do not worry about trying to match + ** unicode characters beyond plane 0 - those are very rare and this is + ** just an optimization. */ + if( pRe->aOp[0]==RE_OP_ANYSTAR && !noCase ){ + for(j=0, i=1; j<(int)sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ + unsigned x = pRe->aArg[i]; + if( x<=0x7f ){ + pRe->zInit[j++] = (unsigned char)x; + }else if( x<=0x7ff ){ + pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); + pRe->zInit[j++] = 0x80 | (x&0x3f); + }else if( x<=0xffff ){ + pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12)); + pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); + pRe->zInit[j++] = 0x80 | (x&0x3f); + }else{ + break; + } + } + if( j>0 && pRe->zInit[j-1]==0 ) j--; + pRe->nInit = j; + } + return pRe->zErr; } /* -** Implementation of the sha3_query(SQL,SIZE) function. -** -** This function compiles and runs the SQL statement(s) given in the -** argument. The results are hashed using a SIZE-bit SHA3. The default -** size is 256. -** -** The format of the byte stream that is hashed is summarized as follows: -** -** S: -** R -** N -** I -** F -** B: -** T: +** Implementation of the regexp() SQL function. This function implements +** the build-in REGEXP operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: ** -** is the original SQL text for each statement run and is -** the size of that text. The SQL text is UTF-8. A single R character -** occurs before the start of each row. N means a NULL value. -** I mean an 8-byte little-endian integer . F is a floating point -** number with an 8-byte little-endian IEEE floating point value . -** B means blobs of bytes. T means text rendered as -** bytes of UTF-8. The and values are expressed as an ASCII -** text integers. +** A REGEXP B ** -** For each SQL statement in the X input, there is one S segment. Each -** S segment is followed by zero or more R segments, one for each row in the -** result set. After each R, there are one or more N, I, F, B, or T segments, -** one for each column in the result set. Segments are concatentated directly -** with no delimiters of any kind. +** is implemented as regexp(B,A). */ -static void sha3QueryFunc( +static void re_sql_func( sqlite3_context *context, int argc, sqlite3_value **argv ){ - sqlite3 *db = sqlite3_context_db_handle(context); - const char *zSql = (const char*)sqlite3_value_text(argv[0]); - sqlite3_stmt *pStmt = 0; - int nCol; /* Number of columns in the result set */ - int i; /* Loop counter */ - int rc; - int n; - const char *z; - SHA3Context cx; - int iSize; - - if( argc==1 ){ - iSize = 256; - }else{ - iSize = sqlite3_value_int(argv[1]); - if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ - sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " - "384 512", -1); - return; - } - } - if( zSql==0 ) return; - SHA3Init(&cx, iSize); - while( zSql[0] ){ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc ){ - char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", - zSql, sqlite3_errmsg(db)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - if( !sqlite3_stmt_readonly(pStmt) ){ - char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); - sqlite3_finalize(pStmt); - sqlite3_result_error(context, zMsg, -1); - sqlite3_free(zMsg); - return; - } - nCol = sqlite3_column_count(pStmt); - z = sqlite3_sql(pStmt); - if( z ){ - n = (int)strlen(z); - hash_step_vformat(&cx,"S%d:",n); - SHA3Update(&cx,(unsigned char*)z,n); - } - - /* Compute a hash over the result of the query */ - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - SHA3Update(&cx,(const unsigned char*)"R",1); - for(i=0; i=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - SHA3Update(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - SHA3Update(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - hash_step_vformat(&cx,"T%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - hash_step_vformat(&cx,"B%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - } - } + ReCompiled *pRe; /* Compiled regular expression */ + const char *zPattern; /* The regular expression */ + const unsigned char *zStr;/* String being searched */ + const char *zErr; /* Compile error message */ + int setAux = 0; /* True to invoke sqlite3_set_auxdata() */ + + (void)argc; /* Unused */ + pRe = sqlite3_get_auxdata(context, 0); + if( pRe==0 ){ + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; } - sqlite3_finalize(pStmt); + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + setAux = 1; } - sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); + zStr = (const unsigned char*)sqlite3_value_text(argv[1]); + if( zStr!=0 ){ + sqlite3_result_int(context, re_match(pRe, zStr, -1)); + } + if( setAux ){ + sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + } +} + +#if defined(SQLITE_DEBUG) +/* +** This function is used for testing and debugging only. It is only available +** if the SQLITE_DEBUG compile-time option is used. +** +** Compile a regular expression and then convert the compiled expression into +** text and return that text. +*/ +static void re_bytecode_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPattern; + const char *zErr; + ReCompiled *pRe; + sqlite3_str *pStr; + int i; + int n; + char *z; + (void)argc; + + zPattern = (const char*)sqlite3_value_text(argv[0]); + if( zPattern==0 ) return; + zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + if( zErr ){ + re_free(pRe); + sqlite3_result_error(context, zErr, -1); + return; + } + if( pRe==0 ){ + sqlite3_result_error_nomem(context); + return; + } + pStr = sqlite3_str_new(0); + if( pStr==0 ) goto re_bytecode_func_err; + if( pRe->nInit>0 ){ + sqlite3_str_appendf(pStr, "INIT "); + for(i=0; inInit; i++){ + sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]); + } + sqlite3_str_appendf(pStr, "\n"); + } + for(i=0; (unsigned)inState; i++){ + sqlite3_str_appendf(pStr, "%-8s %4d\n", + ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); + } + n = sqlite3_str_length(pStr); + z = sqlite3_str_finish(pStr); + if( n==0 ){ + sqlite3_free(z); + }else{ + sqlite3_result_text(context, z, n-1, sqlite3_free); + } + +re_bytecode_func_err: + re_free(pRe); } +#endif /* SQLITE_DEBUG */ + +/* +** Invoke this routine to register the regexp() function with the +** SQLite database connection. +*/ #ifdef _WIN32 #endif -int sqlite3_shathree_init( - sqlite3 *db, - char **pzErrMsg, +int sqlite3_regexp_init( + sqlite3 *db, + char **pzErrMsg, const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "sha3", 1, - SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha3Func, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3", 2, - SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha3Func, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 1, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); - } + (void)pzErrMsg; /* Unused */ + rc = sqlite3_create_function(db, "regexp", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_sql_func, 0, 0); if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sha3_query", 2, - SQLITE_UTF8 | SQLITE_DIRECTONLY, - 0, sha3QueryFunc, 0, 0); + /* The regexpi(PATTERN,STRING) function is a case-insensitive version + ** of regexp(PATTERN,STRING). */ + rc = sqlite3_create_function(db, "regexpi", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + (void*)db, re_sql_func, 0, 0); +#if defined(SQLITE_DEBUG) + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "regexp_bytecode", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, re_bytecode_func, 0, 0); + } +#endif /* SQLITE_DEBUG */ } return rc; } -/************************* End ../ext/misc/shathree.c ********************/ +/************************* End ../ext/misc/regexp.c ********************/ +#ifndef SQLITE_SHELL_FIDDLE /************************* Begin ../ext/misc/fileio.c ******************/ /* ** 2014-06-13 @@ -2519,9 +7520,12 @@ static int writeFile( mode_t mode, /* MODE parameter passed to writefile() */ sqlite3_int64 mtime /* MTIME parameter (or -1 to not set time) */ ){ + if( zFile==0 ) return 1; #if !defined(_WIN32) && !defined(WIN32) if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); + if( zTo==0 ) return 1; + unlink(zFile); if( symlink(zTo, zFile)<0 ) return 1; }else #endif @@ -2608,13 +7612,19 @@ static int writeFile( return 1; } #else - /* Legacy unix */ - struct timeval times[2]; - times[0].tv_usec = times[1].tv_usec = 0; - times[0].tv_sec = time(0); - times[1].tv_sec = mtime; - if( utimes(zFile, times) ){ - return 1; + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } } #endif } @@ -3133,6 +8143,7 @@ static int fsdirRegister(sqlite3 *db){ 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); @@ -3653,7 +8664,8 @@ static sqlite3_module completionModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -4302,177 +9314,68 @@ static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); } -static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ - return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); -} -static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ - return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); -} -static int apndSetSystemCall( - sqlite3_vfs *pVfs, - const char *zName, - sqlite3_syscall_ptr pCall -){ - return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); -} -static sqlite3_syscall_ptr apndGetSystemCall( - sqlite3_vfs *pVfs, - const char *zName -){ - return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); -} -static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ - return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); -} - - -#ifdef _WIN32 - -#endif -/* -** This routine is called when the extension is loaded. -** Register the new VFS. -*/ -int sqlite3_appendvfs_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - sqlite3_vfs *pOrig; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; - (void)db; - pOrig = sqlite3_vfs_find(0); - if( pOrig==0 ) return SQLITE_ERROR; - apnd_vfs.iVersion = pOrig->iVersion; - apnd_vfs.pAppData = pOrig; - apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); - rc = sqlite3_vfs_register(&apnd_vfs, 0); -#ifdef APPENDVFS_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); - } -#endif - if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; - return rc; -} - -/************************* End ../ext/misc/appendvfs.c ********************/ -/************************* Begin ../ext/misc/memtrace.c ******************/ -/* -** 2019-01-21 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements an extension that uses the SQLITE_CONFIG_MALLOC -** mechanism to add a tracing layer on top of SQLite. If this extension -** is registered prior to sqlite3_initialize(), it will cause all memory -** allocation activities to be logged on standard output, or to some other -** FILE specified by the initializer. -** -** This file needs to be compiled into the application that uses it. -** -** This extension is used to implement the --memtrace option of the -** command-line shell. -*/ -#include -#include -#include - -/* The original memory allocation routines */ -static sqlite3_mem_methods memtraceBase; -static FILE *memtraceOut; - -/* Methods that trace memory allocations */ -static void *memtraceMalloc(int n){ - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", - memtraceBase.xRoundup(n)); - } - return memtraceBase.xMalloc(n); -} -static void memtraceFree(void *p){ - if( p==0 ) return; - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); - } - memtraceBase.xFree(p); -} -static void *memtraceRealloc(void *p, int n){ - if( p==0 ) return memtraceMalloc(n); - if( n==0 ){ - memtraceFree(p); - return 0; - } - if( memtraceOut ){ - fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", - memtraceBase.xSize(p), memtraceBase.xRoundup(n)); - } - return memtraceBase.xRealloc(p, n); -} -static int memtraceSize(void *p){ - return memtraceBase.xSize(p); -} -static int memtraceRoundup(int n){ - return memtraceBase.xRoundup(n); -} -static int memtraceInit(void *p){ - return memtraceBase.xInit(p); -} -static void memtraceShutdown(void *p){ - memtraceBase.xShutdown(p); -} - -/* The substitute memory allocator */ -static sqlite3_mem_methods ersaztMethods = { - memtraceMalloc, - memtraceFree, - memtraceRealloc, - memtraceSize, - memtraceRoundup, - memtraceInit, - memtraceShutdown, - 0 -}; - -/* Begin tracing memory allocations to out. */ -int sqlite3MemTraceActivate(FILE *out){ - int rc = SQLITE_OK; - if( memtraceBase.xMalloc==0 ){ - rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); - if( rc==SQLITE_OK ){ - rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); - } - } - memtraceOut = out; - return rc; -} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} -/* Deactivate memory tracing */ -int sqlite3MemTraceDeactivate(void){ + +#ifdef _WIN32 + +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ int rc = SQLITE_OK; - if( memtraceBase.xMalloc!=0 ){ - rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); - if( rc==SQLITE_OK ){ - memset(&memtraceBase, 0, sizeof(memtraceBase)); - } + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + (void)db; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 0); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); } - memtraceOut = 0; +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; return rc; } -/************************* End ../ext/misc/memtrace.c ********************/ -/************************* Begin ../ext/misc/uint.c ******************/ +/************************* End ../ext/misc/appendvfs.c ********************/ +#endif +#ifdef SQLITE_HAVE_ZLIB +/************************* Begin ../ext/misc/zipfile.c ******************/ /* -** 2020-04-14 +** 2017-12-26 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -4483,2262 +9386,2536 @@ int sqlite3MemTraceDeactivate(void){ ** ****************************************************************************** ** -** This SQLite extension implements the UINT collating sequence. +** This file implements a virtual table for reading and writing ZIP archive +** files. ** -** UINT works like BINARY for text, except that embedded strings -** of digits compare in numeric order. +** Usage example: ** -** * Leading zeros are handled properly, in the sense that -** they do not mess of the maginitude comparison of embedded -** strings of digits. "x00123y" is equal to "x123y". +** SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename); ** -** * Only unsigned integers are recognized. Plus and minus -** signs are ignored. Decimal points and exponential notation -** are ignored. +** Current limitations: ** -** * Embedded integers can be of arbitrary length. Comparison -** is *not* limited integers that can be expressed as a -** 64-bit machine integer. +** * No support for encryption +** * No support for ZIP archives spanning multiple files +** * No support for zip64 extensions +** * Only the "inflate/deflate" (zlib) compression method is supported */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 -#include +#include #include -#include +#include +#include + +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +#ifndef SQLITE_AMALGAMATION + +#ifndef UINT32_TYPE +# ifdef HAVE_UINT32_T +# define UINT32_TYPE uint32_t +# else +# define UINT32_TYPE unsigned int +# endif +#endif +#ifndef UINT16_TYPE +# ifdef HAVE_UINT16_T +# define UINT16_TYPE uint16_t +# else +# define UINT16_TYPE unsigned short int +# endif +#endif +/* typedef sqlite3_int64 i64; */ +/* typedef unsigned char u8; */ +/* typedef UINT32_TYPE u32; // 4-byte unsigned integer // */ +/* typedef UINT16_TYPE u16; // 2-byte unsigned integer // */ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +#endif /* SQLITE_AMALGAMATION */ /* -** Compare text in lexicographic order, except strings of digits -** compare in numeric order. +** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. +** +** In some ways it would be better to obtain these values from system +** header files. But, the dependency is undesirable and (a) these +** have been stable for decades, (b) the values are part of POSIX and +** are also made explicit in [man stat], and (c) are part of the +** file format for zip archives. */ -static int uintCollFunc( - void *notUsed, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - const unsigned char *zA = (const unsigned char*)pKey1; - const unsigned char *zB = (const unsigned char*)pKey2; - int i=0, j=0, x; - (void)notUsed; - while( i -#include -#include -#include - -/* Mark a function parameter as unused, to suppress nuisance compiler -** warnings. */ -#ifndef UNUSED_PARAMETER -# define UNUSED_PARAMETER(X) (void)(X) -#endif +#define ZIPFILE_EXTRA_TIMESTAMP 0x5455 +#define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) +#define ZIPFILE_NEWENTRY_REQUIRED 20 +#define ZIPFILE_NEWENTRY_FLAGS 0x800 +#define ZIPFILE_SIGNATURE_CDS 0x02014b50 +#define ZIPFILE_SIGNATURE_LFH 0x04034b50 +#define ZIPFILE_SIGNATURE_EOCD 0x06054b50 +/* +** The sizes of the fixed-size part of each of the three main data +** structures in a zip archive. +*/ +#define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 +#define ZIPFILE_CDS_FIXED_SZ 46 -/* A decimal object */ -typedef struct Decimal Decimal; -struct Decimal { - char sign; /* 0 for positive, 1 for negative */ - char oom; /* True if an OOM is encountered */ - char isNull; /* True if holds a NULL rather than a number */ - char isInit; /* True upon initialization */ - int nDigit; /* Total number of digits */ - int nFrac; /* Number of digits to the right of the decimal point */ - signed char *a; /* Array of digits. Most significant first. */ +/* +*** 4.3.16 End of central directory record: +*** +*** end of central dir signature 4 bytes (0x06054b50) +*** number of this disk 2 bytes +*** number of the disk with the +*** start of the central directory 2 bytes +*** total number of entries in the +*** central directory on this disk 2 bytes +*** total number of entries in +*** the central directory 2 bytes +*** size of the central directory 4 bytes +*** offset of start of central +*** directory with respect to +*** the starting disk number 4 bytes +*** .ZIP file comment length 2 bytes +*** .ZIP file comment (variable size) +*/ +typedef struct ZipfileEOCD ZipfileEOCD; +struct ZipfileEOCD { + u16 iDisk; + u16 iFirstDisk; + u16 nEntry; + u16 nEntryTotal; + u32 nSize; + u32 iOffset; }; /* -** Release memory held by a Decimal, but do not free the object itself. +*** 4.3.12 Central directory structure: +*** +*** ... +*** +*** central file header signature 4 bytes (0x02014b50) +*** version made by 2 bytes +*** version needed to extract 2 bytes +*** general purpose bit flag 2 bytes +*** compression method 2 bytes +*** last mod file time 2 bytes +*** last mod file date 2 bytes +*** crc-32 4 bytes +*** compressed size 4 bytes +*** uncompressed size 4 bytes +*** file name length 2 bytes +*** extra field length 2 bytes +*** file comment length 2 bytes +*** disk number start 2 bytes +*** internal file attributes 2 bytes +*** external file attributes 4 bytes +*** relative offset of local header 4 bytes */ -static void decimal_clear(Decimal *p){ - sqlite3_free(p->a); -} +typedef struct ZipfileCDS ZipfileCDS; +struct ZipfileCDS { + u16 iVersionMadeBy; + u16 iVersionExtract; + u16 flags; + u16 iCompression; + u16 mTime; + u16 mDate; + u32 crc32; + u32 szCompressed; + u32 szUncompressed; + u16 nFile; + u16 nExtra; + u16 nComment; + u16 iDiskStart; + u16 iInternalAttr; + u32 iExternalAttr; + u32 iOffset; + char *zFile; /* Filename (sqlite3_malloc()) */ +}; /* -** Destroy a Decimal object +*** 4.3.7 Local file header: +*** +*** local file header signature 4 bytes (0x04034b50) +*** version needed to extract 2 bytes +*** general purpose bit flag 2 bytes +*** compression method 2 bytes +*** last mod file time 2 bytes +*** last mod file date 2 bytes +*** crc-32 4 bytes +*** compressed size 4 bytes +*** uncompressed size 4 bytes +*** file name length 2 bytes +*** extra field length 2 bytes +*** */ -static void decimal_free(Decimal *p){ - if( p ){ - decimal_clear(p); - sqlite3_free(p); - } -} +typedef struct ZipfileLFH ZipfileLFH; +struct ZipfileLFH { + u16 iVersionExtract; + u16 flags; + u16 iCompression; + u16 mTime; + u16 mDate; + u32 crc32; + u32 szCompressed; + u32 szUncompressed; + u16 nFile; + u16 nExtra; +}; -/* -** Allocate a new Decimal object. Initialize it to the number given -** by the input string. +typedef struct ZipfileEntry ZipfileEntry; +struct ZipfileEntry { + ZipfileCDS cds; /* Parsed CDS record */ + u32 mUnixTime; /* Modification time, in UNIX format */ + u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ + i64 iDataOff; /* Offset to data in file (if aData==0) */ + u8 *aData; /* cds.szCompressed bytes of compressed data */ + ZipfileEntry *pNext; /* Next element in in-memory CDS */ +}; + +/* +** Cursor type for zipfile tables. */ -static Decimal *decimal_new( - sqlite3_context *pCtx, - sqlite3_value *pIn, - int nAlt, - const unsigned char *zAlt -){ - Decimal *p; - int n, i; - const unsigned char *zIn; - int iExp = 0; - p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) goto new_no_mem; - p->sign = 0; - p->oom = 0; - p->isInit = 1; - p->isNull = 0; - p->nDigit = 0; - p->nFrac = 0; - if( zAlt ){ - n = nAlt, - zIn = zAlt; - }else{ - if( sqlite3_value_type(pIn)==SQLITE_NULL ){ - p->a = 0; - p->isNull = 1; - return p; - } - n = sqlite3_value_bytes(pIn); - zIn = sqlite3_value_text(pIn); - } - p->a = sqlite3_malloc64( n+1 ); - if( p->a==0 ) goto new_no_mem; - for(i=0; isspace(zIn[i]); i++){} - if( zIn[i]=='-' ){ - p->sign = 1; - i++; - }else if( zIn[i]=='+' ){ - i++; - } - while( i='0' && c<='9' ){ - p->a[p->nDigit++] = c - '0'; - }else if( c=='.' ){ - p->nFrac = p->nDigit + 1; - }else if( c=='e' || c=='E' ){ - int j = i+1; - int neg = 0; - if( j>=n ) break; - if( zIn[j]=='-' ){ - neg = 1; - j++; - }else if( zIn[j]=='+' ){ - j++; - } - while( j='0' && zIn[j]<='9' ){ - iExp = iExp*10 + zIn[j] - '0'; - } - j++; - } - if( neg ) iExp = -iExp; - break; - } - i++; - } - if( p->nFrac ){ - p->nFrac = p->nDigit - (p->nFrac - 1); - } - if( iExp>0 ){ - if( p->nFrac>0 ){ - if( iExp<=p->nFrac ){ - p->nFrac -= iExp; - iExp = 0; - }else{ - iExp -= p->nFrac; - p->nFrac = 0; - } - } - if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; - memset(p->a+p->nDigit, 0, iExp); - p->nDigit += iExp; - } - }else if( iExp<0 ){ - int nExtra; - iExp = -iExp; - nExtra = p->nDigit - p->nFrac - 1; - if( nExtra ){ - if( nExtra>=iExp ){ - p->nFrac += iExp; - iExp = 0; - }else{ - iExp -= nExtra; - p->nFrac = p->nDigit - 1; - } - } - if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; - memmove(p->a+iExp, p->a, p->nDigit); - memset(p->a, 0, iExp); - p->nDigit += iExp; - p->nFrac += iExp; - } - } - return p; +typedef struct ZipfileCsr ZipfileCsr; +struct ZipfileCsr { + sqlite3_vtab_cursor base; /* Base class - must be first */ + i64 iId; /* Cursor ID */ + u8 bEof; /* True when at EOF */ + u8 bNoop; /* If next xNext() call is no-op */ + + /* Used outside of write transactions */ + FILE *pFile; /* Zip file */ + i64 iNextOff; /* Offset of next record in central directory */ + ZipfileEOCD eocd; /* Parse of central directory record */ + + ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ + ZipfileEntry *pCurrent; /* Current entry */ + ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ +}; + +typedef struct ZipfileTab ZipfileTab; +struct ZipfileTab { + sqlite3_vtab base; /* Base class - must be first */ + char *zFile; /* Zip file this table accesses (may be NULL) */ + sqlite3 *db; /* Host database connection */ + u8 *aBuffer; /* Temporary buffer used for various tasks */ -new_no_mem: - if( pCtx ) sqlite3_result_error_nomem(pCtx); - sqlite3_free(p); - return 0; + ZipfileCsr *pCsrList; /* List of cursors */ + i64 iNextCsrid; + + /* The following are used by write transactions only */ + ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */ + ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */ + FILE *pWriteFd; /* File handle open on zip archive */ + i64 szCurrent; /* Current size of zip archive */ + i64 szOrig; /* Size of archive at start of transaction */ +}; + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); } /* -** Make the given Decimal the result. +** If string zIn is quoted, dequote it in place. Otherwise, if the string +** is not quoted, do nothing. */ -static void decimal_result(sqlite3_context *pCtx, Decimal *p){ - char *z; - int i, j; - int n; - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(pCtx); - return; - } - if( p->isNull ){ - sqlite3_result_null(pCtx); - return; - } - z = sqlite3_malloc( p->nDigit+4 ); - if( z==0 ){ - sqlite3_result_error_nomem(pCtx); - return; - } - i = 0; - if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ - p->sign = 0; - } - if( p->sign ){ - z[0] = '-'; - i = 1; - } - n = p->nDigit - p->nFrac; - if( n<=0 ){ - z[i++] = '0'; +static void zipfileDequote(char *zIn){ + char q = zIn[0]; + if( q=='"' || q=='\'' || q=='`' || q=='[' ){ + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + while( ALWAYS(zIn[iIn]) ){ + char c = zIn[iIn++]; + if( c==q && zIn[iIn++]!=q ) break; + zIn[iOut++] = c; + } + zIn[iOut] = '\0'; } - j = 0; - while( n>1 && p->a[j]==0 ){ - j++; - n--; +} + +/* +** Construct a new ZipfileTab virtual table object. +** +** argv[0] -> module name ("zipfile") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int zipfileConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; + int nFile = 0; + const char *zFile = 0; + ZipfileTab *pNew = 0; + int rc; + (void)pAux; + + /* If the table name is not "zipfile", require that the argument be + ** specified. This stops zipfile tables from being created as: + ** + ** CREATE VIRTUAL TABLE zzz USING zipfile(); + ** + ** It does not prevent: + ** + ** CREATE VIRTUAL TABLE zipfile USING zipfile(); + */ + assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); + if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ + *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); + return SQLITE_ERROR; } - while( n>0 ){ - z[i++] = p->a[j] + '0'; - j++; - n--; + + if( argc>3 ){ + zFile = argv[3]; + nFile = (int)strlen(zFile)+1; } - if( p->nFrac ){ - z[i++] = '.'; - do{ - z[i++] = p->a[j] + '0'; - j++; - }while( jnDigit ); + + rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, nByte+nFile); + pNew->db = db; + pNew->aBuffer = (u8*)&pNew[1]; + if( zFile ){ + pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; + memcpy(pNew->zFile, zFile, nFile); + zipfileDequote(pNew->zFile); + } } - z[i] = 0; - sqlite3_result_text(pCtx, z, i, sqlite3_free); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + *ppVtab = (sqlite3_vtab*)pNew; + return rc; } /* -** SQL Function: decimal(X) -** -** Convert input X into decimal and then back into text +** Free the ZipfileEntry structure indicated by the only argument. */ -static void decimalFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *p = decimal_new(context, argv[0], 0, 0); - UNUSED_PARAMETER(argc); - decimal_result(context, p); - decimal_free(p); +static void zipfileEntryFree(ZipfileEntry *p){ + if( p ){ + sqlite3_free(p->cds.zFile); + sqlite3_free(p); + } } /* -** Compare to Decimal objects. Return negative, 0, or positive if the -** first object is less than, equal to, or greater than the second. -** -** Preconditions for this routine: -** -** pA!=0 -** pA->isNull==0 -** pB!=0 -** pB->isNull==0 +** Release resources that should be freed at the end of a write +** transaction. */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ - int nASig, nBSig, rc, n; - if( pA->sign!=pB->sign ){ - return pA->sign ? -1 : +1; - } - if( pA->sign ){ - const Decimal *pTemp = pA; - pA = pB; - pB = pTemp; - } - nASig = pA->nDigit - pA->nFrac; - nBSig = pB->nDigit - pB->nFrac; - if( nASig!=nBSig ){ - return nASig - nBSig; +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + if( pTab->pWriteFd ){ + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; } - n = pA->nDigit; - if( n>pB->nDigit ) n = pB->nDigit; - rc = memcmp(pA->a, pB->a, n); - if( rc==0 ){ - rc = pA->nDigit - pB->nDigit; + for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + zipfileEntryFree(pEntry); } - return rc; + pTab->pFirstEntry = 0; + pTab->pLastEntry = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; } /* -** SQL Function: decimal_cmp(X, Y) -** -** Return negative, zero, or positive if X is less then, equal to, or -** greater than Y. +** This method is the destructor for zipfile vtab objects. */ -static void decimalCmpFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *pA = 0, *pB = 0; - int rc; - - UNUSED_PARAMETER(argc); - pA = decimal_new(context, argv[0], 0, 0); - if( pA==0 || pA->isNull ) goto cmp_done; - pB = decimal_new(context, argv[1], 0, 0); - if( pB==0 || pB->isNull ) goto cmp_done; - rc = decimal_cmp(pA, pB); - if( rc<0 ) rc = -1; - else if( rc>0 ) rc = +1; - sqlite3_result_int(context, rc); -cmp_done: - decimal_free(pA); - decimal_free(pB); +static int zipfileDisconnect(sqlite3_vtab *pVtab){ + zipfileCleanupTransaction((ZipfileTab*)pVtab); + sqlite3_free(pVtab); + return SQLITE_OK; } /* -** Expand the Decimal so that it has a least nDigit digits and nFrac -** digits to the right of the decimal point. +** Constructor for a new ZipfileCsr object. */ -static void decimal_expand(Decimal *p, int nDigit, int nFrac){ - int nAddSig; - int nAddFrac; - if( p==0 ) return; - nAddFrac = nFrac - p->nFrac; - nAddSig = (nDigit - p->nDigit) - nAddFrac; - if( nAddFrac==0 && nAddSig==0 ) return; - p->a = sqlite3_realloc64(p->a, nDigit+1); - if( p->a==0 ){ - p->oom = 1; - return; - } - if( nAddSig ){ - memmove(p->a+nAddSig, p->a, p->nDigit); - memset(p->a, 0, nAddSig); - p->nDigit += nAddSig; - } - if( nAddFrac ){ - memset(p->a+p->nDigit, 0, nAddFrac); - p->nDigit += nAddFrac; - p->nFrac += nAddFrac; +static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + ZipfileTab *pTab = (ZipfileTab*)p; + ZipfileCsr *pCsr; + pCsr = sqlite3_malloc(sizeof(*pCsr)); + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + if( pCsr==0 ){ + return SQLITE_NOMEM; } + memset(pCsr, 0, sizeof(*pCsr)); + pCsr->iId = ++pTab->iNextCsrid; + pCsr->pCsrNext = pTab->pCsrList; + pTab->pCsrList = pCsr; + return SQLITE_OK; } /* -** Add the value pB into pA. -** -** Both pA and pB might become denormalized by this routine. +** Reset a cursor back to the state it was in when first returned +** by zipfileOpen(). */ -static void decimal_add(Decimal *pA, Decimal *pB){ - int nSig, nFrac, nDigit; - int i, rc; - if( pA==0 ){ - return; - } - if( pA->oom || pB==0 || pB->oom ){ - pA->oom = 1; - return; - } - if( pA->isNull || pB->isNull ){ - pA->isNull = 1; - return; - } - nSig = pA->nDigit - pA->nFrac; - if( nSig && pA->a[0]==0 ) nSig--; - if( nSignDigit-pB->nFrac ){ - nSig = pB->nDigit - pB->nFrac; +static void zipfileResetCursor(ZipfileCsr *pCsr){ + ZipfileEntry *p; + ZipfileEntry *pNext; + + pCsr->bEof = 0; + if( pCsr->pFile ){ + fclose(pCsr->pFile); + pCsr->pFile = 0; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; } - nFrac = pA->nFrac; - if( nFracnFrac ) nFrac = pB->nFrac; - nDigit = nSig + nFrac + 1; - decimal_expand(pA, nDigit, nFrac); - decimal_expand(pB, nDigit, nFrac); - if( pA->oom || pB->oom ){ - pA->oom = 1; - }else{ - if( pA->sign==pB->sign ){ - int carry = 0; - for(i=nDigit-1; i>=0; i--){ - int x = pA->a[i] + pB->a[i] + carry; - if( x>=10 ){ - carry = 1; - pA->a[i] = x - 10; - }else{ - carry = 0; - pA->a[i] = x; - } - } - }else{ - signed char *aA, *aB; - int borrow = 0; - rc = memcmp(pA->a, pB->a, nDigit); - if( rc<0 ){ - aA = pB->a; - aB = pA->a; - pA->sign = !pA->sign; - }else{ - aA = pA->a; - aB = pB->a; - } - for(i=nDigit-1; i>=0; i--){ - int x = aA[i] - aB[i] - borrow; - if( x<0 ){ - pA->a[i] = x+10; - borrow = 1; - }else{ - pA->a[i] = x; - borrow = 0; - } - } - } + + for(p=pCsr->pFreeEntry; p; p=pNext){ + pNext = p->pNext; + zipfileEntryFree(p); } } /* -** Compare text in decimal order. +** Destructor for an ZipfileCsr. */ -static int decimalCollFunc( - void *notUsed, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 -){ - const unsigned char *zA = (const unsigned char*)pKey1; - const unsigned char *zB = (const unsigned char*)pKey2; - Decimal *pA = decimal_new(0, 0, nKey1, zA); - Decimal *pB = decimal_new(0, 0, nKey2, zB); - int rc; - UNUSED_PARAMETER(notUsed); - if( pA==0 || pB==0 ){ - rc = 0; - }else{ - rc = decimal_cmp(pA, pB); - } - decimal_free(pA); - decimal_free(pB); - return rc; +static int zipfileClose(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab); + ZipfileCsr **pp; + zipfileResetCursor(pCsr); + + /* Remove this cursor from the ZipfileTab.pCsrList list. */ + for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); + *pp = pCsr->pCsrNext; + + sqlite3_free(pCsr); + return SQLITE_OK; } +/* +** Set the error message for the virtual table associated with cursor +** pCsr to the results of vprintf(zFmt, ...). +*/ +static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} +static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + sqlite3_free(pCsr->base.pVtab->zErrMsg); + pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} /* -** SQL Function: decimal_add(X, Y) -** decimal_sub(X, Y) +** Read nRead bytes of data from offset iOff of file pFile into buffer +** aRead[]. Return SQLITE_OK if successful, or an SQLite error code +** otherwise. ** -** Return the sum or difference of X and Y. +** If an error does occur, output variable (*pzErrmsg) may be set to point +** to an English language error message. It is the responsibility of the +** caller to eventually free this buffer using +** sqlite3_free(). */ -static void decimalAddFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); - UNUSED_PARAMETER(argc); - decimal_add(pA, pB); - decimal_result(context, pA); - decimal_free(pA); - decimal_free(pB); -} -static void decimalSubFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static int zipfileReadData( + FILE *pFile, /* Read from this file */ + u8 *aRead, /* Read into this buffer */ + int nRead, /* Number of bytes to read */ + i64 iOff, /* Offset to read from */ + char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); - UNUSED_PARAMETER(argc); - if( pB ){ - pB->sign = !pB->sign; - decimal_add(pA, pB); - decimal_result(context, pA); + size_t n; + fseek(pFile, (long)iOff, SEEK_SET); + n = fread(aRead, 1, nRead, pFile); + if( (int)n!=nRead ){ + *pzErrmsg = sqlite3_mprintf("error in fread()"); + return SQLITE_ERROR; } - decimal_free(pA); - decimal_free(pB); + return SQLITE_OK; } -/* Aggregate funcion: decimal_sum(X) -** -** Works like sum() except that it uses decimal arithmetic for unlimited -** precision. -*/ -static void decimalSumStep( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static int zipfileAppendData( + ZipfileTab *pTab, + const u8 *aWrite, + int nWrite ){ - Decimal *p; - Decimal *pArg; - UNUSED_PARAMETER(argc); - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( p==0 ) return; - if( !p->isInit ){ - p->isInit = 1; - p->a = sqlite3_malloc(2); - if( p->a==0 ){ - p->oom = 1; - }else{ - p->a[0] = 0; + if( nWrite>0 ){ + size_t n = nWrite; + fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); + n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); + if( (int)n!=nWrite ){ + pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); + return SQLITE_ERROR; } - p->nDigit = 1; - p->nFrac = 0; + pTab->szCurrent += nWrite; } - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); - decimal_add(p, pArg); - decimal_free(pArg); + return SQLITE_OK; } -static void decimalSumInverse( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *p; - Decimal *pArg; - UNUSED_PARAMETER(argc); - p = sqlite3_aggregate_context(context, sizeof(*p)); - if( p==0 ) return; - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); - if( pArg ) pArg->sign = !pArg->sign; - decimal_add(p, pArg); - decimal_free(pArg); + +/* +** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +*/ +static u16 zipfileGetU16(const u8 *aBuf){ + return (aBuf[1] << 8) + aBuf[0]; } -static void decimalSumValue(sqlite3_context *context){ - Decimal *p = sqlite3_aggregate_context(context, 0); - if( p==0 ) return; - decimal_result(context, p); + +/* +** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +*/ +static u32 zipfileGetU32(const u8 *aBuf){ + if( aBuf==0 ) return 0; + return ((u32)(aBuf[3]) << 24) + + ((u32)(aBuf[2]) << 16) + + ((u32)(aBuf[1]) << 8) + + ((u32)(aBuf[0]) << 0); +} + +/* +** Write a 16-bit little endiate integer into buffer aBuf. +*/ +static void zipfilePutU16(u8 *aBuf, u16 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; } -static void decimalSumFinalize(sqlite3_context *context){ - Decimal *p = sqlite3_aggregate_context(context, 0); - if( p==0 ) return; - decimal_result(context, p); - decimal_clear(p); + +/* +** Write a 32-bit little endiate integer into buffer aBuf. +*/ +static void zipfilePutU32(u8 *aBuf, u32 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; + aBuf[2] = (val>>16) & 0xFF; + aBuf[3] = (val>>24) & 0xFF; } +#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) ) +#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) ) + +#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } +#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } + /* -** SQL Function: decimal_mul(X, Y) -** -** Return the product of X and Y. -** -** All significant digits after the decimal point are retained. -** Trailing zeros after the decimal point are omitted as long as -** the number of digits after the decimal point is no less than -** either the number of digits in either input. +** Magic numbers used to read CDS records. */ -static void decimalMulFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); - signed char *acc = 0; - int i, j, k; - int minFrac; - UNUSED_PARAMETER(argc); - if( pA==0 || pA->oom || pA->isNull - || pB==0 || pB->oom || pB->isNull - ){ - goto mul_end; - } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); - if( acc==0 ){ - sqlite3_result_error_nomem(context); - goto mul_end; - } - memset(acc, 0, pA->nDigit + pB->nDigit + 2); - minFrac = pA->nFrac; - if( pB->nFracnFrac; - for(i=pA->nDigit-1; i>=0; i--){ - signed char f = pA->a[i]; - int carry = 0, x; - for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ - x = acc[k] + f*pB->a[j] + carry; - acc[k] = x%10; - carry = x/10; - } - x = acc[k] + carry; - acc[k] = x%10; - acc[k-1] += x/10; - } - sqlite3_free(pA->a); - pA->a = acc; - acc = 0; - pA->nDigit += pB->nDigit + 2; - pA->nFrac += pB->nFrac; - pA->sign ^= pB->sign; - while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ - pA->nFrac--; - pA->nDigit--; +#define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 + +/* +** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ + u8 *aRead = aBuf; + u32 sig = zipfileRead32(aRead); + int rc = SQLITE_OK; + if( sig!=ZIPFILE_SIGNATURE_CDS ){ + rc = SQLITE_ERROR; + }else{ + pCDS->iVersionMadeBy = zipfileRead16(aRead); + pCDS->iVersionExtract = zipfileRead16(aRead); + pCDS->flags = zipfileRead16(aRead); + pCDS->iCompression = zipfileRead16(aRead); + pCDS->mTime = zipfileRead16(aRead); + pCDS->mDate = zipfileRead16(aRead); + pCDS->crc32 = zipfileRead32(aRead); + pCDS->szCompressed = zipfileRead32(aRead); + pCDS->szUncompressed = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + pCDS->nFile = zipfileRead16(aRead); + pCDS->nExtra = zipfileRead16(aRead); + pCDS->nComment = zipfileRead16(aRead); + pCDS->iDiskStart = zipfileRead16(aRead); + pCDS->iInternalAttr = zipfileRead16(aRead); + pCDS->iExternalAttr = zipfileRead32(aRead); + pCDS->iOffset = zipfileRead32(aRead); + assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] ); } - decimal_result(context, pA); -mul_end: - sqlite3_free(acc); - decimal_free(pA); - decimal_free(pB); + return rc; } -#ifdef _WIN32 - -#endif -int sqlite3_decimal_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi +/* +** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadLFH( + u8 *aBuffer, + ZipfileLFH *pLFH ){ + u8 *aRead = aBuffer; int rc = SQLITE_OK; - static const struct { - const char *zFuncName; - int nArg; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFunc[] = { - { "decimal", 1, decimalFunc }, - { "decimal_cmp", 2, decimalCmpFunc }, - { "decimal_add", 2, decimalAddFunc }, - { "decimal_sub", 2, decimalSubFunc }, - { "decimal_mul", 2, decimalMulFunc }, - }; - unsigned int i; - (void)pzErrMsg; /* Unused parameter */ - SQLITE_EXTENSION_INIT2(pApi); - - for(i=0; iiVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); } return rc; } -/************************* End ../ext/misc/decimal.c ********************/ -/************************* Begin ../ext/misc/ieee754.c ******************/ + /* -** 2013-04-17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This SQLite extension implements functions for the exact display -** and input of IEEE754 Binary64 floating-point numbers. -** -** ieee754(X) -** ieee754(Y,Z) -** -** In the first form, the value X should be a floating-point number. -** The function will return a string of the form 'ieee754(Y,Z)' where -** Y and Z are integers such that X==Y*pow(2,Z). -** -** In the second form, Y and Z are integers which are the mantissa and -** base-2 exponent of a new floating point number. The function returns -** a floating-point value equal to Y*pow(2,Z). -** -** Examples: -** -** ieee754(2.0) -> 'ieee754(2,0)' -** ieee754(45.25) -> 'ieee754(181,-2)' -** ieee754(2, 0) -> 2.0 -** ieee754(181, -2) -> 45.25 -** -** Two additional functions break apart the one-argument ieee754() -** result into separate integer values: -** -** ieee754_mantissa(45.25) -> 181 -** ieee754_exponent(45.25) -> -2 -** -** These functions convert binary64 numbers into blobs and back again. -** -** ieee754_from_blob(x'3ff0000000000000') -> 1.0 -** ieee754_to_blob(1.0) -> x'3ff0000000000000' -** -** In all single-argument functions, if the argument is an 8-byte blob -** then that blob is interpreted as a big-endian binary64 value. -** -** -** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES -** ----------------------------------------------- -** -** This extension in combination with the separate 'decimal' extension -** can be used to compute the exact decimal representation of binary64 -** values. To begin, first compute a table of exponent values: +** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. +** Scan through this buffer to find an "extra-timestamp" field. If one +** exists, extract the 32-bit modification-timestamp from it and store +** the value in output parameter *pmTime. ** -** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); -** WITH RECURSIVE c(x,v) AS ( -** VALUES(0,'1') -** UNION ALL -** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 -** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; -** WITH RECURSIVE c(x,v) AS ( -** VALUES(-1,'0.5') -** UNION ALL -** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 -** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** Zero is returned if no extra-timestamp record could be found (and so +** *pmTime is left unchanged), or non-zero otherwise. ** -** Then, to compute the exact decimal representation of a floating -** point value (the value 47.49 is used in the example) do: +** The general format of an extra field is: ** -** WITH c(n) AS (VALUES(47.49)) -** ---------------^^^^^---- Replace with whatever you want -** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) -** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** Header ID 2 bytes +** Data Size 2 bytes +** Data N bytes +*/ +static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ + int ret = 0; + u8 *p = aExtra; + u8 *pEnd = &aExtra[nExtra]; + + while( p modtime is present */ + *pmTime = zipfileGetU32(&p[1]); + ret = 1; + } + break; + } + } + + p += nByte; + } + return ret; +} + +/* +** Convert the standard MS-DOS timestamp stored in the mTime and mDate +** fields of the CDS structure passed as the only argument to a 32-bit +** UNIX seconds-since-the-epoch timestamp. Return the result. ** -** Here is a query to show various boundry values for the binary64 -** number format: +** "Standard" MS-DOS time format: ** -** WITH c(name,bin) AS (VALUES -** ('minimum positive value', x'0000000000000001'), -** ('maximum subnormal value', x'000fffffffffffff'), -** ('mininum positive nornal value', x'0010000000000000'), -** ('maximum value', x'7fefffffffffffff')) -** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) -** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** File modification time: +** Bits 00-04: seconds divided by 2 +** Bits 05-10: minute +** Bits 11-15: hour +** File modification date: +** Bits 00-04: day +** Bits 05-08: month (1-12) +** Bits 09-15: years from 1980 ** +** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx */ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include +static u32 zipfileMtime(ZipfileCDS *pCDS){ + int Y,M,D,X1,X2,A,B,sec,min,hr; + i64 JDsec; + Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); + M = ((pCDS->mDate >> 5) & 0x0F); + D = (pCDS->mDate & 0x1F); + sec = (pCDS->mTime & 0x1F)*2; + min = (pCDS->mTime >> 5) & 0x3F; + hr = (pCDS->mTime >> 11) & 0x1F; + if( M<=2 ){ + Y--; + M += 12; + } + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + A = Y/100; + B = 2 - A + (A/4); + JDsec = (i64)((X1 + X2 + D + B - 1524.5)*86400) + hr*3600 + min*60 + sec; + return (u32)(JDsec - (i64)24405875*(i64)8640); +} -/* Mark a function parameter as unused, to suppress nuisance compiler -** warnings. */ -#ifndef UNUSED_PARAMETER -# define UNUSED_PARAMETER(X) (void)(X) -#endif +/* +** The opposite of zipfileMtime(). This function populates the mTime and +** mDate fields of the CDS structure passed as the first argument according +** to the UNIX timestamp value passed as the second. +*/ +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ + /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ + i64 JD = (i64)2440588 + mUnixTime / (24*60*60); + + int A, B, C, D, E; + int yr, mon, day; + int hr, min, sec; + + A = (int)((JD - 1867216.25)/36524.25); + A = (int)(JD + 1 + A - (A/4)); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); + + day = B - D - (int)(30.6001*E); + mon = (E<14 ? E-1 : E-13); + yr = mon>2 ? C-4716 : C-4715; + + hr = (mUnixTime % (24*60*60)) / (60*60); + min = (mUnixTime % (60*60)) / 60; + sec = (mUnixTime % 60); + + if( yr>=1980 ){ + pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); + pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); + }else{ + pCds->mDate = pCds->mTime = 0; + } + + assert( mUnixTime<315507600 + || mUnixTime==zipfileMtime(pCds) + || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) + /* || (mUnixTime % 2) */ + ); +} /* -** Implementation of the ieee754() function +** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in +** size) containing an entire zip archive image. Or, if aBlob is NULL, +** then pFile is a file-handle open on a zip file. In either case, this +** function creates a ZipfileEntry object based on the zip archive entry +** for which the CDS record is at offset iOff. +** +** If successful, SQLITE_OK is returned and (*ppEntry) set to point to +** the new object. Otherwise, an SQLite error code is returned and the +** final value of (*ppEntry) undefined. */ -static void ieee754func( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static int zipfileGetEntry( + ZipfileTab *pTab, /* Store any error message here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* If aBlob==0, read from this file */ + i64 iOff, /* Offset of CDS record */ + ZipfileEntry **ppEntry /* OUT: Pointer to new object */ ){ - if( argc==1 ){ - sqlite3_int64 m, a; - double r; - int e; - int isNeg; - char zResult[100]; - assert( sizeof(m)==sizeof(r) ); - if( sqlite3_value_type(argv[0])==SQLITE_BLOB - && sqlite3_value_bytes(argv[0])==sizeof(r) - ){ - const unsigned char *x = sqlite3_value_blob(argv[0]); - unsigned int i; - sqlite3_uint64 v = 0; - for(i=0; ibase.zErrMsg; + int rc = SQLITE_OK; + (void)nBlob; + + if( aBlob==0 ){ + aRead = pTab->aBuffer; + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + }else{ + aRead = (u8*)&aBlob[iOff]; + } + + if( rc==SQLITE_OK ){ + sqlite3_int64 nAlloc; + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + nAlloc = sizeof(ZipfileEntry) + nExtra; + if( aBlob ){ + nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + } + + pNew = (ZipfileEntry*)sqlite3_malloc64(nAlloc); + if( pNew==0 ){ + rc = SQLITE_NOMEM; }else{ - r = sqlite3_value_double(argv[0]); + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else if( aBlob==0 ){ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); + }else{ + aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; + } } - if( r<0.0 ){ - isNeg = 1; - r = -r; + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + pNew->aExtra = (u8*)&pNew[1]; + memcpy(pNew->aExtra, &aRead[nFile], nExtra); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc==SQLITE_OK ){ + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + ZipfileLFH lfh; + if( pFile ){ + rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); + }else{ + aRead = (u8*)&aBlob[pNew->cds.iOffset]; + } + + if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); + if( rc==SQLITE_OK ){ + pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff += lfh.nFile + lfh.nExtra; + if( aBlob && pNew->cds.szCompressed ){ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } + }else{ + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + (int)pNew->cds.iOffset + ); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); }else{ - isNeg = 0; + *ppEntry = pNew; } - memcpy(&a,&r,sizeof(a)); - if( a==0 ){ - e = 0; - m = 0; + } + + return rc; +} + +/* +** Advance an ZipfileCsr to its next row of output. +*/ +static int zipfileNext(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + int rc = SQLITE_OK; + + if( pCsr->pFile ){ + i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + if( pCsr->iNextOff>=iEof ){ + pCsr->bEof = 1; }else{ - e = a>>52; - m = a & ((((sqlite3_int64)1)<<52)-1); - if( e==0 ){ - m <<= 1; - }else{ - m |= ((sqlite3_int64)1)<<52; - } - while( e<1075 && m>0 && (m&1)==0 ){ - m >>= 1; - e++; + ZipfileEntry *p = 0; + ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); + rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); + if( rc==SQLITE_OK ){ + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; } - if( isNeg ) m = -m; + pCsr->pCurrent = p; } - switch( *(int*)sqlite3_user_data(context) ){ - case 0: - sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", - m, e-1075); - sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); - break; - case 1: - sqlite3_result_int64(context, m); - break; - case 2: - sqlite3_result_int(context, e-1075); - break; + }else{ + if( !pCsr->bNoop ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + } + if( pCsr->pCurrent==0 ){ + pCsr->bEof = 1; } + } + + pCsr->bNoop = 0; + return rc; +} + +static void zipfileFree(void *p) { + sqlite3_free(p); +} + +/* +** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the +** size is nOut bytes. This function uncompresses the data and sets the +** return value in context pCtx to the result (a blob). +** +** If an error occurs, an error code is left in pCtx instead. +*/ +static void zipfileInflate( + sqlite3_context *pCtx, /* Store result here */ + const u8 *aIn, /* Compressed data */ + int nIn, /* Size of buffer aIn[] in bytes */ + int nOut /* Expected output size */ +){ + u8 *aRes = sqlite3_malloc(nOut); + if( aRes==0 ){ + sqlite3_result_error_nomem(pCtx); }else{ - sqlite3_int64 m, e, a; - double r; - int isNeg = 0; - m = sqlite3_value_int64(argv[0]); - e = sqlite3_value_int64(argv[1]); + int err; + z_stream str; + memset(&str, 0, sizeof(str)); - /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ - if( e>10000 ){ - e = 10000; - }else if( e<-10000 ){ - e = -10000; + str.next_in = (Byte*)aIn; + str.avail_in = nIn; + str.next_out = (Byte*)aRes; + str.avail_out = nOut; + + err = inflateInit2(&str, -15); + if( err!=Z_OK ){ + zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err); + }else{ + err = inflate(&str, Z_NO_FLUSH); + if( err!=Z_STREAM_END ){ + zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); + }else{ + sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + aRes = 0; + } } + sqlite3_free(aRes); + inflateEnd(&str); + } +} - if( m<0 ){ - isNeg = 1; - m = -m; - if( m<0 ) return; - }else if( m==0 && e>-1000 && e<1000 ){ - sqlite3_result_double(context, 0.0); - return; +/* +** Buffer aIn (size nIn bytes) contains uncompressed data. This function +** compresses it and sets (*ppOut) to point to a buffer containing the +** compressed data. The caller is responsible for eventually calling +** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) +** is set to the size of buffer (*ppOut) in bytes. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned and an error message left in virtual-table handle +** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this +** case. +*/ +static int zipfileDeflate( + const u8 *aIn, int nIn, /* Input */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; + sqlite3_int64 nAlloc; + z_stream str; + u8 *aOut; + + memset(&str, 0, sizeof(str)); + str.next_in = (Bytef*)aIn; + str.avail_in = nIn; + deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + + nAlloc = deflateBound(&str, nIn); + aOut = (u8*)sqlite3_malloc64(nAlloc); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + int res; + str.next_out = aOut; + str.avail_out = nAlloc; + res = deflate(&str, Z_FINISH); + if( res==Z_STREAM_END ){ + *ppOut = aOut; + *pnOut = (int)str.total_out; + }else{ + sqlite3_free(aOut); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); + rc = SQLITE_ERROR; } - while( (m>>32)&0xffe00000 ){ - m >>= 1; - e++; + deflateEnd(&str); + } + + return rc; +} + + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int zipfileColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; + int rc = SQLITE_OK; + switch( i ){ + case 0: /* name */ + sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); + break; + case 1: /* mode */ + /* TODO: Whether or not the following is correct surely depends on + ** the platform on which the archive was created. */ + sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); + break; + case 2: { /* mtime */ + sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); + break; } - while( m!=0 && ((m>>32)&0xfff00000)==0 ){ - m <<= 1; - e--; + case 3: { /* sz */ + if( sqlite3_vtab_nochange(ctx)==0 ){ + sqlite3_result_int64(ctx, pCDS->szUncompressed); + } + break; } - e += 1075; - if( e<=0 ){ - /* Subnormal */ - if( 1-e >= 64 ){ - m = 0; - }else{ - m >>= 1-e; + case 4: /* rawdata */ + if( sqlite3_vtab_nochange(ctx) ) break; + case 5: { /* data */ + if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ + int sz = pCDS->szCompressed; + int szFinal = pCDS->szUncompressed; + if( szFinal>0 ){ + u8 *aBuf; + u8 *aFree = 0; + if( pCsr->pCurrent->aData ){ + aBuf = pCsr->pCurrent->aData; + }else{ + aBuf = aFree = sqlite3_malloc64(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = pCsr->pFile; + if( pFile==0 ){ + pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; + } + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } + } + if( rc==SQLITE_OK ){ + if( i==5 && pCDS->iCompression ){ + zipfileInflate(ctx, aBuf, sz, szFinal); + }else{ + sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); + } + } + sqlite3_free(aFree); + }else{ + /* Figure out if this is a directory or a zero-sized file. Consider + ** it to be a directory either if the mode suggests so, or if + ** the final character in the name is '/'. */ + u32 mode = pCDS->iExternalAttr >> 16; + if( !(mode & S_IFDIR) + && pCDS->nFile>=1 + && pCDS->zFile[pCDS->nFile-1]!='/' + ){ + sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); + } + } } - e = 0; - }else if( e>0x7ff ){ - e = 0x7ff; + break; } - a = m & ((((sqlite3_int64)1)<<52)-1); - a |= e<<52; - if( isNeg ) a |= ((sqlite3_uint64)1)<<63; - memcpy(&r, &a, sizeof(r)); - sqlite3_result_double(context, r); + case 6: /* method */ + sqlite3_result_int(ctx, pCDS->iCompression); + break; + default: /* z */ + assert( i==7 ); + sqlite3_result_int64(ctx, pCsr->iId); + break; } + + return rc; } /* -** Functions to convert between blobs and floats. +** Return TRUE if the cursor is at EOF. */ -static void ieee754func_from_blob( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_BLOB - && sqlite3_value_bytes(argv[0])==sizeof(double) - ){ - double r; - const unsigned char *x = sqlite3_value_blob(argv[0]); - unsigned int i; - sqlite3_uint64 v = 0; - for(i=0; ibEof; } -static void ieee754func_to_blob( - sqlite3_context *context, - int argc, - sqlite3_value **argv + +/* +** If aBlob is not NULL, then it points to a buffer nBlob bytes in size +** containing an entire zip archive image. Or, if aBlob is NULL, then pFile +** is guaranteed to be a file-handle open on a zip file. +** +** This function attempts to locate the EOCD record within the zip archive +** and populate *pEOCD with the results of decoding it. SQLITE_OK is +** returned if successful. Otherwise, an SQLite error code is returned and +** an English language error message may be left in virtual-table pTab. +*/ +static int zipfileReadEOCD( + ZipfileTab *pTab, /* Return errors here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* Read from this file if aBlob==0 */ + ZipfileEOCD *pEOCD /* Object to populate */ ){ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_FLOAT - || sqlite3_value_type(argv[0])==SQLITE_INTEGER - ){ - double r = sqlite3_value_double(argv[0]); - sqlite3_uint64 v; - unsigned char a[sizeof(r)]; - unsigned int i; - memcpy(&v, &r, sizeof(r)); - for(i=1; i<=sizeof(r); i++){ - a[sizeof(r)-i] = v&0xff; - v >>= 8; + u8 *aRead = pTab->aBuffer; /* Temporary buffer */ + int nRead; /* Bytes to read from file */ + int rc = SQLITE_OK; + + memset(pEOCD, 0, sizeof(ZipfileEOCD)); + if( aBlob==0 ){ + i64 iOff; /* Offset to read from */ + i64 szFile; /* Total size of file in bytes */ + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + return SQLITE_OK; } - sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; + rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); + }else{ + nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); + aRead = (u8*)&aBlob[nBlob-nRead]; } -} - -#ifdef _WIN32 + if( rc==SQLITE_OK ){ + int i; -#endif -int sqlite3_ieee_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - static const struct { - char *zFName; - int nArg; - int iAux; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFunc[] = { - { "ieee754", 1, 0, ieee754func }, - { "ieee754", 2, 0, ieee754func }, - { "ieee754_mantissa", 1, 1, ieee754func }, - { "ieee754_exponent", 1, 2, ieee754func }, - { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, - { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + /* Scan backwards looking for the signature bytes */ + for(i=nRead-20; i>=0; i--){ + if( aRead[i]==0x50 && aRead[i+1]==0x4b + && aRead[i+2]==0x05 && aRead[i+3]==0x06 + ){ + break; + } + } + if( i<0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "cannot find end of central directory record" + ); + return SQLITE_ERROR; + } - }; - unsigned int i; - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ - for(i=0; iiDisk = zipfileRead16(aRead); + pEOCD->iFirstDisk = zipfileRead16(aRead); + pEOCD->nEntry = zipfileRead16(aRead); + pEOCD->nEntryTotal = zipfileRead16(aRead); + pEOCD->nSize = zipfileRead32(aRead); + pEOCD->iOffset = zipfileRead32(aRead); } + return rc; } -/************************* End ../ext/misc/ieee754.c ********************/ -/************************* Begin ../ext/misc/series.c ******************/ -/* -** 2015-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file demonstrates how to create a table-valued-function using -** a virtual table. This demo implements the generate_series() function -** which gives similar results to the eponymous function in PostgreSQL. -** Examples: -** -** SELECT * FROM generate_series(0,100,5); -** -** The query above returns integers from 0 through 100 counting by steps -** of 5. -** -** SELECT * FROM generate_series(0,100); -** -** Integers from 0 through 100 with a step size of 1. -** -** SELECT * FROM generate_series(20) LIMIT 10; -** -** Integers 20 through 29. -** -** HOW IT WORKS -** -** The generate_series "function" is really a virtual table with the -** following schema: -** -** CREATE TABLE generate_series( -** value, -** start HIDDEN, -** stop HIDDEN, -** step HIDDEN -** ); -** -** Function arguments in queries against this virtual table are translated -** into equality constraints against successive hidden columns. In other -** words, the following pairs of queries are equivalent to each other: -** -** SELECT * FROM generate_series(0,100,5); -** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; -** -** SELECT * FROM generate_series(0,100); -** SELECT * FROM generate_series WHERE start=0 AND stop=100; -** -** SELECT * FROM generate_series(20) LIMIT 10; -** SELECT * FROM generate_series WHERE start=20 LIMIT 10; -** -** The generate_series virtual table implementation leaves the xCreate method -** set to NULL. This means that it is not possible to do a CREATE VIRTUAL -** TABLE command with "generate_series" as the USING argument. Instead, there -** is a single generate_series virtual table that is always available without -** having to be created first. -** -** The xBestIndex method looks for equality constraints against the hidden -** start, stop, and step columns, and if present, it uses those constraints -** to bound the sequence of generated values. If the equality constraints -** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. -** xBestIndex returns a small cost when both start and stop are available, -** and a very large cost if either start or stop are unavailable. This -** encourages the query planner to order joins such that the bounds of the -** series are well-defined. -*/ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include - -#ifndef SQLITE_OMIT_VIRTUALTABLE - - -/* series_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -typedef struct series_cursor series_cursor; -struct series_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - int isDesc; /* True to count down rather than up */ - sqlite3_int64 iRowid; /* The rowid */ - sqlite3_int64 iValue; /* Current value ("value") */ - sqlite3_int64 mnValue; /* Mimimum value ("start") */ - sqlite3_int64 mxValue; /* Maximum value ("stop") */ - sqlite3_int64 iStep; /* Increment ("step") */ -}; - /* -** The seriesConnect() method is invoked to create a new -** series_vtab that describes the generate_series virtual table. -** -** Think of this routine as the constructor for series_vtab objects. -** -** All this routine needs to do is: -** -** (1) Allocate the series_vtab object and initialize all fields. -** -** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the -** result set of queries against generate_series will look like. +** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry +** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added +** to the end of the list. Otherwise, it is added to the list immediately +** before pBefore (which is guaranteed to be a part of said list). */ -static int seriesConnect( - sqlite3 *db, - void *pUnused, - int argcUnused, const char *const*argvUnused, - sqlite3_vtab **ppVtab, - char **pzErrUnused +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew ){ - sqlite3_vtab *pNew; + assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); + assert( pNew->pNext==0 ); + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } + }else{ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; + } +} + +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ + ZipfileEOCD eocd; int rc; + int i; + i64 iOff; -/* Column numbers */ -#define SERIES_COLUMN_VALUE 0 -#define SERIES_COLUMN_START 1 -#define SERIES_COLUMN_STOP 2 -#define SERIES_COLUMN_STEP 3 + rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); - (void)pUnused; - (void)argcUnused; - (void)argvUnused; - (void)pzErrUnused; - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); - if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); - sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; + } } return rc; } /* -** This method is the destructor for series_cursor objects. +** xFilter callback. */ -static int seriesDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return SQLITE_OK; +static int zipfileFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + const char *zFile = 0; /* Zip file to scan */ + int rc = SQLITE_OK; /* Return Code */ + int bInMemory = 0; /* True for an in-memory zipfile */ + + (void)idxStr; + (void)argc; + + zipfileResetCursor(pCsr); + + if( pTab->zFile ){ + zFile = pTab->zFile; + }else if( idxNum==0 ){ + zipfileCursorErr(pCsr, "zipfile() function requires an argument"); + return SQLITE_ERROR; + }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + static const u8 aEmptyBlob = 0; + const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( pTab->pFirstEntry==0 ); + if( aBlob==0 ){ + aBlob = &aEmptyBlob; + nBlob = 0; + } + rc = zipfileLoadDirectory(pTab, aBlob, nBlob); + pCsr->pFreeEntry = pTab->pFirstEntry; + pTab->pFirstEntry = pTab->pLastEntry = 0; + if( rc!=SQLITE_OK ) return rc; + bInMemory = 1; + }else{ + zFile = (const char*)sqlite3_value_text(argv[0]); + } + + if( 0==pTab->pWriteFd && 0==bInMemory ){ + pCsr->pFile = zFile ? fopen(zFile, "rb") : 0; + if( pCsr->pFile==0 ){ + zipfileCursorErr(pCsr, "cannot open file: %s", zFile); + rc = SQLITE_ERROR; + }else{ + rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); + if( rc==SQLITE_OK ){ + if( pCsr->eocd.nEntry==0 ){ + pCsr->bEof = 1; + }else{ + pCsr->iNextOff = pCsr->eocd.iOffset; + rc = zipfileNext(cur); + } + } + } + }else{ + pCsr->bNoop = 1; + pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; + rc = zipfileNext(cur); + } + + return rc; } /* -** Constructor for a new series_cursor object. +** xBestIndex callback. */ -static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ - series_cursor *pCur; - (void)pUnused; - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - *ppCursor = &pCur->base; +static int zipfileBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int idx = -1; + int unusable = 0; + (void)tab; + + for(i=0; inConstraint; i++){ + const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; + if( pCons->usable==0 ){ + unusable = 1; + }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idx = i; + } + } + pIdxInfo->estimatedCost = 1000.0; + if( idx>=0 ){ + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; + pIdxInfo->idxNum = 1; + }else if( unusable ){ + return SQLITE_CONSTRAINT; + } return SQLITE_OK; } -/* -** Destructor for a series_cursor. -*/ -static int seriesClose(sqlite3_vtab_cursor *cur){ - sqlite3_free(cur); - return SQLITE_OK; +static ZipfileEntry *zipfileNewEntry(const char *zPath){ + ZipfileEntry *pNew; + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + if( pNew ){ + memset(pNew, 0, sizeof(ZipfileEntry)); + pNew->cds.zFile = sqlite3_mprintf("%s", zPath); + if( pNew->cds.zFile==0 ){ + sqlite3_free(pNew); + pNew = 0; + } + } + return pNew; } +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; -/* -** Advance a series_cursor to its next row of output. -*/ -static int seriesNext(sqlite3_vtab_cursor *cur){ - series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - pCur->iValue -= pCur->iStep; - }else{ - pCur->iValue += pCur->iStep; - } - pCur->iRowid++; - return SQLITE_OK; + pCds->nExtra = 9; + + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); + + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; + + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + + return a-aBuf; } -/* -** Return values of columns for the row at which the series_cursor -** is currently pointing. -*/ -static int seriesColumn( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ +static int zipfileAppendEntry( + ZipfileTab *pTab, + ZipfileEntry *pEntry, + const u8 *pData, + int nData ){ - series_cursor *pCur = (series_cursor*)cur; - sqlite3_int64 x = 0; - switch( i ){ - case SERIES_COLUMN_START: x = pCur->mnValue; break; - case SERIES_COLUMN_STOP: x = pCur->mxValue; break; - case SERIES_COLUMN_STEP: x = pCur->iStep; break; - default: x = pCur->iValue; break; + u8 *aBuf = pTab->aBuffer; + int nBuf; + int rc; + + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); + if( rc==SQLITE_OK ){ + pEntry->iDataOff = pTab->szCurrent; + rc = zipfileAppendData(pTab, pData, nData); } - sqlite3_result_int64(ctx, x); - return SQLITE_OK; + + return rc; } -/* -** Return the rowid for the current row. In this implementation, the -** first row returned is assigned rowid value 1, and each subsequent -** row a value 1 more than that of the previous. -*/ -static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->iRowid; +static int zipfileGetMode( + sqlite3_value *pVal, + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ +){ + const char *z = (const char*)sqlite3_value_text(pVal); + u32 mode = 0; + if( z==0 ){ + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); + }else if( z[0]>='0' && z[0]<='9' ){ + mode = (unsigned int)sqlite3_value_int(pVal); + }else{ + const char zTemplate[11] = "-rwxrwxrwx"; + int i; + if( strlen(z)!=10 ) goto parse_error; + switch( z[0] ){ + case '-': mode |= S_IFREG; break; + case 'd': mode |= S_IFDIR; break; + case 'l': mode |= S_IFLNK; break; + default: goto parse_error; + } + for(i=1; i<10; i++){ + if( z[i]==zTemplate[i] ) mode |= 1 << (9-i); + else if( z[i]!='-' ) goto parse_error; + } + } + if( ((mode & S_IFDIR)==0)==bIsDir ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + *pzErr = sqlite3_mprintf("zipfile: mode does not match data"); + return SQLITE_CONSTRAINT; + } + *pMode = mode; return SQLITE_OK; + + parse_error: + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + return SQLITE_ERROR; } /* -** Return TRUE if the cursor has been moved off of the last -** row of output. -*/ -static int seriesEof(sqlite3_vtab_cursor *cur){ - series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - return pCur->iValue < pCur->mnValue; - }else{ - return pCur->iValue > pCur->mxValue; - } +** Both (const char*) arguments point to nul-terminated strings. Argument +** nB is the value of strlen(zB). This function returns 0 if the strings are +** identical, ignoring any trailing '/' character in either path. */ +static int zipfileComparePath(const char *zA, const char *zB, int nB){ + int nA = (int)strlen(zA); + if( nA>0 && zA[nA-1]=='/' ) nA--; + if( nB>0 && zB[nB-1]=='/' ) nB--; + if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; + return 1; } -/* True to cause run-time checking of the start=, stop=, and/or step= -** parameters. The only reason to do this is for testing the -** constraint checking logic for virtual tables in the SQLite core. -*/ -#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY -# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 -#endif +static int zipfileBegin(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; -/* -** This method is called to "rewind" the series_cursor object back -** to the first row of output. This method is always called at least -** once prior to any call to seriesColumn() or seriesRowid() or -** seriesEof(). -** -** The query plan selected by seriesBestIndex is passed in the idxNum -** parameter. (idxStr is not used in this implementation.) idxNum -** is a bitmask showing which constraints are available: -** -** 1: start=VALUE -** 2: stop=VALUE -** 4: step=VALUE -** -** Also, if bit 8 is set, that means that the series should be output -** in descending order rather than in ascending order. If bit 16 is -** set, then output must appear in ascending order. -** -** This routine should initialize the cursor and position it so that it -** is pointing at the first row, or pointing off the end of the table -** (so that seriesEof() will return true) if the table is empty. -*/ -static int seriesFilter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStrUnused, - int argc, sqlite3_value **argv -){ - series_cursor *pCur = (series_cursor *)pVtabCursor; - int i = 0; - (void)idxStrUnused; - if( idxNum & 1 ){ - pCur->mnValue = sqlite3_value_int64(argv[i++]); - }else{ - pCur->mnValue = 0; - } - if( idxNum & 2 ){ - pCur->mxValue = sqlite3_value_int64(argv[i++]); - }else{ - pCur->mxValue = 0xffffffff; + assert( pTab->pWriteFd==0 ); + if( pTab->zFile==0 || pTab->zFile[0]==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); + return SQLITE_ERROR; } - if( idxNum & 4 ){ - pCur->iStep = sqlite3_value_int64(argv[i++]); - if( pCur->iStep==0 ){ - pCur->iStep = 1; - }else if( pCur->iStep<0 ){ - pCur->iStep = -pCur->iStep; - if( (idxNum & 16)==0 ) idxNum |= 8; - } + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; }else{ - pCur->iStep = 1; + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); } - for(i=0; imnValue = 1; - pCur->mxValue = 0; - break; - } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); } - if( idxNum & 8 ){ - pCur->isDesc = 1; - pCur->iValue = pCur->mxValue; - if( pCur->iStep>0 ){ - pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; - } + + return rc; +} + +/* +** Return the current time as a 32-bit timestamp in UNIX epoch format (like +** time(2)). +*/ +static u32 zipfileTime(void){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + u32 ret; + if( pVfs==0 ) return 0; + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + i64 ms; + pVfs->xCurrentTimeInt64(pVfs, &ms); + ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); }else{ - pCur->isDesc = 0; - pCur->iValue = pCur->mnValue; + double day; + pVfs->xCurrentTime(pVfs, &day); + ret = (u32)((day - 2440587.5) * 86400); } - pCur->iRowid = 1; - return SQLITE_OK; + return ret; } /* -** SQLite will invoke this method one or more times while planning a query -** that uses the generate_series virtual table. This routine needs to create -** a query plan for each invocation and compute an estimated cost for that -** plan. -** -** In this implementation idxNum is used to represent the -** query plan. idxStr is unused. -** -** The query plan is represented by bits in idxNum: +** Return a 32-bit timestamp in UNIX epoch format. ** -** (1) start = $value -- constraint exists -** (2) stop = $value -- constraint exists -** (4) step = $value -- constraint exists -** (8) output in descending order +** If the value passed as the only argument is either NULL or an SQL NULL, +** return the current time. Otherwise, return the value stored in (*pVal) +** cast to a 32-bit unsigned integer. */ -static int seriesBestIndex( - sqlite3_vtab *pVTab, - sqlite3_index_info *pIdxInfo -){ - int i, j; /* Loop over constraints */ - int idxNum = 0; /* The query plan bitmask */ - int bStartSeen = 0; /* EQ constraint seen on the START column */ - int unusableMask = 0; /* Mask of unusable constraints */ - int nArg = 0; /* Number of arguments that seriesFilter() expects */ - int aIdx[3]; /* Constraints on start, stop, and step */ - const struct sqlite3_index_constraint *pConstraint; - - /* This implementation assumes that the start, stop, and step columns - ** are the last three columns in the virtual table. */ - assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); - assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); - - aIdx[0] = aIdx[1] = aIdx[2] = -1; - pConstraint = pIdxInfo->aConstraint; - for(i=0; inConstraint; i++, pConstraint++){ - int iCol; /* 0 for start, 1 for stop, 2 for step */ - int iMask; /* bitmask for those column */ - if( pConstraint->iColumniColumn - SERIES_COLUMN_START; - assert( iCol>=0 && iCol<=2 ); - iMask = 1 << iCol; - if( iCol==0 ) bStartSeen = 1; - if( pConstraint->usable==0 ){ - unusableMask |= iMask; - continue; - }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - idxNum |= iMask; - aIdx[iCol] = i; - } - } - for(i=0; i<3; i++){ - if( (j = aIdx[i])>=0 ){ - pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; - } - } - /* The current generate_column() implementation requires at least one - ** argument (the START value). Legacy versions assumed START=0 if the - ** first argument was omitted. Compile with -DZERO_ARGUMENT_GENERATE_SERIES - ** to obtain the legacy behavior */ -#ifndef ZERO_ARGUMENT_GENERATE_SERIES - if( !bStartSeen ){ - sqlite3_free(pVTab->zErrMsg); - pVTab->zErrMsg = sqlite3_mprintf( - "first argument to \"generate_series()\" missing or unusable"); - return SQLITE_ERROR; - } -#endif - if( (unusableMask & ~idxNum)!=0 ){ - /* The start, stop, and step columns are inputs. Therefore if there - ** are unusable constraints on any of start, stop, or step then - ** this plan is unusable */ - return SQLITE_CONSTRAINT; - } - if( (idxNum & 3)==3 ){ - /* Both start= and stop= boundaries are available. This is the - ** the preferred case */ - pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); - pIdxInfo->estimatedRows = 1000; - if( pIdxInfo->nOrderBy==1 ){ - if( pIdxInfo->aOrderBy[0].desc ){ - idxNum |= 8; - }else{ - idxNum |= 16; - } - pIdxInfo->orderByConsumed = 1; - } - }else{ - /* If either boundary is missing, we have to generate a huge span - ** of numbers. Make this case very expensive so that the query - ** planner will work hard to avoid it. */ - pIdxInfo->estimatedRows = 2147483647; +static u32 zipfileGetTime(sqlite3_value *pVal){ + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + return zipfileTime(); } - pIdxInfo->idxNum = idxNum; - return SQLITE_OK; + return (u32)sqlite3_value_int64(pVal); } /* -** This following structure defines all the methods for the -** generate_series virtual table. +** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry +** linked list. Remove it from the list and free the object. */ -static sqlite3_module seriesModule = { - 0, /* iVersion */ - 0, /* xCreate */ - seriesConnect, /* xConnect */ - seriesBestIndex, /* xBestIndex */ - seriesDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - seriesOpen, /* xOpen - open a cursor */ - seriesClose, /* xClose - close a cursor */ - seriesFilter, /* xFilter - configure scan constraints */ - seriesNext, /* xNext - advance a cursor */ - seriesEof, /* xEof - check for end of scan */ - seriesColumn, /* xColumn - read data */ - seriesRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0 /* xShadowName */ -}; - -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -#ifdef _WIN32 - -#endif -int sqlite3_series_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3_libversion_number()<3008012 && pzErrMsg!=0 ){ - *pzErrMsg = sqlite3_mprintf( - "generate_series() requires SQLite 3.8.12 or later"); - return SQLITE_ERROR; +static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ + if( pOld ){ + if( pTab->pFirstEntry==pOld ){ + pTab->pFirstEntry = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = 0; + }else{ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( p->pNext==pOld ){ + p->pNext = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = p; + break; + } + } + } + zipfileEntryFree(pOld); } - rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); -#endif - return rc; } -/************************* End ../ext/misc/series.c ********************/ -/************************* Begin ../ext/misc/regexp.c ******************/ /* -** 2012-11-13 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** The code in this file implements a compact but reasonably -** efficient regular-expression matcher for posix extended regular -** expressions against UTF8 text. -** -** This file is an SQLite extension. It registers a single function -** named "regexp(A,B)" where A is the regular expression and B is the -** string to be matched. By registering this function, SQLite will also -** then implement the "B regexp A" operator. Note that with the function -** the regular expression comes first, but with the operator it comes -** second. -** -** The following regular expression syntax is supported: -** -** X* zero or more occurrences of X -** X+ one or more occurrences of X -** X? zero or one occurrences of X -** X{p,q} between p and q occurrences of X -** (X) match X -** X|Y X or Y -** ^X X occurring at the beginning of the string -** X$ X occurring at the end of the string -** . Match any single character -** \c Character c where c is one of \{}()[]|*+?. -** \c C-language escapes for c in afnrtv. ex: \t or \n -** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX -** \xXX Where XX is exactly 2 hex digits, unicode value XX -** [abc] Any single character from the set abc -** [^abc] Any single character not in the set abc -** [a-z] Any single character in the range a-z -** [^a-z] Any single character not in the range a-z -** \b Word boundary -** \w Word character. [A-Za-z0-9_] -** \W Non-word character -** \d Digit -** \D Non-digit -** \s Whitespace character -** \S Non-whitespace character -** -** A nondeterministic finite automaton (NFA) is used for matching, so the -** performance is bounded by O(N*M) where N is the size of the regular -** expression and M is the size of the input string. The matcher never -** exhibits exponential behavior. Note that the X{p,q} operator expands -** to p copies of X following by q-p copies of X? and that the size of the -** regular expression in the O(N*M) performance bound is computed after -** this expansion. +** xUpdate method. */ -#include -#include -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 +static int zipfileUpdate( + sqlite3_vtab *pVtab, + int nVal, + sqlite3_value **apVal, + sqlite_int64 *pRowid +){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; /* Return Code */ + ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ -/* -** The following #defines change the names of some functions implemented in -** this file to prevent name collisions with C-library functions of the -** same name. -*/ -#define re_match sqlite3re_match -#define re_compile sqlite3re_compile -#define re_free sqlite3re_free + u32 mode = 0; /* Mode for new entry */ + u32 mTime = 0; /* Modification time for new entry */ + i64 sz = 0; /* Uncompressed size */ + const char *zPath = 0; /* Path for new entry */ + int nPath = 0; /* strlen(zPath) */ + const u8 *pData = 0; /* Pointer to buffer containing content */ + int nData = 0; /* Size of pData buffer in bytes */ + int iMethod = 0; /* Compression method for new entry */ + u8 *pFree = 0; /* Free this */ + char *zFree = 0; /* Also free this */ + ZipfileEntry *pOld = 0; + ZipfileEntry *pOld2 = 0; + int bUpdate = 0; /* True for an update that modifies "name" */ + int bIsDir = 0; + u32 iCrc32 = 0; -/* The end-of-input character */ -#define RE_EOF 0 /* End of input */ + (void)pRowid; -/* The NFA is implemented as sequence of opcodes taken from the following -** set. Each opcode has a single integer argument. -*/ -#define RE_OP_MATCH 1 /* Match the one character in the argument */ -#define RE_OP_ANY 2 /* Match any one character. (Implements ".") */ -#define RE_OP_ANYSTAR 3 /* Special optimized version of .* */ -#define RE_OP_FORK 4 /* Continue to both next and opcode at iArg */ -#define RE_OP_GOTO 5 /* Jump to opcode at iArg */ -#define RE_OP_ACCEPT 6 /* Halt and indicate a successful match */ -#define RE_OP_CC_INC 7 /* Beginning of a [...] character class */ -#define RE_OP_CC_EXC 8 /* Beginning of a [^...] character class */ -#define RE_OP_CC_VALUE 9 /* Single value in a character class */ -#define RE_OP_CC_RANGE 10 /* Range of values in a character class */ -#define RE_OP_WORD 11 /* Perl word character [A-Za-z0-9_] */ -#define RE_OP_NOTWORD 12 /* Not a perl word character */ -#define RE_OP_DIGIT 13 /* digit: [0-9] */ -#define RE_OP_NOTDIGIT 14 /* Not a digit */ -#define RE_OP_SPACE 15 /* space: [ \t\n\r\v\f] */ -#define RE_OP_NOTSPACE 16 /* Not a digit */ -#define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ + if( pTab->pWriteFd==0 ){ + rc = zipfileBegin(pVtab); + if( rc!=SQLITE_OK ) return rc; + } -/* Each opcode is a "state" in the NFA */ -typedef unsigned short ReStateNumber; + /* If this is a DELETE or UPDATE, find the archive entry to delete. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); + int nDelete = (int)strlen(zDelete); + if( nVal>1 ){ + const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]); + if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){ + bUpdate = 1; + } + } + for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ + if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ + break; + } + assert( pOld->pNext ); + } + } -/* Because this is an NFA and not a DFA, multiple states can be active at -** once. An instance of the following object records all active states in -** the NFA. The implementation is optimized for the common case where the -** number of actives states is small. -*/ -typedef struct ReStateSet { - unsigned nState; /* Number of current states */ - ReStateNumber *aState; /* Current states */ -} ReStateSet; + if( nVal>1 ){ + /* Check that "sz" and "rawdata" are both NULL: */ + if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "sz must be NULL"); + rc = SQLITE_CONSTRAINT; + } + if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){ + zipfileTableErr(pTab, "rawdata must be NULL"); + rc = SQLITE_CONSTRAINT; + } -/* An input string read one character at a time. -*/ -typedef struct ReInput ReInput; -struct ReInput { - const unsigned char *z; /* All text */ - int i; /* Next byte to read */ - int mx; /* EOF when i>=mx */ -}; + if( rc==SQLITE_OK ){ + if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + /* data=NULL. A directory */ + bIsDir = 1; + }else{ + /* Value specified for "data", and possibly "method". This must be + ** a regular file or a symlink. */ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; -/* A compiled NFA (or an NFA that is in the process of being compiled) is -** an instance of the following object. -*/ -typedef struct ReCompiled ReCompiled; -struct ReCompiled { - ReInput sIn; /* Regular expression text */ - const char *zErr; /* Error message to return */ - char *aOp; /* Operators for the virtual machine */ - int *aArg; /* Arguments to each operator */ - unsigned (*xNextChar)(ReInput*); /* Next character function */ - unsigned char zInit[12]; /* Initial text to match */ - int nInit; /* Number of characters in zInit */ - unsigned nState; /* Number of entries in aOp[] and aArg[] */ - unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ -}; + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + pData = aIn; + nData = nIn; + if( iMethod!=0 && iMethod!=8 ){ + zipfileTableErr(pTab, "unknown compression method: %d", iMethod); + rc = SQLITE_CONSTRAINT; + }else{ + if( bAuto || iMethod ){ + int nCmp; + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); + if( rc==SQLITE_OK ){ + if( iMethod || nCmpbase.zErrMsg); + } + + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); + if( zPath==0 ) zPath = ""; + nPath = (int)strlen(zPath); + mTime = zipfileGetTime(apVal[4]); + } + + if( rc==SQLITE_OK && bIsDir ){ + /* For a directory, check that the last character in the path is a + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ + if( nPath<=0 || zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); + zPath = (const char*)zFree; + if( zFree==0 ){ + rc = SQLITE_NOMEM; + nPath = 0; + }else{ + nPath = (int)strlen(zPath); + } + } + } + + /* Check that we're not inserting a duplicate entry -OR- updating an + ** entry with a path, thereby making it into a duplicate. */ + if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ + switch( sqlite3_vtab_on_conflict(pTab->db) ){ + case SQLITE_IGNORE: { + goto zipfile_update_done; + } + case SQLITE_REPLACE: { + pOld2 = p; + break; + } + default: { + zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath); + rc = SQLITE_CONSTRAINT; + break; + } + } + break; + } + } + } -/* Add a state to the given state set if it is not already there */ -static void re_add_state(ReStateSet *pSet, int newState){ - unsigned i; - for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; - pSet->aState[pSet->nState++] = (ReStateNumber)newState; -} + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + pNew = zipfileNewEntry(zPath); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; + pNew->cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&pNew->cds, mTime); + pNew->cds.crc32 = iCrc32; + pNew->cds.szCompressed = nData; + pNew->cds.szUncompressed = (u32)sz; + pNew->cds.iExternalAttr = (mode<<16); + pNew->cds.iOffset = (u32)pTab->szCurrent; + pNew->cds.nFile = (u16)nPath; + pNew->mUnixTime = (u32)mTime; + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + zipfileAddEntry(pTab, pOld, pNew); + } + } + } -/* Extract the next unicode character from *pzIn and return it. Advance -** *pzIn to the first byte past the end of the character returned. To -** be clear: this routine converts utf8 to unicode. This routine is -** optimized for the common case where the next character is a single byte. -*/ -static unsigned re_next_char(ReInput *p){ - unsigned c; - if( p->i>=p->mx ) return 0; - c = p->z[p->i++]; - if( c>=0x80 ){ - if( (c&0xe0)==0xc0 && p->imx && (p->z[p->i]&0xc0)==0x80 ){ - c = (c&0x1f)<<6 | (p->z[p->i++]&0x3f); - if( c<0x80 ) c = 0xfffd; - }else if( (c&0xf0)==0xe0 && p->i+1mx && (p->z[p->i]&0xc0)==0x80 - && (p->z[p->i+1]&0xc0)==0x80 ){ - c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f); - p->i += 2; - if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd; - }else if( (c&0xf8)==0xf0 && p->i+3mx && (p->z[p->i]&0xc0)==0x80 - && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){ - c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6) - | (p->z[p->i+2]&0x3f); - p->i += 3; - if( c<=0xffff || c>0x10ffff ) c = 0xfffd; - }else{ - c = 0xfffd; + if( rc==SQLITE_OK && (pOld || pOld2) ){ + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + pCsr->bNoop = 1; + } } + + zipfileRemoveEntryFromList(pTab, pOld); + zipfileRemoveEntryFromList(pTab, pOld2); } - return c; -} -static unsigned re_next_char_nocase(ReInput *p){ - unsigned c = re_next_char(p); - if( c>='A' && c<='Z' ) c += 'a' - 'A'; - return c; -} -/* Return true if c is a perl "word" character: [A-Za-z0-9_] */ -static int re_word_char(int c){ - return (c>='0' && c<='9') || (c>='a' && c<='z') - || (c>='A' && c<='Z') || c=='_'; +zipfile_update_done: + sqlite3_free(pFree); + sqlite3_free(zFree); + return rc; } -/* Return true if c is a "digit" character: [0-9] */ -static int re_digit_char(int c){ - return (c>='0' && c<='9'); +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ + + return a-aBuf; } -/* Return true if c is a perl "space" character: [ \t\r\n\v\f] */ -static int re_space_char(int c){ - return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } -/* Run a compiled regular expression on the zero-terminated input -** string zIn[]. Return true on a match and false if there is no match. +/* +** Serialize the CDS structure into buffer aBuf[]. Return the number +** of bytes written. */ -static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ - ReStateSet aStateSet[2], *pThis, *pNext; - ReStateNumber aSpace[100]; - ReStateNumber *pToFree; - unsigned int i = 0; - unsigned int iSwap = 0; - int c = RE_EOF+1; - int cPrev = 0; - int rc = 0; - ReInput in; - - in.z = zIn; - in.i = 0; - in.mx = nIn>=0 ? nIn : (int)strlen((char const*)zIn); +static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ + u8 *a = aBuf; + ZipfileCDS *pCDS = &pEntry->cds; - /* Look for the initial prefix match, if there is one. */ - if( pRe->nInit ){ - unsigned char x = pRe->zInit[0]; - while( in.i+pRe->nInit<=in.mx - && (zIn[in.i]!=x || - strncmp((const char*)zIn+in.i, (const char*)pRe->zInit, pRe->nInit)!=0) - ){ - in.i++; - } - if( in.i+pRe->nInit>in.mx ) return 0; + if( pEntry->aExtra==0 ){ + pCDS->nExtra = 9; } - if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){ - pToFree = 0; - aStateSet[0].aState = aSpace; - }else{ - pToFree = sqlite3_malloc64( sizeof(ReStateNumber)*2*pRe->nState ); - if( pToFree==0 ) return -1; - aStateSet[0].aState = pToFree; - } - aStateSet[1].aState = &aStateSet[0].aState[pRe->nState]; - pNext = &aStateSet[1]; - pNext->nState = 0; - re_add_state(pNext, 0); - while( c!=RE_EOF && pNext->nState>0 ){ - cPrev = c; - c = pRe->xNextChar(&in); - pThis = pNext; - pNext = &aStateSet[iSwap]; - iSwap = 1 - iSwap; - pNext->nState = 0; - for(i=0; inState; i++){ - int x = pThis->aState[i]; - switch( pRe->aOp[x] ){ - case RE_OP_MATCH: { - if( pRe->aArg[x]==c ) re_add_state(pNext, x+1); - break; - } - case RE_OP_ANY: { - if( c!=0 ) re_add_state(pNext, x+1); - break; - } - case RE_OP_WORD: { - if( re_word_char(c) ) re_add_state(pNext, x+1); - break; - } - case RE_OP_NOTWORD: { - if( !re_word_char(c) && c!=0 ) re_add_state(pNext, x+1); - break; - } - case RE_OP_DIGIT: { - if( re_digit_char(c) ) re_add_state(pNext, x+1); - break; - } - case RE_OP_NOTDIGIT: { - if( !re_digit_char(c) && c!=0 ) re_add_state(pNext, x+1); - break; - } - case RE_OP_SPACE: { - if( re_space_char(c) ) re_add_state(pNext, x+1); - break; - } - case RE_OP_NOTSPACE: { - if( !re_space_char(c) && c!=0 ) re_add_state(pNext, x+1); - break; - } - case RE_OP_BOUNDARY: { - if( re_word_char(c)!=re_word_char(cPrev) ) re_add_state(pThis, x+1); - break; - } - case RE_OP_ANYSTAR: { - re_add_state(pNext, x); - re_add_state(pThis, x+1); - break; - } - case RE_OP_FORK: { - re_add_state(pThis, x+pRe->aArg[x]); - re_add_state(pThis, x+1); - break; - } - case RE_OP_GOTO: { - re_add_state(pThis, x+pRe->aArg[x]); - break; - } - case RE_OP_ACCEPT: { - rc = 1; - goto re_match_end; - } - case RE_OP_CC_EXC: { - if( c==0 ) break; - /* fall-through */ goto re_op_cc_inc; - } - case RE_OP_CC_INC: re_op_cc_inc: { - int j = 1; - int n = pRe->aArg[x]; - int hit = 0; - for(j=1; j>0 && jaOp[x+j]==RE_OP_CC_VALUE ){ - if( pRe->aArg[x+j]==c ){ - hit = 1; - j = -1; - } - }else{ - if( pRe->aArg[x+j]<=c && pRe->aArg[x+j+1]>=c ){ - hit = 1; - j = -1; - }else{ - j++; - } - } - } - if( pRe->aOp[x]==RE_OP_CC_EXC ) hit = !hit; - if( hit ) re_add_state(pNext, x+n); - break; - } - } - } + zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(a, pCDS->iVersionMadeBy); + zipfileWrite16(a, pCDS->iVersionExtract); + zipfileWrite16(a, pCDS->flags); + zipfileWrite16(a, pCDS->iCompression); + zipfileWrite16(a, pCDS->mTime); + zipfileWrite16(a, pCDS->mDate); + zipfileWrite32(a, pCDS->crc32); + zipfileWrite32(a, pCDS->szCompressed); + zipfileWrite32(a, pCDS->szUncompressed); + assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + zipfileWrite16(a, pCDS->nFile); + zipfileWrite16(a, pCDS->nExtra); + zipfileWrite16(a, pCDS->nComment); + zipfileWrite16(a, pCDS->iDiskStart); + zipfileWrite16(a, pCDS->iInternalAttr); + zipfileWrite32(a, pCDS->iExternalAttr); + zipfileWrite32(a, pCDS->iOffset); + + memcpy(a, pCDS->zFile, pCDS->nFile); + a += pCDS->nFile; + + if( pEntry->aExtra ){ + int n = (int)pCDS->nExtra + (int)pCDS->nComment; + memcpy(a, pEntry->aExtra, n); + a += n; + }else{ + assert( pCDS->nExtra==9 ); + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); } - for(i=0; inState; i++){ - if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; } + + return a-aBuf; +} + +static int zipfileCommit(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + if( pTab->pWriteFd ){ + i64 iOffset = pTab->szCurrent; + ZipfileEntry *p; + ZipfileEOCD eocd; + int nEntry = 0; + + /* Write out all entries */ + for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ + int n = zipfileSerializeCDS(p, pTab->aBuffer); + rc = zipfileAppendData(pTab, pTab->aBuffer, n); + nEntry++; + } + + /* Write out the EOCD record */ + eocd.iDisk = 0; + eocd.iFirstDisk = 0; + eocd.nEntry = (u16)nEntry; + eocd.nEntryTotal = (u16)nEntry; + eocd.nSize = (u32)(pTab->szCurrent - iOffset); + eocd.iOffset = (u32)iOffset; + rc = zipfileAppendEOCD(pTab, &eocd); + + zipfileCleanupTransaction(pTab); } -re_match_end: - sqlite3_free(pToFree); return rc; } -/* Resize the opcode and argument arrays for an RE under construction. -*/ -static int re_resize(ReCompiled *p, int N){ - char *aOp; - int *aArg; - aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); - if( aOp==0 ) return 1; - p->aOp = aOp; - aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); - if( aArg==0 ) return 1; - p->aArg = aArg; - p->nAlloc = N; - return 0; +static int zipfileRollback(sqlite3_vtab *pVtab){ + return zipfileCommit(pVtab); } -/* Insert a new opcode and argument into an RE under construction. The -** insertion point is just prior to existing opcode iBefore. -*/ -static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ - int i; - if( p->nAlloc<=p->nState && re_resize(p, p->nAlloc*2) ) return 0; - for(i=p->nState; i>iBefore; i--){ - p->aOp[i] = p->aOp[i-1]; - p->aArg[i] = p->aArg[i-1]; +static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){ + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( iId==pCsr->iId ) break; } - p->nState++; - p->aOp[iBefore] = (char)op; - p->aArg[iBefore] = arg; - return iBefore; + return pCsr; } -/* Append a new opcode and argument to the end of the RE under construction. -*/ -static int re_append(ReCompiled *p, int op, int arg){ - return re_insert(p, p->nState, op, arg); +static void zipfileFunctionCds( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ZipfileCsr *pCsr; + ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); + assert( argc>0 ); + + pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); + if( pCsr ){ + ZipfileCDS *p = &pCsr->pCurrent->cds; + char *zRes = sqlite3_mprintf("{" + "\"version-made-by\" : %u, " + "\"version-to-extract\" : %u, " + "\"flags\" : %u, " + "\"compression\" : %u, " + "\"time\" : %u, " + "\"date\" : %u, " + "\"crc32\" : %u, " + "\"compressed-size\" : %u, " + "\"uncompressed-size\" : %u, " + "\"file-name-length\" : %u, " + "\"extra-field-length\" : %u, " + "\"file-comment-length\" : %u, " + "\"disk-number-start\" : %u, " + "\"internal-attr\" : %u, " + "\"external-attr\" : %u, " + "\"offset\" : %u }", + (u32)p->iVersionMadeBy, (u32)p->iVersionExtract, + (u32)p->flags, (u32)p->iCompression, + (u32)p->mTime, (u32)p->mDate, + (u32)p->crc32, (u32)p->szCompressed, + (u32)p->szUncompressed, (u32)p->nFile, + (u32)p->nExtra, (u32)p->nComment, + (u32)p->iDiskStart, (u32)p->iInternalAttr, + (u32)p->iExternalAttr, (u32)p->iOffset + ); + + if( zRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT); + sqlite3_free(zRes); + } + } } -/* Make a copy of N opcodes starting at iStart onto the end of the RE -** under construction. +/* +** xFindFunction method. */ -static void re_copy(ReCompiled *p, int iStart, int N){ - if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; - memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); - memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); - p->nState += N; +static int zipfileFindFunction( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* OUT: User data for *pxFunc */ +){ + (void)nArg; + if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ + *pxFunc = zipfileFunctionCds; + *ppArg = (void*)pVtab; + return 1; + } + return 0; } -/* Return true if c is a hexadecimal digit character: [0-9a-fA-F] -** If c is a hex digit, also set *pV = (*pV)*16 + valueof(c). If -** c is not a hex digit *pV is unchanged. +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNewa, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = (int)nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... */ -static int re_hex(int c, int *pV){ - if( c>='0' && c<='9' ){ - c -= '0'; - }else if( c>='a' && c<='f' ){ - c -= 'a' - 10; - }else if( c>='A' && c<='F' ){ - c -= 'A' - 10; +static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; }else{ - return 0; + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; + } } - *pV = (*pV)*16 + (c & 0xff); - return 1; -} -/* A backslash character has been seen, read the next character and -** return its interpretation. -*/ -static unsigned re_esc_char(ReCompiled *p){ - static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; - static const char zTrans[] = "\a\f\n\r\t\v"; - int i, v = 0; - char c; - if( p->sIn.i>=p->sIn.mx ) return 0; - c = p->sIn.z[p->sIn.i]; - if( c=='u' && p->sIn.i+4sIn.mx ){ - const unsigned char *zIn = p->sIn.z + p->sIn.i; - if( re_hex(zIn[1],&v) - && re_hex(zIn[2],&v) - && re_hex(zIn[3],&v) - && re_hex(zIn[4],&v) - ){ - p->sIn.i += 5; - return v; - } + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; } - if( c=='x' && p->sIn.i+2sIn.mx ){ - const unsigned char *zIn = p->sIn.z + p->sIn.i; - if( re_hex(zIn[1],&v) - && re_hex(zIn[2],&v) - ){ - p->sIn.i += 3; - return v; + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = (int)sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; } } - for(i=0; zEsc[i] && zEsc[i]!=c; i++){} - if( zEsc[i] ){ - if( i<6 ) c = zTrans[i]; - p->sIn.i++; + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; }else{ - p->zErr = "unknown \\ escape"; + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOutsIn.isIn.mx ? p->sIn.z[p->sIn.i] : 0; -} + /* Decode the "mode" argument. */ + rc = zipfileGetMode(pMode, bIsDir, &mode, &zErr); + if( rc ) goto zipfile_step_out; -/* Compile RE text into a sequence of opcodes. Continue up to the -** first unmatched ")" character, then return. If an error is found, -** return a pointer to the error message string. -*/ -static const char *re_subcompile_re(ReCompiled *p){ - const char *zErr; - int iStart, iEnd, iGoto; - iStart = p->nState; - zErr = re_subcompile_string(p); - if( zErr ) return zErr; - while( rePeek(p)=='|' ){ - iEnd = p->nState; - re_insert(p, iStart, RE_OP_FORK, iEnd + 2 - iStart); - iGoto = re_append(p, RE_OP_GOTO, 0); - p->sIn.i++; - zErr = re_subcompile_string(p); - if( zErr ) return zErr; - p->aArg[iGoto] = p->nState - iGoto; - } - return 0; -} + /* Decode the "mtime" argument. */ + e.mUnixTime = zipfileGetTime(pMtime); -/* Compile an element of regular expression text (anything that can be -** an operand to the "|" operator). Return NULL on success or a pointer -** to the error message if there is a problem. -*/ -static const char *re_subcompile_string(ReCompiled *p){ - int iPrev = -1; - int iStart; - unsigned c; - const char *zErr; - while( (c = p->xNextChar(&p->sIn))!=0 ){ - iStart = p->nState; - switch( c ){ - case '|': - case '$': - case ')': { - p->sIn.i--; - return 0; - } - case '(': { - zErr = re_subcompile_re(p); - if( zErr ) return zErr; - if( rePeek(p)!=')' ) return "unmatched '('"; - p->sIn.i++; - break; - } - case '.': { - if( rePeek(p)=='*' ){ - re_append(p, RE_OP_ANYSTAR, 0); - p->sIn.i++; - }else{ - re_append(p, RE_OP_ANY, 0); - } - break; - } - case '*': { - if( iPrev<0 ) return "'*' without operand"; - re_insert(p, iPrev, RE_OP_GOTO, p->nState - iPrev + 1); - re_append(p, RE_OP_FORK, iPrev - p->nState + 1); - break; - } - case '+': { - if( iPrev<0 ) return "'+' without operand"; - re_append(p, RE_OP_FORK, iPrev - p->nState); - break; - } - case '?': { - if( iPrev<0 ) return "'?' without operand"; - re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1); - break; - } - case '{': { - int m = 0, n = 0; - int sz, j; - if( iPrev<0 ) return "'{m,n}' without operand"; - while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } - n = m; - if( c==',' ){ - p->sIn.i++; - n = 0; - while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } - } - if( c!='}' ) return "unmatched '{'"; - if( n>0 && nsIn.i++; - sz = p->nState - iPrev; - if( m==0 ){ - if( n==0 ) return "both m and n are zero in '{m,n}'"; - re_insert(p, iPrev, RE_OP_FORK, sz+1); - n--; - }else{ - for(j=1; j0 ){ - re_append(p, RE_OP_FORK, -sz); - } - break; - } - case '[': { - int iFirst = p->nState; - if( rePeek(p)=='^' ){ - re_append(p, RE_OP_CC_EXC, 0); - p->sIn.i++; - }else{ - re_append(p, RE_OP_CC_INC, 0); - } - while( (c = p->xNextChar(&p->sIn))!=0 ){ - if( c=='[' && rePeek(p)==':' ){ - return "POSIX character classes not supported"; - } - if( c=='\\' ) c = re_esc_char(p); - if( rePeek(p)=='-' ){ - re_append(p, RE_OP_CC_RANGE, c); - p->sIn.i++; - c = p->xNextChar(&p->sIn); - if( c=='\\' ) c = re_esc_char(p); - re_append(p, RE_OP_CC_RANGE, c); - }else{ - re_append(p, RE_OP_CC_VALUE, c); - } - if( rePeek(p)==']' ){ p->sIn.i++; break; } - } - if( c==0 ) return "unclosed '['"; - p->aArg[iFirst] = p->nState - iFirst; - break; - } - case '\\': { - int specialOp = 0; - switch( rePeek(p) ){ - case 'b': specialOp = RE_OP_BOUNDARY; break; - case 'd': specialOp = RE_OP_DIGIT; break; - case 'D': specialOp = RE_OP_NOTDIGIT; break; - case 's': specialOp = RE_OP_SPACE; break; - case 'S': specialOp = RE_OP_NOTSPACE; break; - case 'w': specialOp = RE_OP_WORD; break; - case 'W': specialOp = RE_OP_NOTWORD; break; - } - if( specialOp ){ - p->sIn.i++; - re_append(p, specialOp, 0); - }else{ - c = re_esc_char(p); - re_append(p, RE_OP_MATCH, c); - } - break; - } - default: { - re_append(p, RE_OP_MATCH, c); - break; + /* If this is a directory entry, ensure that there is exactly one '/' + ** at the end of the path. Or, if this is not a directory and the path + ** ends in '/' it is an error. */ + if( bIsDir==0 ){ + if( nName>0 && zName[nName-1]=='/' ){ + zErr = sqlite3_mprintf("non-directory name must not end with /"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + }else{ + if( nName==0 || zName[nName-1]!='/' ){ + zName = zFree = sqlite3_mprintf("%s/", zName); + if( zName==0 ){ + rc = SQLITE_NOMEM; + goto zipfile_step_out; } + nName = (int)strlen(zName); + }else{ + while( nName>1 && zName[nName-2]=='/' ) nName--; } - iPrev = iStart; } - return 0; + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = (u16)nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( nData>0 ){ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + } + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); } -/* Free and reclaim all the memory used by a previously compiled -** regular expression. Applications should invoke this routine once -** for every call to re_compile() to avoid memory leaks. +/* +** xFinalize() callback for zipfile aggregate function. */ -static void re_free(ReCompiled *pRe){ - if( pRe ){ - sqlite3_free(pRe->aOp); - sqlite3_free(pRe->aArg); - sqlite3_free(pRe); +static void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + sqlite3_int64 nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + if( p->nEntry>0 ){ + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = (u16)p->nEntry; + eocd.nEntryTotal = (u16)p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc64(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, (int)nZip, zipfileFree); + } } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); } + /* -** Compile a textual regular expression in zIn[] into a compiled regular -** expression suitable for us by re_match() and return a pointer to the -** compiled regular expression in *ppRe. Return NULL on success or an -** error message if something goes wrong. +** Register the "zipfile" virtual table. */ -static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ - ReCompiled *pRe; - const char *zErr; - int i, j; +static int zipfileRegister(sqlite3 *db){ + static sqlite3_module zipfileModule = { + 1, /* iVersion */ + zipfileConnect, /* xCreate */ + zipfileConnect, /* xConnect */ + zipfileBestIndex, /* xBestIndex */ + zipfileDisconnect, /* xDisconnect */ + zipfileDisconnect, /* xDestroy */ + zipfileOpen, /* xOpen - open a cursor */ + zipfileClose, /* xClose - close a cursor */ + zipfileFilter, /* xFilter - configure scan constraints */ + zipfileNext, /* xNext - advance a cursor */ + zipfileEof, /* xEof - check for end of scan */ + zipfileColumn, /* xColumn - read data */ + 0, /* xRowid - read data */ + zipfileUpdate, /* xUpdate */ + zipfileBegin, /* xBegin */ + 0, /* xSync */ + zipfileCommit, /* xCommit */ + zipfileRollback, /* xRollback */ + zipfileFindFunction, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollback */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; - *ppRe = 0; - pRe = sqlite3_malloc( sizeof(*pRe) ); - if( pRe==0 ){ - return "out of memory"; - } - memset(pRe, 0, sizeof(*pRe)); - pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; - if( re_resize(pRe, 30) ){ - re_free(pRe); - return "out of memory"; - } - if( zIn[0]=='^' ){ - zIn++; - }else{ - re_append(pRe, RE_OP_ANYSTAR, 0); - } - pRe->sIn.z = (unsigned char*)zIn; - pRe->sIn.i = 0; - pRe->sIn.mx = (int)strlen(zIn); - zErr = re_subcompile_re(pRe); - if( zErr ){ - re_free(pRe); - return zErr; - } - if( rePeek(pRe)=='$' && pRe->sIn.i+1>=pRe->sIn.mx ){ - re_append(pRe, RE_OP_MATCH, RE_EOF); - re_append(pRe, RE_OP_ACCEPT, 0); - *ppRe = pRe; - }else if( pRe->sIn.i>=pRe->sIn.mx ){ - re_append(pRe, RE_OP_ACCEPT, 0); - *ppRe = pRe; - }else{ - re_free(pRe); - return "unrecognized character"; + int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); } + assert( sizeof(i64)==8 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(u8)==1 ); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define zipfileRegister(x) SQLITE_OK +#endif - /* The following is a performance optimization. If the regex begins with - ** ".*" (if the input regex lacks an initial "^") and afterwards there are - ** one or more matching characters, enter those matching characters into - ** zInit[]. The re_match() routine can then search ahead in the input - ** string looking for the initial match without having to run the whole - ** regex engine over the string. Do not worry able trying to match - ** unicode characters beyond plane 0 - those are very rare and this is - ** just an optimization. */ - if( pRe->aOp[0]==RE_OP_ANYSTAR && !noCase ){ - for(j=0, i=1; j<(int)sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ - unsigned x = pRe->aArg[i]; - if( x<=127 ){ - pRe->zInit[j++] = (unsigned char)x; - }else if( x<=0xfff ){ - pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); - pRe->zInit[j++] = 0x80 | (x&0x3f); - }else if( x<=0xffff ){ - pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); - pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); - pRe->zInit[j++] = 0x80 | (x&0x3f); +#ifdef _WIN32 + +#endif +int sqlite3_zipfile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return zipfileRegister(db); +} + +/************************* End ../ext/misc/zipfile.c ********************/ +/************************* Begin ../ext/misc/sqlar.c ******************/ +/* +** 2017-12-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Utility functions sqlar_compress() and sqlar_uncompress(). Useful +** for working with sqlar archives and used by the shell tool's built-in +** sqlar support. +*/ +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Implementation of the "sqlar_compress(X)" SQL function. +** +** If the type of X is SQLITE_BLOB, and compressing that blob using +** zlib utility function compress() yields a smaller blob, return the +** compressed blob. Otherwise, return a copy of X. +** +** SQLar uses the "zlib format" for compressed content. The zlib format +** contains a two-byte identification header and a four-byte checksum at +** the end. This is different from ZIP which uses the raw deflate format. +** +** Future enhancements to SQLar might add support for new compression formats. +** If so, those new formats will be identified by alternative headers in the +** compressed data. +*/ +static void sqlarCompressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + const Bytef *pData = sqlite3_value_blob(argv[0]); + uLong nData = sqlite3_value_bytes(argv[0]); + uLongf nOut = compressBound(nData); + Bytef *pOut; + + pOut = (Bytef*)sqlite3_malloc(nOut); + if( pOut==0 ){ + sqlite3_result_error_nomem(context); + return; + }else{ + if( Z_OK!=compress(pOut, &nOut, pData, nData) ){ + sqlite3_result_error(context, "error in compress()", -1); + }else if( nOut0 && pRe->zInit[j-1]==0 ) j--; - pRe->nInit = j; + }else{ + sqlite3_result_value(context, argv[0]); } - return pRe->zErr; } /* -** Implementation of the regexp() SQL function. This function implements -** the build-in REGEXP operator. The first argument to the function is the -** pattern and the second argument is the string. So, the SQL statements: -** -** A REGEXP B +** Implementation of the "sqlar_uncompress(X,SZ)" SQL function ** -** is implemented as regexp(B,A). +** Parameter SZ is interpreted as an integer. If it is less than or +** equal to zero, then this function returns a copy of X. Or, if +** SZ is equal to the size of X when interpreted as a blob, also +** return a copy of X. Otherwise, decompress blob X using zlib +** utility function uncompress() and return the results (another +** blob). */ -static void re_sql_func( +static void sqlarUncompressFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - ReCompiled *pRe; /* Compiled regular expression */ - const char *zPattern; /* The regular expression */ - const unsigned char *zStr;/* String being searched */ - const char *zErr; /* Compile error message */ - int setAux = 0; /* True to invoke sqlite3_set_auxdata() */ + uLong nData; + sqlite3_int64 sz; - (void)argc; /* Unused */ - pRe = sqlite3_get_auxdata(context, 0); - if( pRe==0 ){ - zPattern = (const char*)sqlite3_value_text(argv[0]); - if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); - if( zErr ){ - re_free(pRe); - sqlite3_result_error(context, zErr, -1); - return; - } - if( pRe==0 ){ + assert( argc==2 ); + sz = sqlite3_value_int(argv[1]); + + if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ + sqlite3_result_value(context, argv[0]); + }else{ + uLongf szf = sz; + const Bytef *pData= sqlite3_value_blob(argv[0]); + Bytef *pOut = sqlite3_malloc(sz); + if( pOut==0 ){ sqlite3_result_error_nomem(context); - return; + }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ + sqlite3_result_error(context, "error in uncompress()", -1); + }else{ + sqlite3_result_blob(context, pOut, szf, SQLITE_TRANSIENT); } - setAux = 1; - } - zStr = (const unsigned char*)sqlite3_value_text(argv[1]); - if( zStr!=0 ){ - sqlite3_result_int(context, re_match(pRe, zStr, -1)); + sqlite3_free(pOut); } - if( setAux ){ - sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); +} + +#ifdef _WIN32 + +#endif +int sqlite3_sqlar_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sqlar_compress", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + sqlarCompressFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sqlar_uncompress", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + sqlarUncompressFunc, 0, 0); } + return rc; } +/************************* End ../ext/misc/sqlar.c ********************/ +#endif +/************************* Begin ../ext/expert/sqlite3expert.h ******************/ /* -** Invoke this routine to register the regexp() function with the -** SQLite database connection. +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +#if !defined(SQLITEEXPERT_H) +#define SQLITEEXPERT_H 1 +/* #include "sqlite3.h" */ + +typedef struct sqlite3expert sqlite3expert; + +/* +** Create a new sqlite3expert object. +** +** If successful, a pointer to the new object is returned and (*pzErr) set +** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to +** an English-language error message. In this case it is the responsibility +** of the caller to eventually free the error message buffer using +** sqlite3_free(). +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr); + +/* +** Configure an sqlite3expert object. +** +** EXPERT_CONFIG_SAMPLE: +** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for +** each candidate index. This involves scanning and sorting the entire +** contents of each user database table once for each candidate index +** associated with the table. For large databases, this can be +** prohibitively slow. This option allows the sqlite3expert object to +** be configured so that sqlite_stat1 data is instead generated based on a +** subset of each table, or so that no sqlite_stat1 data is used at all. +** +** A single integer argument is passed to this option. If the value is less +** than or equal to zero, then no sqlite_stat1 data is generated or used by +** the analysis - indexes are recommended based on the database schema only. +** Or, if the value is 100 or greater, complete sqlite_stat1 data is +** generated for each candidate index (this is the default). Finally, if the +** value falls between 0 and 100, then it represents the percentage of user +** table rows that should be considered when generating sqlite_stat1 data. +** +** Examples: +** +** // Do not generate any sqlite_stat1 data +** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0); +** +** // Generate sqlite_stat1 data based on 10% of the rows in each table. +** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10); +*/ +int sqlite3_expert_config(sqlite3expert *p, int op, ...); + +#define EXPERT_CONFIG_SAMPLE 1 /* int */ + +/* +** Specify zero or more SQL statements to be included in the analysis. +** +** Buffer zSql must contain zero or more complete SQL statements. This +** function parses all statements contained in the buffer and adds them +** to the internal list of statements to analyze. If successful, SQLITE_OK +** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example +** due to a error in the SQL - an SQLite error code is returned and (*pzErr) +** may be set to point to an English language error message. In this case +** the caller is responsible for eventually freeing the error message buffer +** using sqlite3_free(). +** +** If an error does occur while processing one of the statements in the +** buffer passed as the second argument, none of the statements in the +** buffer are added to the analysis. +** +** This function must be called before sqlite3_expert_analyze(). If a call +** to this function is made on an sqlite3expert object that has already +** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned +** immediately and no statements are added to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From a successful sqlite3_expert_new() */ + const char *zSql, /* SQL statement(s) to add */ + char **pzErr /* OUT: Error message (if any) */ +); + + +/* +** This function is called after the sqlite3expert object has been configured +** with all SQL statements using sqlite3_expert_sql() to actually perform +** the analysis. Once this function has been called, it is not possible to +** add further SQL statements to the analysis. +** +** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if +** an error occurs, an SQLite error code is returned and (*pzErr) set to +** point to a buffer containing an English language error message. In this +** case it is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** If an error does occur within this function, the sqlite3expert object +** is no longer useful for any purpose. At that point it is no longer +** possible to add further SQL statements to the object or to re-attempt +** the analysis. The sqlite3expert object must still be freed using a call +** sqlite3_expert_destroy(). +*/ +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr); + +/* +** Return the total number of statements loaded using sqlite3_expert_sql(). +** The total number of SQL statements may be different from the total number +** to calls to sqlite3_expert_sql(). +*/ +int sqlite3_expert_count(sqlite3expert*); + +/* +** Return a component of the report. +** +** This function is called after sqlite3_expert_analyze() to extract the +** results of the analysis. Each call to this function returns either a +** NULL pointer or a pointer to a buffer containing a nul-terminated string. +** The value passed as the third argument must be one of the EXPERT_REPORT_* +** #define constants defined below. +** +** For some EXPERT_REPORT_* parameters, the buffer returned contains +** information relating to a specific SQL statement. In these cases that +** SQL statement is identified by the value passed as the second argument. +** SQL statements are numbered from 0 in the order in which they are parsed. +** If an out-of-range value (less than zero or equal to or greater than the +** value returned by sqlite3_expert_count()) is passed as the second argument +** along with such an EXPERT_REPORT_* parameter, NULL is always returned. +** +** EXPERT_REPORT_SQL: +** Return the text of SQL statement iStmt. +** +** EXPERT_REPORT_INDEXES: +** Return a buffer containing the CREATE INDEX statements for all recommended +** indexes for statement iStmt. If there are no new recommeded indexes, NULL +** is returned. +** +** EXPERT_REPORT_PLAN: +** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query +** iStmt after the proposed indexes have been added to the database schema. +** +** EXPERT_REPORT_CANDIDATES: +** Return a pointer to a buffer containing the CREATE INDEX statements +** for all indexes that were tested (for all SQL statements). The iStmt +** parameter is ignored for EXPERT_REPORT_CANDIDATES calls. +*/ +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport); + +/* +** Values for the third argument passed to sqlite3_expert_report(). +*/ +#define EXPERT_REPORT_SQL 1 +#define EXPERT_REPORT_INDEXES 2 +#define EXPERT_REPORT_PLAN 3 +#define EXPERT_REPORT_CANDIDATES 4 + +/* +** Free an (sqlite3expert*) handle and all associated resources. There +** should be one call to this function for each successful call to +** sqlite3-expert_new(). */ -#ifdef _WIN32 +void sqlite3_expert_destroy(sqlite3expert*); -#endif -int sqlite3_regexp_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused */ - rc = sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, - 0, re_sql_func, 0, 0); - if( rc==SQLITE_OK ){ - /* The regexpi(PATTERN,STRING) function is a case-insensitive version - ** of regexp(PATTERN,STRING). */ - rc = sqlite3_create_function(db, "regexpi", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, - (void*)db, re_sql_func, 0, 0); - } - return rc; -} +#endif /* !defined(SQLITEEXPERT_H) */ -/************************* End ../ext/misc/regexp.c ********************/ -#ifdef SQLITE_HAVE_ZLIB -/************************* Begin ../ext/misc/zipfile.c ******************/ +/************************* End ../ext/expert/sqlite3expert.h ********************/ +/************************* Begin ../ext/expert/sqlite3expert.c ******************/ /* -** 2017-12-26 +** 2017 April 09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -6747,54 +11924,14 @@ int sqlite3_regexp_init( ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -****************************************************************************** -** -** This file implements a virtual table for reading and writing ZIP archive -** files. -** -** Usage example: -** -** SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename); -** -** Current limitations: -** -** * No support for encryption -** * No support for ZIP archives spanning multiple files -** * No support for zip64 extensions -** * Only the "inflate/deflate" (zlib) compression method is supported +************************************************************************* */ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include +/* #include "sqlite3expert.h" */ #include +#include +#include -#include - -#ifndef SQLITE_OMIT_VIRTUALTABLE - -#ifndef SQLITE_AMALGAMATION - -#ifndef UINT32_TYPE -# ifdef HAVE_UINT32_T -# define UINT32_TYPE uint32_t -# else -# define UINT32_TYPE unsigned int -# endif -#endif -#ifndef UINT16_TYPE -# ifdef HAVE_UINT16_T -# define UINT16_TYPE uint16_t -# else -# define UINT16_TYPE unsigned short int -# endif -#endif -/* typedef sqlite3_int64 i64; */ -/* typedef unsigned char u8; */ -typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ -typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ -#define MIN(a,b) ((a)<(b) ? (a) : (b)) - +#if !defined(SQLITE_AMALGAMATION) #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 #endif @@ -6808,2271 +11945,2134 @@ typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +#endif /* !defined(SQLITE_AMALGAMATION) */ -#endif /* SQLITE_AMALGAMATION */ - -/* -** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. -** -** In some ways it would be better to obtain these values from system -** header files. But, the dependency is undesirable and (a) these -** have been stable for decades, (b) the values are part of POSIX and -** are also made explicit in [man stat], and (c) are part of the -** file format for zip archives. -*/ -#ifndef S_IFDIR -# define S_IFDIR 0040000 -#endif -#ifndef S_IFREG -# define S_IFREG 0100000 -#endif -#ifndef S_IFLNK -# define S_IFLNK 0120000 -#endif - -static const char ZIPFILE_SCHEMA[] = - "CREATE TABLE y(" - "name PRIMARY KEY," /* 0: Name of file in zip archive */ - "mode," /* 1: POSIX mode for file */ - "mtime," /* 2: Last modification time (secs since 1970)*/ - "sz," /* 3: Size of object */ - "rawdata," /* 4: Raw data */ - "data," /* 5: Uncompressed data */ - "method," /* 6: Compression method (integer) */ - "z HIDDEN" /* 7: Name of zip file */ - ") WITHOUT ROWID;"; - -#define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */ -#define ZIPFILE_BUFFER_SIZE (64*1024) - - -/* -** Magic numbers used to read and write zip files. -** -** ZIPFILE_NEWENTRY_MADEBY: -** Use this value for the "version-made-by" field in new zip file -** entries. The upper byte indicates "unix", and the lower byte -** indicates that the zip file matches pkzip specification 3.0. -** This is what info-zip seems to do. -** -** ZIPFILE_NEWENTRY_REQUIRED: -** Value for "version-required-to-extract" field of new entries. -** Version 2.0 is required to support folders and deflate compression. -** -** ZIPFILE_NEWENTRY_FLAGS: -** Value for "general-purpose-bit-flags" field of new entries. Bit -** 11 means "utf-8 filename and comment". -** -** ZIPFILE_SIGNATURE_CDS: -** First 4 bytes of a valid CDS record. -** -** ZIPFILE_SIGNATURE_LFH: -** First 4 bytes of a valid LFH record. -** -** ZIPFILE_SIGNATURE_EOCD -** First 4 bytes of a valid EOCD record. -*/ -#define ZIPFILE_EXTRA_TIMESTAMP 0x5455 -#define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) -#define ZIPFILE_NEWENTRY_REQUIRED 20 -#define ZIPFILE_NEWENTRY_FLAGS 0x800 -#define ZIPFILE_SIGNATURE_CDS 0x02014b50 -#define ZIPFILE_SIGNATURE_LFH 0x04034b50 -#define ZIPFILE_SIGNATURE_EOCD 0x06054b50 - -/* -** The sizes of the fixed-size part of each of the three main data -** structures in a zip archive. -*/ -#define ZIPFILE_LFH_FIXED_SZ 30 -#define ZIPFILE_EOCD_FIXED_SZ 22 -#define ZIPFILE_CDS_FIXED_SZ 46 - -/* -*** 4.3.16 End of central directory record: -*** -*** end of central dir signature 4 bytes (0x06054b50) -*** number of this disk 2 bytes -*** number of the disk with the -*** start of the central directory 2 bytes -*** total number of entries in the -*** central directory on this disk 2 bytes -*** total number of entries in -*** the central directory 2 bytes -*** size of the central directory 4 bytes -*** offset of start of central -*** directory with respect to -*** the starting disk number 4 bytes -*** .ZIP file comment length 2 bytes -*** .ZIP file comment (variable size) -*/ -typedef struct ZipfileEOCD ZipfileEOCD; -struct ZipfileEOCD { - u16 iDisk; - u16 iFirstDisk; - u16 nEntry; - u16 nEntryTotal; - u32 nSize; - u32 iOffset; -}; - -/* -*** 4.3.12 Central directory structure: -*** -*** ... -*** -*** central file header signature 4 bytes (0x02014b50) -*** version made by 2 bytes -*** version needed to extract 2 bytes -*** general purpose bit flag 2 bytes -*** compression method 2 bytes -*** last mod file time 2 bytes -*** last mod file date 2 bytes -*** crc-32 4 bytes -*** compressed size 4 bytes -*** uncompressed size 4 bytes -*** file name length 2 bytes -*** extra field length 2 bytes -*** file comment length 2 bytes -*** disk number start 2 bytes -*** internal file attributes 2 bytes -*** external file attributes 4 bytes -*** relative offset of local header 4 bytes -*/ -typedef struct ZipfileCDS ZipfileCDS; -struct ZipfileCDS { - u16 iVersionMadeBy; - u16 iVersionExtract; - u16 flags; - u16 iCompression; - u16 mTime; - u16 mDate; - u32 crc32; - u32 szCompressed; - u32 szUncompressed; - u16 nFile; - u16 nExtra; - u16 nComment; - u16 iDiskStart; - u16 iInternalAttr; - u32 iExternalAttr; - u32 iOffset; - char *zFile; /* Filename (sqlite3_malloc()) */ -}; - -/* -*** 4.3.7 Local file header: -*** -*** local file header signature 4 bytes (0x04034b50) -*** version needed to extract 2 bytes -*** general purpose bit flag 2 bytes -*** compression method 2 bytes -*** last mod file time 2 bytes -*** last mod file date 2 bytes -*** crc-32 4 bytes -*** compressed size 4 bytes -*** uncompressed size 4 bytes -*** file name length 2 bytes -*** extra field length 2 bytes -*** -*/ -typedef struct ZipfileLFH ZipfileLFH; -struct ZipfileLFH { - u16 iVersionExtract; - u16 flags; - u16 iCompression; - u16 mTime; - u16 mDate; - u32 crc32; - u32 szCompressed; - u32 szUncompressed; - u16 nFile; - u16 nExtra; -}; - -typedef struct ZipfileEntry ZipfileEntry; -struct ZipfileEntry { - ZipfileCDS cds; /* Parsed CDS record */ - u32 mUnixTime; /* Modification time, in UNIX format */ - u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ - i64 iDataOff; /* Offset to data in file (if aData==0) */ - u8 *aData; /* cds.szCompressed bytes of compressed data */ - ZipfileEntry *pNext; /* Next element in in-memory CDS */ -}; -/* -** Cursor type for zipfile tables. -*/ -typedef struct ZipfileCsr ZipfileCsr; -struct ZipfileCsr { - sqlite3_vtab_cursor base; /* Base class - must be first */ - i64 iId; /* Cursor ID */ - u8 bEof; /* True when at EOF */ - u8 bNoop; /* If next xNext() call is no-op */ +#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Used outside of write transactions */ - FILE *pFile; /* Zip file */ - i64 iNextOff; /* Offset of next record in central directory */ - ZipfileEOCD eocd; /* Parse of central directory record */ +/* typedef sqlite3_int64 i64; */ +/* typedef sqlite3_uint64 u64; */ - ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ - ZipfileEntry *pCurrent; /* Current entry */ - ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ -}; +typedef struct IdxColumn IdxColumn; +typedef struct IdxConstraint IdxConstraint; +typedef struct IdxScan IdxScan; +typedef struct IdxStatement IdxStatement; +typedef struct IdxTable IdxTable; +typedef struct IdxWrite IdxWrite; -typedef struct ZipfileTab ZipfileTab; -struct ZipfileTab { - sqlite3_vtab base; /* Base class - must be first */ - char *zFile; /* Zip file this table accesses (may be NULL) */ - sqlite3 *db; /* Host database connection */ - u8 *aBuffer; /* Temporary buffer used for various tasks */ +#define STRLEN (int)strlen - ZipfileCsr *pCsrList; /* List of cursors */ - i64 iNextCsrid; +/* +** A temp table name that we assume no user database will actually use. +** If this assumption proves incorrect triggers on the table with the +** conflicting name will be ignored. +*/ +#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776" - /* The following are used by write transactions only */ - ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */ - ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */ - FILE *pWriteFd; /* File handle open on zip archive */ - i64 szCurrent; /* Current size of zip archive */ - i64 szOrig; /* Size of archive at start of transaction */ +/* +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or +** any other type of single-ended range constraint on a column). +** +** pLink: +** Used to temporarily link IdxConstraint objects into lists while +** creating candidate indexes. +*/ +struct IdxConstraint { + char *zColl; /* Collation sequence */ + int bRange; /* True for range, false for eq */ + int iCol; /* Constrained table column */ + int bFlag; /* Used by idxFindCompatible() */ + int bDesc; /* True if ORDER BY DESC */ + IdxConstraint *pNext; /* Next constraint in pEq or pRange list */ + IdxConstraint *pLink; /* See above */ }; /* -** Set the error message contained in context ctx to the results of -** vprintf(zFmt, ...). +** A single scan of a single table. */ -static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ - char *zMsg = 0; - va_list ap; - va_start(ap, zFmt); - zMsg = sqlite3_vmprintf(zFmt, ap); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - va_end(ap); -} +struct IdxScan { + IdxTable *pTab; /* Associated table object */ + int iDb; /* Database containing table zTable */ + i64 covering; /* Mask of columns required for cov. index */ + IdxConstraint *pOrder; /* ORDER BY columns */ + IdxConstraint *pEq; /* List of == constraints */ + IdxConstraint *pRange; /* List of < constraints */ + IdxScan *pNextScan; /* Next IdxScan object for same analysis */ +}; /* -** If string zIn is quoted, dequote it in place. Otherwise, if the string -** is not quoted, do nothing. +** Information regarding a single database table. Extracted from +** "PRAGMA table_info" by function idxGetTableInfo(). */ -static void zipfileDequote(char *zIn){ - char q = zIn[0]; - if( q=='"' || q=='\'' || q=='`' || q=='[' ){ - int iIn = 1; - int iOut = 0; - if( q=='[' ) q = ']'; - while( ALWAYS(zIn[iIn]) ){ - char c = zIn[iIn++]; - if( c==q && zIn[iIn++]!=q ) break; - zIn[iOut++] = c; - } - zIn[iOut] = '\0'; - } -} +struct IdxColumn { + char *zName; + char *zColl; + int iPk; +}; +struct IdxTable { + int nCol; + char *zName; /* Table name */ + IdxColumn *aCol; + IdxTable *pNext; /* Next table in linked list of all tables */ +}; /* -** Construct a new ZipfileTab virtual table object. -** -** argv[0] -> module name ("zipfile") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. +** An object of the following type is created for each unique table/write-op +** seen. The objects are stored in a singly-linked list beginning at +** sqlite3expert.pWrite. */ -static int zipfileConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; - int nFile = 0; - const char *zFile = 0; - ZipfileTab *pNew = 0; - int rc; - - /* If the table name is not "zipfile", require that the argument be - ** specified. This stops zipfile tables from being created as: - ** - ** CREATE VIRTUAL TABLE zzz USING zipfile(); - ** - ** It does not prevent: - ** - ** CREATE VIRTUAL TABLE zipfile USING zipfile(); - */ - assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); - if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ - *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); - return SQLITE_ERROR; - } +struct IdxWrite { + IdxTable *pTab; + int eOp; /* SQLITE_UPDATE, DELETE or INSERT */ + IdxWrite *pNext; +}; - if( argc>3 ){ - zFile = argv[3]; - nFile = (int)strlen(zFile)+1; - } +/* +** Each statement being analyzed is represented by an instance of this +** structure. +*/ +struct IdxStatement { + int iId; /* Statement number */ + char *zSql; /* SQL statement */ + char *zIdx; /* Indexes */ + char *zEQP; /* Plan */ + IdxStatement *pNext; +}; - rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); - if( rc==SQLITE_OK ){ - pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); - if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, nByte+nFile); - pNew->db = db; - pNew->aBuffer = (u8*)&pNew[1]; - if( zFile ){ - pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; - memcpy(pNew->zFile, zFile, nFile); - zipfileDequote(pNew->zFile); - } - } - sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); - *ppVtab = (sqlite3_vtab*)pNew; - return rc; -} /* -** Free the ZipfileEntry structure indicated by the only argument. +** A hash table for storing strings. With space for a payload string +** with each entry. Methods are: +** +** idxHashInit() +** idxHashClear() +** idxHashAdd() +** idxHashSearch() */ -static void zipfileEntryFree(ZipfileEntry *p){ - if( p ){ - sqlite3_free(p->cds.zFile); - sqlite3_free(p); - } -} +#define IDX_HASH_SIZE 1023 +typedef struct IdxHashEntry IdxHashEntry; +typedef struct IdxHash IdxHash; +struct IdxHashEntry { + char *zKey; /* nul-terminated key */ + char *zVal; /* nul-terminated value string */ + char *zVal2; /* nul-terminated value string 2 */ + IdxHashEntry *pHashNext; /* Next entry in same hash bucket */ + IdxHashEntry *pNext; /* Next entry in hash */ +}; +struct IdxHash { + IdxHashEntry *pFirst; + IdxHashEntry *aHash[IDX_HASH_SIZE]; +}; /* -** Release resources that should be freed at the end of a write -** transaction. +** sqlite3expert object. */ -static void zipfileCleanupTransaction(ZipfileTab *pTab){ - ZipfileEntry *pEntry; - ZipfileEntry *pNext; +struct sqlite3expert { + int iSample; /* Percentage of tables to sample for stat1 */ + sqlite3 *db; /* User database */ + sqlite3 *dbm; /* In-memory db for this analysis */ + sqlite3 *dbv; /* Vtab schema for this analysis */ + IdxTable *pTable; /* List of all IdxTable objects */ + IdxScan *pScan; /* List of scan objects */ + IdxWrite *pWrite; /* List of write objects */ + IdxStatement *pStatement; /* List of IdxStatement objects */ + int bRun; /* True once analysis has run */ + char **pzErrmsg; + int rc; /* Error code from whereinfo hook */ + IdxHash hIdx; /* Hash containing all candidate indexes */ + char *zCandidates; /* For EXPERT_REPORT_CANDIDATES */ +}; - if( pTab->pWriteFd ){ - fclose(pTab->pWriteFd); - pTab->pWriteFd = 0; - } - for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ - pNext = pEntry->pNext; - zipfileEntryFree(pEntry); + +/* +** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). +** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. +*/ +static void *idxMalloc(int *pRc, int nByte){ + void *pRet; + assert( *pRc==SQLITE_OK ); + assert( nByte>0 ); + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; } - pTab->pFirstEntry = 0; - pTab->pLastEntry = 0; - pTab->szCurrent = 0; - pTab->szOrig = 0; + return pRet; } /* -** This method is the destructor for zipfile vtab objects. +** Initialize an IdxHash hash table. */ -static int zipfileDisconnect(sqlite3_vtab *pVtab){ - zipfileCleanupTransaction((ZipfileTab*)pVtab); - sqlite3_free(pVtab); - return SQLITE_OK; +static void idxHashInit(IdxHash *pHash){ + memset(pHash, 0, sizeof(IdxHash)); } /* -** Constructor for a new ZipfileCsr object. +** Reset an IdxHash hash table. */ -static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ - ZipfileTab *pTab = (ZipfileTab*)p; - ZipfileCsr *pCsr; - pCsr = sqlite3_malloc(sizeof(*pCsr)); - *ppCsr = (sqlite3_vtab_cursor*)pCsr; - if( pCsr==0 ){ - return SQLITE_NOMEM; +static void idxHashClear(IdxHash *pHash){ + int i; + for(i=0; iaHash[i]; pEntry; pEntry=pNext){ + pNext = pEntry->pHashNext; + sqlite3_free(pEntry->zVal2); + sqlite3_free(pEntry); + } } - memset(pCsr, 0, sizeof(*pCsr)); - pCsr->iId = ++pTab->iNextCsrid; - pCsr->pCsrNext = pTab->pCsrList; - pTab->pCsrList = pCsr; - return SQLITE_OK; + memset(pHash, 0, sizeof(IdxHash)); } /* -** Reset a cursor back to the state it was in when first returned -** by zipfileOpen(). +** Return the index of the hash bucket that the string specified by the +** arguments to this function belongs. */ -static void zipfileResetCursor(ZipfileCsr *pCsr){ - ZipfileEntry *p; - ZipfileEntry *pNext; +static int idxHashString(const char *z, int n){ + unsigned int ret = 0; + int i; + for(i=0; ibEof = 0; - if( pCsr->pFile ){ - fclose(pCsr->pFile); - pCsr->pFile = 0; - zipfileEntryFree(pCsr->pCurrent); - pCsr->pCurrent = 0; +/* +** If zKey is already present in the hash table, return non-zero and do +** nothing. Otherwise, add an entry with key zKey and payload string zVal to +** the hash table passed as the second argument. +*/ +static int idxHashAdd( + int *pRc, + IdxHash *pHash, + const char *zKey, + const char *zVal +){ + int nKey = STRLEN(zKey); + int iHash = idxHashString(zKey, nKey); + int nVal = (zVal ? STRLEN(zVal) : 0); + IdxHashEntry *pEntry; + assert( iHash>=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return 1; + } } + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + if( pEntry ){ + pEntry->zKey = (char*)&pEntry[1]; + memcpy(pEntry->zKey, zKey, nKey); + if( zVal ){ + pEntry->zVal = &pEntry->zKey[nKey+1]; + memcpy(pEntry->zVal, zVal, nVal); + } + pEntry->pHashNext = pHash->aHash[iHash]; + pHash->aHash[iHash] = pEntry; - for(p=pCsr->pFreeEntry; p; p=pNext){ - pNext = p->pNext; - zipfileEntryFree(p); + pEntry->pNext = pHash->pFirst; + pHash->pFirst = pEntry; } + return 0; } /* -** Destructor for an ZipfileCsr. +** If zKey/nKey is present in the hash table, return a pointer to the +** hash-entry object. */ -static int zipfileClose(sqlite3_vtab_cursor *cur){ - ZipfileCsr *pCsr = (ZipfileCsr*)cur; - ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab); - ZipfileCsr **pp; - zipfileResetCursor(pCsr); - - /* Remove this cursor from the ZipfileTab.pCsrList list. */ - for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); - *pp = pCsr->pCsrNext; - - sqlite3_free(pCsr); - return SQLITE_OK; +static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){ + int iHash; + IdxHashEntry *pEntry; + if( nKey<0 ) nKey = STRLEN(zKey); + iHash = idxHashString(zKey, nKey); + assert( iHash>=0 ); + for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ + if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ + return pEntry; + } + } + return 0; } /* -** Set the error message for the virtual table associated with cursor -** pCsr to the results of vprintf(zFmt, ...). +** If the hash table contains an entry with a key equal to the string +** passed as the final two arguments to this function, return a pointer +** to the payload string. Otherwise, if zKey/nKey is not present in the +** hash table, return NULL. */ -static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - sqlite3_free(pTab->base.zErrMsg); - pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); -} -static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - sqlite3_free(pCsr->base.pVtab->zErrMsg); - pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); +static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){ + IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey); + if( pEntry ) return pEntry->zVal; + return 0; } /* -** Read nRead bytes of data from offset iOff of file pFile into buffer -** aRead[]. Return SQLITE_OK if successful, or an SQLite error code -** otherwise. -** -** If an error does occur, output variable (*pzErrmsg) may be set to point -** to an English language error message. It is the responsibility of the -** caller to eventually free this buffer using -** sqlite3_free(). +** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl +** variable to point to a copy of nul-terminated string zColl. */ -static int zipfileReadData( - FILE *pFile, /* Read from this file */ - u8 *aRead, /* Read into this buffer */ - int nRead, /* Number of bytes to read */ - i64 iOff, /* Offset to read from */ - char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ -){ - size_t n; - fseek(pFile, (long)iOff, SEEK_SET); - n = fread(aRead, 1, nRead, pFile); - if( (int)n!=nRead ){ - *pzErrmsg = sqlite3_mprintf("error in fread()"); - return SQLITE_ERROR; - } - return SQLITE_OK; -} +static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ + IdxConstraint *pNew; + int nColl = STRLEN(zColl); -static int zipfileAppendData( - ZipfileTab *pTab, - const u8 *aWrite, - int nWrite -){ - if( nWrite>0 ){ - size_t n = nWrite; - fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); - n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); - if( (int)n!=nWrite ){ - pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); - return SQLITE_ERROR; - } - pTab->szCurrent += nWrite; + assert( *pRc==SQLITE_OK ); + pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1); + if( pNew ){ + pNew->zColl = (char*)&pNew[1]; + memcpy(pNew->zColl, zColl, nColl+1); } - return SQLITE_OK; + return pNew; } /* -** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +** An error associated with database handle db has just occurred. Pass +** the error message to callback function xOut. */ -static u16 zipfileGetU16(const u8 *aBuf){ - return (aBuf[1] << 8) + aBuf[0]; +static void idxDatabaseError( + sqlite3 *db, /* Database handle */ + char **pzErrmsg /* Write error here */ +){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } /* -** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +** Prepare an SQL statement. */ -static u32 zipfileGetU32(const u8 *aBuf){ - if( aBuf==0 ) return 0; - return ((u32)(aBuf[3]) << 24) - + ((u32)(aBuf[2]) << 16) - + ((u32)(aBuf[1]) << 8) - + ((u32)(aBuf[0]) << 0); +static int idxPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zSql /* SQL statement to compile */ +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *ppStmt = 0; + idxDatabaseError(db, pzErrmsg); + } + return rc; } /* -** Write a 16-bit little endiate integer into buffer aBuf. +** Prepare an SQL statement using the results of a printf() formatting. */ -static void zipfilePutU16(u8 *aBuf, u16 val){ - aBuf[0] = val & 0xFF; - aBuf[1] = (val>>8) & 0xFF; +static int idxPrintfPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ + const char *zFmt, /* printf() format of SQL statement */ + ... /* Trailing printf() arguments */ +){ + va_list ap; + int rc; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + va_end(ap); + return rc; } -/* -** Write a 32-bit little endiate integer into buffer aBuf. + +/************************************************************************* +** Beginning of virtual table implementation. */ -static void zipfilePutU32(u8 *aBuf, u32 val){ - aBuf[0] = val & 0xFF; - aBuf[1] = (val>>8) & 0xFF; - aBuf[2] = (val>>16) & 0xFF; - aBuf[3] = (val>>24) & 0xFF; -} +typedef struct ExpertVtab ExpertVtab; +struct ExpertVtab { + sqlite3_vtab base; + IdxTable *pTab; + sqlite3expert *pExpert; +}; -#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) ) -#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) ) +typedef struct ExpertCsr ExpertCsr; +struct ExpertCsr { + sqlite3_vtab_cursor base; + sqlite3_stmt *pData; +}; -#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } -#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } +static char *expertDequote(const char *zIn){ + int n = STRLEN(zIn); + char *zRet = sqlite3_malloc(n); -/* -** Magic numbers used to read CDS records. -*/ -#define ZIPFILE_CDS_NFILE_OFF 28 -#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 + assert( zIn[0]=='\'' ); + assert( zIn[n-1]=='\'' ); -/* -** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR -** if the record is not well-formed, or SQLITE_OK otherwise. -*/ -static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ - u8 *aRead = aBuf; - u32 sig = zipfileRead32(aRead); - int rc = SQLITE_OK; - if( sig!=ZIPFILE_SIGNATURE_CDS ){ - rc = SQLITE_ERROR; - }else{ - pCDS->iVersionMadeBy = zipfileRead16(aRead); - pCDS->iVersionExtract = zipfileRead16(aRead); - pCDS->flags = zipfileRead16(aRead); - pCDS->iCompression = zipfileRead16(aRead); - pCDS->mTime = zipfileRead16(aRead); - pCDS->mDate = zipfileRead16(aRead); - pCDS->crc32 = zipfileRead32(aRead); - pCDS->szCompressed = zipfileRead32(aRead); - pCDS->szUncompressed = zipfileRead32(aRead); - assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); - pCDS->nFile = zipfileRead16(aRead); - pCDS->nExtra = zipfileRead16(aRead); - pCDS->nComment = zipfileRead16(aRead); - pCDS->iDiskStart = zipfileRead16(aRead); - pCDS->iInternalAttr = zipfileRead16(aRead); - pCDS->iExternalAttr = zipfileRead32(aRead); - pCDS->iOffset = zipfileRead32(aRead); - assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] ); + if( zRet ){ + int iOut = 0; + int iIn = 0; + for(iIn=1; iIn<(n-1); iIn++){ + if( zIn[iIn]=='\'' ){ + assert( zIn[iIn+1]=='\'' ); + iIn++; + } + zRet[iOut++] = zIn[iIn]; + } + zRet[iOut] = '\0'; } - return rc; + return zRet; } -/* -** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR -** if the record is not well-formed, or SQLITE_OK otherwise. +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... */ -static int zipfileReadLFH( - u8 *aBuffer, - ZipfileLFH *pLFH +static int expertConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr ){ - u8 *aRead = aBuffer; - int rc = SQLITE_OK; + sqlite3expert *pExpert = (sqlite3expert*)pAux; + ExpertVtab *p = 0; + int rc; - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_LFH ){ + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("internal error!"); rc = SQLITE_ERROR; }else{ - pLFH->iVersionExtract = zipfileRead16(aRead); - pLFH->flags = zipfileRead16(aRead); - pLFH->iCompression = zipfileRead16(aRead); - pLFH->mTime = zipfileRead16(aRead); - pLFH->mDate = zipfileRead16(aRead); - pLFH->crc32 = zipfileRead32(aRead); - pLFH->szCompressed = zipfileRead32(aRead); - pLFH->szUncompressed = zipfileRead32(aRead); - pLFH->nFile = zipfileRead16(aRead); - pLFH->nExtra = zipfileRead16(aRead); + char *zCreateTable = expertDequote(argv[3]); + if( zCreateTable ){ + rc = sqlite3_declare_vtab(db, zCreateTable); + if( rc==SQLITE_OK ){ + p = idxMalloc(&rc, sizeof(ExpertVtab)); + } + if( rc==SQLITE_OK ){ + p->pExpert = pExpert; + p->pTab = pExpert->pTable; + assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 ); + } + sqlite3_free(zCreateTable); + }else{ + rc = SQLITE_NOMEM; + } } + + *ppVtab = (sqlite3_vtab*)p; return rc; } +static int expertDisconnect(sqlite3_vtab *pVtab){ + ExpertVtab *p = (ExpertVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} -/* -** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. -** Scan through this buffer to find an "extra-timestamp" field. If one -** exists, extract the 32-bit modification-timestamp from it and store -** the value in output parameter *pmTime. -** -** Zero is returned if no extra-timestamp record could be found (and so -** *pmTime is left unchanged), or non-zero otherwise. -** -** The general format of an extra field is: -** -** Header ID 2 bytes -** Data Size 2 bytes -** Data N bytes -*/ -static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ - int ret = 0; - u8 *p = aExtra; - u8 *pEnd = &aExtra[nExtra]; +static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){ + ExpertVtab *p = (ExpertVtab*)pVtab; + int rc = SQLITE_OK; + int n = 0; + IdxScan *pScan; + const int opmask = + SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT | + SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE | + SQLITE_INDEX_CONSTRAINT_LE; - while( p modtime is present */ - *pmTime = zipfileGetU32(&p[1]); - ret = 1; + /* Link the new scan object into the list */ + pScan->pTab = p->pTab; + pScan->pNextScan = p->pExpert->pScan; + p->pExpert->pScan = pScan; + + /* Add the constraints to the IdxScan object */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + if( pCons->usable + && pCons->iColumn>=0 + && p->pTab->aCol[pCons->iColumn].iPk==0 + && (pCons->op & opmask) + ){ + IdxConstraint *pNew; + const char *zColl = sqlite3_vtab_collation(pIdxInfo, i); + pNew = idxNewConstraint(&rc, zColl); + if( pNew ){ + pNew->iCol = pCons->iColumn; + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pNew->pNext = pScan->pEq; + pScan->pEq = pNew; + }else{ + pNew->bRange = 1; + pNew->pNext = pScan->pRange; + pScan->pRange = pNew; + } + } + n++; + pIdxInfo->aConstraintUsage[i].argvIndex = n; + } + } + + /* Add the ORDER BY to the IdxScan object */ + for(i=pIdxInfo->nOrderBy-1; i>=0; i--){ + int iCol = pIdxInfo->aOrderBy[i].iColumn; + if( iCol>=0 ){ + IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl); + if( pNew ){ + pNew->iCol = iCol; + pNew->bDesc = pIdxInfo->aOrderBy[i].desc; + pNew->pNext = pScan->pOrder; + pNew->pLink = pScan->pOrder; + pScan->pOrder = pNew; + n++; } - break; } } + } + + pIdxInfo->estimatedCost = 1000000.0 / (n+1); + return rc; +} + +static int expertUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **azData, + sqlite_int64 *pRowid +){ + (void)pVtab; + (void)nData; + (void)azData; + (void)pRowid; + return SQLITE_OK; +} + +/* +** Virtual table module xOpen method. +*/ +static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_OK; + ExpertCsr *pCsr; + (void)pVTab; + pCsr = idxMalloc(&rc, sizeof(ExpertCsr)); + *ppCursor = (sqlite3_vtab_cursor*)pCsr; + return rc; +} - p += nByte; - } - return ret; +/* +** Virtual table module xClose method. +*/ +static int expertClose(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_finalize(pCsr->pData); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** Convert the standard MS-DOS timestamp stored in the mTime and mDate -** fields of the CDS structure passed as the only argument to a 32-bit -** UNIX seconds-since-the-epoch timestamp. Return the result. -** -** "Standard" MS-DOS time format: -** -** File modification time: -** Bits 00-04: seconds divided by 2 -** Bits 05-10: minute -** Bits 11-15: hour -** File modification date: -** Bits 00-04: day -** Bits 05-08: month (1-12) -** Bits 09-15: years from 1980 +** Virtual table module xEof method. ** -** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. */ -static u32 zipfileMtime(ZipfileCDS *pCDS){ - int Y,M,D,X1,X2,A,B,sec,min,hr; - i64 JDsec; - Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); - M = ((pCDS->mDate >> 5) & 0x0F); - D = (pCDS->mDate & 0x1F); - sec = (pCDS->mTime & 0x1F)*2; - min = (pCDS->mTime >> 5) & 0x3F; - hr = (pCDS->mTime >> 11) & 0x1F; - if( M<=2 ){ - Y--; - M += 12; - } - X1 = 36525*(Y+4716)/100; - X2 = 306001*(M+1)/10000; - A = Y/100; - B = 2 - A + (A/4); - JDsec = (i64)((X1 + X2 + D + B - 1524.5)*86400) + hr*3600 + min*60 + sec; - return (u32)(JDsec - (i64)24405875*(i64)8640); +static int expertEof(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + return pCsr->pData==0; } -/* -** The opposite of zipfileMtime(). This function populates the mTime and -** mDate fields of the CDS structure passed as the first argument according -** to the UNIX timestamp value passed as the second. +/* +** Virtual table module xNext method. */ -static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ - /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ - i64 JD = (i64)2440588 + mUnixTime / (24*60*60); +static int expertNext(sqlite3_vtab_cursor *cur){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + int rc = SQLITE_OK; - int A, B, C, D, E; - int yr, mon, day; - int hr, min, sec; + assert( pCsr->pData ); + rc = sqlite3_step(pCsr->pData); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + }else{ + rc = SQLITE_OK; + } - A = (int)((JD - 1867216.25)/36524.25); - A = (int)(JD + 1 + A - (A/4)); - B = A + 1524; - C = (int)((B - 122.1)/365.25); - D = (36525*(C&32767))/100; - E = (int)((B-D)/30.6001); + return rc; +} - day = B - D - (int)(30.6001*E); - mon = (E<14 ? E-1 : E-13); - yr = mon>2 ? C-4716 : C-4715; +/* +** Virtual table module xRowid method. +*/ +static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + (void)cur; + *pRowid = 0; + return SQLITE_OK; +} - hr = (mUnixTime % (24*60*60)) / (60*60); - min = (mUnixTime % (60*60)) / 60; - sec = (mUnixTime % 60); +/* +** Virtual table module xColumn method. +*/ +static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + sqlite3_value *pVal; + pVal = sqlite3_column_value(pCsr->pData, i); + if( pVal ){ + sqlite3_result_value(ctx, pVal); + } + return SQLITE_OK; +} - if( yr>=1980 ){ - pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); - pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); - }else{ - pCds->mDate = pCds->mTime = 0; +/* +** Virtual table module xFilter method. +*/ +static int expertFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + ExpertCsr *pCsr = (ExpertCsr*)cur; + ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab); + sqlite3expert *pExpert = pVtab->pExpert; + int rc; + + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; + rc = sqlite3_finalize(pCsr->pData); + pCsr->pData = 0; + if( rc==SQLITE_OK ){ + rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName + ); } - assert( mUnixTime<315507600 - || mUnixTime==zipfileMtime(pCds) - || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) - /* || (mUnixTime % 2) */ - ); + if( rc==SQLITE_OK ){ + rc = expertNext(cur); + } + return rc; } +static int idxRegisterVtab(sqlite3expert *p){ + static sqlite3_module expertModule = { + 2, /* iVersion */ + expertConnect, /* xCreate - create a table */ + expertConnect, /* xConnect - connect to an existing table */ + expertBestIndex, /* xBestIndex - Determine search strategy */ + expertDisconnect, /* xDisconnect - Disconnect from a table */ + expertDisconnect, /* xDestroy - Drop a table */ + expertOpen, /* xOpen - open a cursor */ + expertClose, /* xClose - close a cursor */ + expertFilter, /* xFilter - configure scan constraints */ + expertNext, /* xNext - advance a cursor */ + expertEof, /* xEof */ + expertColumn, /* xColumn - read data */ + expertRowid, /* xRowid - read data */ + expertUpdate, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0, /* xIntegrity */ + }; + + return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); +} /* -** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in -** size) containing an entire zip archive image. Or, if aBlob is NULL, -** then pFile is a file-handle open on a zip file. In either case, this -** function creates a ZipfileEntry object based on the zip archive entry -** for which the CDS record is at offset iOff. +** End of virtual table implementation. +*************************************************************************/ +/* +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function +** is called, set it to the return value of sqlite3_finalize() before +** returning. Otherwise, discard the sqlite3_finalize() return value. +*/ +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +/* +** Attempt to allocate an IdxTable structure corresponding to table zTab +** in the main database of connection db. If successful, set (*ppOut) to +** point to the new object and return SQLITE_OK. Otherwise, return an +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be +** set to point to an error string. ** -** If successful, SQLITE_OK is returned and (*ppEntry) set to point to -** the new object. Otherwise, an SQLite error code is returned and the -** final value of (*ppEntry) undefined. +** It is the responsibility of the caller to eventually free either the +** IdxTable object or error message using sqlite3_free(). */ -static int zipfileGetEntry( - ZipfileTab *pTab, /* Store any error message here */ - const u8 *aBlob, /* Pointer to in-memory file image */ - int nBlob, /* Size of aBlob[] in bytes */ - FILE *pFile, /* If aBlob==0, read from this file */ - i64 iOff, /* Offset of CDS record */ - ZipfileEntry **ppEntry /* OUT: Pointer to new object */ +static int idxGetTableInfo( + sqlite3 *db, /* Database connection to read details from */ + const char *zTab, /* Table name */ + IdxTable **ppOut, /* OUT: New object (if successful) */ + char **pzErrmsg /* OUT: Error message (if not) */ ){ - u8 *aRead; - char **pzErr = &pTab->base.zErrMsg; - int rc = SQLITE_OK; + sqlite3_stmt *p1 = 0; + int nCol = 0; + int nTab; + int nByte; + IdxTable *pNew = 0; + int rc, rc2; + char *pCsr = 0; + int nPk = 0; - if( aBlob==0 ){ - aRead = pTab->aBuffer; - rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); - }else{ - aRead = (u8*)&aBlob[iOff]; + *ppOut = 0; + if( zTab==0 ) return SQLITE_ERROR; + nTab = STRLEN(zTab); + nByte = sizeof(IdxTable) + nTab + 1; + rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + if( zCol==0 ){ + rc = SQLITE_ERROR; + break; + } + nByte += 1 + STRLEN(zCol); + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( zColSeq==0 ) zColSeq = "binary"; + nByte += 1 + STRLEN(zColSeq); + nCol++; + nPk += (sqlite3_column_int(p1, 5)>0); } + rc2 = sqlite3_reset(p1); + if( rc==SQLITE_OK ) rc = rc2; + nByte += sizeof(IdxColumn) * nCol; if( rc==SQLITE_OK ){ - sqlite3_int64 nAlloc; - ZipfileEntry *pNew; + pNew = idxMalloc(&rc, nByte); + } + if( rc==SQLITE_OK ){ + pNew->aCol = (IdxColumn*)&pNew[1]; + pNew->nCol = nCol; + pCsr = (char*)&pNew->aCol[nCol]; + } - int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); - int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); - nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + nCol = 0; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = (const char*)sqlite3_column_text(p1, 1); + const char *zColSeq = 0; + int nCopy; + if( zCol==0 ) continue; + nCopy = STRLEN(zCol) + 1; + pNew->aCol[nCol].zName = pCsr; + pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1); + memcpy(pCsr, zCol, nCopy); + pCsr += nCopy; - nAlloc = sizeof(ZipfileEntry) + nExtra; - if( aBlob ){ - nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + rc = sqlite3_table_column_metadata( + db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + if( zColSeq==0 ) zColSeq = "binary"; + nCopy = STRLEN(zColSeq) + 1; + pNew->aCol[nCol].zColl = pCsr; + memcpy(pCsr, zColSeq, nCopy); + pCsr += nCopy; } - pNew = (ZipfileEntry*)sqlite3_malloc64(nAlloc); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pNew, 0, sizeof(ZipfileEntry)); - rc = zipfileReadCDS(aRead, &pNew->cds); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); - }else if( aBlob==0 ){ - rc = zipfileReadData( - pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr - ); - }else{ - aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; - } - } + nCol++; + } + idxFinalize(&rc, p1); - if( rc==SQLITE_OK ){ - u32 *pt = &pNew->mUnixTime; - pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); - pNew->aExtra = (u8*)&pNew[1]; - memcpy(pNew->aExtra, &aRead[nFile], nExtra); - if( pNew->cds.zFile==0 ){ - rc = SQLITE_NOMEM; - }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ - pNew->mUnixTime = zipfileMtime(&pNew->cds); - } - } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + }else if( ALWAYS(pNew!=0) ){ + pNew->zName = pCsr; + if( ALWAYS(pNew->zName!=0) ) memcpy(pNew->zName, zTab, nTab+1); + } - if( rc==SQLITE_OK ){ - static const int szFix = ZIPFILE_LFH_FIXED_SZ; - ZipfileLFH lfh; - if( pFile ){ - rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); - }else{ - aRead = (u8*)&aBlob[pNew->cds.iOffset]; - } + *ppOut = pNew; + return rc; +} - if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); - if( rc==SQLITE_OK ){ - pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; - pNew->iDataOff += lfh.nFile + lfh.nExtra; - if( aBlob && pNew->cds.szCompressed ){ - pNew->aData = &pNew->aExtra[nExtra]; - memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); - } - }else{ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", - (int)pNew->cds.iOffset - ); - } +/* +** This function is a no-op if *pRc is set to anything other than +** SQLITE_OK when it is called. +** +** If *pRc is initially set to SQLITE_OK, then the text specified by +** the printf() style arguments is appended to zIn and the result returned +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on +** zIn before returning. +*/ +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ + va_list ap; + char *zAppend = 0; + char *zRet = 0; + int nIn = zIn ? STRLEN(zIn) : 0; + int nAppend = 0; + va_start(ap, zFmt); + if( *pRc==SQLITE_OK ){ + zAppend = sqlite3_vmprintf(zFmt, ap); + if( zAppend ){ + nAppend = STRLEN(zAppend); + zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); } - - if( rc!=SQLITE_OK ){ - zipfileEntryFree(pNew); + if( zAppend && zRet ){ + if( nIn ) memcpy(zRet, zIn, nIn); + memcpy(&zRet[nIn], zAppend, nAppend+1); }else{ - *ppEntry = pNew; + sqlite3_free(zRet); + zRet = 0; + *pRc = SQLITE_NOMEM; } + sqlite3_free(zAppend); + sqlite3_free(zIn); } - - return rc; + va_end(ap); + return zRet; } /* -** Advance an ZipfileCsr to its next row of output. +** Return true if zId must be quoted in order to use it as an SQL +** identifier, or false otherwise. */ -static int zipfileNext(sqlite3_vtab_cursor *cur){ - ZipfileCsr *pCsr = (ZipfileCsr*)cur; - int rc = SQLITE_OK; +static int idxIdentifierRequiresQuotes(const char *zId){ + int i; + int nId = STRLEN(zId); + + if( sqlite3_keyword_check(zId, nId) ) return 1; - if( pCsr->pFile ){ - i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; - zipfileEntryFree(pCsr->pCurrent); - pCsr->pCurrent = 0; - if( pCsr->iNextOff>=iEof ){ - pCsr->bEof = 1; - }else{ - ZipfileEntry *p = 0; - ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); - rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); - if( rc==SQLITE_OK ){ - pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; - pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; - } - pCsr->pCurrent = p; + for(i=0; zId[i]; i++){ + if( !(zId[i]=='_') + && !(zId[i]>='0' && zId[i]<='9') + && !(zId[i]>='a' && zId[i]<='z') + && !(zId[i]>='A' && zId[i]<='Z') + ){ + return 1; } + } + return 0; +} + +/* +** This function appends an index column definition suitable for constraint +** pCons to the string passed as zIn and returns the result. +*/ +static char *idxAppendColDefn( + int *pRc, /* IN/OUT: Error code */ + char *zIn, /* Column defn accumulated so far */ + IdxTable *pTab, /* Table index will be created on */ + IdxConstraint *pCons +){ + char *zRet = zIn; + IdxColumn *p = &pTab->aCol[pCons->iCol]; + if( zRet ) zRet = idxAppendText(pRc, zRet, ", "); + + if( idxIdentifierRequiresQuotes(p->zName) ){ + zRet = idxAppendText(pRc, zRet, "%Q", p->zName); }else{ - if( !pCsr->bNoop ){ - pCsr->pCurrent = pCsr->pCurrent->pNext; - } - if( pCsr->pCurrent==0 ){ - pCsr->bEof = 1; - } + zRet = idxAppendText(pRc, zRet, "%s", p->zName); } - pCsr->bNoop = 0; - return rc; -} + if( sqlite3_stricmp(p->zColl, pCons->zColl) ){ + if( idxIdentifierRequiresQuotes(pCons->zColl) ){ + zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl); + }else{ + zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl); + } + } -static void zipfileFree(void *p) { - sqlite3_free(p); + if( pCons->bDesc ){ + zRet = idxAppendText(pRc, zRet, " DESC"); + } + return zRet; } /* -** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the -** size is nOut bytes. This function uncompresses the data and sets the -** return value in context pCtx to the result (a blob). +** Search database dbm for an index compatible with the one idxCreateFromCons() +** would create from arguments pScan, pEq and pTail. If no error occurs and +** such an index is found, return non-zero. Or, if no such index is found, +** return zero. ** -** If an error occurs, an error code is left in pCtx instead. +** If an error occurs, set *pRc to an SQLite error code and return zero. */ -static void zipfileInflate( - sqlite3_context *pCtx, /* Store result here */ - const u8 *aIn, /* Compressed data */ - int nIn, /* Size of buffer aIn[] in bytes */ - int nOut /* Expected output size */ +static int idxFindCompatible( + int *pRc, /* OUT: Error code */ + sqlite3* dbm, /* Database to search */ + IdxScan *pScan, /* Scan for table to search for index on */ + IdxConstraint *pEq, /* List of == constraints */ + IdxConstraint *pTail /* List of range constraints */ ){ - u8 *aRes = sqlite3_malloc(nOut); - if( aRes==0 ){ - sqlite3_result_error_nomem(pCtx); - }else{ - int err; - z_stream str; - memset(&str, 0, sizeof(str)); + const char *zTbl = pScan->pTab->zName; + sqlite3_stmt *pIdxList = 0; + IdxConstraint *pIter; + int nEq = 0; /* Number of elements in pEq */ + int rc; - str.next_in = (Byte*)aIn; - str.avail_in = nIn; - str.next_out = (Byte*)aRes; - str.avail_out = nOut; + /* Count the elements in list pEq */ + for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++; - err = inflateInit2(&str, -15); - if( err!=Z_OK ){ - zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err); - }else{ - err = inflate(&str, Z_NO_FLUSH); - if( err!=Z_STREAM_END ){ - zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); + rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl); + while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){ + int bMatch = 1; + IdxConstraint *pT = pTail; + sqlite3_stmt *pInfo = 0; + const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1); + if( zIdx==0 ) continue; + + /* Zero the IdxConstraint.bFlag values in the pEq list */ + for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0; + + rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx); + while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){ + int iIdx = sqlite3_column_int(pInfo, 0); + int iCol = sqlite3_column_int(pInfo, 1); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + + if( iIdxpLink){ + if( pIter->bFlag ) continue; + if( pIter->iCol!=iCol ) continue; + if( sqlite3_stricmp(pIter->zColl, zColl) ) continue; + pIter->bFlag = 1; + break; + } + if( pIter==0 ){ + bMatch = 0; + break; + } }else{ - sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); - aRes = 0; + if( pT ){ + if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){ + bMatch = 0; + break; + } + pT = pT->pLink; + } } } - sqlite3_free(aRes); - inflateEnd(&str); + idxFinalize(&rc, pInfo); + + if( rc==SQLITE_OK && bMatch ){ + sqlite3_finalize(pIdxList); + return 1; + } + } + idxFinalize(&rc, pIdxList); + + *pRc = rc; + return 0; +} + +/* Callback for sqlite3_exec() with query with leading count(*) column. + * The first argument is expected to be an int*, referent to be incremented + * if that leading column is not exactly '0'. + */ +static int countNonzeros(void* pCount, int nc, + char* azResults[], char* azColumns[]){ + (void)azColumns; /* Suppress unused parameter warning */ + if( nc>0 && (azResults[0][0]!='0' || azResults[0][1]!=0) ){ + *((int *)pCount) += 1; } + return 0; } -/* -** Buffer aIn (size nIn bytes) contains uncompressed data. This function -** compresses it and sets (*ppOut) to point to a buffer containing the -** compressed data. The caller is responsible for eventually calling -** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) -** is set to the size of buffer (*ppOut) in bytes. -** -** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error -** code is returned and an error message left in virtual-table handle -** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this -** case. -*/ -static int zipfileDeflate( - const u8 *aIn, int nIn, /* Input */ - u8 **ppOut, int *pnOut, /* Output */ - char **pzErr /* OUT: Error message */ +static int idxCreateFromCons( + sqlite3expert *p, + IdxScan *pScan, + IdxConstraint *pEq, + IdxConstraint *pTail ){ + sqlite3 *dbm = p->dbm; int rc = SQLITE_OK; - sqlite3_int64 nAlloc; - z_stream str; - u8 *aOut; + if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ + IdxTable *pTab = pScan->pTab; + char *zCols = 0; + char *zIdx = 0; + IdxConstraint *pCons; + unsigned int h = 0; + const char *zFmt; - memset(&str, 0, sizeof(str)); - str.next_in = (Bytef*)aIn; - str.avail_in = nIn; - deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + for(pCons=pEq; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + for(pCons=pTail; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } - nAlloc = deflateBound(&str, nIn); - aOut = (u8*)sqlite3_malloc64(nAlloc); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - int res; - str.next_out = aOut; - str.avail_out = nAlloc; - res = deflate(&str, Z_FINISH); - if( res==Z_STREAM_END ){ - *ppOut = aOut; - *pnOut = (int)str.total_out; - }else{ - sqlite3_free(aOut); - *pzErr = sqlite3_mprintf("zipfile: deflate() error"); - rc = SQLITE_ERROR; + if( rc==SQLITE_OK ){ + /* Hash the list of columns to come up with a name for the index */ + const char *zTable = pScan->pTab->zName; + int quoteTable = idxIdentifierRequiresQuotes(zTable); + char *zName = 0; /* Index name */ + int collisions = 0; + do{ + int i; + char *zFind; + for(i=0; zCols[i]; i++){ + h += ((h<<3) + zCols[i]); + } + sqlite3_free(zName); + zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); + if( zName==0 ) break; + /* Is is unique among table, view and index names? */ + zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q" + " AND type in ('index','table','view')"; + zFind = sqlite3_mprintf(zFmt, zName); + i = 0; + rc = sqlite3_exec(dbm, zFind, countNonzeros, &i, 0); + assert(rc==SQLITE_OK); + sqlite3_free(zFind); + if( i==0 ){ + collisions = 0; + break; + } + ++collisions; + }while( collisions<50 && zName!=0 ); + if( collisions ){ + /* This return means "Gave up trying to find a unique index name." */ + rc = SQLITE_BUSY_TIMEOUT; + }else if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( quoteTable ){ + zFmt = "CREATE INDEX \"%w\" ON \"%w\"(%s)"; + }else{ + zFmt = "CREATE INDEX %s ON %s(%s)"; + } + zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols); + if( !zIdx ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg); + if( rc!=SQLITE_OK ){ + rc = SQLITE_BUSY_TIMEOUT; + }else{ + idxHashAdd(&rc, &p->hIdx, zName, zIdx); + } + } + sqlite3_free(zName); + sqlite3_free(zIdx); + } } - deflateEnd(&str); - } + sqlite3_free(zCols); + } return rc; } - /* -** Return values of columns for the row at which the series_cursor -** is currently pointing. +** Return true if list pList (linked by IdxConstraint.pLink) contains +** a constraint compatible with *p. Otherwise return false. */ -static int zipfileColumn( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ +static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ + IdxConstraint *pCmp; + for(pCmp=pList; pCmp; pCmp=pCmp->pLink){ + if( p->iCol==pCmp->iCol ) return 1; + } + return 0; +} + +static int idxCreateFromWhere( + sqlite3expert *p, + IdxScan *pScan, /* Create indexes for this scan */ + IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ ){ - ZipfileCsr *pCsr = (ZipfileCsr*)cur; - ZipfileCDS *pCDS = &pCsr->pCurrent->cds; - int rc = SQLITE_OK; - switch( i ){ - case 0: /* name */ - sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); - break; - case 1: /* mode */ - /* TODO: Whether or not the following is correct surely depends on - ** the platform on which the archive was created. */ - sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); - break; - case 2: { /* mtime */ - sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); - break; - } - case 3: { /* sz */ - if( sqlite3_vtab_nochange(ctx)==0 ){ - sqlite3_result_int64(ctx, pCDS->szUncompressed); - } - break; + IdxConstraint *p1 = 0; + IdxConstraint *pCon; + int rc; + + /* Gather up all the == constraints. */ + for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){ + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + pCon->pLink = p1; + p1 = pCon; } - case 4: /* rawdata */ - if( sqlite3_vtab_nochange(ctx) ) break; - case 5: { /* data */ - if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ - int sz = pCDS->szCompressed; - int szFinal = pCDS->szUncompressed; - if( szFinal>0 ){ - u8 *aBuf; - u8 *aFree = 0; - if( pCsr->pCurrent->aData ){ - aBuf = pCsr->pCurrent->aData; - }else{ - aBuf = aFree = sqlite3_malloc64(sz); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - FILE *pFile = pCsr->pFile; - if( pFile==0 ){ - pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; - } - rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); - } - } - if( rc==SQLITE_OK ){ - if( i==5 && pCDS->iCompression ){ - zipfileInflate(ctx, aBuf, sz, szFinal); - }else{ - sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); - } - } - sqlite3_free(aFree); - }else{ - /* Figure out if this is a directory or a zero-sized file. Consider - ** it to be a directory either if the mode suggests so, or if - ** the final character in the name is '/'. */ - u32 mode = pCDS->iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ - sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); - } - } + } + + /* Create an index using the == constraints collected above. And the + ** range constraint/ORDER BY terms passed in by the caller, if any. */ + rc = idxCreateFromCons(p, pScan, p1, pTail); + + /* If no range/ORDER BY passed by the caller, create a version of the + ** index for each range constraint. */ + if( pTail==0 ){ + for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ + assert( pCon->pLink==0 ); + if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ + rc = idxCreateFromCons(p, pScan, p1, pCon); } - break; } - case 6: /* method */ - sqlite3_result_int(ctx, pCDS->iCompression); - break; - default: /* z */ - assert( i==7 ); - sqlite3_result_int64(ctx, pCsr->iId); - break; } return rc; } /* -** Return TRUE if the cursor is at EOF. -*/ -static int zipfileEof(sqlite3_vtab_cursor *cur){ - ZipfileCsr *pCsr = (ZipfileCsr*)cur; - return pCsr->bEof; -} - -/* -** If aBlob is not NULL, then it points to a buffer nBlob bytes in size -** containing an entire zip archive image. Or, if aBlob is NULL, then pFile -** is guaranteed to be a file-handle open on a zip file. -** -** This function attempts to locate the EOCD record within the zip archive -** and populate *pEOCD with the results of decoding it. SQLITE_OK is -** returned if successful. Otherwise, an SQLite error code is returned and -** an English language error message may be left in virtual-table pTab. +** Create candidate indexes in database [dbm] based on the data in +** linked-list pScan. */ -static int zipfileReadEOCD( - ZipfileTab *pTab, /* Return errors here */ - const u8 *aBlob, /* Pointer to in-memory file image */ - int nBlob, /* Size of aBlob[] in bytes */ - FILE *pFile, /* Read from this file if aBlob==0 */ - ZipfileEOCD *pEOCD /* Object to populate */ -){ - u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - int nRead; /* Bytes to read from file */ +static int idxCreateCandidates(sqlite3expert *p){ int rc = SQLITE_OK; + IdxScan *pIter; - memset(pEOCD, 0, sizeof(ZipfileEOCD)); - if( aBlob==0 ){ - i64 iOff; /* Offset to read from */ - i64 szFile; /* Total size of file in bytes */ - fseek(pFile, 0, SEEK_END); - szFile = (i64)ftell(pFile); - if( szFile==0 ){ - return SQLITE_OK; + for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + rc = idxCreateFromWhere(p, pIter, 0); + if( rc==SQLITE_OK && pIter->pOrder ){ + rc = idxCreateFromWhere(p, pIter, pIter->pOrder); } - nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); - iOff = szFile - nRead; - rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); - }else{ - nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); - aRead = (u8*)&aBlob[nBlob-nRead]; } - if( rc==SQLITE_OK ){ - int i; + return rc; +} - /* Scan backwards looking for the signature bytes */ - for(i=nRead-20; i>=0; i--){ - if( aRead[i]==0x50 && aRead[i+1]==0x4b - && aRead[i+2]==0x05 && aRead[i+3]==0x06 - ){ - break; - } - } - if( i<0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "cannot find end of central directory record" - ); - return SQLITE_ERROR; - } +/* +** Free all elements of the linked list starting at pConstraint. +*/ +static void idxConstraintFree(IdxConstraint *pConstraint){ + IdxConstraint *pNext; + IdxConstraint *p; - aRead += i+4; - pEOCD->iDisk = zipfileRead16(aRead); - pEOCD->iFirstDisk = zipfileRead16(aRead); - pEOCD->nEntry = zipfileRead16(aRead); - pEOCD->nEntryTotal = zipfileRead16(aRead); - pEOCD->nSize = zipfileRead32(aRead); - pEOCD->iOffset = zipfileRead32(aRead); + for(p=pConstraint; p; p=pNext){ + pNext = p->pNext; + sqlite3_free(p); } - - return rc; } /* -** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry -** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added -** to the end of the list. Otherwise, it is added to the list immediately -** before pBefore (which is guaranteed to be a part of said list). +** Free all elements of the linked list starting from pScan up until pLast +** (pLast is not freed). */ -static void zipfileAddEntry( - ZipfileTab *pTab, - ZipfileEntry *pBefore, - ZipfileEntry *pNew -){ - assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); - assert( pNew->pNext==0 ); - if( pBefore==0 ){ - if( pTab->pFirstEntry==0 ){ - pTab->pFirstEntry = pTab->pLastEntry = pNew; - }else{ - assert( pTab->pLastEntry->pNext==0 ); - pTab->pLastEntry->pNext = pNew; - pTab->pLastEntry = pNew; - } - }else{ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); - pNew->pNext = pBefore; - *pp = pNew; +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ + IdxScan *p; + IdxScan *pNext; + for(p=pScan; p!=pLast; p=pNext){ + pNext = p->pNextScan; + idxConstraintFree(p->pOrder); + idxConstraintFree(p->pEq); + idxConstraintFree(p->pRange); + sqlite3_free(p); } } -static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ - ZipfileEOCD eocd; - int rc; - int i; - i64 iOff; +/* +** Free all elements of the linked list starting from pStatement up +** until pLast (pLast is not freed). +*/ +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ + IdxStatement *p; + IdxStatement *pNext; + for(p=pStatement; p!=pLast; p=pNext){ + pNext = p->pNext; + sqlite3_free(p->zEQP); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} - rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); - iOff = eocd.iOffset; - for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); +/* +** Free the linked list of IdxTable objects starting at pTab. +*/ +static void idxTableFree(IdxTable *pTab){ + IdxTable *pIter; + IdxTable *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} - if( rc==SQLITE_OK ){ - zipfileAddEntry(pTab, 0, pNew); - iOff += ZIPFILE_CDS_FIXED_SZ; - iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; - } +/* +** Free the linked list of IdxWrite objects starting at pTab. +*/ +static void idxWriteFree(IdxWrite *pTab){ + IdxWrite *pIter; + IdxWrite *pNext; + for(pIter=pTab; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); } - return rc; } + + /* -** xFilter callback. +** This function is called after candidate indexes have been created. It +** runs all the queries to see which indexes they prefer, and populates +** IdxStatement.zIdx and IdxStatement.zEQP with the results. */ -static int zipfileFilter( - sqlite3_vtab_cursor *cur, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv +static int idxFindIndexes( + sqlite3expert *p, + char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ - ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; - ZipfileCsr *pCsr = (ZipfileCsr*)cur; - const char *zFile = 0; /* Zip file to scan */ - int rc = SQLITE_OK; /* Return Code */ - int bInMemory = 0; /* True for an in-memory zipfile */ + IdxStatement *pStmt; + sqlite3 *dbm = p->dbm; + int rc = SQLITE_OK; + + IdxHash hIdx; + idxHashInit(&hIdx); + + for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){ + IdxHashEntry *pEntry; + sqlite3_stmt *pExplain = 0; + idxHashClear(&hIdx); + rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr, + "EXPLAIN QUERY PLAN %s", pStmt->zSql + ); + while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ + /* int iId = sqlite3_column_int(pExplain, 0); */ + /* int iParent = sqlite3_column_int(pExplain, 1); */ + /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); + int nDetail; + int i; + + if( !zDetail ) continue; + nDetail = STRLEN(zDetail); + + for(i=0; ihIdx, zIdx, nIdx); + if( zSql ){ + idxHashAdd(&rc, &hIdx, zSql, 0); + if( rc ) goto find_indexes_out; + } + break; + } + } + + if( zDetail[0]!='-' ){ + pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail); + } + } - zipfileResetCursor(pCsr); + for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey); + } - if( pTab->zFile ){ - zFile = pTab->zFile; - }else if( idxNum==0 ){ - zipfileCursorErr(pCsr, "zipfile() function requires an argument"); - return SQLITE_ERROR; - }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); - assert( pTab->pFirstEntry==0 ); - rc = zipfileLoadDirectory(pTab, aBlob, nBlob); - pCsr->pFreeEntry = pTab->pFirstEntry; - pTab->pFirstEntry = pTab->pLastEntry = 0; - if( rc!=SQLITE_OK ) return rc; - bInMemory = 1; - }else{ - zFile = (const char*)sqlite3_value_text(argv[0]); + idxFinalize(&rc, pExplain); } - if( 0==pTab->pWriteFd && 0==bInMemory ){ - pCsr->pFile = fopen(zFile, "rb"); - if( pCsr->pFile==0 ){ - zipfileCursorErr(pCsr, "cannot open file: %s", zFile); - rc = SQLITE_ERROR; - }else{ - rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); - if( rc==SQLITE_OK ){ - if( pCsr->eocd.nEntry==0 ){ - pCsr->bEof = 1; - }else{ - pCsr->iNextOff = pCsr->eocd.iOffset; - rc = zipfileNext(cur); + find_indexes_out: + idxHashClear(&hIdx); + return rc; +} + +static int idxAuthCallback( + void *pCtx, + int eOp, + const char *z3, + const char *z4, + const char *zDb, + const char *zTrigger +){ + int rc = SQLITE_OK; + (void)z4; + (void)zTrigger; + if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){ + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3expert *p = (sqlite3expert*)pCtx; + IdxTable *pTab; + for(pTab=p->pTable; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_stricmp(z3, pTab->zName) ) break; + } + if( pTab ){ + IdxWrite *pWrite; + for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){ + if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break; + } + if( pWrite==0 ){ + pWrite = idxMalloc(&rc, sizeof(IdxWrite)); + if( rc==SQLITE_OK ){ + pWrite->pTab = pTab; + pWrite->eOp = eOp; + pWrite->pNext = p->pWrite; + p->pWrite = pWrite; + } } } } - }else{ - pCsr->bNoop = 1; - pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; - rc = zipfileNext(cur); } - return rc; } -/* -** xBestIndex callback. -*/ -static int zipfileBestIndex( - sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo +static int idxProcessOneTrigger( + sqlite3expert *p, + IdxWrite *pWrite, + char **pzErr ){ - int i; - int idx = -1; - int unusable = 0; + static const char *zInt = UNIQUE_TABLE_NAME; + static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME; + IdxTable *pTab = pWrite->pTab; + const char *zTab = pTab->zName; + const char *zSql = + "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_schema " + "WHERE tbl_name = %Q AND type IN ('table', 'trigger') " + "ORDER BY type;"; + sqlite3_stmt *pSelect = 0; + int rc = SQLITE_OK; + char *zWrite = 0; - for(i=0; inConstraint; i++){ - const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; - if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; - if( pCons->usable==0 ){ - unusable = 1; - }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - idx = i; - } - } - pIdxInfo->estimatedCost = 1000.0; - if( idx>=0 ){ - pIdxInfo->aConstraintUsage[idx].argvIndex = 1; - pIdxInfo->aConstraintUsage[idx].omit = 1; - pIdxInfo->idxNum = 1; - }else if( unusable ){ - return SQLITE_CONSTRAINT; + /* Create the table and its triggers in the temp schema */ + rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0); + if( zCreate==0 ) continue; + rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr); } - return SQLITE_OK; -} + idxFinalize(&rc, pSelect); -static ZipfileEntry *zipfileNewEntry(const char *zPath){ - ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); - if( pNew ){ - memset(pNew, 0, sizeof(ZipfileEntry)); - pNew->cds.zFile = sqlite3_mprintf("%s", zPath); - if( pNew->cds.zFile==0 ){ - sqlite3_free(pNew); - pNew = 0; + /* Rename the table in the temp schema to zInt */ + if( rc==SQLITE_OK ){ + char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt); + if( z==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr); + sqlite3_free(z); } } - return pNew; -} - -static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ - ZipfileCDS *pCds = &pEntry->cds; - u8 *a = aBuf; - - pCds->nExtra = 9; - - /* Write the LFH itself */ - zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); - zipfileWrite16(a, pCds->iVersionExtract); - zipfileWrite16(a, pCds->flags); - zipfileWrite16(a, pCds->iCompression); - zipfileWrite16(a, pCds->mTime); - zipfileWrite16(a, pCds->mDate); - zipfileWrite32(a, pCds->crc32); - zipfileWrite32(a, pCds->szCompressed); - zipfileWrite32(a, pCds->szUncompressed); - zipfileWrite16(a, (u16)pCds->nFile); - zipfileWrite16(a, pCds->nExtra); - assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); - - /* Add the file name */ - memcpy(a, pCds->zFile, (int)pCds->nFile); - a += (int)pCds->nFile; - /* The "extra" data */ - zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(a, 5); - *a++ = 0x01; - zipfileWrite32(a, pEntry->mUnixTime); - - return a-aBuf; -} + switch( pWrite->eOp ){ + case SQLITE_INSERT: { + int i; + zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", "); + } + zWrite = idxAppendText(&rc, zWrite, ")"); + break; + } + case SQLITE_UPDATE: { + int i; + zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt); + for(i=0; inCol; i++){ + zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", + pTab->aCol[i].zName + ); + } + break; + } + default: { + assert( pWrite->eOp==SQLITE_DELETE ); + if( rc==SQLITE_OK ){ + zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt); + if( zWrite==0 ) rc = SQLITE_NOMEM; + } + } + } -static int zipfileAppendEntry( - ZipfileTab *pTab, - ZipfileEntry *pEntry, - const u8 *pData, - int nData -){ - u8 *aBuf = pTab->aBuffer; - int nBuf; - int rc; + if( rc==SQLITE_OK ){ + sqlite3_stmt *pX = 0; + rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0); + idxFinalize(&rc, pX); + if( rc!=SQLITE_OK ){ + idxDatabaseError(p->dbv, pzErr); + } + } + sqlite3_free(zWrite); - nBuf = zipfileSerializeLFH(pEntry, aBuf); - rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ - pEntry->iDataOff = pTab->szCurrent; - rc = zipfileAppendData(pTab, pData, nData); + rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr); } return rc; } -static int zipfileGetMode( - sqlite3_value *pVal, - int bIsDir, /* If true, default to directory */ - u32 *pMode, /* OUT: Mode value */ - char **pzErr /* OUT: Error message */ -){ - const char *z = (const char*)sqlite3_value_text(pVal); - u32 mode = 0; - if( z==0 ){ - mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); - }else if( z[0]>='0' && z[0]<='9' ){ - mode = (unsigned int)sqlite3_value_int(pVal); - }else{ - const char zTemplate[11] = "-rwxrwxrwx"; - int i; - if( strlen(z)!=10 ) goto parse_error; - switch( z[0] ){ - case '-': mode |= S_IFREG; break; - case 'd': mode |= S_IFDIR; break; - case 'l': mode |= S_IFLNK; break; - default: goto parse_error; - } - for(i=1; i<10; i++){ - if( z[i]==zTemplate[i] ) mode |= 1 << (9-i); - else if( z[i]!='-' ) goto parse_error; +static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + IdxWrite *pEnd = 0; + IdxWrite *pFirst = p->pWrite; + + while( rc==SQLITE_OK && pFirst!=pEnd ){ + IdxWrite *pIter; + for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){ + rc = idxProcessOneTrigger(p, pIter, pzErr); } + pEnd = pFirst; + pFirst = p->pWrite; } - if( ((mode & S_IFDIR)==0)==bIsDir ){ - /* The "mode" attribute is a directory, but data has been specified. - ** Or vice-versa - no data but "mode" is a file or symlink. */ - *pzErr = sqlite3_mprintf("zipfile: mode does not match data"); - return SQLITE_CONSTRAINT; - } - *pMode = mode; - return SQLITE_OK; - parse_error: - *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); - return SQLITE_ERROR; + return rc; } -/* -** Both (const char*) arguments point to nul-terminated strings. Argument -** nB is the value of strlen(zB). This function returns 0 if the strings are -** identical, ignoring any trailing '/' character in either path. */ -static int zipfileComparePath(const char *zA, const char *zB, int nB){ - int nA = (int)strlen(zA); - if( nA>0 && zA[nA-1]=='/' ) nA--; - if( nB>0 && zB[nB-1]=='/' ) nB--; - if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; - return 1; -} -static int zipfileBegin(sqlite3_vtab *pVtab){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; +static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ + int rc = idxRegisterVtab(p); + sqlite3_stmt *pSchema = 0; - assert( pTab->pWriteFd==0 ); - if( pTab->zFile==0 || pTab->zFile[0]==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); - return SQLITE_ERROR; - } + /* For each table in the main db schema: + ** + ** 1) Add an entry to the p->pTable list, and + ** 2) Create the equivalent virtual table in dbv. + */ + rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg, + "SELECT type, name, sql, 1 FROM sqlite_schema " + "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' " + " UNION ALL " + "SELECT type, name, sql, 2 FROM sqlite_schema " + "WHERE type = 'trigger'" + " AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') " + "ORDER BY 4, 1" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){ + const char *zType = (const char*)sqlite3_column_text(pSchema, 0); + const char *zName = (const char*)sqlite3_column_text(pSchema, 1); + const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; - }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab, 0, 0); - } + if( zType==0 || zName==0 ) continue; + if( zType[0]=='v' || zType[1]=='r' ){ + if( zSql ) rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg); + }else{ + IdxTable *pTab; + rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); + if( rc==SQLITE_OK ){ + int i; + char *zInner = 0; + char *zOuter = 0; + pTab->pNext = p->pTable; + p->pTable = pTab; - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); - } + /* The statement the vtab will pass to sqlite3_declare_vtab() */ + zInner = idxAppendText(&rc, 0, "CREATE TABLE x("); + for(i=0; inCol; i++){ + zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", + (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl + ); + } + zInner = idxAppendText(&rc, zInner, ")"); + /* The CVT statement to create the vtab */ + zOuter = idxAppendText(&rc, 0, + "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg); + } + sqlite3_free(zInner); + sqlite3_free(zOuter); + } + } + } + idxFinalize(&rc, pSchema); return rc; } -/* -** Return the current time as a 32-bit timestamp in UNIX epoch format (like -** time(2)). -*/ -static u32 zipfileTime(void){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - u32 ret; - if( pVfs==0 ) return 0; - if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ - i64 ms; - pVfs->xCurrentTimeInt64(pVfs, &ms); - ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); +struct IdxSampleCtx { + int iTarget; + double target; /* Target nRet/nRow value */ + double nRow; /* Number of rows seen */ + double nRet; /* Number of rows returned */ +}; + +static void idxSampleFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx); + int bRet; + + (void)argv; + assert( argc==0 ); + if( p->nRow==0.0 ){ + bRet = 1; }else{ - double day; - pVfs->xCurrentTime(pVfs, &day); - ret = (u32)((day - 2440587.5) * 86400); + bRet = (p->nRet / p->nRow) <= p->target; + if( bRet==0 ){ + unsigned short rnd; + sqlite3_randomness(2, (void*)&rnd); + bRet = ((int)rnd % 100) <= p->iTarget; + } } - return ret; -} -/* -** Return a 32-bit timestamp in UNIX epoch format. -** -** If the value passed as the only argument is either NULL or an SQL NULL, -** return the current time. Otherwise, return the value stored in (*pVal) -** cast to a 32-bit unsigned integer. -*/ -static u32 zipfileGetTime(sqlite3_value *pVal){ - if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ - return zipfileTime(); - } - return (u32)sqlite3_value_int64(pVal); + sqlite3_result_int(pCtx, bRet); + p->nRow += 1.0; + p->nRet += (double)bRet; } -/* -** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry -** linked list. Remove it from the list and free the object. -*/ -static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ - if( pOld ){ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; - zipfileEntryFree(pOld); - } -} +struct IdxRemCtx { + int nSlot; + struct IdxRemSlot { + int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ + i64 iVal; /* SQLITE_INTEGER value */ + double rVal; /* SQLITE_FLOAT value */ + int nByte; /* Bytes of space allocated at z */ + int n; /* Size of buffer z */ + char *z; /* SQLITE_TEXT/BLOB value */ + } aSlot[1]; +}; /* -** xUpdate method. +** Implementation of scalar function sqlite_expert_rem(). */ -static int zipfileUpdate( - sqlite3_vtab *pVtab, - int nVal, - sqlite3_value **apVal, - sqlite_int64 *pRowid +static void idxRemFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv ){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; /* Return Code */ - ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ - - u32 mode = 0; /* Mode for new entry */ - u32 mTime = 0; /* Modification time for new entry */ - i64 sz = 0; /* Uncompressed size */ - const char *zPath = 0; /* Path for new entry */ - int nPath = 0; /* strlen(zPath) */ - const u8 *pData = 0; /* Pointer to buffer containing content */ - int nData = 0; /* Size of pData buffer in bytes */ - int iMethod = 0; /* Compression method for new entry */ - u8 *pFree = 0; /* Free this */ - char *zFree = 0; /* Also free this */ - ZipfileEntry *pOld = 0; - ZipfileEntry *pOld2 = 0; - int bUpdate = 0; /* True for an update that modifies "name" */ - int bIsDir = 0; - u32 iCrc32 = 0; - - if( pTab->pWriteFd==0 ){ - rc = zipfileBegin(pVtab); - if( rc!=SQLITE_OK ) return rc; - } + struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx); + struct IdxRemSlot *pSlot; + int iSlot; + assert( argc==2 ); - /* If this is a DELETE or UPDATE, find the archive entry to delete. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); - int nDelete = (int)strlen(zDelete); - if( nVal>1 ){ - const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]); - if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){ - bUpdate = 1; - } - } - for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ - if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ - break; - } - assert( pOld->pNext ); - } - } + iSlot = sqlite3_value_int(argv[0]); + assert( iSlotnSlot ); + pSlot = &p->aSlot[iSlot]; - if( nVal>1 ){ - /* Check that "sz" and "rawdata" are both NULL: */ - if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){ - zipfileTableErr(pTab, "sz must be NULL"); - rc = SQLITE_CONSTRAINT; - } - if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){ - zipfileTableErr(pTab, "rawdata must be NULL"); - rc = SQLITE_CONSTRAINT; - } + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; - if( rc==SQLITE_OK ){ - if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ - /* data=NULL. A directory */ - bIsDir = 1; - }else{ - /* Value specified for "data", and possibly "method". This must be - ** a regular file or a symlink. */ - const u8 *aIn = sqlite3_value_blob(apVal[7]); - int nIn = sqlite3_value_bytes(apVal[7]); - int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + case SQLITE_INTEGER: + sqlite3_result_int64(pCtx, pSlot->iVal); + break; - iMethod = sqlite3_value_int(apVal[8]); - sz = nIn; - pData = aIn; - nData = nIn; - if( iMethod!=0 && iMethod!=8 ){ - zipfileTableErr(pTab, "unknown compression method: %d", iMethod); - rc = SQLITE_CONSTRAINT; - }else{ - if( bAuto || iMethod ){ - int nCmp; - rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); - if( rc==SQLITE_OK ){ - if( iMethod || nCmprVal); + break; - if( rc==SQLITE_OK ){ - rc = zipfileGetMode(apVal[3], bIsDir, &mode, &pTab->base.zErrMsg); - } + case SQLITE_BLOB: + sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; - if( rc==SQLITE_OK ){ - zPath = (const char*)sqlite3_value_text(apVal[2]); - if( zPath==0 ) zPath = ""; - nPath = (int)strlen(zPath); - mTime = zipfileGetTime(apVal[4]); - } + case SQLITE_TEXT: + sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + break; + } - if( rc==SQLITE_OK && bIsDir ){ - /* For a directory, check that the last character in the path is a - ** '/'. This appears to be required for compatibility with info-zip - ** (the unzip command on unix). It does not create directories - ** otherwise. */ - if( nPath<=0 || zPath[nPath-1]!='/' ){ - zFree = sqlite3_mprintf("%s/", zPath); - zPath = (const char*)zFree; - if( zFree==0 ){ - rc = SQLITE_NOMEM; - nPath = 0; - }else{ - nPath = (int)strlen(zPath); - } - } - } + pSlot->eType = sqlite3_value_type(argv[1]); + switch( pSlot->eType ){ + case SQLITE_NULL: + /* no-op */ + break; - /* Check that we're not inserting a duplicate entry -OR- updating an - ** entry with a path, thereby making it into a duplicate. */ - if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ - ZipfileEntry *p; - for(p=pTab->pFirstEntry; p; p=p->pNext){ - if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ - switch( sqlite3_vtab_on_conflict(pTab->db) ){ - case SQLITE_IGNORE: { - goto zipfile_update_done; - } - case SQLITE_REPLACE: { - pOld2 = p; - break; - } - default: { - zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath); - rc = SQLITE_CONSTRAINT; - break; - } - } - break; + case SQLITE_INTEGER: + pSlot->iVal = sqlite3_value_int64(argv[1]); + break; + + case SQLITE_FLOAT: + pSlot->rVal = sqlite3_value_double(argv[1]); + break; + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int nByte = sqlite3_value_bytes(argv[1]); + const void *pData = 0; + if( nByte>pSlot->nByte ){ + char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + if( zNew==0 ){ + sqlite3_result_error_nomem(pCtx); + return; } + pSlot->nByte = nByte*2; + pSlot->z = zNew; } - } - - if( rc==SQLITE_OK ){ - /* Create the new CDS record. */ - pNew = zipfileNewEntry(zPath); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + pSlot->n = nByte; + if( pSlot->eType==SQLITE_BLOB ){ + pData = sqlite3_value_blob(argv[1]); + if( pData ) memcpy(pSlot->z, pData, nByte); }else{ - pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; - pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; - pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; - pNew->cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&pNew->cds, mTime); - pNew->cds.crc32 = iCrc32; - pNew->cds.szCompressed = nData; - pNew->cds.szUncompressed = (u32)sz; - pNew->cds.iExternalAttr = (mode<<16); - pNew->cds.iOffset = (u32)pTab->szCurrent; - pNew->cds.nFile = (u16)nPath; - pNew->mUnixTime = (u32)mTime; - rc = zipfileAppendEntry(pTab, pNew, pData, nData); - zipfileAddEntry(pTab, pOld, pNew); + pData = sqlite3_value_text(argv[1]); + memcpy(pSlot->z, pData, nByte); } + break; } } +} - if( rc==SQLITE_OK && (pOld || pOld2) ){ - ZipfileCsr *pCsr; - for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ - if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){ - pCsr->pCurrent = pCsr->pCurrent->pNext; - pCsr->bNoop = 1; - } - } +static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){ + int rc = SQLITE_OK; + const char *zMax = + "SELECT max(i.seqno) FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l, " + " pragma_index_info(l.name) AS i " + "WHERE s.type = 'table'"; + sqlite3_stmt *pMax = 0; - zipfileRemoveEntryFromList(pTab, pOld); - zipfileRemoveEntryFromList(pTab, pOld2); + *pnMax = 0; + rc = idxPrepareStmt(db, &pMax, pzErr, zMax); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + *pnMax = sqlite3_column_int(pMax, 0) + 1; } + idxFinalize(&rc, pMax); -zipfile_update_done: - sqlite3_free(pFree); - sqlite3_free(zFree); return rc; } -static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ - u8 *a = aBuf; - zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); - zipfileWrite16(a, p->iDisk); - zipfileWrite16(a, p->iFirstDisk); - zipfileWrite16(a, p->nEntry); - zipfileWrite16(a, p->nEntryTotal); - zipfileWrite32(a, p->nSize); - zipfileWrite32(a, p->iOffset); - zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ +static int idxPopulateOneStat1( + sqlite3expert *p, + sqlite3_stmt *pIndexXInfo, + sqlite3_stmt *pWriteStat, + const char *zTab, + const char *zIdx, + char **pzErr +){ + char *zCols = 0; + char *zOrder = 0; + char *zQuery = 0; + int nCol = 0; + int i; + sqlite3_stmt *pQuery = 0; + int *aStat = 0; + int rc = SQLITE_OK; - return a-aBuf; -} + assert( p->iSample>0 ); -static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ - int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); - assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); - return zipfileAppendData(pTab, pTab->aBuffer, nBuf); -} + /* Formulate the query text */ + sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC); + while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){ + const char *zComma = zCols==0 ? "" : ", "; + const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); + const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); + zCols = idxAppendText(&rc, zCols, + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl + ); + zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); + } + sqlite3_reset(pIndexXInfo); + if( rc==SQLITE_OK ){ + if( p->iSample==100 ){ + zQuery = sqlite3_mprintf( + "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder + ); + }else{ + zQuery = sqlite3_mprintf( + "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder + ); + } + } + sqlite3_free(zCols); + sqlite3_free(zOrder); -/* -** Serialize the CDS structure into buffer aBuf[]. Return the number -** of bytes written. -*/ -static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ - u8 *a = aBuf; - ZipfileCDS *pCDS = &pEntry->cds; + /* Formulate the query text */ + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery); + } + sqlite3_free(zQuery); - if( pEntry->aExtra==0 ){ - pCDS->nExtra = 9; + if( rc==SQLITE_OK ){ + aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); } + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + IdxHashEntry *pEntry; + char *zStat = 0; + for(i=0; i<=nCol; i++) aStat[i] = 1; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ + aStat[0]++; + for(i=0; iiVersionMadeBy); - zipfileWrite16(a, pCDS->iVersionExtract); - zipfileWrite16(a, pCDS->flags); - zipfileWrite16(a, pCDS->iCompression); - zipfileWrite16(a, pCDS->mTime); - zipfileWrite16(a, pCDS->mDate); - zipfileWrite32(a, pCDS->crc32); - zipfileWrite32(a, pCDS->szCompressed); - zipfileWrite32(a, pCDS->szUncompressed); - assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); - zipfileWrite16(a, pCDS->nFile); - zipfileWrite16(a, pCDS->nExtra); - zipfileWrite16(a, pCDS->nComment); - zipfileWrite16(a, pCDS->iDiskStart); - zipfileWrite16(a, pCDS->iInternalAttr); - zipfileWrite32(a, pCDS->iExternalAttr); - zipfileWrite32(a, pCDS->iOffset); + if( rc==SQLITE_OK ){ + int s0 = aStat[0]; + zStat = sqlite3_mprintf("%d", s0); + if( zStat==0 ) rc = SQLITE_NOMEM; + for(i=1; rc==SQLITE_OK && i<=nCol; i++){ + zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]); + } + } - memcpy(a, pCDS->zFile, pCDS->nFile); - a += pCDS->nFile; + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC); + sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC); + sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC); + sqlite3_step(pWriteStat); + rc = sqlite3_reset(pWriteStat); + } - if( pEntry->aExtra ){ - int n = (int)pCDS->nExtra + (int)pCDS->nComment; - memcpy(a, pEntry->aExtra, n); - a += n; - }else{ - assert( pCDS->nExtra==9 ); - zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(a, 5); - *a++ = 0x01; - zipfileWrite32(a, pEntry->mUnixTime); + pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx)); + if( pEntry ){ + assert( pEntry->zVal2==0 ); + pEntry->zVal2 = zStat; + }else{ + sqlite3_free(zStat); + } } + sqlite3_free(aStat); + idxFinalize(&rc, pQuery); - return a-aBuf; + return rc; } -static int zipfileCommit(sqlite3_vtab *pVtab){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; - if( pTab->pWriteFd ){ - i64 iOffset = pTab->szCurrent; - ZipfileEntry *p; - ZipfileEOCD eocd; - int nEntry = 0; +static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){ + int rc; + char *zSql; - /* Write out all entries */ - for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ - int n = zipfileSerializeCDS(p, pTab->aBuffer); - rc = zipfileAppendData(pTab, pTab->aBuffer, n); - nEntry++; - } + rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + if( rc!=SQLITE_OK ) return rc; - /* Write out the EOCD record */ - eocd.iDisk = 0; - eocd.iFirstDisk = 0; - eocd.nEntry = (u16)nEntry; - eocd.nEntryTotal = (u16)nEntry; - eocd.nSize = (u32)(pTab->szCurrent - iOffset); - eocd.iOffset = (u32)iOffset; - rc = zipfileAppendEOCD(pTab, &eocd); + zSql = sqlite3_mprintf( + "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab + ); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0); + sqlite3_free(zSql); - zipfileCleanupTransaction(pTab); - } return rc; } -static int zipfileRollback(sqlite3_vtab *pVtab){ - return zipfileCommit(pVtab); -} +/* +** This function is called as part of sqlite3_expert_analyze(). Candidate +** indexes have already been created in database sqlite3expert.dbm, this +** function populates sqlite_stat1 table in the same database. +** +** The stat1 data is generated by querying the +*/ +static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + int nMax =0; + struct IdxRemCtx *pCtx = 0; + struct IdxSampleCtx samplectx; + int i; + i64 iPrev = -100000; + sqlite3_stmt *pAllIndex = 0; + sqlite3_stmt *pIndexXInfo = 0; + sqlite3_stmt *pWrite = 0; -static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){ - ZipfileCsr *pCsr; - for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ - if( iId==pCsr->iId ) break; - } - return pCsr; -} + const char *zAllIndex = + "SELECT s.rowid, s.name, l.name FROM " + " sqlite_schema AS s, " + " pragma_index_list(s.name) AS l " + "WHERE s.type = 'table'"; + const char *zIndexXInfo = + "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key"; + const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)"; -static void zipfileFunctionCds( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - ZipfileCsr *pCsr; - ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); - assert( argc>0 ); + /* If iSample==0, no sqlite_stat1 data is required. */ + if( p->iSample==0 ) return SQLITE_OK; - pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); - if( pCsr ){ - ZipfileCDS *p = &pCsr->pCurrent->cds; - char *zRes = sqlite3_mprintf("{" - "\"version-made-by\" : %u, " - "\"version-to-extract\" : %u, " - "\"flags\" : %u, " - "\"compression\" : %u, " - "\"time\" : %u, " - "\"date\" : %u, " - "\"crc32\" : %u, " - "\"compressed-size\" : %u, " - "\"uncompressed-size\" : %u, " - "\"file-name-length\" : %u, " - "\"extra-field-length\" : %u, " - "\"file-comment-length\" : %u, " - "\"disk-number-start\" : %u, " - "\"internal-attr\" : %u, " - "\"external-attr\" : %u, " - "\"offset\" : %u }", - (u32)p->iVersionMadeBy, (u32)p->iVersionExtract, - (u32)p->flags, (u32)p->iCompression, - (u32)p->mTime, (u32)p->mDate, - (u32)p->crc32, (u32)p->szCompressed, - (u32)p->szUncompressed, (u32)p->nFile, - (u32)p->nExtra, (u32)p->nComment, - (u32)p->iDiskStart, (u32)p->iInternalAttr, - (u32)p->iExternalAttr, (u32)p->iOffset - ); + rc = idxLargestIndex(p->dbm, &nMax, pzErr); + if( nMax<=0 || rc!=SQLITE_OK ) return rc; - if( zRes==0 ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT); - sqlite3_free(zRes); - } + rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); + + if( rc==SQLITE_OK ){ + int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); } -} -/* -** xFindFunction method. -*/ -static int zipfileFindFunction( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* OUT: User data for *pxFunc */ -){ - if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ - *pxFunc = zipfileFunctionCds; - *ppArg = (void*)pVtab; - return 1; + if( rc==SQLITE_OK ){ + sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + ); } - return 0; -} -typedef struct ZipfileBuffer ZipfileBuffer; -struct ZipfileBuffer { - u8 *a; /* Pointer to buffer */ - int n; /* Size of buffer in bytes */ - int nAlloc; /* Byte allocated at a[] */ -}; + if( rc==SQLITE_OK ){ + pCtx->nSlot = nMax+1; + rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo); + } + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite); + } -typedef struct ZipfileCtx ZipfileCtx; -struct ZipfileCtx { - int nEntry; - ZipfileBuffer body; - ZipfileBuffer cds; -}; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){ + i64 iRowid = sqlite3_column_int64(pAllIndex, 0); + const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1); + const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2); + if( zTab==0 || zIdx==0 ) continue; + if( p->iSample<100 && iPrev!=iRowid ){ + samplectx.target = (double)p->iSample / 100.0; + samplectx.iTarget = p->iSample; + samplectx.nRow = 0.0; + samplectx.nRet = 0.0; + rc = idxBuildSampleTable(p, zTab); + if( rc!=SQLITE_OK ) break; + } + rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr); + iPrev = iRowid; + } + if( rc==SQLITE_OK && p->iSample<100 ){ + rc = sqlite3_exec(p->dbv, + "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0 + ); + } -static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ - if( pBuf->n+nByte>pBuf->nAlloc ){ - u8 *aNew; - sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; - int nReq = pBuf->n + nByte; + idxFinalize(&rc, pAllIndex); + idxFinalize(&rc, pIndexXInfo); + idxFinalize(&rc, pWrite); + + if( pCtx ){ + for(i=0; inSlot; i++){ + sqlite3_free(pCtx->aSlot[i].z); + } + sqlite3_free(pCtx); + } - while( nNewa, nNew); - if( aNew==0 ) return SQLITE_NOMEM; - pBuf->a = aNew; - pBuf->nAlloc = (int)nNew; + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } - return SQLITE_OK; + + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); + return rc; } /* -** xStep() callback for the zipfile() aggregate. This can be called in -** any of the following ways: -** -** SELECT zipfile(name,data) ... -** SELECT zipfile(name,mode,mtime,data) ... -** SELECT zipfile(name,mode,mtime,data,method) ... +** Define and possibly pretend to use a useless collation sequence. +** This pretense allows expert to accept SQL using custom collations. */ -void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ - ZipfileCtx *p; /* Aggregate function context */ - ZipfileEntry e; /* New entry to add to zip archive */ - - sqlite3_value *pName = 0; - sqlite3_value *pMode = 0; - sqlite3_value *pMtime = 0; - sqlite3_value *pData = 0; - sqlite3_value *pMethod = 0; - - int bIsDir = 0; - u32 mode; - int rc = SQLITE_OK; - char *zErr = 0; +int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ + (void)up1; + (void)up2; + (void)up3; + (void)up4; + (void)up5; + assert(0); /* VDBE should never be run. */ + return 0; +} +/* And a callback to register above upon actual need */ +void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ + (void)up1; + sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0); +} - int iMethod = -1; /* Compression method to use (0 or 8) */ +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) +/* +** dummy functions for no-op implementation of UDFs during expert's work +*/ +void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ + (void)up1; + (void)up2; + (void)up3; + assert(0); /* VDBE should never be run. */ +} +void dummyUDFvalue(sqlite3_context *up1){ + (void)up1; + assert(0); /* VDBE should never be run. */ +} - const u8 *aData = 0; /* Possibly compressed data for new entry */ - int nData = 0; /* Size of aData[] in bytes */ - int szUncompressed = 0; /* Size of data before compression */ - u8 *aFree = 0; /* Free this before returning */ - u32 iCrc32 = 0; /* crc32 of uncompressed data */ +/* +** Register UDFs from user database with another. +*/ +int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ + sqlite3_stmt *pStmt; + int rc = sqlite3_prepare_v2(dbSrc, + "SELECT name,type,enc,narg,flags " + "FROM pragma_function_list() " + "WHERE builtin==0", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + int nargs = sqlite3_column_int(pStmt,3); + int flags = sqlite3_column_int(pStmt,4); + const char *name = (char*)sqlite3_column_text(pStmt,0); + const char *type = (char*)sqlite3_column_text(pStmt,1); + const char *enc = (char*)sqlite3_column_text(pStmt,2); + if( name==0 || type==0 || enc==0 ){ + /* no-op. Only happens on OOM */ + }else{ + int ienc = SQLITE_UTF8; + int rcf = SQLITE_ERROR; + if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE; + else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE; + ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY)); + if( strcmp(type,"w")==0 ){ + rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0, + dummyUDF,dummyUDFvalue,0,0,0); + }else if( strcmp(type,"a")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + 0,dummyUDF,dummyUDFvalue); + }else if( strcmp(type,"s")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + dummyUDF,0,0); + } + if( rcf!=SQLITE_OK ){ + rc = rcf; + break; + } + } + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + return rc; +} +#endif - char *zName = 0; /* Path (name) of new entry */ - int nName = 0; /* Size of zName in bytes */ - char *zFree = 0; /* Free this before returning */ - int nByte; +/* +** Allocate a new sqlite3expert object. +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ + int rc = SQLITE_OK; + sqlite3expert *pNew; - memset(&e, 0, sizeof(e)); - p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); - if( p==0 ) return; + pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert)); - /* Martial the arguments into stack variables */ - if( nVal!=2 && nVal!=4 && nVal!=5 ){ - zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); - rc = SQLITE_ERROR; - goto zipfile_step_out; + /* Open two in-memory databases to work with. The "vtab database" (dbv) + ** will contain a virtual table corresponding to each real table in + ** the user database schema, and a copy of each view. It is used to + ** collect information regarding the WHERE, ORDER BY and other clauses + ** of the user's query. + */ + if( rc==SQLITE_OK ){ + pNew->db = db; + pNew->iSample = 100; + rc = sqlite3_open(":memory:", &pNew->dbv); } - pName = apVal[0]; - if( nVal==2 ){ - pData = apVal[1]; - }else{ - pMode = apVal[1]; - pMtime = apVal[2]; - pData = apVal[3]; - if( nVal==5 ){ - pMethod = apVal[4]; + if( rc==SQLITE_OK ){ + rc = sqlite3_open(":memory:", &pNew->dbm); + if( rc==SQLITE_OK ){ + sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); } } - /* Check that the 'name' parameter looks ok. */ - zName = (char*)sqlite3_value_text(pName); - nName = sqlite3_value_bytes(pName); - if( zName==0 ){ - zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); - rc = SQLITE_ERROR; - goto zipfile_step_out; - } + /* Allow custom collations to be dealt with through prepare. */ + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS); + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS); - /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use - ** deflate compression) or NULL (choose automatically). */ - if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ - iMethod = (int)sqlite3_value_int64(pMethod); - if( iMethod!=0 && iMethod!=8 ){ - zErr = sqlite3_mprintf("illegal method value: %d", iMethod); - rc = SQLITE_ERROR; - goto zipfile_step_out; - } +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + /* Register UDFs from database [db] with [dbm] and [dbv]. */ + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbm); } - - /* Now inspect the data. If this is NULL, then the new entry must be a - ** directory. Otherwise, figure out whether or not the data should - ** be deflated or simply stored in the zip archive. */ - if( sqlite3_value_type(pData)==SQLITE_NULL ){ - bIsDir = 1; - iMethod = 0; - }else{ - aData = sqlite3_value_blob(pData); - szUncompressed = nData = sqlite3_value_bytes(pData); - iCrc32 = crc32(0, aData, nData); - if( iMethod<0 || iMethod==8 ){ - int nOut = 0; - rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); - if( rc!=SQLITE_OK ){ - goto zipfile_step_out; - } - if( iMethod==8 || nOutdb, pNew->dbv); } +#endif - /* Decode the "mode" argument. */ - rc = zipfileGetMode(pMode, bIsDir, &mode, &zErr); - if( rc ) goto zipfile_step_out; - - /* Decode the "mtime" argument. */ - e.mUnixTime = zipfileGetTime(pMtime); - - /* If this is a directory entry, ensure that there is exactly one '/' - ** at the end of the path. Or, if this is not a directory and the path - ** ends in '/' it is an error. */ - if( bIsDir==0 ){ - if( nName>0 && zName[nName-1]=='/' ){ - zErr = sqlite3_mprintf("non-directory name must not end with /"); - rc = SQLITE_ERROR; - goto zipfile_step_out; - } - }else{ - if( nName==0 || zName[nName-1]!='/' ){ - zName = zFree = sqlite3_mprintf("%s/", zName); - if( zName==0 ){ - rc = SQLITE_NOMEM; - goto zipfile_step_out; - } - nName = (int)strlen(zName); - }else{ - while( nName>1 && zName[nName-2]=='/' ) nName--; + /* Copy the entire schema of database [db] into [dbm]. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSql = 0; + rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, + "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" + " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + if( zSql ) rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); } + idxFinalize(&rc, pSql); } - /* Assemble the ZipfileEntry object for the new zip archive entry */ - e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; - e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; - e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; - e.cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); - e.cds.crc32 = iCrc32; - e.cds.szCompressed = nData; - e.cds.szUncompressed = szUncompressed; - e.cds.iExternalAttr = (mode<<16); - e.cds.iOffset = p->body.n; - e.cds.nFile = (u16)nName; - e.cds.zFile = zName; - - /* Append the LFH to the body of the new archive */ - nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; - if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; - p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); - - /* Append the data to the body of the new archive */ - if( nData>0 ){ - if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; - memcpy(&p->body.a[p->body.n], aData, nData); - p->body.n += nData; + /* Create the vtab schema */ + if( rc==SQLITE_OK ){ + rc = idxCreateVtabSchema(pNew, pzErrmsg); } - /* Append the CDS record to the directory of the new archive */ - nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; - if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; - p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + /* Register the auth callback with dbv */ + if( rc==SQLITE_OK ){ + sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); + } - /* Increment the count of entries in the archive */ - p->nEntry++; + /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + ** return the new sqlite3expert handle. */ + if( rc!=SQLITE_OK ){ + sqlite3_expert_destroy(pNew); + pNew = 0; + } + return pNew; +} - zipfile_step_out: - sqlite3_free(aFree); - sqlite3_free(zFree); - if( rc ){ - if( zErr ){ - sqlite3_result_error(pCtx, zErr, -1); - }else{ - sqlite3_result_error_code(pCtx, rc); +/* +** Configure an sqlite3expert object. +*/ +int sqlite3_expert_config(sqlite3expert *p, int op, ...){ + int rc = SQLITE_OK; + va_list ap; + va_start(ap, op); + switch( op ){ + case EXPERT_CONFIG_SAMPLE: { + int iVal = va_arg(ap, int); + if( iVal<0 ) iVal = 0; + if( iVal>100 ) iVal = 100; + p->iSample = iVal; + break; } + default: + rc = SQLITE_NOTFOUND; + break; } - sqlite3_free(zErr); + + va_end(ap); + return rc; } /* -** xFinalize() callback for zipfile aggregate function. +** Add an SQL statement to the analysis. */ -void zipfileFinal(sqlite3_context *pCtx){ - ZipfileCtx *p; - ZipfileEOCD eocd; - sqlite3_int64 nZip; - u8 *aZip; +int sqlite3_expert_sql( + sqlite3expert *p, /* From sqlite3_expert_new() */ + const char *zSql, /* SQL statement to add */ + char **pzErr /* OUT: Error message (if any) */ +){ + IdxScan *pScanOrig = p->pScan; + IdxStatement *pStmtOrig = p->pStatement; + int rc = SQLITE_OK; + const char *zStmt = zSql; - p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); - if( p==0 ) return; - if( p->nEntry>0 ){ - memset(&eocd, 0, sizeof(eocd)); - eocd.nEntry = (u16)p->nEntry; - eocd.nEntryTotal = (u16)p->nEntry; - eocd.nSize = p->cds.n; - eocd.iOffset = p->body.n; + if( p->bRun ) return SQLITE_MISUSE; - nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; - aZip = (u8*)sqlite3_malloc64(nZip); - if( aZip==0 ){ - sqlite3_result_error_nomem(pCtx); + while( rc==SQLITE_OK && zStmt && zStmt[0] ){ + sqlite3_stmt *pStmt = 0; + /* Ensure that the provided statement compiles against user's DB. */ + rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt); + if( rc!=SQLITE_OK ) break; + sqlite3_finalize(pStmt); + rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); + if( rc==SQLITE_OK ){ + if( pStmt ){ + IdxStatement *pNew; + const char *z = sqlite3_sql(pStmt); + int n = STRLEN(z); + pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); + if( rc==SQLITE_OK ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, z, n+1); + pNew->pNext = p->pStatement; + if( p->pStatement ) pNew->iId = p->pStatement->iId+1; + p->pStatement = pNew; + } + sqlite3_finalize(pStmt); + } }else{ - memcpy(aZip, p->body.a, p->body.n); - memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); - zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); - sqlite3_result_blob(pCtx, aZip, (int)nZip, zipfileFree); + idxDatabaseError(p->dbv, pzErr); } } - sqlite3_free(p->body.a); - sqlite3_free(p->cds.a); + if( rc!=SQLITE_OK ){ + idxScanFree(p->pScan, pScanOrig); + idxStatementFree(p->pStatement, pStmtOrig); + p->pScan = pScanOrig; + p->pStatement = pStmtOrig; + } + + return rc; } +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ + int rc; + IdxHashEntry *pEntry; + + /* Do trigger processing to collect any extra IdxScan structures */ + rc = idxProcessTriggers(p, pzErr); -/* -** Register the "zipfile" virtual table. -*/ -static int zipfileRegister(sqlite3 *db){ - static sqlite3_module zipfileModule = { - 1, /* iVersion */ - zipfileConnect, /* xCreate */ - zipfileConnect, /* xConnect */ - zipfileBestIndex, /* xBestIndex */ - zipfileDisconnect, /* xDisconnect */ - zipfileDisconnect, /* xDestroy */ - zipfileOpen, /* xOpen - open a cursor */ - zipfileClose, /* xClose - close a cursor */ - zipfileFilter, /* xFilter - configure scan constraints */ - zipfileNext, /* xNext - advance a cursor */ - zipfileEof, /* xEof - check for end of scan */ - zipfileColumn, /* xColumn - read data */ - 0, /* xRowid - read data */ - zipfileUpdate, /* xUpdate */ - zipfileBegin, /* xBegin */ - 0, /* xSync */ - zipfileCommit, /* xCommit */ - zipfileRollback, /* xRollback */ - zipfileFindFunction, /* xFindMethod */ - 0, /* xRename */ - }; + /* Create candidate indexes within the in-memory database file */ + if( rc==SQLITE_OK ){ + rc = idxCreateCandidates(p); + }else if ( rc==SQLITE_BUSY_TIMEOUT ){ + if( pzErr ) + *pzErr = sqlite3_mprintf("Cannot find a unique index name to propose."); + return rc; + } - int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); - if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); + /* Generate the stat1 data */ if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, - zipfileStep, zipfileFinal + rc = idxPopulateStat1(p, pzErr); + } + + /* Formulate the EXPERT_REPORT_CANDIDATES text */ + for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ + p->zCandidates = idxAppendText(&rc, p->zCandidates, + "%s;%s%s\n", pEntry->zVal, + pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2 ); } - assert( sizeof(i64)==8 ); - assert( sizeof(u32)==4 ); - assert( sizeof(u16)==2 ); - assert( sizeof(u8)==1 ); - return rc; -} -#else /* SQLITE_OMIT_VIRTUALTABLE */ -# define zipfileRegister(x) SQLITE_OK -#endif -#ifdef _WIN32 + /* Figure out which of the candidate indexes are preferred by the query + ** planner and report the results to the user. */ + if( rc==SQLITE_OK ){ + rc = idxFindIndexes(p, pzErr); + } -#endif -int sqlite3_zipfile_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ - return zipfileRegister(db); + if( rc==SQLITE_OK ){ + p->bRun = 1; + } + return rc; } -/************************* End ../ext/misc/zipfile.c ********************/ -/************************* Begin ../ext/misc/sqlar.c ******************/ -/* -** 2017-12-17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** Utility functions sqlar_compress() and sqlar_uncompress(). Useful -** for working with sqlar archives and used by the shell tool's built-in -** sqlar support. -*/ -/* #include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT1 -#include -#include - /* -** Implementation of the "sqlar_compress(X)" SQL function. -** -** If the type of X is SQLITE_BLOB, and compressing that blob using -** zlib utility function compress() yields a smaller blob, return the -** compressed blob. Otherwise, return a copy of X. -** -** SQLar uses the "zlib format" for compressed content. The zlib format -** contains a two-byte identification header and a four-byte checksum at -** the end. This is different from ZIP which uses the raw deflate format. -** -** Future enhancements to SQLar might add support for new compression formats. -** If so, those new formats will be identified by alternative headers in the -** compressed data. +** Return the total number of statements that have been added to this +** sqlite3expert using sqlite3_expert_sql(). */ -static void sqlarCompressFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - assert( argc==1 ); - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - const Bytef *pData = sqlite3_value_blob(argv[0]); - uLong nData = sqlite3_value_bytes(argv[0]); - uLongf nOut = compressBound(nData); - Bytef *pOut; - - pOut = (Bytef*)sqlite3_malloc(nOut); - if( pOut==0 ){ - sqlite3_result_error_nomem(context); - return; - }else{ - if( Z_OK!=compress(pOut, &nOut, pData, nData) ){ - sqlite3_result_error(context, "error in compress()", -1); - }else if( nOutpStatement ) nRet = p->pStatement->iId+1; + return nRet; } /* -** Implementation of the "sqlar_uncompress(X,SZ)" SQL function -** -** Parameter SZ is interpreted as an integer. If it is less than or -** equal to zero, then this function returns a copy of X. Or, if -** SZ is equal to the size of X when interpreted as a blob, also -** return a copy of X. Otherwise, decompress blob X using zlib -** utility function uncompress() and return the results (another -** blob). +** Return a component of the report. */ -static void sqlarUncompressFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - uLong nData; - uLongf sz; - - assert( argc==2 ); - sz = sqlite3_value_int(argv[1]); +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ + const char *zRet = 0; + IdxStatement *pStmt; - if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ - sqlite3_result_value(context, argv[0]); - }else{ - const Bytef *pData= sqlite3_value_blob(argv[0]); - Bytef *pOut = sqlite3_malloc(sz); - if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){ - sqlite3_result_error(context, "error in uncompress()", -1); - }else{ - sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT); - } - sqlite3_free(pOut); + if( p->bRun==0 ) return 0; + for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext); + switch( eReport ){ + case EXPERT_REPORT_SQL: + if( pStmt ) zRet = pStmt->zSql; + break; + case EXPERT_REPORT_INDEXES: + if( pStmt ) zRet = pStmt->zIdx; + break; + case EXPERT_REPORT_PLAN: + if( pStmt ) zRet = pStmt->zEQP; + break; + case EXPERT_REPORT_CANDIDATES: + zRet = p->zCandidates; + break; } + return zRet; } - -#ifdef _WIN32 - -#endif -int sqlite3_sqlar_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "sqlar_compress", 1, - SQLITE_UTF8|SQLITE_INNOCUOUS, 0, - sqlarCompressFunc, 0, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "sqlar_uncompress", 2, - SQLITE_UTF8|SQLITE_INNOCUOUS, 0, - sqlarUncompressFunc, 0, 0); +/* +** Free an sqlite3expert object. +*/ +void sqlite3_expert_destroy(sqlite3expert *p){ + if( p ){ + sqlite3_close(p->dbm); + sqlite3_close(p->dbv); + idxScanFree(p->pScan, 0); + idxStatementFree(p->pStatement, 0); + idxTableFree(p->pTable); + idxWriteFree(p->pWrite); + idxHashClear(&p->hIdx); + sqlite3_free(p->zCandidates); + sqlite3_free(p); } - return rc; } -/************************* End ../ext/misc/sqlar.c ********************/ -#endif -/************************* Begin ../ext/expert/sqlite3expert.h ******************/ +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +/************************* End ../ext/expert/sqlite3expert.c ********************/ + +/************************* Begin ../ext/intck/sqlite3intck.h ******************/ /* -** 2017 April 07 +** 2024-02-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -9083,3050 +14083,5269 @@ int sqlite3_sqlar_init( ** ************************************************************************* */ -#if !defined(SQLITEEXPERT_H) -#define SQLITEEXPERT_H 1 -/* #include "sqlite3.h" */ - -typedef struct sqlite3expert sqlite3expert; - -/* -** Create a new sqlite3expert object. -** -** If successful, a pointer to the new object is returned and (*pzErr) set -** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to -** an English-language error message. In this case it is the responsibility -** of the caller to eventually free the error message buffer using -** sqlite3_free(). -*/ -sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr); /* -** Configure an sqlite3expert object. -** -** EXPERT_CONFIG_SAMPLE: -** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for -** each candidate index. This involves scanning and sorting the entire -** contents of each user database table once for each candidate index -** associated with the table. For large databases, this can be -** prohibitively slow. This option allows the sqlite3expert object to -** be configured so that sqlite_stat1 data is instead generated based on a -** subset of each table, or so that no sqlite_stat1 data is used at all. -** -** A single integer argument is passed to this option. If the value is less -** than or equal to zero, then no sqlite_stat1 data is generated or used by -** the analysis - indexes are recommended based on the database schema only. -** Or, if the value is 100 or greater, complete sqlite_stat1 data is -** generated for each candidate index (this is the default). Finally, if the -** value falls between 0 and 100, then it represents the percentage of user -** table rows that should be considered when generating sqlite_stat1 data. +** Incremental Integrity-Check Extension +** ------------------------------------- ** -** Examples: +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: ** -** // Do not generate any sqlite_stat1 data -** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0); +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. ** -** // Generate sqlite_stat1 data based on 10% of the rows in each table. -** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10); -*/ -int sqlite3_expert_config(sqlite3expert *p, int op, ...); - -#define EXPERT_CONFIG_SAMPLE 1 /* int */ - -/* -** Specify zero or more SQL statements to be included in the analysis. +** + It is slower. Sometimes up to three times slower. ** -** Buffer zSql must contain zero or more complete SQL statements. This -** function parses all statements contained in the buffer and adds them -** to the internal list of statements to analyze. If successful, SQLITE_OK -** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example -** due to a error in the SQL - an SQLite error code is returned and (*pzErr) -** may be set to point to an English language error message. In this case -** the caller is responsible for eventually freeing the error message buffer -** using sqlite3_free(). +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. ** -** If an error does occur while processing one of the statements in the -** buffer passed as the second argument, none of the statements in the -** buffer are added to the analysis. +** One way to use the API to run integrity-check on the "main" database +** of handle db is: ** -** This function must be called before sqlite3_expert_analyze(). If a call -** to this function is made on an sqlite3expert object that has already -** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned -** immediately and no statements are added to the analysis. -*/ -int sqlite3_expert_sql( - sqlite3expert *p, /* From a successful sqlite3_expert_new() */ - const char *zSql, /* SQL statement(s) to add */ - char **pzErr /* OUT: Error message (if any) */ -); - - -/* -** This function is called after the sqlite3expert object has been configured -** with all SQL statements using sqlite3_expert_sql() to actually perform -** the analysis. Once this function has been called, it is not possible to -** add further SQL statements to the analysis. +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; ** -** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if -** an error occurs, an SQLite error code is returned and (*pzErr) set to -** point to a buffer containing an English language error message. In this -** case it is the responsibility of the caller to eventually free the buffer -** using sqlite3_free(). +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); ** -** If an error does occur within this function, the sqlite3expert object -** is no longer useful for any purpose. At that point it is no longer -** possible to add further SQL statements to the object or to re-attempt -** the analysis. The sqlite3expert object must still be freed using a call -** sqlite3_expert_destroy(). +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). */ -int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr); -/* -** Return the total number of statements loaded using sqlite3_expert_sql(). -** The total number of SQL statements may be different from the total number -** to calls to sqlite3_expert_sql(). -*/ -int sqlite3_expert_count(sqlite3expert*); +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H -/* -** Return a component of the report. -** -** This function is called after sqlite3_expert_analyze() to extract the -** results of the analysis. Each call to this function returns either a -** NULL pointer or a pointer to a buffer containing a nul-terminated string. -** The value passed as the third argument must be one of the EXPERT_REPORT_* -** #define constants defined below. -** -** For some EXPERT_REPORT_* parameters, the buffer returned contains -** information relating to a specific SQL statement. In these cases that -** SQL statement is identified by the value passed as the second argument. -** SQL statements are numbered from 0 in the order in which they are parsed. -** If an out-of-range value (less than zero or equal to or greater than the -** value returned by sqlite3_expert_count()) is passed as the second argument -** along with such an EXPERT_REPORT_* parameter, NULL is always returned. -** -** EXPERT_REPORT_SQL: -** Return the text of SQL statement iStmt. -** -** EXPERT_REPORT_INDEXES: -** Return a buffer containing the CREATE INDEX statements for all recommended -** indexes for statement iStmt. If there are no new recommeded indexes, NULL -** is returned. -** -** EXPERT_REPORT_PLAN: -** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query -** iStmt after the proposed indexes have been added to the database schema. -** -** EXPERT_REPORT_CANDIDATES: -** Return a pointer to a buffer containing the CREATE INDEX statements -** for all indexes that were tested (for all SQL statements). The iStmt -** parameter is ignored for EXPERT_REPORT_CANDIDATES calls. -*/ -const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport); +/* #include "sqlite3.h" */ -/* -** Values for the third argument passed to sqlite3_expert_report(). -*/ -#define EXPERT_REPORT_SQL 1 -#define EXPERT_REPORT_INDEXES 2 -#define EXPERT_REPORT_PLAN 3 -#define EXPERT_REPORT_CANDIDATES 4 +#ifdef __cplusplus +extern "C" { +#endif /* -** Free an (sqlite3expert*) handle and all associated resources. There -** should be one call to this function for each successful call to -** sqlite3-expert_new(). +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. */ -void sqlite3_expert_destroy(sqlite3expert*); - -#endif /* !defined(SQLITEEXPERT_H) */ +typedef struct sqlite3_intck sqlite3_intck; -/************************* End ../ext/expert/sqlite3expert.h ********************/ -/************************* Begin ../ext/expert/sqlite3expert.c ******************/ /* -** 2017 April 09 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). ** -************************************************************************* -*/ -/* #include "sqlite3expert.h" */ -#include -#include -#include - -#if !defined(SQLITE_AMALGAMATION) -#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) -# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 -#endif -#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -#elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -#else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -#endif -#endif /* !defined(SQLITE_AMALGAMATION) */ - - -#ifndef SQLITE_OMIT_VIRTUALTABLE - -/* typedef sqlite3_int64 i64; */ -/* typedef sqlite3_uint64 u64; */ - -typedef struct IdxColumn IdxColumn; -typedef struct IdxConstraint IdxConstraint; -typedef struct IdxScan IdxScan; -typedef struct IdxStatement IdxStatement; -typedef struct IdxTable IdxTable; -typedef struct IdxWrite IdxWrite; - -#define STRLEN (int)strlen - -/* -** A temp table name that we assume no user database will actually use. -** If this assumption proves incorrect triggers on the table with the -** conflicting name will be ignored. +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). */ -#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776" +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); /* -** A single constraint. Equivalent to either "col = ?" or "col < ?" (or -** any other type of single-ended range constraint on a column). -** -** pLink: -** Used to temporarily link IdxConstraint objects into lists while -** creating candidate indexes. +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. */ -struct IdxConstraint { - char *zColl; /* Collation sequence */ - int bRange; /* True for range, false for eq */ - int iCol; /* Constrained table column */ - int bFlag; /* Used by idxFindCompatible() */ - int bDesc; /* True if ORDER BY DESC */ - IdxConstraint *pNext; /* Next constraint in pEq or pRange list */ - IdxConstraint *pLink; /* See above */ -}; +void sqlite3_intck_close(sqlite3_intck *pCk); /* -** A single scan of a single table. +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. */ -struct IdxScan { - IdxTable *pTab; /* Associated table object */ - int iDb; /* Database containing table zTable */ - i64 covering; /* Mask of columns required for cov. index */ - IdxConstraint *pOrder; /* ORDER BY columns */ - IdxConstraint *pEq; /* List of == constraints */ - IdxConstraint *pRange; /* List of < constraints */ - IdxScan *pNextScan; /* Next IdxScan object for same analysis */ -}; +int sqlite3_intck_step(sqlite3_intck *pCk); /* -** Information regarding a single database table. Extracted from -** "PRAGMA table_info" by function idxGetTableInfo(). +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. */ -struct IdxColumn { - char *zName; - char *zColl; - int iPk; -}; -struct IdxTable { - int nCol; - char *zName; /* Table name */ - IdxColumn *aCol; - IdxTable *pNext; /* Next table in linked list of all tables */ -}; +const char *sqlite3_intck_message(sqlite3_intck *pCk); /* -** An object of the following type is created for each unique table/write-op -** seen. The objects are stored in a singly-linked list beginning at -** sqlite3expert.pWrite. +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. */ -struct IdxWrite { - IdxTable *pTab; - int eOp; /* SQLITE_UPDATE, DELETE or INSERT */ - IdxWrite *pNext; -}; +int sqlite3_intck_unlock(sqlite3_intck *pCk); /* -** Each statement being analyzed is represented by an instance of this -** structure. +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. */ -struct IdxStatement { - int iId; /* Statement number */ - char *zSql; /* SQL statement */ - char *zIdx; /* Indexes */ - char *zEQP; /* Plan */ - IdxStatement *pNext; -}; +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ +/************************* End ../ext/intck/sqlite3intck.h ********************/ +/************************* Begin ../ext/intck/sqlite3intck.c ******************/ /* -** A hash table for storing strings. With space for a payload string -** with each entry. Methods are: +** 2024-02-08 ** -** idxHashInit() -** idxHashClear() -** idxHashAdd() -** idxHashSearch() +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* */ -#define IDX_HASH_SIZE 1023 -typedef struct IdxHashEntry IdxHashEntry; -typedef struct IdxHash IdxHash; -struct IdxHashEntry { - char *zKey; /* nul-terminated key */ - char *zVal; /* nul-terminated value string */ - char *zVal2; /* nul-terminated value string 2 */ - IdxHashEntry *pHashNext; /* Next entry in same hash bucket */ - IdxHashEntry *pNext; /* Next entry in hash */ -}; -struct IdxHash { - IdxHashEntry *pFirst; - IdxHashEntry *aHash[IDX_HASH_SIZE]; -}; + +/* #include "sqlite3intck.h" */ +#include +#include + +#include +#include /* -** sqlite3expert object. +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. */ -struct sqlite3expert { - int iSample; /* Percentage of tables to sample for stat1 */ - sqlite3 *db; /* User database */ - sqlite3 *dbm; /* In-memory db for this analysis */ - sqlite3 *dbv; /* Vtab schema for this analysis */ - IdxTable *pTable; /* List of all IdxTable objects */ - IdxScan *pScan; /* List of scan objects */ - IdxWrite *pWrite; /* List of write objects */ - IdxStatement *pStatement; /* List of IdxStatement objects */ - int bRun; /* True once analysis has run */ - char **pzErrmsg; - int rc; /* Error code from whereinfo hook */ - IdxHash hIdx; /* Hash containing all candidate indexes */ - char *zCandidates; /* For EXPERT_REPORT_CANDIDATES */ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ }; /* -** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). -** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. */ -static void *idxMalloc(int *pRc, int nByte){ - void *pRet; - assert( *pRc==SQLITE_OK ); - assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); - if( pRet ){ - memset(pRet, 0, nByte); - }else{ - *pRc = SQLITE_NOMEM; +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } } return pRet; } /* -** Initialize an IdxHash hash table. +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. */ -static void idxHashInit(IdxHash *pHash){ - memset(pHash, 0, sizeof(IdxHash)); +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; } /* -** Reset an IdxHash hash table. +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. */ -static void idxHashClear(IdxHash *pHash){ - int i; - for(i=0; iaHash[i]; pEntry; pEntry=pNext){ - pNext = pEntry->pHashNext; - sqlite3_free(pEntry->zVal2); - sqlite3_free(pEntry); - } +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); } - memset(pHash, 0, sizeof(IdxHash)); } /* -** Return the index of the hash bucket that the string specified by the -** arguments to this function belongs. +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. */ -static int idxHashString(const char *z, int n){ - unsigned int ret = 0; - int i; - for(i=0; irc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; } - return (int)(ret % IDX_HASH_SIZE); + return zRet; } /* -** If zKey is already present in the hash table, return non-zero and do -** nothing. Otherwise, add an entry with key zKey and payload string zVal to -** the hash table passed as the second argument. +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. */ -static int idxHashAdd( - int *pRc, - IdxHash *pHash, - const char *zKey, - const char *zVal -){ - int nKey = STRLEN(zKey); - int iHash = idxHashString(zKey, nKey); - int nVal = (zVal ? STRLEN(zVal) : 0); - IdxHashEntry *pEntry; - assert( iHash>=0 ); - for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ - if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ - return 1; +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); } - pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); - if( pEntry ){ - pEntry->zKey = (char*)&pEntry[1]; - memcpy(pEntry->zKey, zKey, nKey); - if( zVal ){ - pEntry->zVal = &pEntry->zKey[nKey+1]; - memcpy(pEntry->zVal, zVal, nVal); + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); } - pEntry->pHashNext = pHash->aHash[iHash]; - pHash->aHash[iHash] = pEntry; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } - pEntry->pNext = pHash->pFirst; - pHash->pFirst = pEntry; + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } } - return 0; + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); } /* -** If zKey/nKey is present in the hash table, return a pointer to the -** hash-entry object. +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. */ -static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){ - int iHash; - IdxHashEntry *pEntry; - if( nKey<0 ) nKey = STRLEN(zKey); - iHash = idxHashString(zKey, nKey); - assert( iHash>=0 ); - for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){ - if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){ - return pEntry; +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; } } - return 0; + + return iRet; } /* -** If the hash table contains an entry with a key equal to the string -** passed as the final two arguments to this function, return a pointer -** to the payload string. Otherwise, if zKey/nKey is not present in the -** hash table, return NULL. +** Return true if argument c is an ascii whitespace character. */ -static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){ - IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey); - if( pEntry ) return pEntry->zVal; - return 0; +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); } /* -** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl -** variable to point to a copy of nul-terminated string zColl. +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. */ -static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ - IdxConstraint *pNew; - int nColl = STRLEN(zColl); +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; - assert( *pRc==SQLITE_OK ); - pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1); - if( pNew ){ - pNew->zColl = (char*)&pNew[1]; - memcpy(pNew->zColl, zColl, nColl+1); + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; } - return pNew; + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; } /* -** An error associated with database handle db has just occurred. Pass -** the error message to callback function xOut. +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); */ -static void idxDatabaseError( - sqlite3 *db, /* Database handle */ - char **pzErrmsg /* Write error here */ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal ){ - *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); } /* -** Prepare an SQL statement. +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. */ -static int idxPrepareStmt( - sqlite3 *db, /* Database handle to compile against */ - sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ - char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ - const char *zSql /* SQL statement to compile */ -){ - int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); - if( rc!=SQLITE_OK ){ - *ppStmt = 0; - idxDatabaseError(db, pzErrmsg); +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); } - return rc; + intckFinalize(p, pStmt); + return bRet; } /* -** Prepare an SQL statement using the results of a printf() formatting. +** Return true if zObj is an index, or false otherwise. */ -static int idxPrintfPrepareStmt( - sqlite3 *db, /* Database handle to compile against */ - sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ - char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ - const char *zFmt, /* printf() format of SQL statement */ - ... /* Trailing printf() arguments */ -){ - va_list ap; - int rc; - char *zSql; - va_start(ap, zFmt); - zSql = sqlite3_vmprintf(zFmt, ap); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql); - sqlite3_free(zSql); +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; } - va_end(ap); - return rc; + intckFinalize(p, pStmt); + return bRet; } - -/************************************************************************* -** Beginning of virtual table implementation. +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. */ -typedef struct ExpertVtab ExpertVtab; -struct ExpertVtab { - sqlite3_vtab base; - IdxTable *pTab; - sqlite3expert *pExpert; -}; +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; -typedef struct ExpertCsr ExpertCsr; -struct ExpertCsr { - sqlite3_vtab_cursor base; - sqlite3_stmt *pData; -}; + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" -static char *expertDequote(const char *zIn){ - int n = STRLEN(zIn); - char *zRet = sqlite3_malloc(n); + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" - assert( zIn[0]=='\'' ); - assert( zIn[n-1]=='\'' ); + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" - if( zRet ){ - int iOut = 0; - int iIn = 0; - for(iIn=1; iIn<(n-1); iIn++){ - if( zIn[iIn]=='\'' ){ - assert( zIn[iIn+1]=='\'' ); - iIn++; - } - zRet[iOut++] = zIn[iIn]; + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); } - zRet[iOut] = '\0'; } + intckFinalize(p, pStmt); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); return zRet; } -/* -** This function is the implementation of both the xConnect and xCreate -** methods of the r-tree virtual table. -** -** argv[0] -> module name -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> column names... +/* +** Open a new integrity-check object. */ -static int expertConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ ){ - sqlite3expert *pExpert = (sqlite3expert*)pAux; - ExpertVtab *p = 0; - int rc; + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); - if( argc!=4 ){ - *pzErr = sqlite3_mprintf("internal error!"); - rc = SQLITE_ERROR; + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; }else{ - char *zCreateTable = expertDequote(argv[3]); - if( zCreateTable ){ - rc = sqlite3_declare_vtab(db, zCreateTable); - if( rc==SQLITE_OK ){ - p = idxMalloc(&rc, sizeof(ExpertVtab)); - } - if( rc==SQLITE_OK ){ - p->pExpert = pExpert; - p->pTab = pExpert->pTable; - assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 ); - } - sqlite3_free(zCreateTable); - }else{ - rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; } } - *ppVtab = (sqlite3_vtab*)p; + *ppOut = pNew; return rc; } -static int expertDisconnect(sqlite3_vtab *pVtab){ - ExpertVtab *p = (ExpertVtab*)pVtab; - sqlite3_free(p); - return SQLITE_OK; +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } } -static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){ - ExpertVtab *p = (ExpertVtab*)pVtab; - int rc = SQLITE_OK; - int n = 0; - IdxScan *pScan; - const int opmask = - SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT | - SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE | - SQLITE_INDEX_CONSTRAINT_LE; - - pScan = idxMalloc(&rc, sizeof(IdxScan)); - if( pScan ){ - int i; +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ - /* Link the new scan object into the list */ - pScan->pTab = p->pTab; - pScan->pNextScan = p->pExpert->pScan; - p->pExpert->pScan = pScan; + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } - /* Add the constraints to the IdxScan object */ - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; - if( pCons->usable - && pCons->iColumn>=0 - && p->pTab->aCol[pCons->iColumn].iPk==0 - && (pCons->op & opmask) - ){ - IdxConstraint *pNew; - const char *zColl = sqlite3_vtab_collation(pIdxInfo, i); - pNew = idxNewConstraint(&rc, zColl); - if( pNew ){ - pNew->iCol = pCons->iColumn; - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - pNew->pNext = pScan->pEq; - pScan->pEq = pNew; - }else{ - pNew->bRange = 1; - pNew->pNext = pScan->pRange; - pScan->pRange = pNew; - } - } - n++; - pIdxInfo->aConstraintUsage[i].argvIndex = n; + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; } } - /* Add the ORDER BY to the IdxScan object */ - for(i=pIdxInfo->nOrderBy-1; i>=0; i--){ - int iCol = pIdxInfo->aOrderBy[i].iColumn; - if( iCol>=0 ){ - IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl); - if( pNew ){ - pNew->iCol = iCol; - pNew->bDesc = pIdxInfo->aOrderBy[i].desc; - pNew->pNext = pScan->pOrder; - pNew->pLink = pScan->pOrder; - pScan->pOrder = pNew; - n++; + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); } } } } - pIdxInfo->estimatedCost = 1000000.0 / (n+1); - return rc; + return p->rc; } -static int expertUpdate( - sqlite3_vtab *pVtab, - int nData, - sqlite3_value **azData, - sqlite_int64 *pRowid -){ - (void)pVtab; - (void)nData; - (void)azData; - (void)pRowid; - return SQLITE_OK; +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; } -/* -** Virtual table module xOpen method. +/* +** Return the error code and message. */ -static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - int rc = SQLITE_OK; - ExpertCsr *pCsr; - (void)pVTab; - pCsr = idxMalloc(&rc, sizeof(ExpertCsr)); - *ppCursor = (sqlite3_vtab_cursor*)pCsr; - return rc; +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } -/* -** Virtual table module xClose method. +/* +** Close any read transaction the integrity-check object is holding open +** on the database. */ -static int expertClose(sqlite3_vtab_cursor *cur){ - ExpertCsr *pCsr = (ExpertCsr*)cur; - sqlite3_finalize(pCsr->pData); - sqlite3_free(pCsr); - return SQLITE_OK; +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; } /* -** Virtual table module xEof method. -** -** Return non-zero if the cursor does not currently point to a valid -** record (i.e if the scan has finished), or zero otherwise. +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. */ -static int expertEof(sqlite3_vtab_cursor *cur){ - ExpertCsr *pCsr = (ExpertCsr*)cur; - return pCsr->pData==0; +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; } -/* -** Virtual table module xNext method. +/************************* End ../ext/intck/sqlite3intck.c ********************/ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#define SQLITE_SHELL_HAVE_RECOVER 1 +#else +#define SQLITE_SHELL_HAVE_RECOVER 0 +#endif +#if SQLITE_SHELL_HAVE_RECOVER +/************************* Begin ../ext/recover/sqlite3recover.h ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface to the "recover" extension - +** an SQLite extension designed to recover data from corrupted database +** files. */ -static int expertNext(sqlite3_vtab_cursor *cur){ - ExpertCsr *pCsr = (ExpertCsr*)cur; - int rc = SQLITE_OK; - assert( pCsr->pData ); - rc = sqlite3_step(pCsr->pData); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_finalize(pCsr->pData); - pCsr->pData = 0; - }else{ - rc = SQLITE_OK; - } +/* +** OVERVIEW: +** +** To use the API to recover data from a corrupted database, an +** application: +** +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). +** +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on +** the handle until it returns something other than SQLITE_OK. If it +** returns SQLITE_DONE, then the recovery operation completed without +** error. If it returns some other non-SQLITE_OK value, then an error +** has occurred. +** +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. +** +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). +** +** The application may abandon the recovery operation at any point +** before it is finished by passing the sqlite3_recover handle to +** sqlite3_recover_finish(). This is not an error, but the final state +** of the output database, or the results of running the partial script +** delivered to the SQL callback, are undefined. +*/ - return rc; -} +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H -/* -** Virtual table module xRowid method. +/* #include "sqlite3.h" */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An instance of the sqlite3_recover object represents a recovery +** operation in progress. +** +** Constructors: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** Destructor: +** +** sqlite3_recover_finish() +** +** Methods: +** +** sqlite3_recover_config() +** sqlite3_recover_errcode() +** sqlite3_recover_errmsg() +** sqlite3_recover_run() +** sqlite3_recover_step() */ -static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - (void)cur; - *pRowid = 0; - return SQLITE_OK; -} +typedef struct sqlite3_recover sqlite3_recover; /* -** Virtual table module xColumn method. +** These two APIs attempt to create and return a new sqlite3_recover object. +** In both cases the first two arguments identify the (possibly +** corrupt) database to recover data from. The first argument is an open +** database handle and the second the name of a database attached to that +** handle (i.e. "main", "temp" or the name of an attached database). +** +** If sqlite3_recover_init() is used to create the new sqlite3_recover +** handle, then data is recovered into a new database, identified by +** string parameter zUri. zUri may be an absolute or relative file path, +** or may be an SQLite URI. If the identified database file already exists, +** it is overwritten. +** +** If sqlite3_recover_init_sql() is invoked, then any recovered data will +** be returned to the user as a series of SQL statements. Executing these +** SQL statements results in the same database as would have been created +** had sqlite3_recover_init() been used. For each SQL statement in the +** output, the callback function passed as the third argument (xSql) is +** invoked once. The first parameter is a passed a copy of the fourth argument +** to this function (pCtx) as its first parameter, and a pointer to a +** nul-terminated buffer containing the SQL statement formated as UTF-8 as +** the second. If the xSql callback returns any value other than SQLITE_OK, +** then processing is immediately abandoned and the value returned used as +** the recover handle error code (see below). +** +** If an out-of-memory error occurs, NULL may be returned instead of +** a valid handle. In all other cases, it is the responsibility of the +** application to avoid resource leaks by ensuring that +** sqlite3_recover_finish() is called on all allocated handles. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pCtx +); + +/* +** Configure an sqlite3_recover object that has just been created using +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function +** may only be called before the first call to sqlite3_recover_step() +** or sqlite3_recover_run() on the object. +** +** The second argument passed to this function must be one of the +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument +** depend on the specific SQLITE_RECOVER_* symbol in use. +** +** SQLITE_OK is returned if the configuration operation was successful, +** or an SQLite error code otherwise. +*/ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +/* +** SQLITE_RECOVER_LOST_AND_FOUND: +** The pArg argument points to a string buffer containing the name +** of a "lost-and-found" table in the output database, or NULL. If +** the argument is non-NULL and the database contains seemingly +** valid pages that cannot be associated with any table in the +** recovered part of the schema, data is extracted from these +** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1) and a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. The default +** value is 0 (clear). +** +** SQLITE_RECOVER_ROWIDS: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1), then an attempt is made to recover rowid values +** that are not also INTEGER PRIMARY KEY values. If this option is +** clear, then new rowids are assigned to all recovered rows. The +** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. +*/ +#define SQLITE_RECOVER_LOST_AND_FOUND 1 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2 +#define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 + +/* +** Perform a unit of work towards the recovery operation. This function +** must normally be called multiple times to complete database recovery. +** +** If no error occurs but the recovery operation is not completed, this +** function returns SQLITE_OK. If recovery has been completed successfully +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not +** considered an error if some or all of the data cannot be recovered +** due to database corruption. +** +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, +** all further such calls on the same recover handle are no-ops that return +** the same non-SQLITE_OK value. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +/* +** Run the recovery operation to completion. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. Calling this function is the same +** as executing: +** +** while( SQLITE_OK==sqlite3_recover_step(p) ); +** return sqlite3_recover_errcode(p); */ -static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ - ExpertCsr *pCsr = (ExpertCsr*)cur; - sqlite3_value *pVal; - pVal = sqlite3_column_value(pCsr->pData, i); - if( pVal ){ - sqlite3_result_value(ctx, pVal); - } - return SQLITE_OK; -} +int sqlite3_recover_run(sqlite3_recover*); + +/* +** If an error has been encountered during a prior call to +** sqlite3_recover_step(), then this function attempts to return a +** pointer to a buffer containing an English language explanation of +** the error. If no error message is available, or if an out-of memory +** error occurs while attempting to allocate a buffer in which to format +** the error message, NULL is returned. +** +** The returned buffer remains valid until the sqlite3_recover handle is +** destroyed using sqlite3_recover_finish(). +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +/* +** If this function is called on an sqlite3_recover handle after +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3_recover_errcode(sqlite3_recover*); /* -** Virtual table module xFilter method. +** Clean up a recovery object created by a call to sqlite3_recover_init(). +** The results of using a recovery object with any API after it has been +** passed to this function are undefined. +** +** This function returns the same value as sqlite3_recover_errcode(). */ -static int expertFilter( - sqlite3_vtab_cursor *cur, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - ExpertCsr *pCsr = (ExpertCsr*)cur; - ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab); - sqlite3expert *pExpert = pVtab->pExpert; - int rc; +int sqlite3_recover_finish(sqlite3_recover*); - (void)idxNum; - (void)idxStr; - (void)argc; - (void)argv; - rc = sqlite3_finalize(pCsr->pData); - pCsr->pData = 0; - if( rc==SQLITE_OK ){ - rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName - ); - } - if( rc==SQLITE_OK ){ - rc = expertNext(cur); - } - return rc; -} +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ + +/************************* End ../ext/recover/sqlite3recover.h ********************/ +# ifndef SQLITE_HAVE_SQLITE3R +/************************* Begin ../ext/recover/dbdata.c ******************/ +/* +** 2019-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of two eponymous virtual tables, +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the +** "sqlite_dbpage" eponymous virtual table be available. +** +** SQLITE_DBDATA: +** sqlite_dbdata is used to extract data directly from a database b-tree +** page and its associated overflow pages, bypassing the b-tree layer. +** The table schema is equivalent to: +** +** CREATE TABLE sqlite_dbdata( +** pgno INTEGER, +** cell INTEGER, +** field INTEGER, +** value ANY, +** schema TEXT HIDDEN +** ); +** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND +** "schema". +** +** Each page of the database is inspected. If it cannot be interpreted as +** a b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the +** table contains one row for each field in the record associated with +** each cell on the page. For intkey b-trees, the key value is stored in +** field -1. +** +** For example, for the database: +** +** CREATE TABLE t1(a, b); -- root page is page 2 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: +** +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES +** (2, 0, -1, 5 ), +** (2, 0, 0, 'v' ), +** (2, 0, 1, 'five'), +** (2, 1, -1, 10 ), +** (2, 1, 0, 'x' ), +** (2, 1, 1, 'ten' ); +** +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. +** +** SQLITE_DBPTR: +** The sqlite_dbptr table has the following schema: +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); +** +** It contains one entry for each b-tree pointer between a parent and +** child page in the database. +*/ + +#if !defined(SQLITEINT_H) +/* #include "sqlite3.h" */ + +/* typedef unsigned char u8; */ +/* typedef unsigned int u32; */ + +#endif +#include +#include -static int idxRegisterVtab(sqlite3expert *p){ - static sqlite3_module expertModule = { - 2, /* iVersion */ - expertConnect, /* xCreate - create a table */ - expertConnect, /* xConnect - connect to an existing table */ - expertBestIndex, /* xBestIndex - Determine search strategy */ - expertDisconnect, /* xDisconnect - Disconnect from a table */ - expertDisconnect, /* xDestroy - Drop a table */ - expertOpen, /* xOpen - open a cursor */ - expertClose, /* xClose - close a cursor */ - expertFilter, /* xFilter - configure scan constraints */ - expertNext, /* xNext - advance a cursor */ - expertEof, /* xEof */ - expertColumn, /* xColumn - read data */ - expertRowid, /* xRowid - read data */ - expertUpdate, /* xUpdate - write data */ - 0, /* xBegin - begin transaction */ - 0, /* xSync - sync transaction */ - 0, /* xCommit - commit transaction */ - 0, /* xRollback - rollback transaction */ - 0, /* xFindFunction - function overloading */ - 0, /* xRename - rename the table */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - }; +#ifndef SQLITE_OMIT_VIRTUALTABLE - return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); -} -/* -** End of virtual table implementation. -*************************************************************************/ -/* -** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function -** is called, set it to the return value of sqlite3_finalize() before -** returning. Otherwise, discard the sqlite3_finalize() return value. -*/ -static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ - int rc = sqlite3_finalize(pStmt); - if( *pRc==SQLITE_OK ) *pRc = rc; -} +#define DBDATA_PADDING_BYTES 100 + +typedef struct DbdataTable DbdataTable; +typedef struct DbdataCursor DbdataCursor; +typedef struct DbdataBuffer DbdataBuffer; /* -** Attempt to allocate an IdxTable structure corresponding to table zTab -** in the main database of connection db. If successful, set (*ppOut) to -** point to the new object and return SQLITE_OK. Otherwise, return an -** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be -** set to point to an error string. -** -** It is the responsibility of the caller to eventually free either the -** IdxTable object or error message using sqlite3_free(). +** Buffer type. */ -static int idxGetTableInfo( - sqlite3 *db, /* Database connection to read details from */ - const char *zTab, /* Table name */ - IdxTable **ppOut, /* OUT: New object (if successful) */ - char **pzErrmsg /* OUT: Error message (if not) */ -){ - sqlite3_stmt *p1 = 0; - int nCol = 0; - int nTab = STRLEN(zTab); - int nByte = sizeof(IdxTable) + nTab + 1; - IdxTable *pNew = 0; - int rc, rc2; - char *pCsr = 0; - int nPk = 0; - - rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_xinfo=%Q", zTab); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ - const char *zCol = (const char*)sqlite3_column_text(p1, 1); - const char *zColSeq = 0; - nByte += 1 + STRLEN(zCol); - rc = sqlite3_table_column_metadata( - db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 - ); - if( zColSeq==0 ) zColSeq = "binary"; - nByte += 1 + STRLEN(zColSeq); - nCol++; - nPk += (sqlite3_column_int(p1, 5)>0); - } - rc2 = sqlite3_reset(p1); - if( rc==SQLITE_OK ) rc = rc2; +struct DbdataBuffer { + u8 *aBuf; + sqlite3_int64 nBuf; +}; - nByte += sizeof(IdxColumn) * nCol; - if( rc==SQLITE_OK ){ - pNew = idxMalloc(&rc, nByte); - } - if( rc==SQLITE_OK ){ - pNew->aCol = (IdxColumn*)&pNew[1]; - pNew->nCol = nCol; - pCsr = (char*)&pNew->aCol[nCol]; - } +/* Cursor object */ +struct DbdataCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* For fetching database pages */ - nCol = 0; - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ - const char *zCol = (const char*)sqlite3_column_text(p1, 1); - const char *zColSeq = 0; - int nCopy = STRLEN(zCol) + 1; - pNew->aCol[nCol].zName = pCsr; - pNew->aCol[nCol].iPk = (sqlite3_column_int(p1, 5)==1 && nPk==1); - memcpy(pCsr, zCol, nCopy); - pCsr += nCopy; + int iPgno; /* Current page number */ + u8 *aPage; /* Buffer containing page */ + int nPage; /* Size of aPage[] in bytes */ + int nCell; /* Number of cells on aPage[] */ + int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + int szDb; + sqlite3_int64 iRowid; - rc = sqlite3_table_column_metadata( - db, "main", zTab, zCol, 0, &zColSeq, 0, 0, 0 - ); - if( rc==SQLITE_OK ){ - if( zColSeq==0 ) zColSeq = "binary"; - nCopy = STRLEN(zColSeq) + 1; - pNew->aCol[nCol].zColl = pCsr; - memcpy(pCsr, zColSeq, nCopy); - pCsr += nCopy; - } + /* Only for the sqlite_dbdata table */ + DbdataBuffer rec; + sqlite3_int64 nRec; /* Size of pRec[] in bytes */ + sqlite3_int64 nHdr; /* Size of header in bytes */ + int iField; /* Current field number */ + u8 *pHdrPtr; + u8 *pPtr; + u32 enc; /* Text encoding */ + + sqlite3_int64 iIntkey; /* Integer key value */ +}; - nCol++; - } - idxFinalize(&rc, p1); +/* Table object */ +struct DbdataTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + int bPtr; /* True for sqlite3_dbptr table */ +}; - if( rc!=SQLITE_OK ){ - sqlite3_free(pNew); - pNew = 0; - }else if( ALWAYS(pNew!=0) ){ - pNew->zName = pCsr; - if( ALWAYS(pNew->zName!=0) ) memcpy(pNew->zName, zTab, nTab+1); - } +/* Column and schema definitions for sqlite_dbdata */ +#define DBDATA_COLUMN_PGNO 0 +#define DBDATA_COLUMN_CELL 1 +#define DBDATA_COLUMN_FIELD 2 +#define DBDATA_COLUMN_VALUE 3 +#define DBDATA_COLUMN_SCHEMA 4 +#define DBDATA_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " cell INTEGER," \ + " field INTEGER," \ + " value ANY," \ + " schema TEXT HIDDEN" \ + ")" - *ppOut = pNew; - return rc; -} +/* Column and schema definitions for sqlite_dbptr */ +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" /* -** This function is a no-op if *pRc is set to anything other than -** SQLITE_OK when it is called. -** -** If *pRc is initially set to SQLITE_OK, then the text specified by -** the printf() style arguments is appended to zIn and the result returned -** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on -** zIn before returning. +** Ensure the buffer passed as the first argument is at least nMin bytes +** in size. If an error occurs while attempting to resize the buffer, +** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ - va_list ap; - char *zAppend = 0; - char *zRet = 0; - int nIn = zIn ? STRLEN(zIn) : 0; - int nAppend = 0; - va_start(ap, zFmt); - if( *pRc==SQLITE_OK ){ - zAppend = sqlite3_vmprintf(zFmt, ap); - if( zAppend ){ - nAppend = STRLEN(zAppend); - zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); - } - if( zAppend && zRet ){ - if( nIn ) memcpy(zRet, zIn, nIn); - memcpy(&zRet[nIn], zAppend, nAppend+1); - }else{ - sqlite3_free(zRet); - zRet = 0; - *pRc = SQLITE_NOMEM; - } - sqlite3_free(zAppend); - sqlite3_free(zIn); +static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ + if( nMin>pBuf->nBuf ){ + sqlite3_int64 nNew = nMin+16384; + u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); + + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->aBuf = aNew; + pBuf->nBuf = nNew; } - va_end(ap); - return zRet; + return SQLITE_OK; } /* -** Return true if zId must be quoted in order to use it as an SQL -** identifier, or false otherwise. +** Release the allocation managed by buffer pBuf. */ -static int idxIdentifierRequiresQuotes(const char *zId){ - int i; - for(i=0; zId[i]; i++){ - if( !(zId[i]=='_') - && !(zId[i]>='0' && zId[i]<='9') - && !(zId[i]>='a' && zId[i]<='z') - && !(zId[i]>='A' && zId[i]<='Z') - ){ - return 1; - } - } - return 0; +static void dbdataBufferFree(DbdataBuffer *pBuf){ + sqlite3_free(pBuf->aBuf); + memset(pBuf, 0, sizeof(*pBuf)); } /* -** This function appends an index column definition suitable for constraint -** pCons to the string passed as zIn and returns the result. +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual +** table. */ -static char *idxAppendColDefn( - int *pRc, /* IN/OUT: Error code */ - char *zIn, /* Column defn accumulated so far */ - IdxTable *pTab, /* Table index will be created on */ - IdxConstraint *pCons +static int dbdataConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr ){ - char *zRet = zIn; - IdxColumn *p = &pTab->aCol[pCons->iCol]; - if( zRet ) zRet = idxAppendText(pRc, zRet, ", "); - - if( idxIdentifierRequiresQuotes(p->zName) ){ - zRet = idxAppendText(pRc, zRet, "%Q", p->zName); - }else{ - zRet = idxAppendText(pRc, zRet, "%s", p->zName); - } + DbdataTable *pTab = 0; + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); - if( sqlite3_stricmp(p->zColl, pCons->zColl) ){ - if( idxIdentifierRequiresQuotes(pCons->zColl) ){ - zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl); + (void)argc; + (void)argv; + (void)pzErr; + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); + if( rc==SQLITE_OK ){ + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; }else{ - zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl); + memset(pTab, 0, sizeof(DbdataTable)); + pTab->db = db; + pTab->bPtr = (pAux!=0); } } - if( pCons->bDesc ){ - zRet = idxAppendText(pRc, zRet, " DESC"); + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. +*/ +static int dbdataDisconnect(sqlite3_vtab *pVtab){ + DbdataTable *pTab = (DbdataTable*)pVtab; + if( pTab ){ + sqlite3_finalize(pTab->pStmt); + sqlite3_free(pVtab); } - return zRet; + return SQLITE_OK; } /* -** Search database dbm for an index compatible with the one idxCreateFromCons() -** would create from arguments pScan, pEq and pTail. If no error occurs and -** such an index is found, return non-zero. Or, if no such index is found, -** return zero. +** This function interprets two types of constraints: ** -** If an error occurs, set *pRc to an SQLite error code and return zero. +** schema=? +** pgno=? +** +** If neither are present, idxNum is set to 0. If schema=? is present, +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit +** in idxNum is set. +** +** If both parameters are present, schema is in position 0 and pgno in +** position 1. */ -static int idxFindCompatible( - int *pRc, /* OUT: Error code */ - sqlite3* dbm, /* Database to search */ - IdxScan *pScan, /* Scan for table to search for index on */ - IdxConstraint *pEq, /* List of == constraints */ - IdxConstraint *pTail /* List of range constraints */ -){ - const char *zTbl = pScan->pTab->zName; - sqlite3_stmt *pIdxList = 0; - IdxConstraint *pIter; - int nEq = 0; /* Number of elements in pEq */ - int rc; - - /* Count the elements in list pEq */ - for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++; - - rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl); - while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){ - int bMatch = 1; - IdxConstraint *pT = pTail; - sqlite3_stmt *pInfo = 0; - const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1); +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ + DbdataTable *pTab = (DbdataTable*)tab; + int i; + int iSchema = -1; + int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); - /* Zero the IdxConstraint.bFlag values in the pEq list */ - for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( p->iColumn==colSchema ){ + if( p->usable==0 ) return SQLITE_CONSTRAINT; + iSchema = i; + } + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ + iPgno = i; + } + } + } - rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx); - while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){ - int iIdx = sqlite3_column_int(pInfo, 0); - int iCol = sqlite3_column_int(pInfo, 1); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + if( iSchema>=0 ){ + pIdx->aConstraintUsage[iSchema].argvIndex = 1; + pIdx->aConstraintUsage[iSchema].omit = 1; + } + if( iPgno>=0 ){ + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdx->aConstraintUsage[iPgno].omit = 1; + pIdx->estimatedCost = 100; + pIdx->estimatedRows = 50; - if( iIdxpLink){ - if( pIter->bFlag ) continue; - if( pIter->iCol!=iCol ) continue; - if( sqlite3_stricmp(pIter->zColl, zColl) ) continue; - pIter->bFlag = 1; - break; - } - if( pIter==0 ){ - bMatch = 0; - break; - } - }else{ - if( pT ){ - if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){ - bMatch = 0; - break; - } - pT = pT->pLink; - } + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ + int iCol = pIdx->aOrderBy[0].iColumn; + if( pIdx->nOrderBy==1 ){ + pIdx->orderByConsumed = (iCol==0 || iCol==1); + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); } } - idxFinalize(&rc, pInfo); - if( rc==SQLITE_OK && bMatch ){ - sqlite3_finalize(pIdxList); - return 1; - } + }else{ + pIdx->estimatedCost = 100000000; + pIdx->estimatedRows = 1000000000; } - idxFinalize(&rc, pIdxList); + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + return SQLITE_OK; +} - *pRc = rc; - return 0; +/* +** Open a new sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbdataCursor *pCsr; + + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(DbdataCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; } -/* Callback for sqlite3_exec() with query with leading count(*) column. - * The first argument is expected to be an int*, referent to be incremented - * if that leading column is not exactly '0'. - */ -static int countNonzeros(void* pCount, int nc, - char* azResults[], char* azColumns[]){ - (void)azColumns; /* Suppress unused parameter warning */ - if( nc>0 && (azResults[0][0]!='0' || azResults[0][1]!=0) ){ - *((int *)pCount) += 1; +/* +** Restore a cursor object to the state it was in when first allocated +** by dbdataOpen(). +*/ +static void dbdataResetCursor(DbdataCursor *pCsr){ + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); + if( pTab->pStmt==0 ){ + pTab->pStmt = pCsr->pStmt; + }else{ + sqlite3_finalize(pCsr->pStmt); } - return 0; + pCsr->pStmt = 0; + pCsr->iPgno = 1; + pCsr->iCell = 0; + pCsr->iField = 0; + pCsr->bOnePage = 0; + sqlite3_free(pCsr->aPage); + dbdataBufferFree(&pCsr->rec); + pCsr->aPage = 0; + pCsr->nRec = 0; } -static int idxCreateFromCons( - sqlite3expert *p, - IdxScan *pScan, - IdxConstraint *pEq, - IdxConstraint *pTail -){ - sqlite3 *dbm = p->dbm; - int rc = SQLITE_OK; - if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ - IdxTable *pTab = pScan->pTab; - char *zCols = 0; - char *zIdx = 0; - IdxConstraint *pCons; - unsigned int h = 0; - const char *zFmt; +/* +** Close an sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataClose(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + dbdataResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} - for(pCons=pEq; pCons; pCons=pCons->pLink){ - zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); - } - for(pCons=pTail; pCons; pCons=pCons->pLink){ - zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); - } +/* +** Utility methods to decode 16 and 32-bit big-endian unsigned integers. +*/ +static u32 get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); +} - if( rc==SQLITE_OK ){ - /* Hash the list of columns to come up with a name for the index */ - const char *zTable = pScan->pTab->zName; - int quoteTable = idxIdentifierRequiresQuotes(zTable); - char *zName = 0; /* Index name */ - int collisions = 0; - do{ - int i; - char *zFind; - for(i=0; zCols[i]; i++){ - h += ((h<<3) + zCols[i]); - } - sqlite3_free(zName); - zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); - if( zName==0 ) break; - /* Is is unique among table, view and index names? */ - zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q" - " AND type in ('index','table','view')"; - zFind = sqlite3_mprintf(zFmt, zName); - i = 0; - rc = sqlite3_exec(dbm, zFind, countNonzeros, &i, 0); - assert(rc==SQLITE_OK); - sqlite3_free(zFind); - if( i==0 ){ - collisions = 0; - break; - } - ++collisions; - }while( collisions<50 && zName!=0 ); - if( collisions ){ - /* This return means "Gave up trying to find a unique index name." */ - rc = SQLITE_BUSY_TIMEOUT; - }else if( zName==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( quoteTable ){ - zFmt = "CREATE INDEX \"%w\" ON \"%w\"(%s)"; - }else{ - zFmt = "CREATE INDEX %s ON %s(%s)"; - } - zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols); - if( !zIdx ){ +/* +** Load page pgno from the database via the sqlite_dbpage virtual table. +** If successful, set (*ppPage) to point to a buffer containing the page +** data, (*pnPage) to the size of that buffer in bytes and return +** SQLITE_OK. In this case it is the responsibility of the caller to +** eventually free the buffer using sqlite3_free(). +** +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and +** return an SQLite error code. +*/ +static int dbdataLoadPage( + DbdataCursor *pCsr, /* Cursor object */ + u32 pgno, /* Page number of page to load */ + u8 **ppPage, /* OUT: pointer to page buffer */ + int *pnPage /* OUT: Size of (*ppPage) in bytes */ +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = pCsr->pStmt; + + *ppPage = 0; + *pnPage = 0; + if( pgno>0 ){ + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); + if( pPage==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg); - if( rc!=SQLITE_OK ){ - rc = SQLITE_BUSY_TIMEOUT; - }else{ - idxHashAdd(&rc, &p->hIdx, zName, zIdx); - } + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); } - sqlite3_free(zName); - sqlite3_free(zIdx); + *ppPage = pPage; + *pnPage = nCopy; } } - - sqlite3_free(zCols); + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; } + return rc; } /* -** Return true if list pList (linked by IdxConstraint.pLink) contains -** a constraint compatible with *p. Otherwise return false. +** Read a varint. Put the value in *pVal and return the number of bytes. */ -static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ - IdxConstraint *pCmp; - for(pCmp=pList; pCmp; pCmp=pCmp->pLink){ - if( p->iCol==pCmp->iCol ) return 1; +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_uint64 u = 0; + int i; + for(i=0; i<8; i++){ + u = (u<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } - return 0; + u = (u<<8) + (z[i]&0xff); + *pVal = (sqlite3_int64)u; + return 9; } -static int idxCreateFromWhere( - sqlite3expert *p, - IdxScan *pScan, /* Create indexes for this scan */ - IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ -){ - IdxConstraint *p1 = 0; - IdxConstraint *pCon; - int rc; +/* +** Like dbdataGetVarint(), but set the output to 0 if it is less than 0 +** or greater than 0xFFFFFFFF. This can be used for all varints in an +** SQLite database except for key values in intkey tables. +*/ +static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 val; + int nRet = dbdataGetVarint(z, &val); + if( val<0 || val>0xFFFFFFFF ) val = 0; + *pVal = val; + return nRet; +} - /* Gather up all the == constraints. */ - for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){ - if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ - pCon->pLink = p1; - p1 = pCon; +/* +** Return the number of bytes of space used by an SQLite value of type +** eType. +*/ +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + if( eType>0 ){ + return ((eType-12) / 2); + } + return 0; + } +} + +/* +** Load a value of type eType from buffer pData and use it to set the +** result of context object pCtx. +*/ +static void dbdataValue( + sqlite3_context *pCtx, + u32 enc, + int eType, + u8 *pData, + sqlite3_int64 nData +){ + if( eType>=0 ){ + if( dbdataValueBytes(eType)<=nData ){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + switch( enc ){ + #ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + #endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } + }else{ + if( eType==7 ){ + sqlite3_result_double(pCtx, 0.0); + }else if( eType<7 ){ + sqlite3_result_int(pCtx, 0); + }else if( eType%2 ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + }else{ + sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); + } } } +} - /* Create an index using the == constraints collected above. And the - ** range constraint/ORDER BY terms passed in by the caller, if any. */ - rc = idxCreateFromCons(p, pScan, p1, pTail); +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) - /* If no range/ORDER BY passed by the caller, create a version of the - ** index for each range constraint. */ - if( pTail==0 ){ - for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ - assert( pCon->pLink==0 ); - if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){ - rc = idxCreateFromCons(p, pScan, p1, pCon); +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + +/* +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. +*/ +static int dbdataNext(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + + pCsr->iRowid++; + while( 1 ){ + int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); + int bNextPage = 0; + + if( pCsr->aPage==0 ){ + while( 1 ){ + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + if( pCsr->aPage && pCsr->nPage>=256 ) break; + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + } + + assert( iOff+3+2<=pCsr->nPage ); + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); } } - } - return rc; -} + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; + } + pCsr->iCell++; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ + return SQLITE_OK; + } + }else{ + /* If there is no record loaded, load it now. */ + assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); + if( pCsr->nRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } + + if( pCsr->iCell>=pCsr->nCell ){ + bNextPage = 1; + }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; + + if( iCellPtr>pCsr->nPage ){ + bNextPage = 1; + }else{ + iOff = get_uint16(&pCsr->aPage[iCellPtr]); + } + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ + bNextPage = 1; + }else{ + iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); + if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; + if( nPayload==0 ) nPayload = 1; + } + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid && !bNextPage && iOffnPage ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Figure out how much data to read from the local page */ + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; + }else{ + X = ((U-12)*64/255)-23; + } + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } + } + + if( bNextPage || nLocal+iOff>pCsr->nPage ){ + bNextPage = 1; + }else{ + + /* Allocate space for payload. And a bit more to catch small buffer + ** overruns caused by attempting to read a varint or similar from + ** near the end of a corrupt record. */ + rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); + if( rc!=SQLITE_OK ) return rc; + assert( nPayload!=0 ); + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + if( aOvfl==0 ) break; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; -/* -** Create candidate indexes in database [dbm] based on the data in -** linked-list pScan. -*/ -static int idxCreateCandidates(sqlite3expert *p){ - int rc = SQLITE_OK; - IdxScan *pIter; + pgnoOvfl = get_uint32(aOvfl); + sqlite3_free(aOvfl); + } + nPayload -= nRem; + } + memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); + pCsr->nRec = nPayload; + + iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); + if( nHdr>nPayload ) nHdr = 0; + pCsr->nHdr = nHdr; + pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; + pCsr->iField = (bHasRowid ? -1 : 0); + } + } + }else{ + pCsr->iField++; + if( pCsr->iField>0 ){ + sqlite3_int64 iType; + if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ + bNextPage = 1; + }else{ + int szField = 0; + pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); + szField = dbdataValueBytes(iType); + if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))pPtr = &pCsr->rec.aBuf[pCsr->nRec]; + }else{ + pCsr->pPtr += szField; + } + } + } + } - for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ - rc = idxCreateFromWhere(p, pIter, 0); - if( rc==SQLITE_OK && pIter->pOrder ){ - rc = idxCreateFromWhere(p, pIter, pIter->pOrder); + if( bNextPage ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + pCsr->nRec = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ + return SQLITE_OK; + } + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + pCsr->nRec = 0; + pCsr->iCell++; + } } } - return rc; + assert( !"can't get here" ); + return SQLITE_OK; } -/* -** Free all elements of the linked list starting at pConstraint. +/* +** Return true if the cursor is at EOF. */ -static void idxConstraintFree(IdxConstraint *pConstraint){ - IdxConstraint *pNext; - IdxConstraint *p; - - for(p=pConstraint; p; p=pNext){ - pNext = p->pNext; - sqlite3_free(p); - } +static int dbdataEof(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + return pCsr->aPage==0; } /* -** Free all elements of the linked list starting from pScan up until pLast -** (pLast is not freed). +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. */ -static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ - IdxScan *p; - IdxScan *pNext; - for(p=pScan; p!=pLast; p=pNext){ - pNext = p->pNextScan; - idxConstraintFree(p->pOrder); - idxConstraintFree(p->pEq); - idxConstraintFree(p->pRange); - sqlite3_free(p); +static int dbdataIsFunction(const char *zSchema){ + size_t n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return (int)n-2; } + return 0; } -/* -** Free all elements of the linked list starting from pStatement up -** until pLast (pLast is not freed). +/* +** Determine the size in pages of database zSchema (where zSchema is +** "main", "temp" or the name of an attached database) and set +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, +** an SQLite error code. */ -static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ - IdxStatement *p; - IdxStatement *pNext; - for(p=pStatement; p!=pLast; p=pNext){ - pNext = p->pNext; - sqlite3_free(p->zEQP); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; + char *zSql = 0; + int rc, rc2; + int nFunc = 0; + sqlite3_stmt *pStmt = 0; -/* -** Free the linked list of IdxTable objects starting at pTab. -*/ -static void idxTableFree(IdxTable *pTab){ - IdxTable *pIter; - IdxTable *pNext; - for(pIter=pTab; pIter; pIter=pNext){ - pNext = pIter->pNext; - sqlite3_free(pIter); + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); } -} + if( zSql==0 ) return SQLITE_NOMEM; -/* -** Free the linked list of IdxWrite objects starting at pTab. -*/ -static void idxWriteFree(IdxWrite *pTab){ - IdxWrite *pIter; - IdxWrite *pNext; - for(pIter=pTab; pIter; pIter=pNext){ - pNext = pIter->pNext; - sqlite3_free(pIter); + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + pCsr->szDb = sqlite3_column_int(pStmt, 0); } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + return rc; } - - /* -** This function is called after candidate indexes have been created. It -** runs all the queries to see which indexes they prefer, and populates -** IdxStatement.zIdx and IdxStatement.zEQP with the results. +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. */ -int idxFindIndexes( - sqlite3expert *p, - char **pzErr /* OUT: Error message (sqlite3_malloc) */ -){ - IdxStatement *pStmt; - sqlite3 *dbm = p->dbm; +static int dbdataGetEncoding(DbdataCursor *pCsr){ int rc = SQLITE_OK; - - IdxHash hIdx; - idxHashInit(&hIdx); - - for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){ - IdxHashEntry *pEntry; - sqlite3_stmt *pExplain = 0; - idxHashClear(&hIdx); - rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr, - "EXPLAIN QUERY PLAN %s", pStmt->zSql - ); - while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ - /* int iId = sqlite3_column_int(pExplain, 0); */ - /* int iParent = sqlite3_column_int(pExplain, 1); */ - /* int iNotUsed = sqlite3_column_int(pExplain, 2); */ - const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); - int nDetail; - int i; - - if( !zDetail ) continue; - nDetail = STRLEN(zDetail); - - for(i=0; ihIdx, zIdx, nIdx); - if( zSql ){ - idxHashAdd(&rc, &hIdx, zSql, 0); - if( rc ) goto find_indexes_out; - } - break; - } - } - - if( zDetail[0]!='-' ){ - pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail); - } - } - - for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ - pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey); - } - - idxFinalize(&rc, pExplain); + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + if( rc==SQLITE_OK && nPg1>=(56+4) ){ + pCsr->enc = get_uint32(&aPg1[56]); } - - find_indexes_out: - idxHashClear(&hIdx); + sqlite3_free(aPg1); return rc; } -static int idxAuthCallback( - void *pCtx, - int eOp, - const char *z3, - const char *z4, - const char *zDb, - const char *zTrigger -){ - int rc = SQLITE_OK; - (void)z4; - (void)zTrigger; - if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){ - if( sqlite3_stricmp(zDb, "main")==0 ){ - sqlite3expert *p = (sqlite3expert*)pCtx; - IdxTable *pTab; - for(pTab=p->pTable; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_stricmp(z3, pTab->zName) ) break; - } - if( pTab ){ - IdxWrite *pWrite; - for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){ - if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break; - } - if( pWrite==0 ){ - pWrite = idxMalloc(&rc, sizeof(IdxWrite)); - if( rc==SQLITE_OK ){ - pWrite->pTab = pTab; - pWrite->eOp = eOp; - pWrite->pNext = p->pWrite; - p->pWrite = pWrite; - } - } - } - } - } - return rc; -} -static int idxProcessOneTrigger( - sqlite3expert *p, - IdxWrite *pWrite, - char **pzErr -){ - static const char *zInt = UNIQUE_TABLE_NAME; - static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME; - IdxTable *pTab = pWrite->pTab; - const char *zTab = pTab->zName; - const char *zSql = - "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_schema " - "WHERE tbl_name = %Q AND type IN ('table', 'trigger') " - "ORDER BY type;"; - sqlite3_stmt *pSelect = 0; +/* +** xFilter method for sqlite_dbdata and sqlite_dbptr. +*/ +static int dbdataFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; int rc = SQLITE_OK; - char *zWrite = 0; + const char *zSchema = "main"; + (void)idxStr; + (void)argc; - /* Create the table and its triggers in the temp schema */ - rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){ - const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0); - rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr); + dbdataResetCursor(pCsr); + assert( pCsr->iPgno==1 ); + if( idxNum & 0x01 ){ + zSchema = (const char*)sqlite3_value_text(argv[0]); + if( zSchema==0 ) zSchema = ""; + } + if( idxNum & 0x02 ){ + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; + }else{ + rc = dbdataDbsize(pCsr, zSchema); } - idxFinalize(&rc, pSelect); - /* Rename the table in the temp schema to zInt */ if( rc==SQLITE_OK ){ - char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt); - if( z==0 ){ - rc = SQLITE_NOMEM; + int nFunc = 0; + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } }else{ - rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr); - sqlite3_free(z); + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); } } - - switch( pWrite->eOp ){ - case SQLITE_INSERT: { - int i; - zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt); - for(i=0; inCol; i++){ - zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", "); - } - zWrite = idxAppendText(&rc, zWrite, ")"); - break; - } - case SQLITE_UPDATE: { - int i; - zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt); - for(i=0; inCol; i++){ - zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ", - pTab->aCol[i].zName - ); - } - break; - } - default: { - assert( pWrite->eOp==SQLITE_DELETE ); - if( rc==SQLITE_OK ){ - zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt); - if( zWrite==0 ) rc = SQLITE_NOMEM; - } - } + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); } + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ if( rc==SQLITE_OK ){ - sqlite3_stmt *pX = 0; - rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0); - idxFinalize(&rc, pX); - if( rc!=SQLITE_OK ){ - idxDatabaseError(p->dbv, pzErr); - } + rc = dbdataGetEncoding(pCsr); } - sqlite3_free(zWrite); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr); + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } + if( rc==SQLITE_OK ){ + rc = dbdataNext(pCursor); + } return rc; } -static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ - int rc = SQLITE_OK; - IdxWrite *pEnd = 0; - IdxWrite *pFirst = p->pWrite; - - while( rc==SQLITE_OK && pFirst!=pEnd ){ - IdxWrite *pIter; - for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){ - rc = idxProcessOneTrigger(p, pIter, pzErr); +/* +** Return a column for the sqlite_dbdata or sqlite_dbptr table. +*/ +static int dbdataColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + if( iOff>pCsr->nPage ) return SQLITE_OK; + iOff = get_uint16(&pCsr->aPage[iOff]); + } + if( iOff<=pCsr->nPage ){ + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + } + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ + sqlite3_int64 iType; + dbdataGetVarintU32(pCsr->pHdrPtr, &iType); + dbdataValue( + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr + ); + } + break; + } } - pEnd = pFirst; - pFirst = p->pWrite; } - - return rc; + return SQLITE_OK; } +/* +** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +*/ +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} -static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ - int rc = idxRegisterVtab(p); - sqlite3_stmt *pSchema = 0; - - /* For each table in the main db schema: - ** - ** 1) Add an entry to the p->pTable list, and - ** 2) Create the equivalent virtual table in dbv. - */ - rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg, - "SELECT type, name, sql, 1 FROM sqlite_schema " - "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' " - " UNION ALL " - "SELECT type, name, sql, 2 FROM sqlite_schema " - "WHERE type = 'trigger'" - " AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') " - "ORDER BY 4, 1" - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){ - const char *zType = (const char*)sqlite3_column_text(pSchema, 0); - const char *zName = (const char*)sqlite3_column_text(pSchema, 1); - const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); - - if( zType[0]=='v' || zType[1]=='r' ){ - rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg); - }else{ - IdxTable *pTab; - rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); - if( rc==SQLITE_OK ){ - int i; - char *zInner = 0; - char *zOuter = 0; - pTab->pNext = p->pTable; - p->pTable = pTab; - /* The statement the vtab will pass to sqlite3_declare_vtab() */ - zInner = idxAppendText(&rc, 0, "CREATE TABLE x("); - for(i=0; inCol; i++){ - zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s", - (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl - ); - } - zInner = idxAppendText(&rc, zInner, ")"); +/* +** Invoke this routine to register the "sqlite_dbdata" virtual table module +*/ +static int sqlite3DbdataRegister(sqlite3 *db){ + static sqlite3_module dbdata_module = { + 0, /* iVersion */ + 0, /* xCreate */ + dbdataConnect, /* xConnect */ + dbdataBestIndex, /* xBestIndex */ + dbdataDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + dbdataOpen, /* xOpen - open a cursor */ + dbdataClose, /* xClose - close a cursor */ + dbdataFilter, /* xFilter - configure scan constraints */ + dbdataNext, /* xNext - advance a cursor */ + dbdataEof, /* xEof - check for end of scan */ + dbdataColumn, /* xColumn - read data */ + dbdataRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; - /* The CVT statement to create the vtab */ - zOuter = idxAppendText(&rc, 0, - "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg); - } - sqlite3_free(zInner); - sqlite3_free(zOuter); - } - } + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); } - idxFinalize(&rc, pSchema); return rc; } -struct IdxSampleCtx { - int iTarget; - double target; /* Target nRet/nRow value */ - double nRow; /* Number of rows seen */ - double nRet; /* Number of rows returned */ -}; +int sqlite3_dbdata_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + (void)pzErrMsg; + return sqlite3DbdataRegister(db); +} + +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ -static void idxSampleFunc( - sqlite3_context *pCtx, - int argc, - sqlite3_value **argv -){ - struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx); - int bRet; +/************************* End ../ext/recover/dbdata.c ********************/ +/************************* Begin ../ext/recover/sqlite3recover.c ******************/ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ - (void)argv; - assert( argc==0 ); - if( p->nRow==0.0 ){ - bRet = 1; - }else{ - bRet = (p->nRet / p->nRow) <= p->target; - if( bRet==0 ){ - unsigned short rnd; - sqlite3_randomness(2, (void*)&rnd); - bRet = ((int)rnd % 100) <= p->iTarget; - } - } - sqlite3_result_int(pCtx, bRet); - p->nRow += 1.0; - p->nRet += (double)bRet; -} +/* #include "sqlite3recover.h" */ +#include +#include -struct IdxRemCtx { - int nSlot; - struct IdxRemSlot { - int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ - i64 iVal; /* SQLITE_INTEGER value */ - double rVal; /* SQLITE_FLOAT value */ - int nByte; /* Bytes of space allocated at z */ - int n; /* Size of buffer z */ - char *z; /* SQLITE_TEXT/BLOB value */ - } aSlot[1]; +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 + +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* typedef unsigned int u32; */ +/* typedef unsigned char u8; */ +/* typedef sqlite3_int64 i64; */ + +typedef struct RecoverTable RecoverTable; +typedef struct RecoverColumn RecoverColumn; + +/* +** When recovering rows of data that can be associated with table +** definitions recovered from the sqlite_schema table, each table is +** represented by an instance of the following object. +** +** iRoot: +** The root page in the original database. Not necessarily (and usually +** not) the same in the recovered database. +** +** zTab: +** Name of the table. +** +** nCol/aCol[]: +** aCol[] is an array of nCol columns. In the order in which they appear +** in the table. +** +** bIntkey: +** Set to true for intkey tables, false for WITHOUT ROWID. +** +** iRowidBind: +** Each column in the aCol[] array has associated with it the index of +** the bind parameter its values will be bound to in the INSERT statement +** used to construct the output database. If the table does has a rowid +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the +** index of the bind paramater to which the rowid value should be bound. +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY +** KEY column, then the rowid value should be bound to the index associated +** with the column. +** +** pNext: +** All RecoverTable objects used by the recovery operation are allocated +** and populated as part of creating the recovered database schema in +** the output database, before any non-schema data are recovered. They +** are then stored in a singly-linked list linked by this variable beginning +** at sqlite3_recover.pTblList. +*/ +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ + RecoverTable *pNext; }; /* -** Implementation of scalar function rem(). +** Each database column is represented by an instance of the following object +** stored in the RecoverTable.aCol[] array of the associated table. +** +** iField: +** The index of the associated field within database records. Or -1 if +** there is no associated field (e.g. for virtual generated columns). +** +** iBind: +** The bind index of the INSERT statement to bind this columns values +** to. Or 0 if there is no such index (iff (iField<0)). +** +** bIPK: +** True if this is the INTEGER PRIMARY KEY column. +** +** zCol: +** Name of column. +** +** eHidden: +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). */ -static void idxRemFunc( - sqlite3_context *pCtx, - int argc, - sqlite3_value **argv -){ - struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx); - struct IdxRemSlot *pSlot; - int iSlot; - assert( argc==2 ); +struct RecoverColumn { + int iField; /* Field in record on disk */ + int iBind; /* Binding to use in INSERT */ + int bIPK; /* True for IPK column */ + char *zCol; + int eHidden; +}; - iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); - pSlot = &p->aSlot[iSlot]; +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */ +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */ +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */ +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */ - switch( pSlot->eType ){ - case SQLITE_NULL: - /* no-op */ - break; +/* +** Bitmap object used to track pages in the input database. Allocated +** and manipulated only by the following functions: +** +** recoverBitmapAlloc() +** recoverBitmapFree() +** recoverBitmapSet() +** recoverBitmapQuery() +** +** nPg: +** Largest page number that may be stored in the bitmap. The range +** of valid keys is 1 to nPg, inclusive. +** +** aElem[]: +** Array large enough to contain a bit for each key. For key value +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. +** In other words, the following is true if bit iKey is set, or +** false if it is clear: +** +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 +*/ +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[1]; /* Array of 32-bit bitmasks */ +}; - case SQLITE_INTEGER: - sqlite3_result_int64(pCtx, pSlot->iVal); - break; +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data for tables identified in the recovered schema (state +** RECOVER_STATE_WRITING). +*/ +typedef struct RecoverStateW1 RecoverStateW1; +struct RecoverStateW1 { + sqlite3_stmt *pTbls; + sqlite3_stmt *pSel; + sqlite3_stmt *pInsert; + int nInsert; + + RecoverTable *pTab; /* Table currently being written */ + int nMax; /* Max column count in any schema table */ + sqlite3_value **apVal; /* Array of nMax values */ + int nVal; /* Number of valid entries in apVal[] */ + int bHaveRowid; + i64 iRowid; + i64 iPrevPage; + int iPrevCell; +}; - case SQLITE_FLOAT: - sqlite3_result_double(pCtx, pSlot->rVal); - break; +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data destined for the lost and found table (states +** RECOVER_STATE_LOSTANDFOUND[123]). +*/ +typedef struct RecoverStateLAF RecoverStateLAF; +struct RecoverStateLAF { + RecoverBitmap *pUsed; + i64 nPg; /* Size of db in pages */ + sqlite3_stmt *pAllAndParent; + sqlite3_stmt *pMapInsert; + sqlite3_stmt *pMaxField; + sqlite3_stmt *pUsedPages; + sqlite3_stmt *pFindRoot; + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ + sqlite3_stmt *pAllPage; + sqlite3_stmt *pPageData; + sqlite3_value **apVal; + int nMaxField; +}; - case SQLITE_BLOB: - sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); - break; +/* +** Main recover handle structure. +*/ +struct sqlite3_recover { + /* Copies of sqlite3_recover_init[_sql]() parameters */ + sqlite3 *dbIn; /* Input database */ + char *zDb; /* Name of input db ("main" etc.) */ + char *zUri; /* URI for output database */ + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ - case SQLITE_TEXT: - sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); - break; - } + /* Values configured by sqlite3_recover_config() */ + char *zStateDb; /* State database to use (or NULL) */ + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ - pSlot->eType = sqlite3_value_type(argv[1]); - switch( pSlot->eType ){ - case SQLITE_NULL: - /* no-op */ - break; + int pgsz; + int detected_pgsz; + int nReserve; + u8 *pPage1Disk; + u8 *pPage1Cache; - case SQLITE_INTEGER: - pSlot->iVal = sqlite3_value_int64(argv[1]); - break; + /* Error code and error message */ + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ - case SQLITE_FLOAT: - pSlot->rVal = sqlite3_value_double(argv[1]); - break; + int eState; + int bCloseTransaction; - case SQLITE_BLOB: - case SQLITE_TEXT: { - int nByte = sqlite3_value_bytes(argv[1]); - if( nByte>pSlot->nByte ){ - char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); - if( zNew==0 ){ - sqlite3_result_error_nomem(pCtx); - return; - } - pSlot->nByte = nByte*2; - pSlot->z = zNew; - } - pSlot->n = nByte; - if( pSlot->eType==SQLITE_BLOB ){ - memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte); - }else{ - memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte); - } - break; - } - } -} + /* Variables used with eState==RECOVER_STATE_WRITING */ + RecoverStateW1 w1; -static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){ - int rc = SQLITE_OK; - const char *zMax = - "SELECT max(i.seqno) FROM " - " sqlite_schema AS s, " - " pragma_index_list(s.name) AS l, " - " pragma_index_info(l.name) AS i " - "WHERE s.type = 'table'"; - sqlite3_stmt *pMax = 0; + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ + RecoverStateLAF laf; - *pnMax = 0; - rc = idxPrepareStmt(db, &pMax, pzErr, zMax); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ - *pnMax = sqlite3_column_int(pMax, 0) + 1; - } - idxFinalize(&rc, pMax); + /* Fields used within sqlite3_recover_run() */ + sqlite3 *dbOut; /* Output database */ + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ + RecoverTable *pTblList; /* List of tables recovered from schema */ +}; - return rc; -} +/* +** The various states in which an sqlite3_recover object may exist: +** +** RECOVER_STATE_INIT: +** The object is initially created in this state. sqlite3_recover_step() +** has yet to be called. This is the only state in which it is permitted +** to call sqlite3_recover_config(). +** +** RECOVER_STATE_WRITING: +** +** RECOVER_STATE_LOSTANDFOUND1: +** State to populate the bitmap of pages used by other tables or the +** database freelist. +** +** RECOVER_STATE_LOSTANDFOUND2: +** Populate the recovery.map table - used to figure out a "root" page +** for each lost page from in the database from which records are +** extracted. +** +** RECOVER_STATE_LOSTANDFOUND3: +** Populate the lost-and-found table itself. +*/ +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND1 2 +#define RECOVER_STATE_LOSTANDFOUND2 3 +#define RECOVER_STATE_LOSTANDFOUND3 4 +#define RECOVER_STATE_SCHEMA2 5 +#define RECOVER_STATE_DONE 6 -static int idxPopulateOneStat1( - sqlite3expert *p, - sqlite3_stmt *pIndexXInfo, - sqlite3_stmt *pWriteStat, - const char *zTab, - const char *zIdx, - char **pzErr -){ - char *zCols = 0; - char *zOrder = 0; - char *zQuery = 0; - int nCol = 0; - int i; - sqlite3_stmt *pQuery = 0; - int *aStat = 0; - int rc = SQLITE_OK; - assert( p->iSample>0 ); +/* +** Global variables used by this extension. +*/ +typedef struct RecoverGlobal RecoverGlobal; +struct RecoverGlobal { + const sqlite3_io_methods *pMethods; + sqlite3_recover *p; +}; +static RecoverGlobal recover_g; - /* Formulate the query text */ - sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC); - while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){ - const char *zComma = zCols==0 ? "" : ", "; - const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); - const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); - zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl - ); - zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); - } - sqlite3_reset(pIndexXInfo); - if( rc==SQLITE_OK ){ - if( p->iSample==100 ){ - zQuery = sqlite3_mprintf( - "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder - ); - }else{ - zQuery = sqlite3_mprintf( - "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder - ); - } - } - sqlite3_free(zCols); - sqlite3_free(zOrder); +/* +** Use this static SQLite mutex to protect the globals during the +** first call to sqlite3_recover_step(). +*/ +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 - /* Formulate the query text */ - if( rc==SQLITE_OK ){ - sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery); - } - sqlite3_free(zQuery); - if( rc==SQLITE_OK ){ - aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); - } - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ - IdxHashEntry *pEntry; - char *zStat = 0; - for(i=0; i<=nCol; i++) aStat[i] = 1; - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ - aStat[0]++; - for(i=0; i=1 && defined(SQLITE_DEBUG) +static void recoverAssertMutexHeld(void){ + assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); +} +#else +# define recoverAssertMutexHeld() +#endif - if( rc==SQLITE_OK ){ - int s0 = aStat[0]; - zStat = sqlite3_mprintf("%d", s0); - if( zStat==0 ) rc = SQLITE_NOMEM; - for(i=1; rc==SQLITE_OK && i<=nCol; i++){ - zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]); - } - } - if( rc==SQLITE_OK ){ - sqlite3_bind_text(pWriteStat, 1, zTab, -1, SQLITE_STATIC); - sqlite3_bind_text(pWriteStat, 2, zIdx, -1, SQLITE_STATIC); - sqlite3_bind_text(pWriteStat, 3, zStat, -1, SQLITE_STATIC); - sqlite3_step(pWriteStat); - rc = sqlite3_reset(pWriteStat); - } +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + if( zStr==0 ) return 0; + return (int)(strlen(zStr)&0x7fffffff); +} - pEntry = idxHashFind(&p->hIdx, zIdx, STRLEN(zIdx)); - if( pEntry ){ - assert( pEntry->zVal2==0 ); - pEntry->zVal2 = zStat; +/* +** This function is a no-op if the recover handle passed as the first +** argument already contains an error (if p->errCode!=SQLITE_OK). +** +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte +** bytes in size. If successful, a pointer to the new buffer is returned. Or, +** if an OOM error occurs, NULL is returned and the handle error code +** (p->errCode) set to SQLITE_NOMEM. +*/ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); }else{ - sqlite3_free(zStat); + p->errCode = SQLITE_NOMEM; } } - sqlite3_free(aStat); - idxFinalize(&rc, pQuery); - - return rc; + return pRet; } -static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){ - int rc; - char *zSql; +/* +** Set the error code and error message for the recover handle passed as +** the first argument. The error code is set to the value of parameter +** errCode. +** +** Parameter zFmt must be a printf() style formatting string. The handle +** error message is set to the result of using any trailing arguments for +** parameter substitutions in the formatting string. +** +** For example: +** +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); +*/ +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + char *z = 0; + va_list ap; + va_start(ap, zFmt); + if( zFmt ){ + z = sqlite3_vmprintf(zFmt, ap); + } + va_end(ap); + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} - rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); - if( rc!=SQLITE_OK ) return rc; - zSql = sqlite3_mprintf( - "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab - ); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0); - sqlite3_free(zSql); +/* +** This function is a no-op if p->errCode is initially other than SQLITE_OK. +** In this case it returns NULL. +** +** Otherwise, an attempt is made to allocate and return a bitmap object +** large enough to store a bit for all page numbers between 1 and nPg, +** inclusive. The bitmap is initially zeroed. +*/ +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); - return rc; + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; } /* -** This function is called as part of sqlite3_expert_analyze(). Candidate -** indexes have already been created in database sqlite3expert.dbm, this -** function populates sqlite_stat1 table in the same database. -** -** The stat1 data is generated by querying the +** Free a bitmap object allocated by recoverBitmapAlloc(). */ -static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ - int rc = SQLITE_OK; - int nMax =0; - struct IdxRemCtx *pCtx = 0; - struct IdxSampleCtx samplectx; - int i; - i64 iPrev = -100000; - sqlite3_stmt *pAllIndex = 0; - sqlite3_stmt *pIndexXInfo = 0; - sqlite3_stmt *pWrite = 0; +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} - const char *zAllIndex = - "SELECT s.rowid, s.name, l.name FROM " - " sqlite_schema AS s, " - " pragma_index_list(s.name) AS l " - "WHERE s.type = 'table'"; - const char *zIndexXInfo = - "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key"; - const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)"; +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} - /* If iSample==0, no sqlite_stat1 data is required. */ - if( p->iSample==0 ) return SQLITE_OK; +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg && iPg>0 ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} - rc = idxLargestIndex(p->dbm, &nMax, pzErr); - if( nMax<=0 || rc!=SQLITE_OK ) return rc; +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} - rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} - if( rc==SQLITE_OK ){ - int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); - pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } } + return pStmt; +} - if( rc==SQLITE_OK ){ - sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 - ); +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 - ); + return pStmt; +} + +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); } +} - if( rc==SQLITE_OK ){ - pCtx->nSlot = nMax+1; - rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. +** Or, if an error occurs, leave an error code and message in the recover +** handle and return a copy of the error code. +*/ +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, db); + } } - if( rc==SQLITE_OK ){ - rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo); + return p->errCode; +} + +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); } - if( rc==SQLITE_OK ){ - rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite); +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; } + return z; +} - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){ - i64 iRowid = sqlite3_column_int64(pAllIndex, 0); - const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1); - const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2); - if( p->iSample<100 && iPrev!=iRowid ){ - samplectx.target = (double)p->iSample / 100.0; - samplectx.iTarget = p->iSample; - samplectx.nRow = 0.0; - samplectx.nRet = 0.0; - rc = idxBuildSampleTable(p, zTab); - if( rc!=SQLITE_OK ) break; +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt ){ + sqlite3_step(pStmt); + nPg = sqlite3_column_int64(pStmt, 0); } - rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr); - iPrev = iRowid; + recoverFinalize(p, pStmt); } - if( rc==SQLITE_OK && p->iSample<100 ){ - rc = sqlite3_exec(p->dbv, - "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0 - ); + return nPg; +} + +/* +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +** +** SELECT read_i32(, ) +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; + + if( (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); } +} - idxFinalize(&rc, pAllIndex); - idxFinalize(&rc, pIndexXInfo); - idxFinalize(&rc, pWrite); +/* +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.laf.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + assert( nArg==1 ); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); +} - if( pCtx ){ - for(i=0; inSlot; i++){ - sqlite3_free(pCtx->aSlot[i].z); +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the argument +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + i64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); + return; + }else{ + if( p->pGetPage==0 ){ + pStmt = p->pGetPage = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else if( p->errCode==SQLITE_OK ){ + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const u8 *aPg; + int nPg; + assert( p->errCode==SQLITE_OK ); + aPg = sqlite3_column_blob(pStmt, 0); + nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; + } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); + } + recoverReset(p, pStmt); } - sqlite3_free(pCtx); } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); } - - sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); - return rc; } /* -** Allocate a new sqlite3expert object. +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. */ -sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ - int rc = SQLITE_OK; - sqlite3expert *pNew; +static const char *recoverUnusedString( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} - pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert)); +/* +** Implementation of scalar SQL function "escape_crnl". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. +*/ +static void recoverEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + (void)argc; + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; - /* Open two in-memory databases to work with. The "vtab database" (dbv) - ** will contain a virtual table corresponding to each real table in - ** the user database schema, and a copy of each view. It is used to - ** collect information regarding the WHERE, ORDER BY and other clauses - ** of the user's query. - */ - if( rc==SQLITE_OK ){ - pNew->db = db; - pNew->iSample = 100; - rc = sqlite3_open(":memory:", &pNew->dbv); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_open(":memory:", &pNew->dbm); - if( rc==SQLITE_OK ){ - sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } } - } - - /* Copy the entire schema of database [db] into [dbm]. */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSql; - rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, - "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" - " AND sql NOT LIKE 'CREATE VIRTUAL %%'" - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - const char *zSql = (const char*)sqlite3_column_text(pSql, 0); - rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); - } - idxFinalize(&rc, pSql); - } + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } - /* Create the vtab schema */ - if( rc==SQLITE_OK ){ - rc = idxCreateVtabSchema(pNew, pzErrmsg); - } + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } - /* Register the auth callback with dbv */ - if( rc==SQLITE_OK ){ - sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); - } + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } - /* If an error has occurred, free the new object and reutrn NULL. Otherwise, - ** return the new sqlite3expert handle. */ - if( rc!=SQLITE_OK ){ - sqlite3_expert_destroy(pNew); - pNew = 0; + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } } - return pNew; + + sqlite3_result_value(context, argv[0]); } /* -** Configure an sqlite3expert object. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. */ -int sqlite3_expert_config(sqlite3expert *p, int op, ...){ - int rc = SQLITE_OK; - va_list ap; - va_start(ap, op); - switch( op ){ - case EXPERT_CONFIG_SAMPLE: { - int iVal = va_arg(ap, int); - if( iVal<0 ) iVal = 0; - if( iVal>100 ) iVal = 100; - p->iSample = iVal; - break; +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); } - default: - rc = SQLITE_NOTFOUND; - break; } - - va_end(ap); - return rc; } /* -** Add an SQL statement to the analysis. +** Transfer the following settings from the input database to the output +** database: +** +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and */ -int sqlite3_expert_sql( - sqlite3expert *p, /* From sqlite3_expert_new() */ - const char *zSql, /* SQL statement to add */ - char **pzErr /* OUT: Error message (if any) */ -){ - IdxScan *pScanOrig = p->pScan; - IdxStatement *pStmtOrig = p->pStatement; - int rc = SQLITE_OK; - const char *zStmt = zSql; - - if( p->bRun ) return SQLITE_MISUSE; +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } - while( rc==SQLITE_OK && zStmt && zStmt[0] ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); - if( rc==SQLITE_OK ){ - if( pStmt ){ - IdxStatement *pNew; - const char *z = sqlite3_sql(pStmt); - int n = STRLEN(z); - pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); - if( rc==SQLITE_OK ){ - pNew->zSql = (char*)&pNew[1]; - memcpy(pNew->zSql, z, n+1); - pNew->pNext = p->pStatement; - if( p->pStatement ) pNew->iId = p->pStatement->iId+1; - p->pStatement = pNew; + for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){ + const char *zPrag = aPragma[ii]; + sqlite3_stmt *p1 = 0; + p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); } - sqlite3_finalize(pStmt); } - }else{ - idxDatabaseError(p->dbv, pzErr); + recoverFinalize(p, p1); } - } - - if( rc!=SQLITE_OK ){ - idxScanFree(p->pScan, pScanOrig); - idxStatementFree(p->pStatement, pStmtOrig); - p->pScan = pScanOrig; - p->pStatement = pStmtOrig; - } - - return rc; -} - -int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ - int rc; - IdxHashEntry *pEntry; + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); - /* Do trigger processing to collect any extra IdxScan structures */ - rc = idxProcessTriggers(p, pzErr); + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } + } - /* Create candidate indexes within the in-memory database file */ - if( rc==SQLITE_OK ){ - rc = idxCreateCandidates(p); - }else if ( rc==SQLITE_BUSY_TIMEOUT ){ - if( pzErr ) - *pzErr = sqlite3_mprintf("Cannot find a unique index name to propose."); - return rc; + sqlite3_close(db2); } +} - /* Generate the stat1 data */ - if( rc==SQLITE_OK ){ - rc = idxPopulateStat1(p, pzErr); - } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. +*/ +static int recoverOpenOutput(sqlite3_recover *p){ + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crnl", 1, recoverEscapeCrnl }, + }; - /* Formulate the EXPERT_REPORT_CANDIDATES text */ - for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){ - p->zCandidates = idxAppendText(&rc, p->zCandidates, - "%s;%s%s\n", pEntry->zVal, - pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2 - ); + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ + + assert( p->dbOut==0 ); + + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); } - /* Figure out which of the candidate indexes are preferred by the query - ** planner and report the results to the user. */ - if( rc==SQLITE_OK ){ - rc = idxFindIndexes(p, pzErr); + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ + if( p->errCode==SQLITE_OK ){ + p->errCode = sqlite3_dbdata_init(db, 0, 0); } - if( rc==SQLITE_OK ){ - p->bRun = 1; + /* Register the custom user-functions with the output handle. */ + for(ii=0; + p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); + ii++){ + p->errCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); } - return rc; + + p->dbOut = db; + return p->errCode; } /* -** Return the total number of statements that have been added to this -** sqlite3expert using sqlite3_expert_sql(). +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. */ -int sqlite3_expert_count(sqlite3expert *p){ - int nRet = 0; - if( p->pStatement ) nRet = p->pStatement->iId+1; - return nRet; +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + recoverExec(p, p->dbOut, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); } + /* -** Return a component of the report. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. +** +** Parameter iRoot must be the root page of table zName in the INPUT +** database. */ -const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ - const char *zRet = 0; - IdxStatement *pStmt; +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); - if( p->bRun==0 ) return 0; - for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext); - switch( eReport ){ - case EXPERT_REPORT_SQL: - if( pStmt ) zRet = pStmt->zSql; - break; - case EXPERT_REPORT_INDEXES: - if( pStmt ) zRet = pStmt->zIdx; - break; - case EXPERT_REPORT_PLAN: - if( pStmt ) zRet = pStmt->zEQP; - break; - case EXPERT_REPORT_CANDIDATES: - zRet = p->zCandidates; - break; + if( pStmt ){ + int iPk = -1; + int iBind = 1; + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + int iField = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int iPKF = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + pNew->bIntkey = 1; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; + } + recoverFinalize(p, pStmt); + + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } + } } - return zRet; } /* -** Free an sqlite3expert object. +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: +** +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. */ -void sqlite3_expert_destroy(sqlite3expert *p){ - if( p ){ - sqlite3_close(p->dbm); - sqlite3_close(p->dbv); - idxScanFree(p->pScan, 0); - idxStatementFree(p->pStatement, 0); - idxTableFree(p->pTable); - idxWriteFree(p->pWrite); - idxHashClear(&p->hIdx); - sqlite3_free(p->zCandidates); - sqlite3_free(p); - } -} +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND (sql LIKE '%unique%' OR ?1))" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE tbl OR isIndex" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" + ); -#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); -/************************* End ../ext/expert/sqlite3expert.c ********************/ + if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; + + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + if( bTable && !bVirtual ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + sqlite3_free(zFree); + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/************************* Begin ../ext/misc/dbdata.c ******************/ /* -** 2019-04-17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains an implementation of two eponymous virtual tables, -** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the -** "sqlite_dbpage" eponymous virtual table be available. -** -** SQLITE_DBDATA: -** sqlite_dbdata is used to extract data directly from a database b-tree -** page and its associated overflow pages, bypassing the b-tree layer. -** The table schema is equivalent to: +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: ** -** CREATE TABLE sqlite_dbdata( -** pgno INTEGER, -** cell INTEGER, -** field INTEGER, -** value ANY, -** schema TEXT HIDDEN -** ); +** * views, +** * triggers, +** * non-UNIQUE indexes. ** -** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE -** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND -** "schema". +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. +*/ +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. ** -** Each page of the database is inspected. If it cannot be interpreted as -** a b-tree page, or if it is a b-tree page containing 0 entries, the -** sqlite_dbdata table contains no rows for that page. Otherwise, the -** table contains one row for each field in the record associated with -** each cell on the page. For intkey b-trees, the key value is stored in -** field -1. +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. ** -** For example, for the database: +** For example, if table pTab is: ** -** CREATE TABLE t1(a, b); -- root page is page 2 -** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); -** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); ** -** the sqlite_dbdata table contains, as well as from entries related to -** page 1, content equivalent to: +** And nField is 4, then the SQL statement prepared and returned is: ** -** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES -** (2, 0, -1, 5 ), -** (2, 0, 0, 'v' ), -** (2, 0, 1, 'five'), -** (2, 1, -1, 10 ), -** (2, 1, 0, 'x' ), -** (2, 1, 1, 'ten' ); +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); ** -** If database corruption is encountered, this module does not report an -** error. Instead, it attempts to extract as much data as possible and -** ignores the corruption. +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. ** -** SQLITE_DBPTR: -** The sqlite_dbptr table has the following schema: +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: ** -** CREATE TABLE sqlite_dbptr( -** pgno INTEGER, -** child INTEGER, -** schema TEXT HIDDEN -** ); +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; ** -** It contains one entry for each b-tree pointer between a parent and -** child page in the database. +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). */ -#if !defined(SQLITEINT_H) -/* #include "sqlite3ext.h" */ +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + sqlite3_stmt *pRet = 0; + const char *zSep = ""; + const char *zSqlSep = ""; + char *zSql = 0; + char *zFinal = 0; + char *zBind = 0; + int ii; + int bSql = p->xSql ? 1 : 0; -/* typedef unsigned char u8; */ + if( nField<=0 ) return 0; -#endif -SQLITE_EXTENSION_INIT1 -#include -#include + assert( nField<=pTab->nCol ); -#define DBDATA_PADDING_BYTES 100 + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); -typedef struct DbdataTable DbdataTable; -typedef struct DbdataCursor DbdataCursor; + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + } + zSqlSep = "||', '||"; + zSep = ", "; + } -/* Cursor object */ -struct DbdataCursor { - sqlite3_vtab_cursor base; /* Base class. Must be first */ - sqlite3_stmt *pStmt; /* For fetching database pages */ + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); - int iPgno; /* Current page number */ - u8 *aPage; /* Buffer containing page */ - int nPage; /* Size of aPage[] in bytes */ - int nCell; /* Number of cells on aPage[] */ - int iCell; /* Current cell number */ - int bOnePage; /* True to stop after one page */ - int szDb; - sqlite3_int64 iRowid; + if( bSql ){ + zBind = recoverMPrintf(p, + "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + } + zSep = ", "; + } + } - /* Only for the sqlite_dbdata table */ - u8 *pRec; /* Buffer containing current record */ - int nRec; /* Size of pRec[] in bytes */ - int nHdr; /* Size of header in bytes */ - int iField; /* Current field number */ - u8 *pHdrPtr; - u8 *pPtr; + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + } + + pRet = recoverPrepare(p, p->dbOut, zFinal); + sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); - sqlite3_int64 iIntkey; /* Integer key value */ -}; + return pRet; +} -/* Table object */ -struct DbdataTable { - sqlite3_vtab base; /* Base class. Must be first */ - sqlite3 *db; /* The database connection */ - sqlite3_stmt *pStmt; /* For fetching database pages */ - int bPtr; /* True for sqlite3_dbptr table */ -}; -/* Column and schema definitions for sqlite_dbdata */ -#define DBDATA_COLUMN_PGNO 0 -#define DBDATA_COLUMN_CELL 1 -#define DBDATA_COLUMN_FIELD 2 -#define DBDATA_COLUMN_VALUE 3 -#define DBDATA_COLUMN_SCHEMA 4 -#define DBDATA_SCHEMA \ - "CREATE TABLE x(" \ - " pgno INTEGER," \ - " cell INTEGER," \ - " field INTEGER," \ - " value ANY," \ - " schema TEXT HIDDEN" \ - ")" +/* +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. +*/ +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +/* +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). +** +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. +*/ +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ +){ + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; + + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); + } + + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; + } + recoverReset(p, pProbe); + } + + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); + + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + recoverSqlCallback(p, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); + } + + return zTbl; +} /* -** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual -** table. +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. */ -static int dbdataConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField ){ - DbdataTable *pTab = 0; - int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); + int nTotal = nField + 4; + int ii; + char *zBind = 0; + sqlite3_stmt *pRet = 0; - if( rc==SQLITE_OK ){ - pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); - if( pTab==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pTab, 0, sizeof(DbdataTable)); - pTab->db = db; - pTab->bPtr = (pAux!=0); + if( p->xSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); } - *ppVtab = (sqlite3_vtab*)pTab; - return rc; + sqlite3_free(zBind); + return pRet; } /* -** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. */ -static int dbdataDisconnect(sqlite3_vtab *pVtab){ - DbdataTable *pTab = (DbdataTable*)pVtab; - if( pTab ){ - sqlite3_finalize(pTab->pStmt); - sqlite3_free(pVtab); +static int recoverLostAndFoundFindRoot( + sqlite3_recover *p, + i64 iPg, + i64 *piRoot +){ + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); } - return SQLITE_OK; + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; + } + recoverReset(p, pLaf->pFindRoot); + } + return p->errCode; } /* -** This function interprets two types of constraints: -** -** schema=? -** pgno=? -** -** If neither are present, idxNum is set to 0. If schema=? is present, -** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit -** in idxNum is set. -** -** If both parameters are present, schema is in position 0 and pgno in -** position 1. +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. */ -static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ - DbdataTable *pTab = (DbdataTable*)tab; - int i; - int iSchema = -1; - int iPgno = -1; - int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; - if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - if( p->iColumn==colSchema ){ - if( p->usable==0 ) return SQLITE_CONSTRAINT; - iSchema = i; + int nVal = -1; + int iPrevCell = 0; + i64 iRoot = 0; + int bHaveRowid = 0; + i64 iRowid = 0; + int ii = 0; + + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); + + if( iPrevCell!=iCell && nVal>=0 ){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ } - if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ - iPgno = i; + for(ii=0; iinMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); + apVal[iField] = sqlite3_value_dup(pVal); + assert( iField==nVal || (nVal==-1 && iField==0) ); + nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); } } + + iPrevCell = iCell; } + recoverReset(p, pPageData); - if( iSchema>=0 ){ - pIdx->aConstraintUsage[iSchema].argvIndex = 1; - pIdx->aConstraintUsage[iSchema].omit = 1; + for(ii=0; ii=0 ){ - pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); - pIdx->aConstraintUsage[iPgno].omit = 1; - pIdx->estimatedCost = 100; - pIdx->estimatedRows = 50; +} - if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ - int iCol = pIdx->aOrderBy[0].iColumn; - if( pIdx->nOrderBy==1 ){ - pIdx->orderByConsumed = (iCol==0 || iCol==1); - }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ - pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found +** table of the output database is populated with recovered data that can +** not be assigned to any recovered schema object. +*/ +static int recoverLostAndFound3Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } } } - - }else{ - pIdx->estimatedCost = 100000000; - pIdx->estimatedRows = 1000000000; } - pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); return SQLITE_OK; } /* -** Open a new sqlite_dbdata or sqlite_dbptr cursor. -*/ -static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - DbdataCursor *pCsr; +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; - pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); - if( pCsr==0 ){ - return SQLITE_NOMEM; - }else{ - memset(pCsr, 0, sizeof(DbdataCursor)); - pCsr->base.pVtab = pVTab; + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ + + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); + + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" + ); + + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) + ); } +} - *ppCursor = (sqlite3_vtab_cursor *)pCsr; - return SQLITE_OK; +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + } + + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; + + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + p1->pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0" + ); + + return p->errCode; } /* -** Restore a cursor object to the state it was in when first allocated -** by dbdataOpen(). +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). */ -static void dbdataResetCursor(DbdataCursor *pCsr){ - DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); - if( pTab->pStmt==0 ){ - pTab->pStmt = pCsr->pStmt; - }else{ - sqlite3_finalize(pCsr->pStmt); +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); } - pCsr->pStmt = 0; - pCsr->iPgno = 1; - pCsr->iCell = 0; - pCsr->iField = 0; - pCsr->bOnePage = 0; - sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; - pCsr->aPage = 0; + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); } /* -** Close an sqlite_dbdata or sqlite_dbptr cursor. -*/ -static int dbdataClose(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - dbdataResetCursor(pCsr); - sqlite3_free(pCsr); - return SQLITE_OK; +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; + + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); + + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; + + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; + + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } + + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; + } + } + assert( p->errCode!=SQLITE_OK || p1->pTab ); + + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; + + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); + + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; + } + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); + } + } + } + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); + } + if( SQLITE_ROW==sqlite3_step(pInsert) ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; + } + + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; + } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; + } + + return p->errCode; } -/* -** Utility methods to decode 16 and 32-bit big-endian unsigned integers. -*/ -static unsigned int get_uint16(unsigned char *a){ - return (a[0]<<8)|a[1]; +/* +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; + + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" + ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; } -static unsigned int get_uint32(unsigned char *a){ - return ((unsigned int)a[0]<<24) - | ((unsigned int)a[1]<<16) - | ((unsigned int)a[2]<<8) - | ((unsigned int)a[3]); + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; + } + } + return rc; } /* -** Load page pgno from the database via the sqlite_dbpage virtual table. -** If successful, set (*ppPage) to point to a buffer containing the page -** data, (*pnPage) to the size of that buffer in bytes and return -** SQLITE_OK. In this case it is the responsibility of the caller to -** eventually free the buffer using sqlite3_free(). -** -** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and -** return an SQLite error code. -*/ -static int dbdataLoadPage( - DbdataCursor *pCsr, /* Cursor object */ - unsigned int pgno, /* Page number of page to load */ - u8 **ppPage, /* OUT: pointer to page buffer */ - int *pnPage /* OUT: Size of (*ppPage) in bytes */ -){ - int rc2; - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = pCsr->pStmt; +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; - *ppPage = 0; - *pnPage = 0; - sqlite3_bind_int64(pStmt, 2, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - int nCopy = sqlite3_column_bytes(pStmt, 0); - if( nCopy>0 ){ - u8 *pPage; - pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); - if( pPage==0 ){ - rc = SQLITE_NOMEM; - }else{ - const u8 *pCopy = sqlite3_column_blob(pStmt, 0); - memcpy(pPage, pCopy, nCopy); - memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg + ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} + +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); } - *ppPage = pPage; - *pnPage = nCopy; + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; } } - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = rc2; + return p->errCode; +} - return rc; +/* +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +*/ +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; } /* -** Read a varint. Put the value in *pVal and return the number of bytes. +** Free all resources allocated as part of sqlite3_recover_step() calls. */ -static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ - sqlite3_int64 v = 0; +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + + recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + + { +#ifndef NDEBUG + int res = +#endif + sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); + } + p->dbOut = 0; +} + +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} + +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} + +/* +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. +*/ +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ - v = (v<<7) + (z[i]&0x7f); - if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } - v = (v<<8) + (z[i]&0xff); - *pVal = v; + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; return 9; } /* -** Return the number of bytes of space used by an SQLite value of type -** eType. -*/ -static int dbdataValueBytes(int eType){ - switch( eType ){ - case 0: case 8: case 9: - case 10: case 11: +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. +*/ +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; + int ii = 0; + + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n || nByte<4 ) return 0; + if( iNext && iNextiContent ) return 0; + for(ii=0; iin ){ return 0; - case 1: - return 1; - case 2: - return 2; - case 3: - return 3; - case 4: - return 4; - case 5: - return 6; - case 6: - case 7: - return 8; - default: - if( eType>0 ){ - return ((eType-12) / 2); + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); + } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); + + if( nPayloadn ){ return 0; - } + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; + } + } + + nActual = 0; + for(ii=0; iipMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); } /* -** Load a value of type eType from buffer pData and use it to set the -** result of context object pCtx. +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. */ -static void dbdataValue( - sqlite3_context *pCtx, - int eType, - u8 *pData, - int nData +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} + +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ +static void recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; +} + +/* +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. +*/ +static int recoverVfsDetectPagesize( + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ + u32 nReserve, /* Possible nReserve value */ + i64 nSz /* Size of database file in bytes */ ){ - if( eType>=0 && dbdataValueBytes(eType)<=nData ){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); - } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + int nBlk = 0; + + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + do { + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOff(u32)p->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); + + p->detected_pgsz = pgsz; + sqlite3_free(aPg); + return rc; } /* -** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. */ -static int dbdataNext(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; - - pCsr->iRowid++; - while( 1 ){ - int rc; - int iOff = (pCsr->iPgno==1 ? 100 : 0); - int bNextPage = 0; +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + the wal-mode flags (16-bits at offset 18) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[108] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30, + + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 + }; + u8 *a = (u8*)aBuf; + + u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + sqlite3_recover *p = recover_g.p; - if( pCsr->aPage==0 ){ - while( 1 ){ - if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; - rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK ) return rc; - if( pCsr->aPage ) break; - pCsr->iPgno++; - } - pCsr->iCell = pTab->bPtr ? -2 : 0; - pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); - } + if( pgsz==0x01 ) pgsz = 65536; + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); - if( pTab->bPtr ){ - if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ - pCsr->iCell = pCsr->nCell; + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); } - pCsr->iCell++; - if( pCsr->iCell>=pCsr->nCell ){ - sqlite3_free(pCsr->aPage); - pCsr->aPage = 0; - if( pCsr->bOnePage ) return SQLITE_OK; - pCsr->iPgno++; - }else{ - return SQLITE_OK; + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; + nReserve = p->nReserve; } - }else{ - /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ - int bHasRowid = 0; - int nPointer = 0; - sqlite3_int64 nPayload = 0; - sqlite3_int64 nHdr = 0; - int iHdr; - int U, X; - int nLocal; - - switch( pCsr->aPage[iOff] ){ - case 0x02: - nPointer = 4; - break; - case 0x0a: - break; - case 0x0d: - bHasRowid = 1; - break; - default: - /* This is not a b-tree page with records on it. Continue. */ - pCsr->iCell = pCsr->nCell; - break; - } - - if( pCsr->iCell>=pCsr->nCell ){ - bNextPage = 1; - }else{ - - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ - bNextPage = 1; - }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); - } - - /* For an interior node cell, skip past the child-page number */ - iOff += nPointer; - - /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ - bNextPage = 1; - }else{ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); - } - - /* If this is a leaf intkey cell, load the rowid */ - if( bHasRowid && !bNextPage && iOffnPage ){ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); - } - - /* Figure out how much data to read from the local page */ - U = pCsr->nPage; - if( bHasRowid ){ - X = U-35; - }else{ - X = ((U-12)*64/255)-23; - } - if( nPayload<=X ){ - nLocal = nPayload; - }else{ - int M, K; - M = ((U-12)*32/255)-23; - K = M+((nPayload-M)%(U-4)); - if( K<=X ){ - nLocal = K; - }else{ - nLocal = M; - } - } - - if( bNextPage || nLocal+iOff>pCsr->nPage ){ - bNextPage = 1; - }else{ - - /* Allocate space for payload. And a bit more to catch small buffer - ** overruns caused by attempting to read a varint or similar from - ** near the end of a corrupt record. */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES); - pCsr->nRec = nPayload; - - /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); - iOff += nLocal; - - /* Load content from overflow pages */ - if( nPayload>nLocal ){ - sqlite3_int64 nRem = nPayload - nLocal; - unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); - while( nRem>0 ){ - u8 *aOvfl = 0; - int nOvfl = 0; - int nCopy; - rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); - assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage ); - if( rc!=SQLITE_OK ) return rc; - if( aOvfl==0 ) break; - - nCopy = U-4; - if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); - nRem -= nCopy; - - pgnoOvfl = get_uint32(aOvfl); - sqlite3_free(aOvfl); - } - } - - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - pCsr->nHdr = nHdr; - pCsr->pHdrPtr = &pCsr->pRec[iHdr]; - pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; - pCsr->iField = (bHasRowid ? -1 : 0); - } - } - }else{ - pCsr->iField++; - if( pCsr->iField>0 ){ - sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ - bNextPage = 1; - }else{ - pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); - pCsr->pPtr += dbdataValueBytes(iType); - } - } + + if( pgsz ){ + dbsz = dbFileSize / pgsz; + } + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; } - if( bNextPage ){ - sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); - pCsr->aPage = 0; - pCsr->pRec = 0; - if( pCsr->bOnePage ) return SQLITE_OK; - pCsr->iPgno++; - }else{ - if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ - return SQLITE_OK; + sqlite3_free(p->pPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; + + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + aHdr[18] = a[18]; + aHdr[19] = a[19]; + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz-nReserve); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; + for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){ + memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4); } + memcpy(aBuf, aHdr, sizeof(aHdr)); + memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr)); - /* Advance to the next cell. The next iteration of the loop will load - ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; - pCsr->iCell++; + memcpy(p->pPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; } + } + pFd->pMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); } + return rc; +} - assert( !"can't get here" ); +/* +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; + +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); + } +} +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} + +static int recoverVfsFetch( + sqlite3_file *pFd, + sqlite3_int64 iOff, + int iAmt, + void **pp +){ + (void)pFd; + (void)iOff; + (void)iAmt; + *pp = 0; + return SQLITE_OK; +} +static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ + (void)pFd; + (void)iOff; + (void)p; return SQLITE_OK; } -/* -** Return true if the cursor is at EOF. +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. */ -static int dbdataEof(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - return pCsr->aPage==0; +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + assert( pFd==0 || pFd->pMethods!=&recover_methods ); + if( pFd && pFd->pMethods ){ + int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); + recover_g.pMethods = pFd->pMethods; + recover_g.p = p; + recover_methods.iVersion = iVersion; + pFd->pMethods = &recover_methods; + } } -/* -** Determine the size in pages of database zSchema (where zSchema is -** "main", "temp" or the name of an attached database) and set -** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, -** an SQLite error code. +/* +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. */ -static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ - DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; - char *zSql = 0; - int rc, rc2; - sqlite3_stmt *pStmt = 0; - - zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - pCsr->szDb = sqlite3_column_int(pStmt, 0); +static void recoverUninstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + recoverAssertMutexHeld(); + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + if( pFd && pFd->pMethods ){ + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + recover_g.p = 0; } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - return rc; } -/* -** xFilter method for sqlite_dbdata and sqlite_dbptr. +/* +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. */ -static int dbdataFilter( - sqlite3_vtab_cursor *pCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; - int rc = SQLITE_OK; - const char *zSchema = "main"; +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); - dbdataResetCursor(pCsr); - assert( pCsr->iPgno==1 ); - if( idxNum & 0x01 ){ - zSchema = (const char*)sqlite3_value_text(argv[0]); - } - if( idxNum & 0x02 ){ - pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); - pCsr->bOnePage = 1; - }else{ - pCsr->nPage = dbdataDbsize(pCsr, zSchema); - rc = dbdataDbsize(pCsr, zSchema); - } + recoverEnterMutex(); + recoverInstallWrapper(p); - if( rc==SQLITE_OK ){ - if( pTab->pStmt ){ - pCsr->pStmt = pTab->pStmt; - pTab->pStmt = 0; - }else{ - rc = sqlite3_prepare_v2(pTab->db, - "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, - &pCsr->pStmt, 0 - ); + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); + recoverCacheSchema(p); + + recoverUninstallWrapper(p); + recoverLeaveMutex(); + + recoverExec(p, p->dbOut, "BEGIN"); + + recoverWriteSchema1(p); + p->eState = RECOVER_STATE_WRITING; + break; + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND1; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; + } + } + break; } + + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); + } + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); + } + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } + break; + } + + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; + + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; + + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; } - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); - }else{ - pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); - } - if( rc==SQLITE_OK ){ - rc = dbdataNext(pCursor); +} + + +/* +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ +sqlite3_recover *recoverInit( + sqlite3* db, + const char *zDb, + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; } - return rc; + + return pRet; } -/* -** Return a column for the sqlite_dbdata or sqlite_dbptr table. +/* +** Initialize a recovery handle that creates a new database containing +** the recovered data. */ -static int dbdataColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, - int i +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri ){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; - if( pTab->bPtr ){ - switch( i ){ - case DBPTR_COLUMN_PGNO: - sqlite3_result_int64(ctx, pCsr->iPgno); + return recoverInit(db, zDb, zUri, 0, 0); +} + +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, 0, xSql, pSqlCtx); +} + +/* +** Return the handle error message, if any. +*/ +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; +} + +/* +** Return the handle error code. +*/ +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +/* +** Configure the handle. +*/ +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + /* This undocumented magic configuration option is used to set the + ** name of the auxiliary database that is ATTACH-ed to the database + ** connection and used to hold state information during the + ** recovery process. This option is for debugging use only and + ** is subject to change or removal at any time. */ + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); break; - case DBPTR_COLUMN_CHILD: { - int iOff = pCsr->iPgno==1 ? 100 : 0; - if( pCsr->iCell<0 ){ - iOff += 8; + + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); }else{ - iOff += 12 + pCsr->iCell*2; - if( iOff>pCsr->nPage ) return SQLITE_OK; - iOff = get_uint16(&pCsr->aPage[iOff]); - } - if( iOff<=pCsr->nPage ){ - sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + p->zLostAndFound = 0; } break; } - } - }else{ - switch( i ){ - case DBDATA_COLUMN_PGNO: - sqlite3_result_int64(ctx, pCsr->iPgno); + + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; break; - case DBDATA_COLUMN_CELL: - sqlite3_result_int(ctx, pCsr->iCell); + + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; break; - case DBDATA_COLUMN_FIELD: - sqlite3_result_int(ctx, pCsr->iField); + + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; break; - case DBDATA_COLUMN_VALUE: { - if( pCsr->iField<0 ){ - sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ - sqlite3_int64 iType; - dbdataGetVarint(pCsr->pHdrPtr, &iType); - dbdataValue( - ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr - ); - } + + default: + rc = SQLITE_NOTFOUND; break; - } } } - return SQLITE_OK; + + return rc; } -/* -** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +/* +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; +} + +/* +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. */ -static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - DbdataCursor *pCsr = (DbdataCursor*)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; +int sqlite3_recover_run(sqlite3_recover *p){ + while( SQLITE_OK==sqlite3_recover_step(p) ); + return sqlite3_recover_errcode(p); } /* -** Invoke this routine to register the "sqlite_dbdata" virtual table module +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. */ -static int sqlite3DbdataRegister(sqlite3 *db){ - static sqlite3_module dbdata_module = { - 0, /* iVersion */ - 0, /* xCreate */ - dbdataConnect, /* xConnect */ - dbdataBestIndex, /* xBestIndex */ - dbdataDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - dbdataOpen, /* xOpen - open a cursor */ - dbdataClose, /* xClose - close a cursor */ - dbdataFilter, /* xFilter - configure scan constraints */ - dbdataNext, /* xNext - advance a cursor */ - dbdataEof, /* xEof - check for end of scan */ - dbdataColumn, /* xColumn - read data */ - dbdataRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0 /* xShadowName */ - }; - - int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); +int sqlite3_recover_finish(sqlite3_recover *p){ + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); + sqlite3_free(p); } return rc; } -#ifdef _WIN32 +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ +/************************* End ../ext/recover/sqlite3recover.c ********************/ +# endif /* SQLITE_HAVE_SQLITE3R */ #endif -int sqlite3_dbdata_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return sqlite3DbdataRegister(db); -} - -/************************* End ../ext/misc/dbdata.c ********************/ +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) #endif #if defined(SQLITE_ENABLE_SESSION) @@ -12165,6 +19384,15 @@ struct EQPGraph { char zPrefix[100]; /* Graph prefix */ }; +/* Parameters affecting columnar mode result display (defaulting together) */ +typedef struct ColModeOpts { + int iWrap; /* In columnar modes, wrap lines reaching this limit */ + u8 bQuote; /* Quote results for .mode box and table */ + u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ +} ColModeOpts; +#define ColModeOpts_default { 60, 0, 0 } +#define ColModeOpts_default_qbox { 60, 1, 0 } + /* ** State information about the database connection is contained in an ** instance of the following structure. @@ -12173,7 +19401,7 @@ typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ u8 autoEQPtest; /* autoEQP is in test mode */ u8 autoEQPtrace; /* autoEQP is in trace mode */ u8 scanstatsOn; /* True to display scan stats before each finalize */ @@ -12183,8 +19411,11 @@ struct ShellState { u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ + u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ - unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ + unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + int inputNesting; /* Track nesting level of .read and other redirects */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ int lineno; /* Line number of last line read from in */ @@ -12234,11 +19465,22 @@ struct ShellState { int *aiIndent; /* Array of indents used in MODE_Explain */ int nIndent; /* Size of array aiIndent[] */ int iIndent; /* Index of current op in aiIndent[] */ - char *zNonce; /* Nonce for temporary safe-mode excapes */ + char *zNonce; /* Nonce for temporary safe-mode escapes */ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#ifdef SQLITE_SHELL_FIDDLE + struct { + const char * zInput; /* Input string from wasm/JS proxy */ + const char * zPos; /* Cursor pos into zInput */ + const char * zDefaultDbName; /* Default name for db file */ + } wasm; +#endif }; +#ifdef SQLITE_SHELL_FIDDLE +static ShellState shellState; +#endif + /* Allowed values for ShellState.autoEQP */ @@ -12265,7 +19507,7 @@ struct ShellState { /* Bits in the ShellState.flgProgress variable */ #define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ -#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progres +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress ** callback limit is reached, and for each ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ @@ -12279,10 +19521,11 @@ struct ShellState { #define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ #define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ -#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ +#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ #define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ /* ** Macros for testing and setting shellFlgs @@ -12311,6 +19554,9 @@ struct ShellState { #define MODE_Markdown 14 /* Markdown formatting */ #define MODE_Table 15 /* MySQL-style table formatting */ #define MODE_Box 16 /* Unicode box-drawing characters */ +#define MODE_Count 17 /* Output only a count of the rows of output */ +#define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ static const char *modeDescr[] = { "line", @@ -12329,7 +19575,9 @@ static const char *modeDescr[] = { "json", "markdown", "table", - "box" + "box", + "count", + "off" }; /* @@ -12345,13 +19593,19 @@ static const char *modeDescr[] = { #define SEP_Unit "\x1F" #define SEP_Record "\x1E" +/* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + /* ** A callback for the sqlite3_log() interface. */ static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - utf8_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + sputf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -12366,9 +19620,9 @@ static void shellPutsFunc( int nVal, sqlite3_value **apVal ){ - ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + /* Unused: (ShellState*)sqlite3_user_data(pCtx); */ (void)nVal; - utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + oputf("%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } @@ -12387,8 +19641,7 @@ static void failIfSafeMode( va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - raw_printf(stderr, "line %d: ", p->lineno); - utf8_printf(stderr, "%s\n", zMsg); + eputf("line %d: %s\n", p->lineno, zMsg); exit(1); } } @@ -12516,13 +19769,14 @@ static void editFunc( }else{ /* If the file did not originally contain \r\n then convert any new ** \r\n back into \n */ + p[sz] = 0; for(i=j=0; i> 4) ]; + zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; + } + zStr[i*2] = '\0'; + + oputf("X'%s'", zStr); + sqlite3_free(zStr); } /* @@ -12589,24 +19856,28 @@ static const char *unused_string( ** ** See also: output_quoted_escaped_string() */ -static void output_quoted_string(FILE *out, const char *z){ +static void output_quoted_string(const char *z){ int i; char c; - setBinaryMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + FILE *pfO = setOutputStream(invalidFileStream); + setBinaryMode(pfO, 1); +#endif + if( z==0 ) return; for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c==0 ){ - utf8_printf(out,"'%s'",z); + oputf("'%s'",z); }else{ - raw_printf(out, "'"); + oputz("'"); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - utf8_printf(out, "%.*s", i, z); + oputf("%.*s", i, z); z += i; } if( c=='\'' ){ - raw_printf(out, "'"); + oputz("'"); continue; } if( c==0 ){ @@ -12614,9 +19885,13 @@ static void output_quoted_string(FILE *out, const char *z){ } z++; } - raw_printf(out, "'"); + oputz("'"); } - setTextMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + setTextMode(pfO, 1); +#else + setTextMode(stdout, 1); +#endif } /* @@ -12628,13 +19903,16 @@ static void output_quoted_string(FILE *out, const char *z){ ** This is like output_quoted_string() but with the addition of the \r\n ** escape mechanism. */ -static void output_quoted_escaped_string(FILE *out, const char *z){ +static void output_quoted_escaped_string(const char *z){ int i; char c; - setBinaryMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + FILE *pfO = setOutputStream(invalidFileStream); + setBinaryMode(pfO, 1); +#endif for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} if( c==0 ){ - utf8_printf(out,"'%s'",z); + oputf("'%s'",z); }else{ const char *zNL = 0; const char *zCR = 0; @@ -12646,23 +19924,23 @@ static void output_quoted_escaped_string(FILE *out, const char *z){ if( z[i]=='\r' ) nCR++; } if( nNL ){ - raw_printf(out, "replace("); + oputz("replace("); zNL = unused_string(z, "\\n", "\\012", zBuf1); } if( nCR ){ - raw_printf(out, "replace("); + oputz("replace("); zCR = unused_string(z, "\\r", "\\015", zBuf2); } - raw_printf(out, "'"); + oputz("'"); while( *z ){ for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} if( c=='\'' ) i++; if( i ){ - utf8_printf(out, "%.*s", i, z); + oputf("%.*s", i, z); z += i; } if( c=='\'' ){ - raw_printf(out, "'"); + oputz("'"); continue; } if( c==0 ){ @@ -12670,92 +19948,139 @@ static void output_quoted_escaped_string(FILE *out, const char *z){ } z++; if( c=='\n' ){ - raw_printf(out, "%s", zNL); + oputz(zNL); continue; } - raw_printf(out, "%s", zCR); + oputz(zCR); } - raw_printf(out, "'"); + oputz("'"); if( nCR ){ - raw_printf(out, ",'%s',char(13))", zCR); + oputf(",'%s',char(13))", zCR); } if( nNL ){ - raw_printf(out, ",'%s',char(10))", zNL); + oputf(",'%s',char(10))", zNL); } } - setTextMode(out, 1); +#ifndef SQLITE_SHELL_FIDDLE + setTextMode(pfO, 1); +#else + setTextMode(stdout, 1); +#endif } +/* +** Find earliest of chars within s specified in zAny. +** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. +*/ +static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ + const char *pcFirst = 0; + if( ns == ~(size_t)0 ) ns = strlen(s); + while(*zAny){ + const char *pc = (const char*)memchr(s, *zAny&0xff, ns); + if( pc ){ + pcFirst = pc; + ns = pcFirst - s; + } + ++zAny; + } + return pcFirst; +} /* ** Output the given string as a quoted according to C or TCL quoting rules. */ -static void output_c_string(FILE *out, const char *z){ - unsigned int c; - fputc('"', out); - while( (c = *(z++))!=0 ){ - if( c=='\\' ){ - fputc(c, out); - fputc(c, out); - }else if( c=='"' ){ - fputc('\\', out); - fputc('"', out); - }else if( c=='\t' ){ - fputc('\\', out); - fputc('t', out); - }else if( c=='\n' ){ - fputc('\\', out); - fputc('n', out); - }else if( c=='\r' ){ - fputc('\\', out); - fputc('r', out); +static void output_c_string(const char *z){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ + char ace[3] = "\\?"; + char cbsSay; + oputz(zq); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; + if( pcEnd > z ) oputb(z, (int)(pcEnd-z)); + if( (c = *pcEnd)==0 ) break; + ++pcEnd; + switch( c ){ + case '\\': case '"': + cbsSay = (char)c; + break; + case '\t': cbsSay = 't'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\f': cbsSay = 'f'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + oputz(ace); }else if( !isprint(c&0xff) ){ - raw_printf(out, "\\%03o", c&0xff); + oputf("\\%03o", c&0xff); }else{ - fputc(c, out); + ace[1] = (char)c; + oputz(ace+1); } + z = pcEnd; } - fputc('"', out); + oputz(zq); } /* ** Output the given string as a quoted according to JSON quoting rules. */ -static void output_json_string(FILE *out, const char *z, int n){ - unsigned int c; - if( n<0 ) n = (int)strlen(z); - fputc('"', out); - while( n-- ){ +static void output_json_string(const char *z, i64 n){ + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBS = "\"\\"; + const char *pcLimit; + char ace[3] = "\\?"; + char cbsSay; + + if( z==0 ) z = ""; + pcLimit = z + ((n<0)? strlen(z) : (size_t)n); + oputz(zq); + while( z < pcLimit ){ + const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); + const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); + const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + if( pcEnd > z ){ + oputb(z, (int)(pcEnd-z)); + z = pcEnd; + } + if( z >= pcLimit ) break; c = *(z++); - if( c=='\\' || c=='"' ){ - fputc('\\', out); - fputc(c, out); + switch( c ){ + case '"': case '\\': + cbsSay = (char)c; + break; + case '\b': cbsSay = 'b'; break; + case '\f': cbsSay = 'f'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\t': cbsSay = 't'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + oputz(ace); }else if( c<=0x1f ){ - fputc('\\', out); - if( c=='\b' ){ - fputc('b', out); - }else if( c=='\f' ){ - fputc('f', out); - }else if( c=='\n' ){ - fputc('n', out); - }else if( c=='\r' ){ - fputc('r', out); - }else if( c=='\t' ){ - fputc('t', out); - }else{ - raw_printf(out, "u%04x",c); - } + oputf("u%04x", c); }else{ - fputc(c, out); + ace[1] = (char)c; + oputz(ace+1); } } - fputc('"', out); + oputz(zq); } /* ** Output the given string with characters that are special to ** HTML escaped. */ -static void output_html_string(FILE *out, const char *z){ +static void output_html_string(const char *z){ int i; if( z==0 ) z = ""; while( *z ){ @@ -12767,18 +20092,18 @@ static void output_html_string(FILE *out, const char *z){ && z[i]!='\''; i++){} if( i>0 ){ - utf8_printf(out,"%.*s",i,z); + oputf("%.*s",i,z); } if( z[i]=='<' ){ - raw_printf(out,"<"); + oputz("<"); }else if( z[i]=='&' ){ - raw_printf(out,"&"); + oputz("&"); }else if( z[i]=='>' ){ - raw_printf(out,">"); + oputz(">"); }else if( z[i]=='\"' ){ - raw_printf(out,"""); + oputz("""); }else if( z[i]=='\'' ){ - raw_printf(out,"'"); + oputz("'"); }else{ break; } @@ -12816,9 +20141,8 @@ static const char needCsvQuote[] = { ** is only issued if bSep is true. */ static void output_csv(ShellState *p, const char *z, int bSep){ - FILE *out = p->out; if( z==0 ){ - utf8_printf(out,"%s",p->nullValue); + oputf("%s",p->nullValue); }else{ unsigned i; for(i=0; z[i]; i++){ @@ -12829,14 +20153,15 @@ static void output_csv(ShellState *p, const char *z, int bSep){ } if( i==0 || strstr(z, p->colSeparator)!=0 ){ char *zQuoted = sqlite3_mprintf("\"%w\"", z); - utf8_printf(out, "%s", zQuoted); + shell_check_oom(zQuoted); + oputz(zQuoted); sqlite3_free(zQuoted); }else{ - utf8_printf(out, "%s", z); + oputz(z); } } if( bSep ){ - utf8_printf(p->out, "%s", p->colSeparator); + oputz(p->colSeparator); } } @@ -12845,8 +20170,7 @@ static void output_csv(ShellState *p, const char *z, int bSep){ */ static void interrupt_handler(int NotUsed){ UNUSED_PARAMETER(NotUsed); - seenInterrupt++; - if( seenInterrupt>2 ) exit(1); + if( ++seenInterrupt>1 ) exit(1); if( globalDb ) sqlite3_interrupt(globalDb); } @@ -12892,7 +20216,11 @@ static int safeModeAuth( UNUSED_PARAMETER(zA4); switch( op ){ case SQLITE_ATTACH: { +#ifndef SQLITE_SHELL_FIDDLE + /* In WASM builds the filesystem is a virtual sandbox, so + ** there's no harm in using ATTACH. */ failIfSafeMode(p, "cannot run ATTACH in safe mode"); +#endif break; } case SQLITE_FUNCTION: { @@ -12941,16 +20269,16 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - utf8_printf(p->out, "authorizer: %s", azAction[op]); + oputf("authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - raw_printf(p->out, " "); + oputz(" "); if( az[i] ){ - output_c_string(p->out, az[i]); + output_c_string(az[i]); }else{ - raw_printf(p->out, "NULL"); + oputz("NULL"); } } - raw_printf(p->out, "\n"); + oputz("\n"); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } @@ -12961,20 +20289,43 @@ static int shellAuth( ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +** +** If the schema statement in z[] contains a start-of-comment and if +** sqlite3_complete() returns false, try to terminate the comment before +** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ -static void printSchemaLine(FILE *out, const char *z, const char *zTail){ +static void printSchemaLine(const char *z, const char *zTail){ + char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; + if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){ + const char *zOrig = z; + static const char *azTerm[] = { "", "*/", "\n" }; + int i; + for(i=0; iautoEQPtest ){ - utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + oputf("%d,%d,%s\n", iEqpId, p2, zText); } pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); - if( pNew==0 ) shell_out_of_memory(); + shell_check_oom(pNew); pNew->iEqpId = iEqpId; pNew->iParentId = p2; memcpy(pNew->zText, zText, nText+1); @@ -13043,14 +20396,13 @@ static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ */ static void eqp_render_level(ShellState *p, int iEqpId){ EQPGraphRow *pRow, *pNext; - int n = strlen30(p->sGraph.zPrefix); + i64 n = strlen(p->sGraph.zPrefix); char *z; for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; - utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); - if( n<(int)sizeof(p->sGraph.zPrefix)-7 ){ + oputf("%s%s%s\n", p->sGraph.zPrefix, pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); p->sGraph.zPrefix[n] = 0; @@ -13061,7 +20413,7 @@ static void eqp_render_level(ShellState *p, int iEqpId){ /* ** Display and reset the EXPLAIN QUERY PLAN data */ -static void eqp_render(ShellState *p){ +static void eqp_render(ShellState *p, i64 nCycle){ EQPGraphRow *pRow = p->sGraph.pRow; if( pRow ){ if( pRow->zText[0]=='-' ){ @@ -13069,11 +20421,13 @@ static void eqp_render(ShellState *p){ eqp_reset(p); return; } - utf8_printf(p->out, "%s\n", pRow->zText+3); + oputf("%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); + }else if( nCycle>0 ){ + oputf("QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ - utf8_printf(p->out, "QUERY PLAN\n"); + oputz("QUERY PLAN\n"); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); @@ -13089,13 +20443,13 @@ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - raw_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + oputf("Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - raw_printf(p->out, "Progress %u\n", p->nProgress); + oputf("Progress %u\n", p->nProgress); } return 0; } @@ -13104,14 +20458,14 @@ static int progress_handler(void *pClientData) { /* ** Print N dashes */ -static void print_dashes(FILE *out, int N){ +static void print_dashes(int N){ const char zDash[] = "--------------------------------------------------"; const int nDash = sizeof(zDash) - 1; while( N>nDash ){ - fputs(zDash, out); + oputz(zDash); N -= nDash; } - raw_printf(out, "%.*s", N, zDash); + oputf("%.*s", N, zDash); } /* @@ -13124,15 +20478,15 @@ static void print_row_separator( ){ int i; if( nArg>0 ){ - fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[0]+2); + oputz(zSep); + print_dashes(p->actualWidth[0]+2); for(i=1; iout); - print_dashes(p->out, p->actualWidth[i]+2); + oputz(zSep); + print_dashes(p->actualWidth[i]+2); } - fputs(zSep, p->out); + oputz(zSep); } - fputs("\n", p->out); + oputz("\n"); } /* @@ -13151,6 +20505,10 @@ static int shell_callback( if( azArg==0 ) return 0; switch( p->cMode ){ + case MODE_Count: + case MODE_Off: { + break; + } case MODE_Line: { int w = 5; if( azArg==0 ) break; @@ -13158,50 +20516,70 @@ static int shell_callback( int len = strlen30(azCol[i] ? azCol[i] : ""); if( len>w ) w = len; } - if( p->cnt++>0 ) utf8_printf(p->out, "%s", p->rowSeparator); + if( p->cnt++>0 ) oputz(p->rowSeparator); for(i=0; iout,"%*s = %s%s", w, azCol[i], - azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + oputf("%*s = %s%s", w, azCol[i], + azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); } break; } + case MODE_ScanExp: case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - if( nArg>ArraySize(aExplainWidth) ){ - nArg = ArraySize(aExplainWidth); + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, w, azCol[i]); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(aWidth[i], azCol[ aMap[i] ]); + oputz(i==nArg-1 ? "\n" : " "); } for(i=0; iout, w); - fputs(i==nArg-1 ? "\n" : " ", p->out); + print_dashes(aWidth[i]); + oputz(i==nArg-1 ? "\n" : " "); } } + + /* If there is no data, exit early. */ if( azArg==0 ) break; + for(i=0; iw ){ - w = strlenChar(azArg[i]); + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; } - if( i==1 && p->aiIndent && p->pStmt ){ + if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ - utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + oputf("%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(w, zVal ? zVal : p->nullValue); + oputz(i==nArg-1 ? "\n" : zSep); } break; } case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(p->out, azArg[0], ";\n"); + printSchemaLine(azArg[0], ";\n"); break; } case MODE_Pretty: { /* .schema and .fullschema with --indent */ @@ -13216,10 +20594,11 @@ static int shell_callback( if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 ){ - utf8_printf(p->out, "%s;\n", azArg[0]); + oputf("%s;\n", azArg[0]); break; } z = sqlite3_mprintf("%s", azArg[0]); + shell_check_oom(z); j = 0; for(i=0; IsSpace(z[i]); i++){} for(; (c = z[i])!=0; i++){ @@ -13248,7 +20627,7 @@ static int shell_callback( }else if( c==')' ){ nParen--; if( nLine>0 && nParen==0 && j>0 ){ - printSchemaLineN(p->out, z, j, "\n"); + printSchemaLineN(z, j, "\n"); j = 0; } } @@ -13257,7 +20636,7 @@ static int shell_callback( && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) ){ if( c=='\n' ) j--; - printSchemaLineN(p->out, z, j, "\n "); + printSchemaLineN(z, j, "\n "); j = 0; nLine++; while( IsSpace(z[i+1]) ){ i++; } @@ -13265,64 +20644,59 @@ static int shell_callback( } z[j] = 0; } - printSchemaLine(p->out, z, ";\n"); + printSchemaLine(z, ";\n"); sqlite3_free(z); break; } case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,"%s%s",azCol[i], - i==nArg-1 ? p->rowSeparator : p->colSeparator); + oputf("%s%s",azCol[i], i==nArg-1 ? p->rowSeparator : p->colSeparator); } } if( azArg==0 ) break; for(i=0; inullValue; - utf8_printf(p->out, "%s", z); - if( iout, "%s", p->colSeparator); - }else{ - utf8_printf(p->out, "%s", p->rowSeparator); - } + oputz(z); + oputz((icolSeparator : p->rowSeparator); } break; } case MODE_Html: { if( p->cnt++==0 && p->showHeader ){ - raw_printf(p->out,""); + oputz(""); for(i=0; iout,""); - output_html_string(p->out, azCol[i]); - raw_printf(p->out,"\n"); + oputz(""); + output_html_string(azCol[i]); + oputz("\n"); } - raw_printf(p->out,"\n"); + oputz("\n"); } if( azArg==0 ) break; - raw_printf(p->out,""); + oputz(""); for(i=0; iout,""); - output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - raw_printf(p->out,"\n"); + oputz(""); + output_html_string(azArg[i] ? azArg[i] : p->nullValue); + oputz("\n"); } - raw_printf(p->out,"\n"); + oputz("\n"); break; } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - if(iout, "%s", p->colSeparator); + output_c_string(azCol[i] ? azCol[i] : ""); + if(icolSeparator); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); - if(iout, "%s", p->colSeparator); + output_c_string(azArg[i] ? azArg[i] : p->nullValue); + if(icolSeparator); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); break; } case MODE_Csv: { @@ -13331,73 +20705,79 @@ static int shell_callback( for(i=0; iout, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( nArg>0 ){ for(i=0; iout, "%s", p->rowSeparator); + oputz(p->rowSeparator); } setTextMode(p->out, 1); break; } case MODE_Insert: { if( azArg==0 ) break; - utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + oputf("INSERT INTO %s",p->zDestTable); if( p->showHeader ){ - raw_printf(p->out,"("); + oputz("("); for(i=0; i0 ) raw_printf(p->out, ","); + if( i>0 ) oputz(","); if( quoteChar(azCol[i]) ){ char *z = sqlite3_mprintf("\"%w\"", azCol[i]); - utf8_printf(p->out, "%s", z); + shell_check_oom(z); + oputz(z); sqlite3_free(z); }else{ - raw_printf(p->out, "%s", azCol[i]); + oputf("%s", azCol[i]); } } - raw_printf(p->out,")"); + oputz(")"); } p->cnt++; for(i=0; iout, i>0 ? "," : " VALUES("); + oputz(i>0 ? "," : " VALUES("); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + oputz("NULL"); }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(azArg[i]); } }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "1e999"); + oputz("9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + oputz("-9.0e+999"); }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + sqlite3_int64 ir = (sqlite3_int64)r; + if( r==(double)ir ){ + sqlite3_snprintf(50,z,"%lld.0", ir); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + } + oputz(z); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); + output_hex_blob(pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(azArg[i]); } } - raw_printf(p->out,");\n"); + oputz(");\n"); break; } case MODE_Json: { @@ -13409,37 +20789,37 @@ static int shell_callback( } p->cnt++; for(i=0; iout, azCol[i], -1); - putc(':', p->out); + output_json_string(azCol[i], -1); + oputz(":"); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - fputs("null",p->out); + oputz("null"); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "1e999"); + oputz("9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + oputz("-9.0e+999"); }else{ sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + oputz(z); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(p->out, pBlob, nBlob); + output_json_string(pBlob, nBlob); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(p->out, azArg[i], -1); + output_json_string(azArg[i], -1); }else{ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); } if( iout); + oputz(","); } } - putc('}', p->out); + oputz("}"); break; } case MODE_Quote: { @@ -13447,7 +20827,7 @@ static int shell_callback( if( p->cnt==0 && p->showHeader ){ for(i=0; i0 ) fputs(p->colSeparator, p->out); - output_quoted_string(p->out, azCol[i]); + output_quoted_string(azCol[i]); } fputs(p->rowSeparator, p->out); } @@ -13455,24 +20835,24 @@ static int shell_callback( for(i=0; i0 ) fputs(p->colSeparator, p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + oputz("NULL"); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + oputz(z); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); + output_hex_blob(pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + oputz(azArg[i]); }else{ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(azArg[i]); } } fputs(p->rowSeparator, p->out); @@ -13481,17 +20861,17 @@ static int shell_callback( case MODE_Ascii: { if( p->cnt++==0 && p->showHeader ){ for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : ""); + if( i>0 ) oputz(p->colSeparator); + oputz(azCol[i] ? azCol[i] : ""); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); } if( azArg==0 ) break; for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue); + if( i>0 ) oputz(p->colSeparator); + oputz(azArg[i] ? azArg[i] : p->nullValue); } - utf8_printf(p->out, "%s", p->rowSeparator); + oputz(p->rowSeparator); break; } case MODE_EQP: { @@ -13570,7 +20950,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + eputf("SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -13596,7 +20976,7 @@ static void set_table_name(ShellState *p, const char *zName){ n = strlen30(zName); if( cQuote ) n += n+2; z = p->zDestTable = malloc( n+1 ); - if( z==0 ) shell_out_of_memory(); + shell_check_oom(z); n = 0; if( cQuote ) z[n++] = cQuote; for(i=0; zName[i]; i++){ @@ -13607,6 +20987,49 @@ static void set_table_name(ShellState *p, const char *zName){ z[n] = 0; } +/* +** Maybe construct two lines of text that point out the position of a +** syntax error. Return a pointer to the text, in memory obtained from +** sqlite3_malloc(). Or, if the most recent error does not involve a +** specific token that we can point to, return an empty string. +** +** In all cases, the memory returned is obtained from sqlite3_malloc64() +** and should be released by the caller invoking sqlite3_free(). +*/ +static char *shell_error_context(const char *zSql, sqlite3 *db){ + int iOffset; + size_t len; + char *zCode; + char *zMsg; + int i; + if( db==0 + || zSql==0 + || (iOffset = sqlite3_error_offset(db))<0 + || iOffset>=(int)strlen(zSql) + ){ + return sqlite3_mprintf(""); + } + while( iOffset>50 ){ + iOffset--; + zSql++; + while( (zSql[0]&0xc0)==0x80 ){ zSql++; iOffset--; } + } + len = strlen(zSql); + if( len>78 ){ + len = 78; + while( len>0 && (zSql[len]&0xc0)==0x80 ) len--; + } + zCode = sqlite3_mprintf("%.*s", len, zSql); + shell_check_oom(zCode); + for(i=0; zCode[i]; i++){ if( IsSpace(zSql[i]) ) zCode[i] = ' '; } + if( iOffset<25 ){ + zMsg = sqlite3_mprintf("\n %z\n %*s^--- error here", zCode,iOffset,""); + }else{ + zMsg = sqlite3_mprintf("\n %z\n %*serror here ---^", zCode,iOffset-14,""); + } + return zMsg; +} + /* ** Execute a query statement that will generate SQL output. Print @@ -13629,8 +21052,10 @@ static int run_table_dump_query( const char *z; rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, - sqlite3_errmsg(p->db)); + char *zContext = shell_error_context(zSelect, p->db); + oputf("/**** ERROR: (%d) %s *****/\n%s", + rc, sqlite3_errmsg(p->db), zContext); + sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; return rc; } @@ -13638,23 +21063,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - utf8_printf(p->out, "%s", z); + oputf("%s", z); for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + oputf(",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - raw_printf(p->out, "\n;\n"); + oputz("\n;\n"); }else{ - raw_printf(p->out, ";\n"); + oputz(";\n"); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, - sqlite3_errmsg(p->db)); + oputf("/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } return rc; @@ -13665,19 +21089,32 @@ static int run_table_dump_query( */ static char *save_err_msg( sqlite3 *db, /* Database to query */ - const char *zWhen, /* Qualifier (format) wrapper */ - int rc /* Error code returned from API */ + const char *zPhase, /* When the error occurs */ + int rc, /* Error code returned from API */ + const char *zSql /* SQL string, or NULL */ ){ - if( zWhen==0 ) - zWhen = "%s (%d)"; - return sqlite3_mprintf(zWhen, sqlite3_errmsg(db), rc); + char *zErr; + char *zContext; + sqlite3_str *pStr = sqlite3_str_new(0); + sqlite3_str_appendf(pStr, "%s, %s", zPhase, sqlite3_errmsg(db)); + if( rc>1 ){ + sqlite3_str_appendf(pStr, " (%d)", rc); + } + zContext = shell_error_context(zSql, db); + if( zContext ){ + sqlite3_str_appendall(pStr, zContext); + sqlite3_free(zContext); + } + zErr = sqlite3_str_finish(pStr); + shell_check_oom(zErr); + return zErr; } #ifdef __linux__ /* ** Attempt to display I/O stats on Linux using /proc/PID/io */ -static void displayLinuxIoStats(FILE *out){ +static void displayLinuxIoStats(void){ FILE *in; char z[200]; sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); @@ -13699,8 +21136,8 @@ static void displayLinuxIoStats(FILE *out){ int i; for(i=0; iout, "%-36s %s\n", zLabel, zLine); + oputf("%-36s %s\n", zLabel, zLine); } /* @@ -13745,58 +21181,56 @@ static int display_stats( ){ int iCur; int iHiwtr; - FILE *out; if( pArg==0 || pArg->out==0 ) return 0; - out = pArg->out; if( pArg->pStmt && pArg->statsOn==2 ){ int nCol, i, x; sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + oputf("%-36s %d\n", "Number of output columns:", nCol); for(i=0; istatsOn==3 ){ if( pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - raw_printf(pArg->out, "VM-steps: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); + oputf("VM-steps: %d\n", iCur); } return 0; } - displayStatLine(pArg, "Memory Used:", + displayStatLine("Memory Used:", "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine(pArg, "Number of Outstanding Allocations:", + displayStatLine("Number of Outstanding Allocations:", "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine(pArg, "Number of Pcache Pages Used:", + displayStatLine("Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + displayStatLine("Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine(pArg, "Largest Allocation:", + displayStatLine("Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine(pArg, "Largest Pcache Allocation:", + displayStatLine("Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine(pArg, "Deepest Parser Stack:", + displayStatLine("Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif @@ -13805,68 +21239,68 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Lookaside Slots Used: %d (max %d)\n", - iCur, iHiwtr); + oputf("Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Successful lookaside attempts: %d\n", - iHiwtr); + oputf("Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to size: %d\n", - iHiwtr); + oputf("Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to OOM: %d\n", - iHiwtr); + oputf("Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Pager Heap Usage: %d bytes\n", - iCur); + oputf("Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache hits: %d\n", iCur); + oputf("Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + oputf("Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + oputf("Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + oputf("Page cache spills: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", - iCur); + oputf("Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", - iCur); + oputf("Statement Heap/Lookaside Usage: %d bytes\n", iCur); } if( pArg->pStmt ){ + int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); + oputf("Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - raw_printf(pArg->out, "Sort Operations: %d\n", iCur); + oputf("Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + oputf("Autoindex Inserts: %d\n", iCur); + iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, + bReset); + iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, + bReset); + if( iHit || iMiss ){ + oputf("Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); + } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + oputf("Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + oputf("Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - raw_printf(pArg->out, "Number of times run: %d\n", iCur); + oputf("Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + oputf("Memory used by prepared stmt: %d\n", iCur); } #ifdef __linux__ - displayLinuxIoStats(pArg->out); + displayLinuxIoStats(); #endif /* Do not remove this machine readable comment: extra-stats-output-here */ @@ -13874,53 +21308,114 @@ static int display_stats( return 0; } -/* -** Display scan stats. -*/ -static void display_scanstats( + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else - int i, k, n, mx; - raw_printf(pArg->out, "-------- scanstats --------\n"); - mx = 0; - for(k=0; k<=mx; k++){ - double rEstLoop = 1.0; - for(i=n=0; 1; i++){ - sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nLoop, nVisit; - double rEst; - int iSid; - const char *zExplain; - if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ - break; + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *p = pArg->pStmt; + int ii = 0; + i64 nTotal = 0; + int nWidth = 0; + eqp_reset(pArg); + + for(ii=0; 1; ii++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + scanStatsHeight(p, ii)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 4; + + sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(ii=0; 1; ii++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + char *zText = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + zText = sqlite3_mprintf("%s", zo); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + char *z = 0; + if( nCycle>=0 && nTotal>0 ){ + z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, + nCycle, ((nCycle*100)+nTotal/2) / nTotal + ); } - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); - if( iSid>mx ) mx = iSid; - if( iSid!=k ) continue; - if( n==0 ){ - rEstLoop = (double)nLoop; - if( k>0 ) raw_printf(pArg->out, "-------- subquery %d -------\n", k); + if( nLoop>=0 ){ + z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); } - n++; - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); - rEstLoop *= rEst; - raw_printf(pArg->out, - " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst + if( nRow>=0 ){ + z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); + } + + if( zName && pArg->scanstatsOn>1 ){ + double rpl = (double)nRow / (double)nLoop; + z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); + } + + zText = sqlite3_mprintf( + "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z ); } + + eqp_append(pArg, iId, iPid, zText); + sqlite3_free(zText); } - raw_printf(pArg->out, "---------------------------\n"); -#endif + + eqp_render(pArg, nTotal); } +#endif + /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -13931,7 +21426,7 @@ static void display_scanstats( static int str_in_array(const char *zStr, const char **azArray){ int i; for(i=0; azArray[i]; i++){ - if( 0==strcmp(zStr, azArray[i]) ) return 1; + if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; } return 0; } @@ -13947,6 +21442,9 @@ static int str_in_array(const char *zStr, const char **azArray){ ** all opcodes that occur between the p2 jump destination and the opcode ** itself by 2 spaces. ** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: ** Yield SeekGt SeekLt RowSetRead Rewind @@ -13955,76 +21453,55 @@ static int str_in_array(const char *zStr, const char **azArray){ ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - const char *zSql; /* The text of the SQL statement */ - const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; - /* Try to figure out if this is really an EXPLAIN statement. If this - ** cannot be verified, return early. */ - if( sqlite3_column_count(pSql)!=8 ){ - p->cMode = p->mode; - return; - } - zSql = sqlite3_sql(pSql); - if( zSql==0 ) return; - for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); - if( sqlite3_strnicmp(z, "explain", 7) ){ - p->cMode = p->mode; - return; - } + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ int i; int iAddr = sqlite3_column_int(pSql, 0); const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - - /* Set p2 to the P2 field of the current opcode. Then, assuming that - ** p2 is an instruction address, set variable p2op to the index of that - ** instruction in the aiIndent[] array. p2 and p2op may be different if - ** the current instruction is part of a sub-program generated by an - ** SQL trigger or foreign key. */ + int p1 = sqlite3_column_int(pSql, 2); int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ - if( iOp==0 ){ - /* Do further verfication that this is explain output. Abort if - ** it is not */ - static const char *explainCols[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; - int jj; - for(jj=0; jjcMode = p->mode; - sqlite3_reset(pSql); - return; - } - } - } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); - if( p->aiIndent==0 ) shell_out_of_memory(); + shell_check_oom(p->aiIndent); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); - if( abYield==0 ) shell_out_of_memory(); + shell_check_oom(abYield); } + abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; - - if( str_in_array(zOp, azNext) ){ + if( str_in_array(zOp, azNext) && p2op>0 ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent - && (abYield[p2op] || sqlite3_column_int(pSql, 2)) - ){ + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; } } @@ -14044,8 +21521,50 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + /* -** Disable and restore .wheretrace and .selecttrace settings. +** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ static unsigned int savedSelectTrace; static unsigned int savedWhereTrace; @@ -14101,12 +21620,13 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ if( nVar==0 ) return; /* Nothing to do */ if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ - return; /* Parameter table does not exist */ + rc = SQLITE_NOTFOUND; + pQ = 0; + }else{ + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); } - rc = sqlite3_prepare_v2(pArg->db, - "SELECT value FROM temp.sqlite_parameters" - " WHERE key=?1", -1, &pQ, 0); - if( rc || pQ==0 ) return; for(i=1; i<=nVar; i++){ char zNum[30]; const char *zVar = sqlite3_bind_parameter_name(pStmt, i); @@ -14115,8 +21635,16 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ zVar = zNum; } sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); - if( sqlite3_step(pQ)==SQLITE_ROW ){ + if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){ sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); +#ifdef NAN + }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, NAN); +#endif +#ifdef INFINITY + }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, INFINITY); +#endif }else{ sqlite3_bind_null(pStmt, i); } @@ -14153,44 +21681,171 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ /* Draw horizontal line N characters long using unicode box ** characters */ -static void print_box_line(FILE *out, int N){ - const char zDash[] = +static void print_box_line(int N){ + const char zDash[] = BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; const int nDash = sizeof(zDash) - 1; N *= 3; while( N>nDash ){ - utf8_printf(out, zDash); + oputz(zDash); N -= nDash; } - utf8_printf(out, "%.*s", N, zDash); + oputf("%.*s", N, zDash); +} + +/* +** Draw a horizontal separator for a MODE_Box table. +*/ +static void print_box_row_separator( + ShellState *p, + int nArg, + const char *zSep1, + const char *zSep2, + const char *zSep3 +){ + int i; + if( nArg>0 ){ + oputz(zSep1); + print_box_line(p->actualWidth[0]+2); + for(i=1; iactualWidth[i]+2); + } + oputz(zSep3); + } + oputz("\n"); +} + +/* +** z[] is a line of text that is to be displayed the .mode box or table or +** similar tabular formats. z[] might contain control characters such +** as \n, \t, \f, or \r. +** +** Compute characters to display on the first line of z[]. Stop at the +** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained +** from malloc()) of that first line, which caller should free sometime. +** Write anything to display on the next line into *pzTail. If this is +** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) +*/ +static char *translateForDisplayAndDup( + const unsigned char *z, /* Input text to be transformed */ + const unsigned char **pzTail, /* OUT: Tail of the input for next line */ + int mxWidth, /* Max width. 0 means no limit */ + u8 bWordWrap /* If true, avoid breaking mid-word */ +){ + int i; /* Input bytes consumed */ + int j; /* Output bytes generated */ + int k; /* Input bytes to be displayed */ + int n; /* Output column number */ + unsigned char *zOut; /* Output text */ + + if( z==0 ){ + *pzTail = 0; + return 0; + } + if( mxWidth<0 ) mxWidth = -mxWidth; + if( mxWidth==0 ) mxWidth = 1000000; + i = j = n = 0; + while( n=' ' ){ + n++; + do{ i++; j++; }while( (z[i]&0xc0)==0x80 ); + continue; + } + if( z[i]=='\t' ){ + do{ + n++; + j++; + }while( (n&7)!=0 && n=mxWidth && bWordWrap ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i; k>i/2; k--){ + if( isspace(z[k-1]) ) break; + } + if( k<=i/2 ){ + for(k=i; k>i/2; k--){ + if( isalnum(z[k-1])!=isalnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k<=i/2 ){ + k = i; + }else{ + i = k; + while( z[i]==' ' ) i++; + } + }else{ + k = i; + } + if( n>=mxWidth && z[i]>=' ' ){ + *pzTail = &z[i]; + }else if( z[i]=='\r' && z[i+1]=='\n' ){ + *pzTail = z[i+2] ? &z[i+2] : 0; + }else if( z[i]==0 || z[i+1]==0 ){ + *pzTail = 0; + }else{ + *pzTail = &z[i+1]; + } + zOut = malloc( j+1 ); + shell_check_oom(zOut); + i = j = n = 0; + while( i=' ' ){ + n++; + do{ zOut[j++] = z[i++]; }while( (z[i]&0xc0)==0x80 ); + continue; + } + if( z[i]=='\t' ){ + do{ + n++; + zOut[j++] = ' '; + }while( (n&7)!=0 && n0 ){ - utf8_printf(p->out, "%s", zSep1); - print_box_line(p->out, p->actualWidth[0]+2); - for(i=1; iout, "%s", zSep2); - print_box_line(p->out, p->actualWidth[i]+2); +static char *quoted_column(sqlite3_stmt *pStmt, int i){ + switch( sqlite3_column_type(pStmt, i) ){ + case SQLITE_NULL: { + return sqlite3_mprintf("NULL"); + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); + } + case SQLITE_TEXT: { + return sqlite3_mprintf("%Q",sqlite3_column_text(pStmt,i)); + } + case SQLITE_BLOB: { + int j; + sqlite3_str *pStr = sqlite3_str_new(0); + const unsigned char *a = sqlite3_column_blob(pStmt,i); + int n = sqlite3_column_bytes(pStmt,i); + sqlite3_str_append(pStr, "x'", 2); + for(j=0; jout, "%s", zSep3); } - fputs("\n", p->out); + return 0; /* Not reached */ } - - /* ** Run a prepared statement and output the result in one of the ** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, @@ -14203,44 +21858,49 @@ static void print_box_row_separator( */ static void exec_prepared_stmt_columnar( ShellState *p, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statment to run */ + sqlite3_stmt *pStmt /* Statement to run */ ){ sqlite3_int64 nRow = 0; int nColumn = 0; char **azData = 0; sqlite3_int64 nAlloc = 0; + char *abRowDiv = 0; + const unsigned char *uz; const char *z; + char **azQuoted = 0; int rc; sqlite3_int64 i, nData; int j, nTotal, w, n; const char *colSep = 0; const char *rowSep = 0; + const unsigned char **azNextLine = 0; + int bNextLine = 0; + int bMultiLineRowExists = 0; + int bw = p->cmOpts.bWordWrap; + const char *zEmpty = ""; + const char *zShowNull = p->nullValue; rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); - if( azData==0 ) shell_out_of_memory(); - for(i=0; i= nAlloc ){ - nAlloc *= 2; - azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); - if( azData==0 ) shell_out_of_memory(); - } - nRow++; - for(i=0; icmOpts.bQuote ){ + azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); + shell_check_oom(azQuoted); + memset(azQuoted, 0, nColumn*sizeof(char*) ); + } + abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); + shell_check_oom(abRowDiv); if( nColumn>p->nWidth ){ p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); - if( p->colWidth==0 ) shell_out_of_memory(); + shell_check_oom(p->colWidth); for(i=p->nWidth; icolWidth[i] = 0; p->nWidth = nColumn; p->actualWidth = &p->colWidth[nColumn]; @@ -14251,16 +21911,64 @@ static void exec_prepared_stmt_columnar( if( w<0 ) w = -w; p->actualWidth[i] = w; } + for(i=0; icolWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + uz = (const unsigned char*)sqlite3_column_name(pStmt,i); + if( uz==0 ) uz = (u8*)""; + azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); + } + do{ + int useNextLine = bNextLine; + bNextLine = 0; + if( (nRow+2)*nColumn >= nAlloc ){ + nAlloc *= 2; + azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); + shell_check_oom(azData); + abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); + shell_check_oom(abRowDiv); + } + abRowDiv[nRow] = 1; + nRow++; + for(i=0; icolWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + if( useNextLine ){ + uz = azNextLine[i]; + if( uz==0 ) uz = (u8*)zEmpty; + }else if( p->cmOpts.bQuote ){ + sqlite3_free(azQuoted[i]); + azQuoted[i] = quoted_column(pStmt,i); + uz = (const unsigned char*)azQuoted[i]; + }else{ + uz = (const unsigned char*)sqlite3_column_text(pStmt,i); + if( uz==0 ) uz = (u8*)zShowNull; + } + azData[nRow*nColumn + i] + = translateForDisplayAndDup(uz, &azNextLine[i], wx, bw); + if( azNextLine[i] ){ + bNextLine = 1; + abRowDiv[nRow-1] = 0; + bMultiLineRowExists = 1; + } + } + }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); nTotal = nColumn*(nRow+1); for(i=0; inullValue; + if( z==0 ) z = (char*)zEmpty; n = strlenChar(z); j = i%nColumn; if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; @@ -14269,11 +21977,11 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(p->out, w, azData[i]); + utf8_width_print(w, azData[i]); fputs(i==nColumn-1?"\n":" ", p->out); } for(i=0; iout, p->actualWidth[i]); + print_dashes(p->actualWidth[i]); fputs(i==nColumn-1?"\n":" ", p->out); } } @@ -14287,8 +21995,8 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + oputz(i==nColumn-1?" |\n":" | "); } print_row_separator(p, nColumn, "+"); break; @@ -14300,8 +22008,8 @@ static void exec_prepared_stmt_columnar( for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + oputf("%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); + oputz(i==nColumn-1?" |\n":" | "); } print_row_separator(p, nColumn, "|"); break; @@ -14310,13 +22018,13 @@ static void exec_prepared_stmt_columnar( colSep = " " BOX_13 " "; rowSep = " " BOX_13 "\n"; print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - utf8_printf(p->out, BOX_13 " "); + oputz(BOX_13 " "); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + oputf("%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); } print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); break; @@ -14324,19 +22032,28 @@ static void exec_prepared_stmt_columnar( } for(i=nColumn, j=0; icMode!=MODE_Column ){ - utf8_printf(p->out, "%s", p->cMode==MODE_Box?BOX_13" ":"| "); + oputz(p->cMode==MODE_Box?BOX_13" ":"| "); } z = azData[i]; if( z==0 ) z = p->nullValue; w = p->actualWidth[j]; if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(p->out, w, z); + utf8_width_print(w, z); if( j==nColumn-1 ){ - utf8_printf(p->out, "%s", rowSep); + oputz(rowSep); + if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + }else if( p->cMode==MODE_Column ){ + oputz("\n"); + } + } j = -1; if( seenInterrupt ) goto columnar_end; }else{ - utf8_printf(p->out, "%s", colSep); + oputz(colSep); } } if( p->cMode==MODE_Table ){ @@ -14346,11 +22063,20 @@ static void exec_prepared_stmt_columnar( } columnar_end: if( seenInterrupt ){ - utf8_printf(p->out, "Interrupt\n"); + oputz("Interrupt\n"); } nData = (nRow+1)*nColumn; - for(i=0; icMode==MODE_Column || pArg->cMode==MODE_Table @@ -14393,10 +22120,14 @@ static void exec_prepared_stmt( azCols[i] = (char *)sqlite3_column_name(pStmt, i); } do{ + nRow++; /* extract the data and data types */ for(i=0; icMode==MODE_Insert ){ + if( x==SQLITE_BLOB + && pArg + && (pArg->cMode==MODE_Insert || pArg->cMode==MODE_Quote) + ){ azVals[i] = ""; }else{ azVals[i] = (char*)sqlite3_column_text(pStmt, i); @@ -14420,6 +22151,11 @@ static void exec_prepared_stmt( sqlite3_free(pData); if( pArg->cMode==MODE_Json ){ fputs("]\n", pArg->out); + }else if( pArg->cMode==MODE_Count ){ + char zBuf[200]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n", + nRow, nRow!=1 ? "s" : ""); + printf("%s", zBuf); } } } @@ -14437,8 +22173,8 @@ static void exec_prepared_stmt( ** caller to eventually free this buffer using sqlite3_free(). */ static int expertHandleSQL( - ShellState *pState, - const char *zSql, + ShellState *pState, + const char *zSql, char **pzErr ){ assert( pState->expert.pExpert ); @@ -14448,7 +22184,7 @@ static int expertHandleSQL( /* ** This function is called either to silently clean up the object -** created by the ".expert" command (if bCancel==1), or to generate a +** created by the ".expert" command (if bCancel==1), or to generate a ** report from it and then clean it up (if bCancel==0). ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error @@ -14466,7 +22202,6 @@ static int expertFinish( assert( p ); assert( bCancel || pzErr==0 || *pzErr==0 ); if( bCancel==0 ){ - FILE *out = pState->out; int bVerbose = pState->expert.bVerbose; rc = sqlite3_expert_analyze(p, pzErr); @@ -14476,8 +22211,8 @@ static int expertFinish( if( bVerbose ){ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); - raw_printf(out, "-- Candidates -----------------------------\n"); - raw_printf(out, "%s\n", zCand); + oputz("-- Candidates -----------------------------\n"); + oputf("%s\n", zCand); } for(i=0; i=2 && 0==strncmp(z, "-verbose", n) ){ + if( n>=2 && 0==cli_strncmp(z, "-verbose", n) ){ pState->expert.bVerbose = 1; } - else if( n>=2 && 0==strncmp(z, "-sample", n) ){ + else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ - raw_printf(stderr, "option requires an argument: %s\n", z); + eputf("option requires an argument: %s\n", z); rc = SQLITE_ERROR; }else{ iSample = (int)integerValue(azArg[++i]); if( iSample<0 || iSample>100 ){ - raw_printf(stderr, "value out of range: %s\n", azArg[i]); + eputf("value out of range: %s\n", azArg[i]); rc = SQLITE_ERROR; } } } else{ - raw_printf(stderr, "unknown option: %s\n", z); + eputf("unknown option: %s\n", z); rc = SQLITE_ERROR; } } @@ -14543,7 +22278,7 @@ static int expertDotCommand( if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - raw_printf(stderr, "sqlite3_expert_new: %s\n", zErr); + eputf("sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ sqlite3_expert_config( @@ -14551,6 +22286,7 @@ static int expertDotCommand( ); } } + sqlite3_free(zErr); return rc; } @@ -14592,7 +22328,7 @@ static int shell_exec( rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ if( pzErrMsg ){ - *pzErrMsg = save_err_msg(db, "in prepare, %s (%d)", rc); + *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql); } }else{ if( !pStmt ){ @@ -14605,69 +22341,60 @@ static int shell_exec( if( zStmtSql==0 ) zStmtSql = ""; while( IsSpace(zStmtSql[0]) ) zStmtSql++; - /* save off the prepared statment handle and reset row count */ + /* save off the prepared statement handle and reset row count */ if( pArg ){ pArg->pStmt = pStmt; pArg->cnt = 0; } - /* echo the sql statement if echo on */ - if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ - utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); - } - /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ sqlite3_stmt *pExplain; - char *zEQP; int triggerEQP = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); if( pArg->autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + pExplain = pStmt; + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 2); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); int iEqpId = sqlite3_column_int(pExplain, 0); int iParentId = sqlite3_column_int(pExplain, 1); if( zEQPLine==0 ) zEQPLine = ""; - if( zEQPLine[0]=='-' ) eqp_render(pArg); + if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); eqp_append(pArg, iEqpId, iParentId, zEQPLine); } - eqp_render(pArg); + eqp_render(pArg, 0); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); if( pArg->autoEQP>=AUTOEQP_full ){ /* Also do an EXPLAIN for ".eqp full" mode */ - zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 1); if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; + assert( sqlite3_stmt_isexplain(pExplain)==1 ); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); } if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); - /* Reprepare pStmt before reactiving trace modes */ - sqlite3_finalize(pStmt); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( pArg ) pArg->pStmt = pStmt; } + sqlite3_reset(pStmt); + sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); } if( pArg ){ + int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); pArg->cMode = pArg->mode; if( pArg->autoExplain ){ - if( sqlite3_stmt_isexplain(pStmt)==1 ){ + if( bIsExplain ){ pArg->cMode = MODE_Explain; } if( sqlite3_stmt_isexplain(pStmt)==2 ){ @@ -14677,7 +22404,7 @@ static int shell_exec( /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain ){ + if( pArg->cMode==MODE_Explain && bIsExplain ){ explain_data_prepare(pArg, pStmt); } } @@ -14685,7 +22412,7 @@ static int shell_exec( bind_prepared_stmt(pArg, pStmt); exec_prepared_stmt(pArg, pStmt); explain_data_delete(pArg); - eqp_render(pArg); + eqp_render(pArg, 0); /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ @@ -14706,7 +22433,7 @@ static int shell_exec( zSql = zLeftover; while( IsSpace(zSql[0]) ) zSql++; }else if( pzErrMsg ){ - *pzErrMsg = save_err_msg(db, "stepping, %s (%d)", rc); + *pzErrMsg = save_err_msg(db, "stepping", rc, 0); } /* clear saved stmt handle */ @@ -14756,6 +22483,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ int rc; zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + shell_check_oom(zSql); rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc ) return 0; @@ -14763,9 +22491,10 @@ static char **tableColumnList(ShellState *p, const char *zTab){ if( nCol>=nAlloc-2 ){ nAlloc = nAlloc*2 + nCol + 10; azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); - if( azCol==0 ) shell_out_of_memory(); + shell_check_oom(azCol); } azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + shell_check_oom(azCol[nCol]); if( sqlite3_column_int(pStmt, 5) ){ nPK++; if( nPK==1 @@ -14791,7 +22520,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ */ if( preserveRowid && isIPK ){ /* If a single PRIMARY KEY column with type INTEGER was seen, then it - ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** might be an alias for the ROWID. But it might also be a WITHOUT ROWID ** table or a INTEGER PRIMARY KEY DESC column, neither of which are ** ROWID aliases. To distinguish these cases, check to see if ** there is a "pk" entry in "PRAGMA index_list". There will be @@ -14799,6 +22528,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ */ zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" " WHERE origin='pk'", zTab); + shell_check_oom(zSql); rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc ){ @@ -14869,35 +22599,38 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; + if( zTable==0 ) return 0; + if( zType==0 ) return 0; dataOnly = (p->shellFlgs & SHFLG_DumpDataOnly)!=0; noSys = (p->shellFlgs & SHFLG_DumpNoSys)!=0; - if( strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ - if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); + if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){ + if( !dataOnly ) oputz("DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ - if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n"); - }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + if( !dataOnly ) oputz("ANALYZE sqlite_schema;\n"); + }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ /* no-op */ - }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ - raw_printf(p->out, "PRAGMA writable_schema=ON;\n"); + oputz("PRAGMA writable_schema=ON;\n"); p->writableSchema = 1; } zIns = sqlite3_mprintf( "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)" "VALUES('table','%q','%q',0,'%q');", zTable, zTable, zSql); - utf8_printf(p->out, "%s\n", zIns); + shell_check_oom(zIns); + oputf("%s\n", zIns); sqlite3_free(zIns); return 0; }else{ - printSchemaLine(p->out, zSql, ";\n"); + printSchemaLine(zSql, ";\n"); } - if( strcmp(zType, "table")==0 ){ + if( cli_strcmp(zType, "table")==0 ){ ShellText sSelect; ShellText sTable; char **azCol; @@ -14952,7 +22685,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ p->mode = p->cMode = MODE_Insert; rc = shell_exec(p, sSelect.z, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ - raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + oputz("/****** CORRUPTION ERROR *******/\n"); toggleSelectOrder(p->db); shell_exec(p, sSelect.z, 0); toggleSelectOrder(p->db); @@ -14983,9 +22716,9 @@ static int run_schema_dump_query( if( rc==SQLITE_CORRUPT ){ char *zQ2; int len = strlen30(zQuery); - raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + oputz("/****** CORRUPTION ERROR *******/\n"); if( zErr ){ - utf8_printf(p->out, "/****** %s ******/\n", zErr); + oputf("/****** %s ******/\n", zErr); sqlite3_free(zErr); zErr = 0; } @@ -14994,7 +22727,7 @@ static int run_schema_dump_query( sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ - utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr); + oputf("/****** ERROR: %s ******/\n", zErr); }else{ rc = SQLITE_CORRUPT; } @@ -15008,13 +22741,14 @@ static int run_schema_dump_query( ** Text of help messages. ** ** The help text for each individual command begins with a line that starts -** with ".". Subsequent lines are supplimental information. +** with ".". Subsequent lines are supplemental information. ** ** There must be two or more spaces between the end of the command and the ** start of the description of what that command does. */ static const char *(azHelp[]) = { -#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) +#if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) \ + && !defined(SQLITE_SHELL_FIDDLE) ".archive ... Manage SQL archives", " Each command must have exactly one of the following options:", " -c, --create Create a new archive", @@ -15040,19 +22774,30 @@ static const char *(azHelp[]) = { #ifndef SQLITE_OMIT_AUTHORIZATION ".auth ON|OFF Show authorizer callbacks", #endif +#ifndef SQLITE_SHELL_FIDDLE ".backup ?DB? FILE Backup DB (default \"main\") to FILE", + " Options:", " --append Use the appendvfs", " --async Write to FILE without journal and fsync()", +#endif ".bail on|off Stop after hitting an error. Default OFF", - ".binary on|off Turn binary output on or off. Default OFF", +#ifndef SQLITE_SHELL_FIDDLE ".cd DIRECTORY Change the working directory to DIRECTORY", +#endif ".changes on|off Show number of rows changed by SQL", +#ifndef SQLITE_SHELL_FIDDLE ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", +#endif ".connection [close] [#] Open or close an auxiliary database connection", +#if defined(_WIN32) || defined(WIN32) + ".crnl on|off Translate \\n to \\r\\n. Default ON", +#endif ".databases List names and files of attached databases", ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", +#if SQLITE_SHELL_HAVE_RECOVER ".dbinfo ?DB? Show status information about the database", +#endif ".dump ?OBJECTS? Render database content as SQL", " Options:", " --data-only Output only INSERT statements", @@ -15069,9 +22814,13 @@ static const char *(azHelp[]) = { " trace Like \"full\" but enable \"PRAGMA vdbe_trace\"", #endif " trigger Like \"full\" but also show trigger bytecode", +#ifndef SQLITE_SHELL_FIDDLE ".excel Display the output of next command in spreadsheet", " --bom Put a UTF8 byte-order mark on intermediate file", +#endif +#ifndef SQLITE_SHELL_FIDDLE ".exit ?CODE? Exit this program with return-code CODE", +#endif ".expert EXPERIMENTAL. Suggest indexes for queries", ".explain ?on|off|auto? Change the EXPLAIN formatting mode. Default: auto", ".filectrl CMD ... Run various sqlite3_file_control() operations", @@ -15080,11 +22829,13 @@ static const char *(azHelp[]) = { ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", +#ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", " Options:", " --ascii Use \\037 and \\036 as column and row separators", " --csv Use , and \\n as column and row separators", " --skip N Skip the first N rows of input", + " --schema S Target table to be S.TABLE", " -v \"Verbose\" - increase auxiliary output", " Notes:", " * If TABLE does not exist, it is created. The first row of input", @@ -15093,52 +22844,70 @@ static const char *(azHelp[]) = { " from the \".mode\" output mode", " * If FILE begins with \"|\" then it is a command that generates the", " input text.", +#endif #ifndef SQLITE_OMIT_TEST_CONTROL - ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", + ",imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", + ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE - ".iotrace FILE Enable I/O diagnostic logging to FILE", + ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", ".lint OPTIONS Report potential schema issues.", " Options:", " fkey-indexes Find missing foreign key indexes", -#ifndef SQLITE_OMIT_LOAD_EXTENSION +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) ".load FILE ?ENTRY? Load an extension library", #endif - ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", - ".mode MODE ?TABLE? Set output mode", +#if !defined(SQLITE_SHELL_FIDDLE) + ".log FILE|on|off Turn logging on or off. FILE can be stderr/stdout", +#else + ".log on|off Turn logging on or off.", +#endif + ".mode MODE ?OPTIONS? Set output mode", " MODE is one of:", - " ascii Columns/rows delimited by 0x1F and 0x1E", - " box Tables using unicode box-drawing characters", - " csv Comma-separated values", - " column Output in columns. (See .width)", - " html HTML code", - " insert SQL insert statements for TABLE", - " json Results in a JSON array", - " line One value per line", - " list Values delimited by \"|\"", - " markdown Markdown table format", - " quote Escape answers as for SQL", - " table ASCII-art table", - " tabs Tab-separated values", - " tcl TCL list elements", - ".nonce STRING Disable safe mode for one command if the nonce matches", + " ascii Columns/rows delimited by 0x1F and 0x1E", + " box Tables using unicode box-drawing characters", + " csv Comma-separated values", + " column Output in columns. (See .width)", + " html HTML
    code", + " insert SQL insert statements for TABLE", + " json Results in a JSON array", + " line One value per line", + " list Values delimited by \"|\"", + " markdown Markdown table format", + " qbox Shorthand for \"box --wrap 60 --quote\"", + " quote Escape answers as for SQL", + " table ASCII-art table", + " tabs Tab-separated values", + " tcl TCL list elements", + " OPTIONS: (for columnar modes or insert mode):", + " --wrap N Wrap output lines to no longer than N characters", + " --wordwrap B Wrap or not at word boundaries per B (on/off)", + " --ww Shorthand for \"--wordwrap 1\"", + " --quote Quote output text as SQL literals", + " --noquote Do not quote output text", + " TABLE The name of SQL table used for \"insert\" mode", +#ifndef SQLITE_SHELL_FIDDLE + ".nonce STRING Suspend safe mode for one command if nonce matches", +#endif ".nullvalue STRING Use STRING in place of NULL values", +#ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", " If FILE begins with '|' then open as a pipe", " --bom Put a UTF8 byte-order mark at the beginning", " -e Send output to the system text editor", " -x Send output as CSV to a spreadsheet (same as \".excel\")", -#ifdef SQLITE_DEBUG - ".oom ?--repeat M? ?N? Simulate an OOM error on the N-th allocation", -#endif + /* Note that .open is (partially) available in WASM builds but is + ** currently only intended to be used by the fiddle tool, not + ** end users, so is "undocumented." */ ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE", " Options:", " --append Use appendvfs to append database to the end of FILE", +#endif #ifndef SQLITE_OMIT_DESERIALIZE " --deserialize Load into memory using sqlite3_deserialize()", " --hexdb Load the output of \"dbtotxt\" as an in-memory db", @@ -15148,12 +22917,14 @@ static const char *(azHelp[]) = { " --nofollow Do not follow symbolic links", " --readonly Open FILE readonly", " --zip FILE is a ZIP archive", +#ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", " Options:", " --bom Prefix output with a UTF8 byte-order mark", " -e Send output to the system text editor", " -x Send output as CSV to a spreadsheet", +#endif ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", " init Initialize the TEMP table that holds bindings", @@ -15170,24 +22941,28 @@ static const char *(azHelp[]) = { " --reset Reset the count for each input and interrupt", #endif ".prompt MAIN CONTINUE Replace the standard prompts", - ".quit Exit this program", - ".read FILE Read input from FILE", -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#ifndef SQLITE_SHELL_FIDDLE + ".quit Stop interpreting input stream, exit if primary.", + ".read FILE Read input from FILE or command output", + " If FILE begins with \"|\", it is a command that generates the input.", +#endif +#if SQLITE_SHELL_HAVE_RECOVER ".recover Recover as much data as possible from corrupt db.", - " --freelist-corrupt Assume the freelist is corrupt", - " --recovery-db NAME Store recovery metadata in database file NAME", + " --ignore-freelist Ignore pages that appear to be on db freelist", " --lost-and-found TABLE Alternative name for the lost-and-found table", " --no-rowids Do not attempt to recover rowid values", " that are not also INTEGER PRIMARY KEYs", #endif +#ifndef SQLITE_SHELL_FIDDLE ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", - ".save FILE Write in-memory database into FILE", - ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", + ".save ?OPTIONS? FILE Write database to FILE (an alias for .backup ...)", +#endif + ".scanstats on|off|est Turn sqlite3_stmt_scanstatus() metrics on or off", ".schema ?PATTERN? Show the CREATE statements matching PATTERN", " Options:", " --indent Try to pretty-print the schema", " --nosys Omit objects whose names start with \"sqlite_\"", - ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + ",selftest ?OPTIONS? Run tests defined in the SELFTEST table", " Options:", " --init Create a new SELFTEST table", " -v Verbose output", @@ -15215,7 +22990,7 @@ static const char *(azHelp[]) = { " --sha3-384 Use the sha3-384 algorithm", " --sha3-512 Use the sha3-512 algorithm", " Any other argument is a LIKE pattern for tables to hash", -#ifndef SQLITE_NOHAVE_SYSTEM +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif ".show Show the current values for various settings", @@ -15224,12 +22999,14 @@ static const char *(azHelp[]) = { " on Turn on automatic stat display", " stmt Show statement stats", " vmstep Show the virtual machine step count only", -#ifndef SQLITE_NOHAVE_SYSTEM +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", - ".testcase NAME Begin redirecting output to 'testcase-out.txt'", - ".testctrl CMD ... Run various sqlite3_test_control() operations", +#ifndef SQLITE_SHELL_FIDDLE + ",testcase NAME Begin redirecting output to 'testcase-out.txt'", +#endif + ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", @@ -15253,6 +23030,7 @@ static const char *(azHelp[]) = { ".unmodule NAME ... Unregister virtual table modules", " --allexcept Unregister everything except those named", #endif + ".version Show source, library and compiler versions", ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", @@ -15276,24 +23054,50 @@ static int showHelp(FILE *out, const char *zPattern){ char *zPat; if( zPattern==0 || zPattern[0]=='0' - || strcmp(zPattern,"-a")==0 - || strcmp(zPattern,"-all")==0 - || strcmp(zPattern,"--all")==0 + || cli_strcmp(zPattern,"-a")==0 + || cli_strcmp(zPattern,"-all")==0 + || cli_strcmp(zPattern,"--all")==0 ){ - /* Show all commands, but only one line per command */ - if( zPattern==0 ) zPattern = ""; + enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 }; + enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 }; + /* Show all or most commands + ** *zPattern==0 => summary of documented commands only + ** *zPattern=='0' => whole help for undocumented commands + ** Otherwise => whole help for documented commands + */ + enum HelpWanted hw = HW_SummaryOnly; + enum HelpHave hh = HH_More; + if( zPattern!=0 ){ + hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull; + } for(i=0; i65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error; n = (n+pgsz-1)&~(pgsz-1); /* Round n up to the next multiple of pgsz */ a = sqlite3_malloc( n ? n : 1 ); - if( a==0 ){ - utf8_printf(stderr, "Out of memory!\n"); - goto readHexDb_error; - } + shell_check_oom(a); memset(a, 0, n); if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ - utf8_printf(stderr, "invalid pagesize\n"); + eputz("invalid pagesize\n"); goto readHexDb_error; } for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ @@ -15516,7 +23333,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ iOffset = k; continue; } - if( strncmp(zLine, "| end ", 6)==0 ){ + if( cli_strncmp(zLine, "| end ", 6)==0 ){ break; } rc = sscanf(zLine,"| %d: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", @@ -15524,186 +23341,48 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); if( rc==17 ){ k = iOffset+j; - if( k+16<=n && k>=0 ){ - int ii; - for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; - } - } - } - *pnData = n; - if( in!=p->in ){ - fclose(in); - }else{ - p->lineno = nLine; - } - return a; - -readHexDb_error: - if( in!=p->in ){ - fclose(in); - }else{ - while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ - nLine++; - if(strncmp(zLine, "| end ", 6)==0 ) break; - } - p->lineno = nLine; - } - sqlite3_free(a); - utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); - return 0; -} -#endif /* SQLITE_OMIT_DESERIALIZE */ - -/* -** Scalar function "shell_int32". The first argument to this function -** must be a blob. The second a non-negative integer. This function -** reads and returns a 32-bit big-endian integer from byte -** offset (4*) of the blob. -*/ -static void shellInt32( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *pBlob; - int nBlob; - int iInt; - - UNUSED_PARAMETER(argc); - nBlob = sqlite3_value_bytes(argv[0]); - pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]); - - if( iInt>=0 && (iInt+1)*4<=nBlob ){ - const unsigned char *a = &pBlob[iInt*4]; - sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) - + ((sqlite3_int64)a[1]<<16) - + ((sqlite3_int64)a[2]<< 8) - + ((sqlite3_int64)a[3]<< 0); - sqlite3_result_int64(context, iVal); - } -} - -/* -** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, -** using "..." with internal double-quote characters doubled. -*/ -static void shellIdQuote( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zName ){ - char *z = sqlite3_mprintf("\"%w\"", zName); - sqlite3_result_text(context, z, -1, sqlite3_free); - } -} - -/* -** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. -*/ -static void shellUSleepFunc( - sqlite3_context *context, - int argcUnused, - sqlite3_value **argv -){ - int sleep = sqlite3_value_int(argv[0]); - (void)argcUnused; - sqlite3_sleep(sleep/1000); - sqlite3_result_int(context, sleep); -} - -/* -** Scalar function "shell_escape_crnl" used by the .recover command. -** The argument passed to this function is the output of built-in -** function quote(). If the first character of the input is "'", -** indicating that the value passed to quote() was a text value, -** then this function searches the input for "\n" and "\r" characters -** and adds a wrapper similar to the following: -** -** replace(replace(, '\n', char(10), '\r', char(13)); -** -** Or, if the first character of the input is not "'", then a copy -** of the input is returned. -*/ -static void shellEscapeCrnl( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zText = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zText[0]=='\'' ){ - int nText = sqlite3_value_bytes(argv[0]); - int i; - char zBuf1[20]; - char zBuf2[20]; - const char *zNL = 0; - const char *zCR = 0; - int nCR = 0; - int nNL = 0; - - for(i=0; zText[i]; i++){ - if( zNL==0 && zText[i]=='\n' ){ - zNL = unused_string(zText, "\\n", "\\012", zBuf1); - nNL = (int)strlen(zNL); - } - if( zCR==0 && zText[i]=='\r' ){ - zCR = unused_string(zText, "\\r", "\\015", zBuf2); - nCR = (int)strlen(zCR); - } - } - - if( zNL || zCR ){ - int iOut = 0; - i64 nMax = (nNL > nCR) ? nNL : nCR; - i64 nAlloc = nMax * nText + (nMax+64)*2; - char *zOut = (char*)sqlite3_malloc64(nAlloc); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - return; - } - - if( zNL && zCR ){ - memcpy(&zOut[iOut], "replace(replace(", 16); - iOut += 16; - }else{ - memcpy(&zOut[iOut], "replace(", 8); - iOut += 8; - } - for(i=0; zText[i]; i++){ - if( zText[i]=='\n' ){ - memcpy(&zOut[iOut], zNL, nNL); - iOut += nNL; - }else if( zText[i]=='\r' ){ - memcpy(&zOut[iOut], zCR, nCR); - iOut += nCR; - }else{ - zOut[iOut] = zText[i]; - iOut++; - } - } - - if( zNL ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; - memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; - } - if( zCR ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; - memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + if( k+16<=n && k>=0 ){ + int ii; + for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff; } + } + } + *pnData = n; + if( in!=p->in ){ + fclose(in); + }else{ + p->lineno = nLine; + } + return a; - sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); - sqlite3_free(zOut); - return; +readHexDb_error: + if( in!=p->in ){ + fclose(in); + }else{ + while( fgets(zLine, sizeof(zLine), p->in)!=0 ){ + nLine++; + if(cli_strncmp(zLine, "| end ", 6)==0 ) break; } + p->lineno = nLine; } + sqlite3_free(a); + eputf("Error on line %d of --hexdb input\n", nLine); + return 0; +} +#endif /* SQLITE_OMIT_DESERIALIZE */ - sqlite3_result_value(context, argv[0]); +/* +** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. +*/ +static void shellUSleepFunc( + sqlite3_context *context, + int argcUnused, + sqlite3_value **argv +){ + int sleep = sqlite3_value_int(argv[0]); + (void)argcUnused; + sqlite3_sleep(sleep/1000); + sqlite3_result_int(context, sleep); } /* Flags for open_db(). @@ -15730,13 +23409,13 @@ static void open_db(ShellState *p, int openFlags){ if( zDbFilename==0 || zDbFilename[0]==0 ){ p->openMode = SHELL_OPEN_NORMAL; }else{ - p->openMode = (u8)deduceDatabaseType(zDbFilename, + p->openMode = (u8)deduceDatabaseType(zDbFilename, (openFlags & OPEN_DB_ZIPFILE)!=0); } } switch( p->openMode ){ case SHELL_OPEN_APPENDVFS: { - sqlite3_open_v2(zDbFilename, &p->db, + sqlite3_open_v2(zDbFilename, &p->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs"); break; } @@ -15761,46 +23440,93 @@ static void open_db(ShellState *p, int openFlags){ break; } } - globalDb = p->db; if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", - zDbFilename, sqlite3_errmsg(p->db)); - if( openFlags & OPEN_DB_KEEPALIVE ){ - sqlite3_open(":memory:", &p->db); - return; + eputf("Error: unable to open database \"%s\": %s\n", + zDbFilename, sqlite3_errmsg(p->db)); + if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ + exit(1); + } + sqlite3_close(p->db); + sqlite3_open(":memory:", &p->db); + if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ + eputz("Also: unable to open substitute in-memory database.\n"); + exit(1); + }else{ + eputf("Notice: using substitute in-memory database instead of \"%s\"\n", + zDbFilename); } - exit(1); } + globalDb = p->db; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); + + /* Reflect the use or absence of --unsafe-testing invocation. */ + { + int testmode_on = ShellHasFlag(p,SHFLG_TestingMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_TRUSTED_SCHEMA, testmode_on,0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, !testmode_on,0); + } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif - sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); - sqlite3_completion_init(p->db, 0, 0); sqlite3_uint_init(p->db, 0, 0); sqlite3_decimal_init(p->db, 0, 0); + sqlite3_base64_init(p->db, 0, 0); + sqlite3_base85_init(p->db, 0, 0); sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) - sqlite3_dbdata_init(p->db, 0, 0); +#ifndef SQLITE_SHELL_FIDDLE + sqlite3_fileio_init(p->db, 0, 0); + sqlite3_completion_init(p->db, 0, 0); #endif #ifdef SQLITE_HAVE_ZLIB - sqlite3_zipfile_init(p->db, 0, 0); - sqlite3_sqlar_init(p->db, 0, 0); + if( !p->bSafeModePersist ){ + sqlite3_zipfile_init(p->db, 0, 0); + sqlite3_sqlar_init(p->db, 0, 0); + } +#endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * initialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } #endif + + sqlite3_create_function(p->db, "strtod", 1, SQLITE_UTF8, 0, + shellStrtod, 0, 0); + sqlite3_create_function(p->db, "dtostr", 1, SQLITE_UTF8, 0, + shellDtostr, 0, 0); + sqlite3_create_function(p->db, "dtostr", 2, SQLITE_UTF8, 0, + shellDtostr, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); - sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, - shellEscapeCrnl, 0, 0); - sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, - shellInt32, 0, 0); - sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM @@ -15809,9 +23535,11 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename); + shell_check_oom(zSql); sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); } @@ -15825,15 +23553,15 @@ static void open_db(ShellState *p, int openFlags){ aData = (unsigned char*)readFile(zDbFilename, &nData); }else{ aData = readHexDb(p, &nData); - if( aData==0 ){ - return; - } + } + if( aData==0 ){ + return; } rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); if( rc ){ - utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc); + eputf("Error: sqlite3_deserialize() returns %d\n", rc); } if( p->szMax>0 ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); @@ -15841,20 +23569,24 @@ static void open_db(ShellState *p, int openFlags){ } #endif } - if( p->bSafeModePersist && p->db!=0 ){ - sqlite3_set_authorizer(p->db, safeModeAuth, p); + if( p->db!=0 ){ + if( p->bSafeModePersist ){ + sqlite3_set_authorizer(p->db, safeModeAuth, p); + } + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + ); } } /* -** Attempt to close the databaes connection. Report errors. +** Attempt to close the database connection. Report errors. */ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ - utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", - rc, sqlite3_errmsg(db)); - } + eputf("Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); + } } #if HAVE_READLINE || HAVE_EDITLINE @@ -15869,11 +23601,13 @@ static char *readline_completion_generator(const char *text, int state){ sqlite3_finalize(pStmt); zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" " FROM completion(%Q) ORDER BY 1", text); + shell_check_oom(zSql); sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); sqlite3_free(zSql); } if( sqlite3_step(pStmt)==SQLITE_ROW ){ - zRet = strdup((const char*)sqlite3_column_text(pStmt, 0)); + const char *z = (const char*)sqlite3_column_text(pStmt,0); + zRet = z ? strdup(z) : 0; }else{ sqlite3_finalize(pStmt); pStmt = 0; @@ -15882,6 +23616,8 @@ static char *readline_completion_generator(const char *text, int state){ return zRet; } static char **readline_completion(const char *zText, int iStart, int iEnd){ + (void)iStart; + (void)iEnd; rl_attempted_completion_over = 1; return rl_completion_matches(zText, readline_completion_generator); } @@ -15891,13 +23627,13 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){ ** Linenoise completion callback */ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ - int nLine = strlen30(zLine); - int i, iStart; + i64 nLine = strlen(zLine); + i64 i, iStart; sqlite3_stmt *pStmt = 0; char *zSql; char zBuf[1000]; - if( nLine>sizeof(zBuf)-30 ) return; + if( nLine>(i64)sizeof(zBuf)-30 ) return; if( zLine[0]=='.' || zLine[0]=='#') return; for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} if( i==nLine-1 ) return; @@ -15906,13 +23642,14 @@ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ zSql = sqlite3_mprintf("SELECT DISTINCT candidate COLLATE nocase" " FROM completion(%Q,%Q) ORDER BY 1", &zLine[iStart], zLine); + shell_check_oom(zSql); sqlite3_prepare_v2(globalDb, zSql, -1, &pStmt, 0); sqlite3_free(zSql); sqlite3_exec(globalDb, "PRAGMA page_count", 0, 0, 0); /* Load the schema */ while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zCompletion = (const char*)sqlite3_column_text(pStmt, 0); int nCompletion = sqlite3_column_bytes(pStmt, 0); - if( iStart+nCompletion < sizeof(zBuf)-1 ){ + if( iStart+nCompletion < (i64)sizeof(zBuf)-1 && zCompletion ){ memcpy(zBuf+iStart, zCompletion, nCompletion+1); linenoiseAddCompletion(lc, zBuf); } @@ -15936,6 +23673,7 @@ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ ** \' -> ' ** \\ -> backslash ** \NNN -> ascii character NNN in octal +** \xHH -> ascii character HH in hexadecimal */ static void resolve_backslashes(char *z){ int i, j; @@ -15964,6 +23702,15 @@ static void resolve_backslashes(char *z){ c = '\''; }else if( c=='\\' ){ c = '\\'; + }else if( c=='x' ){ + int nhd = 0, hdv; + u8 hv = 0; + while( nhd<2 && (c=z[i+1+nhd])!=0 && (hdv=hexDigitValue(c))>=0 ){ + hv = (u8)((hv<<4)|hdv); + ++nhd; + } + i += nhd; + c = (u8)hv; }else if( c>='0' && c<='7' ){ c -= '0'; if( z[i+1]>='0' && z[i+1]<='7' ){ @@ -15999,8 +23746,7 @@ static int booleanValue(const char *zArg){ if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ return 0; } - utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", - zArg); + eputf("ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg); return 0; } @@ -16029,16 +23775,16 @@ static void output_file_close(FILE *f){ */ static FILE *output_file_open(const char *zFile, int bTextMode){ FILE *f; - if( strcmp(zFile,"stdout")==0 ){ + if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; - }else if( strcmp(zFile, "stderr")==0 ){ + }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( strcmp(zFile, "off")==0 ){ + }else if( cli_strcmp(zFile, "off")==0 ){ f = 0; }else{ f = fopen(zFile, bTextMode ? "w" : "wb"); if( f==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); + eputf("Error: cannot open \"%s\"\n", zFile); } } return f; @@ -16057,13 +23803,13 @@ static int sql_trace_callback( ShellState *p = (ShellState*)pArg; sqlite3_stmt *pStmt; const char *zSql; - int nSql; + i64 nSql; if( p->traceOut==0 ) return 0; if( mType==SQLITE_TRACE_CLOSE ){ - utf8_printf(p->traceOut, "-- closing database connection\n"); + sputz(p->traceOut, "-- closing database connection\n"); return 0; } - if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){ zSql = (const char*)pX; }else{ pStmt = (sqlite3_stmt*)pP; @@ -16085,17 +23831,18 @@ static int sql_trace_callback( } } if( zSql==0 ) return 0; - nSql = strlen30(zSql); + nSql = strlen(zSql); + if( nSql>1000000000 ) nSql = 1000000000; while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + sputf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { - sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; - utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; + sputf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } } @@ -16106,10 +23853,13 @@ static int sql_trace_callback( /* ** A no-op routine that runs with the ".breakpoint" doc-command. This is ** a useful spot to set a debugger breakpoint. +** +** This routine does not do anything practical. The code are there simply +** to prevent the compiler from optimizing this routine out. */ static void test_breakpoint(void){ - static int nCall = 0; - nCall++; + static unsigned int nCall = 0; + if( (nCall++)==0xffffffff ) printf("Many .breakpoints have run\n"); } /* @@ -16147,7 +23897,7 @@ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; p->z = sqlite3_realloc64(p->z, p->nAlloc); - if( p->z==0 ) shell_out_of_memory(); + shell_check_oom(p->z); } p->z[p->n++] = (char)c; } @@ -16167,8 +23917,8 @@ static void import_append_char(ImportCtx *p, int c){ */ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int c; - int cSep = p->cColSep; - int rSep = p->cRowSep; + int cSep = (u8)p->cColSep; + int rSep = (u8)p->cRowSep; p->n = 0; c = fgetc(p->in); if( c==EOF || seenInterrupt ){ @@ -16199,12 +23949,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ break; } if( pc==cQuote && c!='\r' ){ - utf8_printf(stderr, "%s:%d: unescaped %c character\n", - p->zFile, p->nLine, cQuote); + eputf("%s:%d: unescaped %c character\n", p->zFile, p->nLine, cQuote); } if( c==EOF ){ - utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n", - p->zFile, startLine, cQuote); + eputf("%s:%d: unterminated %c-quoted field\n", + p->zFile, startLine, cQuote); p->cTerm = c; break; } @@ -16257,8 +24006,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ */ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ int c; - int cSep = p->cColSep; - int rSep = p->cRowSep; + int cSep = (u8)p->cColSep; + int rSep = (u8)p->cRowSep; p->n = 0; c = fgetc(p->in); if( c==EOF || seenInterrupt ){ @@ -16299,16 +24048,16 @@ static void tryToCloneData( const int spinRate = 10000; zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); + shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error %d: %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error %d: %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } n = sqlite3_column_count(pQuery); zInsert = sqlite3_malloc64(200 + nTable + n*3); - if( zInsert==0 ) shell_out_of_memory(); + shell_check_oom(zInsert); sqlite3_snprintf(200+nTable,zInsert, "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable); i = strlen30(zInsert); @@ -16319,9 +24068,8 @@ static void tryToCloneData( memcpy(zInsert+i, ");", 3); rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0); if( rc ){ - utf8_printf(stderr, "Error %d: %s on [%s]\n", - sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), - zQuery); + eputf("Error %d: %s on [%s]\n", + sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert); goto end_data_xfer; } for(k=0; k<2; k++){ @@ -16356,8 +24104,8 @@ static void tryToCloneData( } /* End for */ rc = sqlite3_step(pInsert); if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - utf8_printf(stderr, "Error %d: %s\n", sqlite3_extended_errcode(newDb), - sqlite3_errmsg(newDb)); + eputf("Error %d: %s\n", + sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb)); } sqlite3_reset(pInsert); cnt++; @@ -16371,9 +24119,10 @@ static void tryToCloneData( sqlite3_free(zQuery); zQuery = sqlite3_mprintf("SELECT * FROM \"%w\" ORDER BY rowid DESC;", zTable); + shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable); + eputf("Warning: cannot step \"%s\" backwards", zTable); break; } } /* End for(k=0...) */ @@ -16406,55 +24155,60 @@ static void tryToCloneSchema( char *zErrMsg = 0; zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" - " WHERE %s", zWhere); + " WHERE %s ORDER BY rowid ASC", zWhere); + shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error: (%d) %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), + sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){ zName = sqlite3_column_text(pQuery, 0); zSql = sqlite3_column_text(pQuery, 1); - printf("%s... ", zName); fflush(stdout); - sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); - if( zErrMsg ){ - utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); - sqlite3_free(zErrMsg); - zErrMsg = 0; + if( zName==0 || zSql==0 ) continue; + if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){ + sputf(stdout, "%s... ", zName); fflush(stdout); + sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); + if( zErrMsg ){ + eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_free(zErrMsg); + zErrMsg = 0; + } } if( xForEach ){ xForEach(p, newDb, (const char*)zName); } - printf("done\n"); + sputz(stdout, "done\n"); } if( rc!=SQLITE_DONE ){ sqlite3_finalize(pQuery); sqlite3_free(zQuery); zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema" " WHERE %s ORDER BY rowid DESC", zWhere); + shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - utf8_printf(stderr, "Error: (%d) %s on [%s]\n", - sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), - zQuery); + eputf("Error: (%d) %s on [%s]\n", + sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } while( sqlite3_step(pQuery)==SQLITE_ROW ){ zName = sqlite3_column_text(pQuery, 0); zSql = sqlite3_column_text(pQuery, 1); - printf("%s... ", zName); fflush(stdout); + if( zName==0 || zSql==0 ) continue; + if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; + sputf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + eputf("Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } if( xForEach ){ xForEach(p, newDb, (const char*)zName); } - printf("done\n"); + sputz(stdout, "done\n"); } } end_schema_xfer: @@ -16471,13 +24225,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ - utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb); + eputf("File \"%s\" already exists.\n", zNewDb); return; } rc = sqlite3_open(zNewDb, &newDb); if( rc ){ - utf8_printf(stderr, "Cannot create output database: %s\n", - sqlite3_errmsg(newDb)); + eputf("Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0); @@ -16489,6 +24242,18 @@ static void tryToClone(ShellState *p, const char *zNewDb){ close_db(newDb); } +#ifndef SQLITE_SHELL_FIDDLE +/* +** Change the output stream (file or pipe or console) to something else. +*/ +static void output_redir(ShellState *p, FILE *pfNew){ + if( p->out != stdout ) eputz("Output already redirected.\n"); + else{ + p->out = pfNew; + setOutputStream(pfNew); + } +} + /* ** Change the output file back to stdout. ** @@ -16516,7 +24281,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - utf8_printf(stderr, "Failed: [%s]\n", zCmd); + eputf("Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -16531,15 +24296,20 @@ static void output_reset(ShellState *p){ } p->outfile[0] = 0; p->out = stdout; + setOutputStream(stdout); } +#else +# define output_redir(SS,pfO) +# define output_reset(SS) +#endif /* ** Run an SQL command and return the single integer result. */ -static int db_int(ShellState *p, const char *zSql){ +static int db_int(sqlite3 *db, const char *zSql){ sqlite3_stmt *pStmt; int res = 0; - sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ res = sqlite3_column_int(pStmt,0); } @@ -16547,6 +24317,7 @@ static int db_int(ShellState *p, const char *zSql){ return res; } +#if SQLITE_SHELL_HAVE_RECOVER /* ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ @@ -16601,7 +24372,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db)); + eputf("error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -16609,57 +24380,60 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ if( sqlite3_step(pStmt)==SQLITE_ROW && sqlite3_column_bytes(pStmt,0)>100 ){ - memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + const u8 *pb = sqlite3_column_blob(pStmt,0); + shell_check_oom(pb); + memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - raw_printf(stderr, "unable to read database header\n"); + eputz("unable to read database header\n"); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - utf8_printf(p->out, "%-20s %d\n", "database page size:", i); - utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + oputf("%-20s %d\n", "database page size:", i); + oputf("%-20s %d\n", "write format:", aHdr[18]); + oputf("%-20s %d\n", "read format:", aHdr[19]); + oputf("%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; iout, "%-20s %u", aField[i].zName, val); + oputf("%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) raw_printf(p->out, " (utf8)"); - if( val==2 ) raw_printf(p->out, " (utf16le)"); - if( val==3 ) raw_printf(p->out, " (utf16be)"); + if( val==1 ) oputz(" (utf8)"); + if( val==2 ) oputz(" (utf16le)"); + if( val==3 ) oputz(" (utf16be)"); } } - raw_printf(p->out, "\n"); + oputz("\n"); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); - }else if( strcmp(zDb,"temp")==0 ){ + }else if( cli_strcmp(zDb,"temp")==0 ){ zSchemaTab = sqlite3_mprintf("%s", "sqlite_temp_schema"); }else{ zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb); } for(i=0; idb, zSql); sqlite3_free(zSql); - utf8_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); + oputf("%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion); + oputf("%-20s %u\n", "data version", iDataVersion); return 0; } +#endif /* SQLITE_SHELL_HAVE_RECOVER */ /* ** Print the current sqlite3_errmsg() value to stderr and return 1. */ static int shellDatabaseError(sqlite3 *db){ const char *zErr = sqlite3_errmsg(db); - utf8_printf(stderr, "Error: %s\n", zErr); + eputf("Error: %s\n", zErr); return 1; } @@ -16769,7 +24543,7 @@ static int optionMatch(const char *zStr, const char *zOpt){ if( zStr[0]!='-' ) return 0; zStr++; if( zStr[0]=='-' ) zStr++; - return strcmp(zStr, zOpt)==0; + return cli_strcmp(zStr, zOpt)==0; } /* @@ -16828,9 +24602,7 @@ static void newTempFile(ShellState *p, const char *zSuffix){ }else{ p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); } - if( p->zTempFile==0 ){ - shell_out_of_memory(); - } + shell_check_oom(p->zTempFile); } @@ -16896,7 +24668,6 @@ static int lintFkeyIndexes( int nArg /* Number of entries in azArg[] */ ){ sqlite3 *db = pState->db; /* Database handle to query "main" db of */ - FILE *out = pState->out; /* Stream to write non-error output to */ int bVerbose = 0; /* If -verbose is present */ int bGroupByParent = 0; /* If -groupbyparent is present */ int i; /* To iterate through azArg[] */ @@ -16978,9 +24749,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", - azArg[0], azArg[1] - ); + eputf("Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } } @@ -17011,36 +24780,36 @@ static int lintFkeyIndexes( const char *zCI = (const char*)sqlite3_column_text(pSql, 4); const char *zParent = (const char*)sqlite3_column_text(pSql, 5); + if( zEQP==0 ) continue; + if( zGlob==0 ) continue; rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc!=SQLITE_OK ) break; if( SQLITE_ROW==sqlite3_step(pExplain) ){ const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3); - res = ( - 0==sqlite3_strglob(zGlob, zPlan) - || 0==sqlite3_strglob(zGlobIPK, zPlan) - ); + res = zPlan!=0 && ( 0==sqlite3_strglob(zGlob, zPlan) + || 0==sqlite3_strglob(zGlobIPK, zPlan)); } rc = sqlite3_finalize(pExplain); if( rc!=SQLITE_OK ) break; if( res<0 ){ - raw_printf(stderr, "Error: internal error"); + eputz("Error: internal error"); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - raw_printf(out, "-- Parent table %s\n", zParent); + oputf("-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + oputf("%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n", - zIndent, zFrom, zTarget + oputf("%s/* no extra indexes required for %s -> %s */\n", + zIndent, zFrom, zTarget ); } } @@ -17048,16 +24817,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } }else{ - raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + eputf("%s\n", sqlite3_errmsg(db)); } return rc; @@ -17077,26 +24846,23 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]); - raw_printf(stderr, "Where sub-commands are:\n"); - raw_printf(stderr, " fkey-indexes\n"); + eputf("Usage %s sub-command ?switches...?\n", azArg[0]); + eputz("Where sub-commands are:\n"); + eputz(" fkey-indexes\n"); return SQLITE_ERROR; } -#if !defined SQLITE_OMIT_VIRTUALTABLE static void shellPrepare( - sqlite3 *db, - int *pRc, - const char *zSql, + sqlite3 *db, + int *pRc, + const char *zSql, sqlite3_stmt **ppStmt ){ *ppStmt = 0; if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - raw_printf(stderr, "sql error: %s (%d)\n", - sqlite3_errmsg(db), sqlite3_errcode(db) - ); + eputf("sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } } @@ -17104,16 +24870,12 @@ static void shellPrepare( /* ** Create a prepared statement using printf-style arguments for the SQL. -** -** This routine is could be marked "static". But it is not always used, -** depending on compile-time options. By omitting the "static", we avoid -** nuisance compiler warnings about "defined but not used". */ -void shellPreparePrintf( - sqlite3 *db, - int *pRc, +static void shellPreparePrintf( + sqlite3 *db, + int *pRc, sqlite3_stmt **ppStmt, - const char *zFmt, + const char *zFmt, ... ){ *ppStmt = 0; @@ -17132,14 +24894,11 @@ void shellPreparePrintf( } } -/* Finalize the prepared statement created using shellPreparePrintf(). -** -** This routine is could be marked "static". But it is not always used, -** depending on compile-time options. By omitting the "static", we avoid -** nuisance compiler warnings about "defined but not used". +/* +** Finalize the prepared statement created using shellPreparePrintf(). */ -void shellFinalize( - int *pRc, +static void shellFinalize( + int *pRc, sqlite3_stmt *pStmt ){ if( pStmt ){ @@ -17147,13 +24906,14 @@ void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + eputf("SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } } } +#if !defined SQLITE_OMIT_VIRTUALTABLE /* Reset the prepared statement created using shellPreparePrintf(). ** ** This routine is could be marked "static". But it is not always used, @@ -17161,14 +24921,14 @@ void shellFinalize( ** nuisance compiler warnings about "defined but not used". */ void shellReset( - int *pRc, + int *pRc, sqlite3_stmt *pStmt ){ int rc = sqlite3_reset(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db)); + eputf("SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -17209,7 +24969,7 @@ static int arUsage(FILE *f){ } /* -** Print an error message for the .ar command to stderr and return +** Print an error message for the .ar command to stderr and return ** SQLITE_ERROR. */ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ @@ -17218,11 +24978,11 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); - utf8_printf(stderr, "Error: %s\n", z); + eputf("Error: %s\n", z); if( pAr->fromCmdLine ){ - utf8_printf(stderr, "Use \"-A\" for more help\n"); + eputz("Use \"-A\" for more help\n"); }else{ - utf8_printf(stderr, "Use \".archive --help\" for more help\n"); + eputz("Use \".archive --help\" for more help\n"); } sqlite3_free(z); return SQLITE_ERROR; @@ -17275,7 +25035,7 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ break; case AR_SWITCH_APPEND: pAr->bAppend = 1; - /* Fall thru into --file */ + deliberate_fall_through; case AR_SWITCH_FILE: pAr->zFile = zArg; break; @@ -17290,7 +25050,7 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){ /* ** Parse the command line for an ".ar" command. The results are written into ** structure (*pAr). SQLITE_OK is returned if the command line is parsed -** successfully, otherwise an error message is written to stderr and +** successfully, otherwise an error message is written to stderr and ** SQLITE_ERROR returned. */ static int arParseCommand( @@ -17322,7 +25082,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - utf8_printf(stderr, "Wrong number of arguments. Usage:\n"); + eputz("Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -17427,7 +25187,10 @@ static int arParseCommand( } } } - + if( pAr->eCmd==0 ){ + eputz("Required argument missing. Usage:\n"); + return arUsage(stderr); + } return SQLITE_OK; } @@ -17468,7 +25231,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - utf8_printf(stderr, "not found in archive: %s\n", z); + eputf("not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -17486,7 +25249,7 @@ static int arCheckEntries(ArCommand *pAr){ ** when pAr->bGlob is false and GLOB match when pAr->bGlob is true. */ static void arWhereClause( - int *pRc, + int *pRc, ArCommand *pAr, char **pzWhere /* OUT: New WHERE clause */ ){ @@ -17501,7 +25264,7 @@ static void arWhereClause( for(i=0; inArg; i++){ const char *z = pAr->azArg[i]; zWhere = sqlite3_mprintf( - "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'", + "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'", zWhere, zSep, zSameOp, z, strlen30(z)+1, zSameOp, z ); if( zWhere==0 ){ @@ -17516,10 +25279,10 @@ static void arWhereClause( } /* -** Implementation of .ar "lisT" command. +** Implementation of .ar "lisT" command. */ static int arListCommand(ArCommand *pAr){ - const char *zSql = "SELECT %s FROM %s WHERE %s"; + const char *zSql = "SELECT %s FROM %s WHERE %s"; const char *azCols[] = { "name", "lsmode(mode), sz, datetime(mtime, 'unixepoch'), name" @@ -17535,18 +25298,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + oputf("%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - utf8_printf(pAr->p->out, "%s % 10d %s %s\n", - sqlite3_column_text(pSql, 0), - sqlite3_column_int(pSql, 1), - sqlite3_column_text(pSql, 2), - sqlite3_column_text(pSql, 3) - ); + oputf("%s % 10d %s %s\n", + sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), + sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + oputf("%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -17555,7 +25315,6 @@ static int arListCommand(ArCommand *pAr){ return rc; } - /* ** Implementation of .ar "Remove" command. */ @@ -17574,7 +25333,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", zSql); + oputf("%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -17587,7 +25346,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - utf8_printf(stdout, "ERROR: %s\n", zErr); + sputf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -17598,17 +25357,17 @@ static int arRemoveCommand(ArCommand *pAr){ } /* -** Implementation of .ar "eXtract" command. +** Implementation of .ar "eXtract" command. */ static int arExtractCommand(ArCommand *pAr){ - const char *zSql1 = + const char *zSql1 = "SELECT " " ($dir || name)," " writefile(($dir || name), %s, mode, mtime) " "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" " AND name NOT GLOB '*..[/\\]*'"; - const char *azExtraArg[] = { + const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", "data" }; @@ -17634,7 +25393,7 @@ static int arExtractCommand(ArCommand *pAr){ if( zDir==0 ) rc = SQLITE_NOMEM; } - shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, + shellPreparePrintf(pAr->db, &rc, &pSql, zSql1, azExtraArg[pAr->bZip], pAr->zSrcTable, zWhere ); @@ -17651,11 +25410,11 @@ static int arExtractCommand(ArCommand *pAr){ j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql)); + oputf("%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( i==0 && pAr->bVerbose ){ - utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0)); + oputf("%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -17675,13 +25434,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - utf8_printf(pAr->p->out, "%s\n", zSql); + oputf("%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - utf8_printf(stdout, "ERROR: %s\n", zErr); + sputf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -17712,7 +25471,7 @@ static int arCreateOrUpdateCommand( int bUpdate, /* true for a --create. */ int bOnlyIfChanged /* Only update if file has changed */ ){ - const char *zCreate = + const char *zCreate = "CREATE TABLE IF NOT EXISTS sqlar(\n" " name TEXT PRIMARY KEY, -- name of the file\n" " mode INT, -- access permissions\n" @@ -17754,7 +25513,7 @@ static int arCreateOrUpdateCommand( arExecSql(pAr, "PRAGMA page_size=512"); rc = arExecSql(pAr, "SAVEPOINT ar;"); if( rc!=SQLITE_OK ) return rc; - zTemp[0] = 0; + zTemp[0] = 0; if( pAr->bZip ){ /* Initialize the zipfile virtual table, if necessary */ if( pAr->zFile ){ @@ -17848,7 +25607,7 @@ static int arDotCommand( }else if( cmd.zFile ){ int flags; if( cmd.bAppend ) eDbType = SHELL_OPEN_APPENDVFS; - if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT + if( cmd.eCmd==AR_CMD_CREATE || cmd.eCmd==AR_CMD_INSERT || cmd.eCmd==AR_CMD_REMOVE || cmd.eCmd==AR_CMD_UPDATE ){ flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; }else{ @@ -17856,15 +25615,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile, - eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); + oputf("-- open database '%s'%s\n", cmd.zFile, + eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } - rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, + rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "cannot open file: %s (%s)\n", - cmd.zFile, sqlite3_errmsg(cmd.db) - ); + eputf("cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } sqlite3_fileio_init(cmd.db, 0, 0); @@ -17877,7 +25634,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - utf8_printf(stderr, "database does not contain an 'sqlar' table\n"); + eputz("database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -17899,388 +25656,44 @@ static int arDotCommand( case AR_CMD_HELP: arUsage(pState->out); - break; - - case AR_CMD_INSERT: - rc = arCreateOrUpdateCommand(&cmd, 1, 0); - break; - - case AR_CMD_REMOVE: - rc = arRemoveCommand(&cmd); - break; - - default: - assert( cmd.eCmd==AR_CMD_UPDATE ); - rc = arCreateOrUpdateCommand(&cmd, 1, 1); - break; - } - } -end_ar_command: - if( cmd.db!=pState->db ){ - close_db(cmd.db); - } - sqlite3_free(cmd.zSrcTable); - - return rc; -} -/* End of the ".archive" or ".ar" command logic -*******************************************************************************/ -#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ - -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/* -** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, the SQL statement or statements in zSql are executed using -** database connection db and the error code written to *pRc before -** this function returns. -*/ -static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ - int rc = *pRc; - if( rc==SQLITE_OK ){ - char *zErr = 0; - rc = sqlite3_exec(db, zSql, 0, 0, &zErr); - if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", zErr); - } - sqlite3_free(zErr); - *pRc = rc; - } -} - -/* -** Like shellExec(), except that zFmt is a printf() style format string. -*/ -static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - shellExec(db, pRc, z); - } - sqlite3_free(z); - } -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, an attempt is made to allocate, zero and return a pointer -** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set -** to SQLITE_NOMEM and NULL returned. -*/ -static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ - void *pRet = 0; - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, zFmt is treated as a printf() style string. The result of -** formatting it along with any trailing arguments is written into a -** buffer obtained from sqlite3_malloc(), and pointer to which is returned. -** It is the responsibility of the caller to eventually free this buffer -** using a call to sqlite3_free(). -** -** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL -** pointer returned. -*/ -static char *shellMPrintf(int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - } - } - return z; -} - -/* -** When running the ".recover" command, each output table, and the special -** orphaned row table if it is required, is represented by an instance -** of the following struct. -*/ -typedef struct RecoverTable RecoverTable; -struct RecoverTable { - char *zQuoted; /* Quoted version of table name */ - int nCol; /* Number of columns in table */ - char **azlCol; /* Array of column lists */ - int iPk; /* Index of IPK column */ -}; - -/* -** Free a RecoverTable object allocated by recoverFindTable() or -** recoverOrphanTable(). -*/ -static void recoverFreeTable(RecoverTable *pTab){ - if( pTab ){ - sqlite3_free(pTab->zQuoted); - if( pTab->azlCol ){ - int i; - for(i=0; i<=pTab->nCol; i++){ - sqlite3_free(pTab->azlCol[i]); - } - sqlite3_free(pTab->azlCol); - } - sqlite3_free(pTab); - } -} - -/* -** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. -** Otherwise, it allocates and returns a RecoverTable object based on the -** final four arguments passed to this function. It is the responsibility -** of the caller to eventually free the returned object using -** recoverFreeTable(). -*/ -static RecoverTable *recoverNewTable( - int *pRc, /* IN/OUT: Error code */ - const char *zName, /* Name of table */ - const char *zSql, /* CREATE TABLE statement */ - int bIntkey, - int nCol -){ - sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ - int rc = *pRc; - RecoverTable *pTab = 0; - - pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); - if( rc==SQLITE_OK ){ - int nSqlCol = 0; - int bSqlIntkey = 0; - sqlite3_stmt *pStmt = 0; - - rc = sqlite3_open("", &dbtmp); - if( rc==SQLITE_OK ){ - sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); - if( rc==SQLITE_ERROR ){ - rc = SQLITE_OK; - goto finished; - } - } - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT count(*) FROM pragma_table_info(%Q)", zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - nSqlCol = sqlite3_column_int(pStmt, 0); - } - shellFinalize(&rc, pStmt); - - if( rc!=SQLITE_OK || nSqlColiPk to the index - ** of the column, where columns are 0-numbered from left to right. - ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, - ** leave zPk as "_rowid_" and pTab->iPk at -2. */ - pTab->iPk = -2; - if( bIntkey ){ - shellPreparePrintf(dbtmp, &rc, &pPkFinder, - "SELECT cid, name FROM pragma_table_info(%Q) " - " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" - , zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); - } - } - - pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); - pTab->nCol = nSqlCol; - - if( bIntkey ){ - pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); - }else{ - pTab->azlCol[0] = shellMPrintf(&rc, ""); - } - i = 1; - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT %Q || group_concat(shell_idquote(name), ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " - "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, - bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", - zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zText = (const char*)sqlite3_column_text(pStmt, 0); - pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); - i++; - } - shellFinalize(&rc, pStmt); - - shellFinalize(&rc, pPkFinder); - } - } - - finished: - sqlite3_close(dbtmp); - *pRc = rc; - if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ - recoverFreeTable(pTab); - pTab = 0; - } - return pTab; -} + break; -/* -** This function is called to search the schema recovered from the -** sqlite_schema table of the (possibly) corrupt database as part -** of a ".recover" command. Specifically, for a table with root page -** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the -** table must be a WITHOUT ROWID table, or if non-zero, not one of -** those. -** -** If a table is found, a (RecoverTable*) object is returned. Or, if -** no such table is found, but bIntkey is false and iRoot is the -** root page of an index in the recovered schema, then (*pbNoop) is -** set to true and NULL returned. Or, if there is no such table or -** index, NULL is returned and (*pbNoop) set to 0, indicating that -** the caller should write data to the orphans table. -*/ -static RecoverTable *recoverFindTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - int iRoot, /* Root page of table */ - int bIntkey, /* True for an intkey table */ - int nCol, /* Number of columns in table */ - int *pbNoop /* OUT: True if iRoot is root of index */ -){ - sqlite3_stmt *pStmt = 0; - RecoverTable *pRet = 0; - int bNoop = 0; - const char *zSql = 0; - const char *zName = 0; + case AR_CMD_INSERT: + rc = arCreateOrUpdateCommand(&cmd, 1, 0); + break; - /* Search the recovered schema for an object with root page iRoot. */ - shellPreparePrintf(pState->db, pRc, &pStmt, - "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot - ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zType = (const char*)sqlite3_column_text(pStmt, 0); - if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ - bNoop = 1; - break; - } - if( sqlite3_stricmp(zType, "table")==0 ){ - zName = (const char*)sqlite3_column_text(pStmt, 1); - zSql = (const char*)sqlite3_column_text(pStmt, 2); - pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); - break; + case AR_CMD_REMOVE: + rc = arRemoveCommand(&cmd); + break; + + default: + assert( cmd.eCmd==AR_CMD_UPDATE ); + rc = arCreateOrUpdateCommand(&cmd, 1, 1); + break; } } +end_ar_command: + if( cmd.db!=pState->db ){ + close_db(cmd.db); + } + sqlite3_free(cmd.zSrcTable); - shellFinalize(pRc, pStmt); - *pbNoop = bNoop; - return pRet; + return rc; } +/* End of the ".archive" or ".ar" command logic +*******************************************************************************/ +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ + +#if SQLITE_SHELL_HAVE_RECOVER /* -** Return a RecoverTable object representing the orphans table. +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. */ -static RecoverTable *recoverOrphanTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - const char *zLostAndFound, /* Base name for orphans table */ - int nCol /* Number of user data columns */ -){ - RecoverTable *pTab = 0; - if( nCol>=0 && *pRc==SQLITE_OK ){ - int i; - - /* This block determines the name of the orphan table. The prefered - ** name is zLostAndFound. But if that clashes with another name - ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 - ** and so on until a non-clashing name is found. */ - int iTab = 0; - char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); - sqlite3_stmt *pTest = 0; - shellPrepare(pState->db, pRc, - "SELECT 1 FROM recovery.schema WHERE name=?", &pTest - ); - if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ - shellReset(pRc, pTest); - sqlite3_free(zTab); - zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); - sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - } - shellFinalize(pRc, pTest); - - pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pTab ){ - pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); - pTab->nCol = nCol; - pTab->iPk = -2; - if( nCol>0 ){ - pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); - if( pTab->azlCol ){ - pTab->azlCol[nCol] = shellMPrintf(pRc, ""); - for(i=nCol-1; i>=0; i--){ - pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); - } - } - } - - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; - }else{ - raw_printf(pState->out, - "CREATE TABLE %s(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted - ); - for(i=0; iout, ", c%d", i); - } - raw_printf(pState->out, ");\n"); - } - } - sqlite3_free(zTab); - } - return pTab; +static int recoverSqlCb(void *pCtx, const char *zSql){ + ShellState *pState = (ShellState*)pCtx; + sputf(pState->out, "%s;\n", zSql); + return SQLITE_OK; } /* @@ -18290,318 +25703,379 @@ static RecoverTable *recoverOrphanTable( */ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; - sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ - sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ - sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ - const char *zRecoveryDb = ""; /* Name of "recovery" database */ - const char *zLostAndFound = "lost_and_found"; - int i; - int nOrphan = -1; - RecoverTable *pOrphan = 0; - - int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + const char *zRecoveryDb = ""; /* Name of "recovery" database. Debug only */ + const char *zLAF = "lost_and_found"; + int bFreelist = 1; /* 0 if --ignore-freelist is specified */ int bRowids = 1; /* 0 if --no-rowids */ + sqlite3_recover *p = 0; + int i = 0; + for(i=1; iout, azArg[0]); return 1; } } - shellExecPrintf(pState->db, &rc, - /* Attach an in-memory database named 'recovery'. Create an indexed - ** cache of the sqlite_dbptr virtual table. */ - "PRAGMA writable_schema = on;" - "ATTACH %Q AS recovery;" - "DROP TABLE IF EXISTS recovery.dbptr;" - "DROP TABLE IF EXISTS recovery.freelist;" - "DROP TABLE IF EXISTS recovery.map;" - "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb - ); - - if( bFreelist ){ - shellExec(pState->db, &rc, - "WITH trunk(pgno) AS (" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " - " WHERE x>0" - " UNION" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " - " FROM trunk WHERE x>0" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " - " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" - " UNION ALL" - " SELECT data, n-1, shell_int32(data, 2+n) " - " FROM freelist WHERE n>=0" - ")" - "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" - ); - } - - /* If this is an auto-vacuum database, add all pointer-map pages to - ** the freelist table. Do this regardless of whether or not - ** --freelist-corrupt was specified. */ - shellExec(pState->db, &rc, - "WITH ptrmap(pgno) AS (" - " SELECT 2 WHERE shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" - " )" - " UNION ALL " - " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " - " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" - ")" - "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" + p = sqlite3_recover_init_sql( + pState->db, "main", recoverSqlCb, (void*)pState ); - shellExec(pState->db, &rc, - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr" - " WHERE pgno NOT IN freelist AND child NOT IN freelist;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - - /* Create the "map" table that will (eventually) contain instructions - ** for dealing with each page in the db that contains one or more - ** records. */ - "CREATE TABLE recovery.map(" - "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" - ");" + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - /* Populate table [map]. If there are circular loops of pages in the - ** database, the following adds all pages in such a loop to the map - ** as individual root pages. This could be handled better. */ - "WITH pages(i, maxlen) AS (" - " SELECT page_count, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" - " ) FROM pragma_page_count WHERE page_count>0" - " UNION ALL" - " SELECT i-1, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" - " ) FROM pages WHERE i>=2" - ")" - "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " - " SELECT i, maxlen, NULL, (" - " WITH p(orig, pgno, parent) AS (" - " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" - " UNION " - " SELECT i, p.parent, " - " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" - " )" - " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" - ") " - "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" - "UPDATE recovery.map AS o SET intkey = (" - " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" - ");" + sqlite3_recover_run(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + eputf("sql error: %s (%d)\n", zErr, errCode); + } + rc = sqlite3_recover_finish(p); + return rc; +} +#endif /* SQLITE_SHELL_HAVE_RECOVER */ - /* Extract data from page 1 and any linked pages into table - ** recovery.schema. With the same schema as an sqlite_schema table. */ - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - "INSERT INTO recovery.schema SELECT " - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=1" - ")" - "GROUP BY pgno, cell;" - "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" - ); +/* +** Implementation of ".intck STEPS_PER_UNLOCK" command. +*/ +static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ + sqlite3_intck *p = 0; + int rc = SQLITE_OK; - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" - ** CREATE TABLE statements that extracted from the existing schema. */ + rc = sqlite3_intck_open(pState->db, "main", &p); if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - /* ".recover" might output content in an order which causes immediate - ** foreign key constraints to be violated. So disable foreign-key - ** constraint enforcement to prevent problems when running the output - ** script. */ - raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(pState->out, "BEGIN;\n"); - raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); - shellPrepare(pState->db, &rc, - "SELECT sql FROM recovery.schema " - "WHERE type='table' AND sql LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", - &zCreateTable[12] - ); + i64 nStep = 0; + i64 nError = 0; + const char *zErr = 0; + while( SQLITE_OK==sqlite3_intck_step(p) ){ + const char *zMsg = sqlite3_intck_message(p); + if( zMsg ){ + oputf("%s\n", zMsg); + nError++; + } + nStep++; + if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){ + sqlite3_intck_unlock(p); + } } - shellFinalize(&rc, pStmt); - } + rc = sqlite3_intck_error(p, &zErr); + if( zErr ){ + eputf("%s\n", zErr); + } + sqlite3_intck_close(p); - /* Figure out if an orphan table will be required. And if so, how many - ** user columns it should contain */ - shellPrepare(pState->db, &rc, - "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" - , &pLoop - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - nOrphan = sqlite3_column_int(pLoop, 0); + oputf("%lld steps, %lld errors\n", nStep, nError); } - shellFinalize(&rc, pLoop); - pLoop = 0; - shellPrepare(pState->db, &rc, - "SELECT pgno FROM recovery.map WHERE root=?", &pPages - ); + return rc; +} - shellPrepare(pState->db, &rc, - "SELECT max(field), group_concat(shell_escape_crnl(quote" - "(case when (? AND field<0) then NULL else value end)" - "), ', ')" - ", min(field) " - "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" - "GROUP BY cell", &pCells - ); +/* + * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it. + * zAutoColumn(0, &db, ?) => (db!=0) Form columns spec for CREATE TABLE, + * close db and set it to 0, and return the columns spec, to later + * be sqlite3_free()'ed by the caller. + * The return is 0 when either: + * (a) The db was not initialized and zCol==0 (There are no columns.) + * (b) zCol!=0 (Column was added, db initialized as needed.) + * The 3rd argument, pRenamed, references an out parameter. If the + * pointer is non-zero, its referent will be set to a summary of renames + * done if renaming was necessary, or set to 0 if none was done. The out + * string (if any) must be sqlite3_free()'ed by the caller. + */ +#ifdef SHELL_DEBUG +#define rc_err_oom_die(rc) \ + if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ + else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ + eputf("E:%d\n",rc), assert(0) +#else +static void rc_err_oom_die(int rc){ + if( rc==SQLITE_NOMEM ) shell_check_oom(0); + assert(rc==SQLITE_OK||rc==SQLITE_DONE); +} +#endif - /* Loop through each root page. */ - shellPrepare(pState->db, &rc, - "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" - " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" - ")", &pLoop - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - int iRoot = sqlite3_column_int(pLoop, 0); - int bIntkey = sqlite3_column_int(pLoop, 1); - int nCol = sqlite3_column_int(pLoop, 2); - int bNoop = 0; - RecoverTable *pTab; - - assert( bIntkey==0 || bIntkey==1 ); - pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); - if( bNoop || rc ) continue; - if( pTab==0 ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab = pOrphan; - if( pTab==0 ) break; - } +#ifdef SHELL_COLFIX_DB /* If this is set, the DB can be in a file. */ +static char zCOL_DB[] = SHELL_STRINGIFY(SHELL_COLFIX_DB); +#else /* Otherwise, memory is faster/better for the transient DB. */ +static const char *zCOL_DB = ":memory:"; +#endif + +/* Define character (as C string) to separate generated column ordinal + * from protected part of incoming column names. This defaults to "_" + * so that incoming column identifiers that did not need not be quoted + * remain usable without being quoted. It must be one character. + */ +#ifndef SHELL_AUTOCOLUMN_SEP +# define AUTOCOLUMN_SEP "_" +#else +# define AUTOCOLUMN_SEP SHELL_STRINGIFY(SHELL_AUTOCOLUMN_SEP) +#endif - if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); +static char *zAutoColumn(const char *zColNew, sqlite3 **pDb, char **pzRenamed){ + /* Queries and D{D,M}L used here */ + static const char * const zTabMake = "\ +CREATE TABLE ColNames(\ + cpos INTEGER PRIMARY KEY,\ + name TEXT, nlen INT, chop INT, reps INT, suff TEXT);\ +CREATE VIEW RepeatedNames AS \ +SELECT DISTINCT t.name FROM ColNames t \ +WHERE t.name COLLATE NOCASE IN (\ + SELECT o.name FROM ColNames o WHERE o.cpos<>t.cpos\ +);\ +"; + static const char * const zTabFill = "\ +INSERT INTO ColNames(name,nlen,chop,reps,suff)\ + VALUES(iif(length(?1)>0,?1,'?'),max(length(?1),1),0,0,'')\ +"; + static const char * const zHasDupes = "\ +SELECT count(DISTINCT (substring(name,1,nlen-chop)||suff) COLLATE NOCASE)\ + 1, printf('%c%0*d', '"AUTOCOLUMN_SEP"', $1, cpos), '')" +#else /* ...RENAME_MINIMAL_ONE_PASS */ +"WITH Lzn(nlz) AS (" /* Find minimum extraneous leading 0's for uniqueness */ +" SELECT 0 AS nlz" +" UNION" +" SELECT nlz+1 AS nlz FROM Lzn" +" WHERE EXISTS(" +" SELECT 1" +" FROM ColNames t, ColNames o" +" WHERE" +" iif(t.name IN (SELECT * FROM RepeatedNames)," +" printf('%s"AUTOCOLUMN_SEP"%s'," +" t.name, substring(printf('%.*c%0.*d',nlz+1,'0',$1,t.cpos),2))," +" t.name" +" )" +" =" +" iif(o.name IN (SELECT * FROM RepeatedNames)," +" printf('%s"AUTOCOLUMN_SEP"%s'," +" o.name, substring(printf('%.*c%0.*d',nlz+1,'0',$1,o.cpos),2))," +" o.name" +" )" +" COLLATE NOCASE" +" AND o.cpos<>t.cpos" +" GROUP BY t.cpos" +" )" +") UPDATE Colnames AS t SET" +" chop = 0," /* No chopping, never touch incoming names. */ +" suff = iif(name IN (SELECT * FROM RepeatedNames)," +" printf('"AUTOCOLUMN_SEP"%s', substring(" +" printf('%.*c%0.*d',(SELECT max(nlz) FROM Lzn)+1,'0',1,t.cpos),2))," +" ''" +" )" +#endif + ; + static const char * const zCollectVar = "\ +SELECT\ + '('||x'0a'\ + || group_concat(\ + cname||' TEXT',\ + ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\ + ||')' AS ColsSpec \ +FROM (\ + SELECT cpos, printf('\"%w\"',printf('%!.*s%s', nlen-chop,name,suff)) AS cname \ + FROM ColNames ORDER BY cpos\ +)"; + static const char * const zRenamesDone = + "SELECT group_concat(" + " printf('\"%w\" to \"%w\"',name,printf('%!.*s%s', nlen-chop, name, suff))," + " ','||x'0a')" + "FROM ColNames WHERE suff<>'' OR chop!=0" + ; + int rc; + sqlite3_stmt *pStmt = 0; + assert(pDb!=0); + if( zColNew ){ + /* Add initial or additional column. Init db if necessary. */ + if( *pDb==0 ){ + if( SQLITE_OK!=sqlite3_open(zCOL_DB, pDb) ) return 0; +#ifdef SHELL_COLFIX_DB + if(*zCOL_DB!=':') + sqlite3_exec(*pDb,"drop table if exists ColNames;" + "drop view if exists RepeatedNames;",0,0,0); +#endif +#undef SHELL_COLFIX_DB + rc = sqlite3_exec(*pDb, zTabMake, 0, 0, 0); + rc_err_oom_die(rc); + } + assert(*pDb!=0); + rc = sqlite3_prepare_v2(*pDb, zTabFill, -1, &pStmt, 0); + rc_err_oom_die(rc); + rc = sqlite3_bind_text(pStmt, 1, zColNew, -1, 0); + rc_err_oom_die(rc); + rc = sqlite3_step(pStmt); + rc_err_oom_die(rc); + sqlite3_finalize(pStmt); + return 0; + }else if( *pDb==0 ){ + return 0; + }else{ + /* Formulate the columns spec, close the DB, zero *pDb. */ + char *zColsSpec = 0; + int hasDupes = db_int(*pDb, zHasDupes); + int nDigits = (hasDupes)? db_int(*pDb, zColDigits) : 0; + if( hasDupes ){ +#ifdef SHELL_COLUMN_RENAME_CLEAN + rc = sqlite3_exec(*pDb, zDedoctor, 0, 0, 0); + rc_err_oom_die(rc); +#endif + rc = sqlite3_exec(*pDb, zSetReps, 0, 0, 0); + rc_err_oom_die(rc); + rc = sqlite3_prepare_v2(*pDb, zRenameRank, -1, &pStmt, 0); + rc_err_oom_die(rc); + sqlite3_bind_int(pStmt, 1, nDigits); + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM); } - sqlite3_bind_int(pPages, 1, iRoot); - if( bRowids==0 && pTab->iPk<0 ){ - sqlite3_bind_int(pCells, 1, 1); + assert(db_int(*pDb, zHasDupes)==0); /* Consider: remove this */ + rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0); + rc_err_oom_die(rc); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + zColsSpec = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); }else{ - sqlite3_bind_int(pCells, 1, 0); - } - sqlite3_bind_int(pCells, 3, pTab->iPk); - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - int iPgno = sqlite3_column_int(pPages, 0); - sqlite3_bind_int(pCells, 2, iPgno); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int nField = sqlite3_column_int(pCells, 0); - int iMin = sqlite3_column_int(pCells, 2); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); - - RecoverTable *pTab2 = pTab; - if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab2 = pOrphan; - if( pTab2==0 ) break; - } - - nField = nField+1; - if( pTab2==pOrphan ){ - raw_printf(pState->out, - "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", - pTab2->zQuoted, iRoot, iPgno, nField, - iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] - ); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab2->zQuoted, pTab2->azlCol[nField], zVal - ); - } + zColsSpec = 0; + } + if( pzRenamed!=0 ){ + if( !hasDupes ) *pzRenamed = 0; + else{ + sqlite3_finalize(pStmt); + if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0) + && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + }else + *pzRenamed = 0; } - shellReset(&rc, pCells); } - shellReset(&rc, pPages); - if( pTab!=pOrphan ) recoverFreeTable(pTab); + sqlite3_finalize(pStmt); + sqlite3_close(*pDb); + *pDb = 0; + return zColsSpec; } - shellFinalize(&rc, pLoop); - shellFinalize(&rc, pPages); - shellFinalize(&rc, pCells); - recoverFreeTable(pOrphan); +} - /* The rest of the schema */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - shellPrepare(pState->db, &rc, - "SELECT sql, name FROM recovery.schema " - "WHERE sql NOT LIKE 'create table%'", &pStmt +/* +** Check if the sqlite_schema table contains one or more virtual tables. If +** parameter zLike is not NULL, then it is an SQL expression that the +** sqlite_schema row must also match. If one or more such rows are found, +** print the following warning to the output: +** +** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled +*/ +static int outputDumpWarning(ShellState *p, const char *zLike){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + shellPreparePrintf(p->db, &rc, &pStmt, + "SELECT 1 FROM sqlite_schema o WHERE " + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" + ); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + oputz("/* WARNING: " + "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n" ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); - if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - char *zPrint = shellMPrintf(&rc, - "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", - zName, zName, zSql - ); - raw_printf(pState->out, "%s;\n", zPrint); - sqlite3_free(zPrint); - }else{ - raw_printf(pState->out, "%s;\n", zSql); - } - } - shellFinalize(&rc, pStmt); } + shellFinalize(&rc, pStmt); + return rc; +} - if( rc==SQLITE_OK ){ - raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); - raw_printf(pState->out, "COMMIT;\n"); +/* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; } - sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); - return rc; + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + oputf("FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + oputf("FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; } -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ /* ** If an input line begins with "." then invoke this routine to @@ -18642,7 +26116,6 @@ static int do_meta_command(char *zLine, ShellState *p){ azArg[nArg++] = &zLine[h]; while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } if( zLine[h] ) zLine[h++] = 0; - resolve_backslashes(azArg[nArg-1]); } } azArg[nArg] = 0; @@ -18655,9 +26128,9 @@ static int do_meta_command(char *zLine, ShellState *p){ clearTempFile(p); #ifndef SQLITE_OMIT_AUTHORIZATION - if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){ + if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .auth ON|OFF\n"); + eputz("Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -18672,16 +26145,18 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) - if( c=='a' && strncmp(azArg[0], "archive", n)==0 ){ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) \ + && !defined(SQLITE_SHELL_FIDDLE) + if( c=='a' && cli_strncmp(azArg[0], "archive", n)==0 ){ open_db(p, 0); failIfSafeMode(p, "cannot run .archive in safe mode"); rc = arDotCommand(p, 0, azArg, nArg); }else #endif - if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) - || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) +#ifndef SQLITE_SHELL_FIDDLE + if( (c=='b' && n>=3 && cli_strncmp(azArg[0], "backup", n)==0) + || (c=='s' && n>=3 && cli_strncmp(azArg[0], "save", n)==0) ){ const char *zDestFile = 0; const char *zDb = 0; @@ -18695,14 +26170,14 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *z = azArg[j]; if( z[0]=='-' ){ if( z[1]=='-' ) z++; - if( strcmp(z, "-append")==0 ){ + if( cli_strcmp(z, "-append")==0 ){ zVfs = "apndvfs"; }else - if( strcmp(z, "-async")==0 ){ + if( cli_strcmp(z, "-async")==0 ){ bAsync = 1; }else { - utf8_printf(stderr, "unknown option: %s\n", azArg[j]); + eputf("unknown option: %s\n", azArg[j]); return 1; } }else if( zDestFile==0 ){ @@ -18711,19 +26186,19 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + eputz("Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - raw_printf(stderr, "missing FILENAME argument on .backup\n"); + eputz("missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; - rc = sqlite3_open_v2(zDestFile, &pDest, + rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile); + eputf("Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -18734,7 +26209,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + eputf("Error: %s\n", sqlite3_errmsg(pDest)); close_db(pDest); return 1; } @@ -18743,22 +26218,24 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else{ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest)); + eputf("Error: %s\n", sqlite3_errmsg(pDest)); rc = 1; } close_db(pDest); }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='b' && n>=3 && strncmp(azArg[0], "bail", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "bail", n)==0 ){ if( nArg==2 ){ bail_on_error = booleanValue(azArg[1]); }else{ - raw_printf(stderr, "Usage: .bail on|off\n"); + eputz("Usage: .bail on|off\n"); rc = 1; } }else - if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ + /* Undocumented. Legacy only. See "crnl" below */ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ setBinaryMode(p->out, 1); @@ -18766,7 +26243,8 @@ static int do_meta_command(char *zLine, ShellState *p){ setTextMode(p->out, 1); } }else{ - raw_printf(stderr, "Usage: .binary on|off\n"); + eputz("The \".binary\" command is deprecated. Use \".crnl\" instead.\n" + "Usage: .binary on|off\n"); rc = 1; } }else @@ -18774,11 +26252,12 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The undocumented ".breakpoint" command causes a call to the no-op ** routine named test_breakpoint(). */ - if( c=='b' && n>=3 && strncmp(azArg[0], "breakpoint", n)==0 ){ + if( c=='b' && n>=3 && cli_strncmp(azArg[0], "breakpoint", n)==0 ){ test_breakpoint(); }else - if( c=='c' && strcmp(azArg[0],"cd")==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='c' && cli_strcmp(azArg[0],"cd")==0 ){ failIfSafeMode(p, "cannot run .cd in safe mode"); if( nArg==2 ){ #if defined(_WIN32) || defined(WIN32) @@ -18789,60 +26268,63 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]); + eputf("Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ - raw_printf(stderr, "Usage: .cd DIRECTORY\n"); + eputz("Usage: .cd DIRECTORY\n"); rc = 1; } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ - raw_printf(stderr, "Usage: .changes on|off\n"); + eputz("Usage: .changes on|off\n"); rc = 1; } }else +#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ - if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ char *zRes = 0; output_reset(p); if( nArg!=2 ){ - raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + eputz("Usage: .check GLOB-PATTERN\n"); rc = 2; }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); rc = 2; }else if( testcase_glob(azArg[1],zRes)==0 ){ - utf8_printf(stderr, - "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", - p->zTestcase, azArg[1], zRes); + eputf("testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); rc = 1; }else{ - utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + oputf("testcase-%s ok\n", p->zTestcase); p->nCheck++; } sqlite3_free(zRes); }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ failIfSafeMode(p, "cannot run .clone in safe mode"); if( nArg==2 ){ tryToClone(p, azArg[1]); }else{ - raw_printf(stderr, "Usage: .clone FILENAME\n"); + eputz("Usage: .clone FILENAME\n"); rc = 1; } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='c' && strncmp(azArg[0], "connection", n)==0 ){ + if( c=='c' && cli_strncmp(azArg[0], "connection", n)==0 ){ if( nArg==1 ){ /* List available connections */ int i; @@ -18856,9 +26338,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - utf8_printf(stdout, "ACTIVE %d: %s\n", i, zFile); + sputf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - utf8_printf(stdout, " %d: %s\n", i, zFile); + sputf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -18869,13 +26351,13 @@ static int do_meta_command(char *zLine, ShellState *p){ globalDb = p->db = p->pAuxDb->db; p->pAuxDb->db = 0; } - }else if( nArg==3 && strcmp(azArg[1], "close")==0 + }else if( nArg==3 && cli_strcmp(azArg[1], "close")==0 && IsDigit(azArg[2][0]) && azArg[2][1]==0 ){ int i = azArg[2][0] - '0'; if( i<0 || i>=ArraySize(p->aAuxDb) ){ /* No-op */ }else if( p->pAuxDb == &p->aAuxDb[i] ){ - raw_printf(stderr, "cannot close the active database connection\n"); + eputz("cannot close the active database connection\n"); rc = 1; }else if( p->aAuxDb[i].db ){ session_close_all(p, i); @@ -18883,12 +26365,28 @@ static int do_meta_command(char *zLine, ShellState *p){ p->aAuxDb[i].db = 0; } }else{ - raw_printf(stderr, "Usage: .connection [close] [CONNECTION-NUMBER]\n"); + eputz("Usage: .connection [close] [CONNECTION-NUMBER]\n"); + rc = 1; + } + }else + + if( c=='c' && n==4 && cli_strncmp(azArg[0], "crnl", n)==0 ){ + if( nArg==2 ){ + if( booleanValue(azArg[1]) ){ + setTextMode(p->out, 1); + }else{ + setBinaryMode(p->out, 1); + } + }else{ +#if !defined(_WIN32) && !defined(WIN32) + eputz("The \".crnl\" is a no-op on non-Windows machines.\n"); +#endif + eputz("Usage: .crnl on|off\n"); rc = 1; } }else - if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ char **azName = 0; int nName = 0; sqlite3_stmt *pStmt; @@ -18896,14 +26394,15 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); rc = 1; }else{ while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zSchema = (const char *)sqlite3_column_text(pStmt,1); const char *zFile = (const char*)sqlite3_column_text(pStmt,2); + if( zSchema==0 || zFile==0 ) continue; azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*)); - if( azName==0 ){ shell_out_of_memory(); /* Does not return */ } + shell_check_oom(azName); azName[nName*2] = strdup(zSchema); azName[nName*2+1] = strdup(zFile); nName++; @@ -18914,11 +26413,9 @@ static int do_meta_command(char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - utf8_printf(p->out, "%s: %s %s%s\n", - azName[i*2], - z && z[0] ? z : "\"\"", - bRdonly ? "r/o" : "r/w", - eTxn==SQLITE_TXN_NONE ? "" : + oputf("%s: %s %s%s\n", + azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", + eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); free(azName[i*2]); free(azName[i*2+1]); @@ -18926,7 +26423,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azName); }else - if( c=='d' && n>=3 && strncmp(azArg[0], "dbconfig", n)==0 ){ + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbconfig", n)==0 ){ static const struct DbConfigChoices { const char *zName; int op; @@ -18944,6 +26441,8 @@ static int do_meta_command(char *zLine, ShellState *p){ { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "reverse_scanorder", SQLITE_DBCONFIG_REVERSE_SCANORDER }, + { "stmt_scanstatus", SQLITE_DBCONFIG_STMT_SCANSTATUS }, { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, @@ -18951,48 +26450,48 @@ static int do_meta_command(char *zLine, ShellState *p){ int ii, v; open_db(p, 0); for(ii=0; ii1 && strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; + if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); + oputf("%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off"); if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]); - utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n"); - } + eputf("Error: unknown dbconfig \"%s\"\n", azArg[1]); + eputz("Enter \".dbconfig\" with no arguments for a list\n"); + } }else - if( c=='d' && n>=3 && strncmp(azArg[0], "dbinfo", n)==0 ){ +#if SQLITE_SHELL_HAVE_RECOVER + if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbinfo", n)==0 ){ rc = shell_dbinfo_command(p, nArg, azArg); }else -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) - if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + if( c=='r' && cli_strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); rc = recoverDatabaseCmd(p, nArg, azArg); }else -#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ +#endif /* SQLITE_SHELL_HAVE_RECOVER */ - if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + if( c=='d' && cli_strncmp(azArg[0], "dump", n)==0 ){ char *zLike = 0; char *zSql; int i; int savedShowHeader = p->showHeader; int savedShellFlags = p->shellFlgs; - ShellClearFlag(p, + ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); for(i=1; ishellFlgs & SHFLG_DumpDataOnly)==0 ){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(p->out, "BEGIN TRANSACTION;\n"); + oputz("PRAGMA foreign_keys=OFF;\n"); + oputz("BEGIN TRANSACTION;\n"); } p->writableSchema = 0; p->showHeader = 0; @@ -19068,7 +26568,8 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "SELECT sql FROM sqlite_schema AS o " "WHERE (%s) AND sql NOT NULL" - " AND type IN ('index','trigger','view')", + " AND type IN ('index','trigger','view') " + "ORDER BY type COLLATE NOCASE DESC", zLike ); run_table_dump_query(p, zSql); @@ -19076,43 +26577,43 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); + oputz("PRAGMA writable_schema=OFF;\n"); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); + oputz(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); } p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; }else - if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ - raw_printf(stderr, "Usage: .echo on|off\n"); + eputz("Usage: .echo on|off\n"); rc = 1; } }else - if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ p->autoEQPtest = 0; if( p->autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); p->autoEQPtrace = 0; } - if( strcmp(azArg[1],"full")==0 ){ + if( cli_strcmp(azArg[1],"full")==0 ){ p->autoEQP = AUTOEQP_full; - }else if( strcmp(azArg[1],"trigger")==0 ){ + }else if( cli_strcmp(azArg[1],"trigger")==0 ){ p->autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( strcmp(azArg[1],"test")==0 ){ + }else if( cli_strcmp(azArg[1],"test")==0 ){ p->autoEQP = AUTOEQP_on; p->autoEQPtest = 1; - }else if( strcmp(azArg[1],"trace")==0 ){ + }else if( cli_strcmp(azArg[1],"trace")==0 ){ p->autoEQP = AUTOEQP_full; p->autoEQPtrace = 1; open_db(p, 0); @@ -19123,22 +26624,24 @@ static int do_meta_command(char *zLine, ShellState *p){ p->autoEQP = (u8)booleanValue(azArg[1]); } }else{ - raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n"); + eputz("Usage: .eqp off|on|trace|trigger|full\n"); rc = 1; } }else - if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; }else +#endif /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ - if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ int val = 1; if( nArg>=2 ){ - if( strcmp(azArg[1],"auto")==0 ){ + if( cli_strcmp(azArg[1],"auto")==0 ){ val = 99; }else{ val = booleanValue(azArg[1]); @@ -19158,13 +26661,19 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #ifndef SQLITE_OMIT_VIRTUALTABLE - if( c=='e' && strncmp(azArg[0], "expert", n)==0 ){ - open_db(p, 0); - expertDotCommand(p, azArg, nArg); + if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ + if( p->bSafeMode ){ + eputf("Cannot run experimental commands such as \"%s\" in safe mode\n", + azArg[0]); + rc = 1; + }else{ + open_db(p, 0); + expertDotCommand(p, azArg, nArg); + } }else #endif - if( c=='f' && strncmp(azArg[0], "filectrl", n)==0 ){ + if( c=='f' && cli_strncmp(azArg[0], "filectrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ @@ -19172,7 +26681,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } aCtrl[] = { { "chunk_size", SQLITE_FCNTL_CHUNK_SIZE, "SIZE" }, { "data_version", SQLITE_FCNTL_DATA_VERSION, "" }, - { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, + { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, { "persist_wal", SQLITE_FCNTL_PERSIST_WAL, "[BOOLEAN]" }, /* { "pragma", SQLITE_FCNTL_PRAGMA, "NAME ARG" },*/ @@ -19193,8 +26702,8 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; - if( zCmd[0]=='-' - && (strcmp(zCmd,"--schema")==0 || strcmp(zCmd,"-schema")==0) + if( zCmd[0]=='-' + && (cli_strcmp(zCmd,"--schema")==0 || cli_strcmp(zCmd,"-schema")==0) && nArg>=4 ){ zSchema = azArg[2]; @@ -19210,11 +26719,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all file-controls */ - if( strcmp(zCmd,"help")==0 ){ - utf8_printf(p->out, "Available file-controls:\n"); + if( cli_strcmp(zCmd,"help")==0 ){ + oputz("Available file-controls:\n"); for(i=0; iout, " .filectrl %s %s\n", - aCtrl[i].zCtrlName, aCtrl[i].zUsage); + oputf(" .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; goto meta_command_exit; @@ -19224,21 +26732,21 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; idb, zSchema, filectrl, &z); if( z ){ - utf8_printf(p->out, "%s\n", z); + oputf("%s\n", z); sqlite3_free(z); } isOk = 2; @@ -19295,23 +26803,23 @@ static int do_meta_command(char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - utf8_printf(p->out,"%d\n", x); + oputf("%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + oputf("Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - raw_printf(p->out, "%s\n", zBuf); + oputf("%s\n", zBuf); } }else - if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){ ShellState data; int doStats = 0; memcpy(&data, p, sizeof(data)); @@ -19322,7 +26830,7 @@ static int do_meta_command(char *zLine, ShellState *p){ nArg = 1; } if( nArg!=1 ){ - raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); + eputz("Usage: .fullschema ?--indent?\n"); rc = 1; goto meta_command_exit; } @@ -19342,71 +26850,72 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT rowid FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); - doStats = sqlite3_step(pStmt)==SQLITE_ROW; - sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } } if( doStats==0 ){ - raw_printf(p->out, "/* No STAT tables available */\n"); + oputz("/* No STAT tables available */\n"); }else{ - raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + oputz("ANALYZE sqlite_schema;\n"); data.cMode = data.mode = MODE_Insert; data.zDestTable = "sqlite_stat1"; shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); data.zDestTable = "sqlite_stat4"; shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - raw_printf(p->out, "ANALYZE sqlite_schema;\n"); + oputz("ANALYZE sqlite_schema;\n"); } }else - if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){ + if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ if( nArg==2 ){ p->showHeader = booleanValue(azArg[1]); p->shellFlgs |= SHFLG_HeaderSet; }else{ - raw_printf(stderr, "Usage: .headers on|off\n"); + eputz("Usage: .headers on|off\n"); rc = 1; } }else - if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ - utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + oputf("Nothing matches '%s'\n", azArg[1]); } }else{ showHelp(p->out, 0); } }else - if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ + char *zSchema = 0; /* Schema of zTable */ char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ - int nByte; /* Number of bytes in an SQL string */ + i64 nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int needCommit; /* True to COMMIT or ROLLBACK at end */ int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql; /* An SQL statement */ + char *zSql = 0; /* An SQL statement */ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int eVerbose = 0; /* Larger for more console output */ int nSkip = 0; /* Initial lines to skip */ int useOutputMode = 1; /* Use output mode to determine separators */ + char *zCreate = 0; /* CREATE TABLE statement text */ failIfSafeMode(p, "cannot run .import in safe mode"); memset(&sCtx, 0, sizeof(sCtx)); - sCtx.z = sqlite3_malloc64(120); - if( sCtx.z==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } if( p->mode==MODE_Ascii ){ xRead = ascii_read_one_field; }else{ xRead = csv_read_one_field; } + rc = 1; for(i=1; iout, "ERROR: extra argument: \"%s\". Usage:\n", z); + oputf("ERROR: extra argument: \"%s\". Usage:\n", z); showHelp(p->out, "import"); - rc = 1; goto meta_command_exit; } - }else if( strcmp(z,"-v")==0 ){ + }else if( cli_strcmp(z,"-v")==0 ){ eVerbose++; - }else if( strcmp(z,"-skip")==0 && iout, "ERROR: unknown option: \"%s\". Usage:\n", z); + oputf("ERROR: unknown option: \"%s\". Usage:\n", z); showHelp(p->out, "import"); - rc = 1; goto meta_command_exit; } } if( zTable==0 ){ - utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", - zFile==0 ? "FILE" : "TABLE"); + oputf("ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); showHelp(p->out, "import"); - rc = 1; goto meta_command_exit; } seenInterrupt = 0; @@ -19456,26 +26964,22 @@ static int do_meta_command(char *zLine, ShellState *p){ ** the column and row separator characters from the output mode. */ nSep = strlen30(p->colSeparator); if( nSep==0 ){ - raw_printf(stderr, - "Error: non-null column separator required for import\n"); - rc = 1; + eputz("Error: non-null column separator required for import\n"); goto meta_command_exit; } if( nSep>1 ){ - raw_printf(stderr, - "Error: multi-character column separators not allowed" + eputz("Error: multi-character column separators not allowed" " for import\n"); - rc = 1; goto meta_command_exit; } nSep = strlen30(p->rowSeparator); if( nSep==0 ){ - raw_printf(stderr, - "Error: non-null row separator required for import\n"); - rc = 1; + eputz("Error: non-null row separator required for import\n"); goto meta_command_exit; } - if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + if( nSep==2 && p->mode==MODE_Csv + && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 + ){ /* When importing CSV (only), if the row separator is set to the ** default output row separator, change it to the default input ** row separator. This avoids having to maintain different input @@ -19484,20 +26988,18 @@ static int do_meta_command(char *zLine, ShellState *p){ nSep = strlen30(p->rowSeparator); } if( nSep>1 ){ - raw_printf(stderr, "Error: multi-character row separators not allowed" - " for import\n"); - rc = 1; + eputz("Error: multi-character row separators not allowed" + " for import\n"); goto meta_command_exit; } - sCtx.cColSep = p->colSeparator[0]; - sCtx.cRowSep = p->rowSeparator[0]; + sCtx.cColSep = (u8)p->colSeparator[0]; + sCtx.cRowSep = (u8)p->rowSeparator[0]; } sCtx.zFile = zFile; sCtx.nLine = 1; if( sCtx.zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); - rc = 1; + eputz("Error: pipes are not supported in this OS\n"); goto meta_command_exit; #else sCtx.in = popen(sCtx.zFile+1, "r"); @@ -19509,81 +27011,113 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.xCloser = fclose; } if( sCtx.in==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); - rc = 1; - import_cleanup(&sCtx); + eputf("Error: cannot open \"%s\"\n", zFile); goto meta_command_exit; } if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ char zSep[2]; zSep[1] = 0; zSep[0] = sCtx.cColSep; - utf8_printf(p->out, "Column separator "); - output_c_string(p->out, zSep); - utf8_printf(p->out, ", row separator "); + oputz("Column separator "); + output_c_string(zSep); + oputz(", row separator "); zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - utf8_printf(p->out, "\n"); + output_c_string(zSep); + oputz("\n"); } - while( (nSkip--)>0 ){ - while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} - } - zSql = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable); - if( zSql==0 ){ + sCtx.z = sqlite3_malloc64(120); + if( sCtx.z==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - nByte = strlen30(zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + /* Below, resources must be freed before exit. */ + while( (nSkip--)>0 ){ + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ - char *zCreate = sqlite3_mprintf("CREATE TABLE \"%w\"", zTable); - char cSep = '('; + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){ + /* Table does not exist. Create it. */ + sqlite3 *dbCols = 0; + char *zRenames = 0; + char *zColDefs; + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); while( xRead(&sCtx) ){ - zCreate = sqlite3_mprintf("%z%c\n \"%w\" TEXT", zCreate, cSep, sCtx.z); - cSep = ','; + zAutoColumn(sCtx.z, &dbCols, 0); if( sCtx.cTerm!=sCtx.cColSep ) break; } - if( cSep=='(' ){ - sqlite3_free(zCreate); + zColDefs = zAutoColumn(0, &dbCols, &zRenames); + if( zRenames!=0 ){ + sputf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); + sqlite3_free(zRenames); + } + assert(dbCols==0); + if( zColDefs==0 ){ + eputf("%s: empty file\n", sCtx.zFile); import_cleanup(&sCtx); - utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); rc = 1; goto meta_command_exit; } - zCreate = sqlite3_mprintf("%z\n)", zCreate); + zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } if( eVerbose>=1 ){ - utf8_printf(p->out, "%s\n", zCreate); + oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); + zCreate = 0; if( rc ){ - utf8_printf(stderr, "CREATE TABLE \"%s\"(...) failed: %s\n", zTable, - sqlite3_errmsg(p->db)); + eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); + zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); - utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); import_cleanup(&sCtx); rc = 1; goto meta_command_exit; } - nCol = sqlite3_column_count(pStmt); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } j = strlen30(zSql); for(i=1; i=2 ){ - utf8_printf(p->out, "Insert using: %s\n", zSql); + oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); + zSql = 0; if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); import_cleanup(&sCtx); rc = 1; @@ -19620,11 +27156,19 @@ static int do_meta_command(char *zLine, ShellState *p){ ** the remaining columns. */ if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); if( i=nCol ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, - startLine, sqlite3_errmsg(p->db)); + eputf("%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); sCtx.nErr++; }else{ sCtx.nRow++; @@ -19655,14 +27198,14 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); if( eVerbose>0 ){ - utf8_printf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + oputf("Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "imposter", n)==0 ){ char *zSql; char *zCollist = 0; sqlite3_stmt *pStmt; @@ -19670,9 +27213,15 @@ static int do_meta_command(char *zLine, ShellState *p){ int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ int i; + if( !ShellHasFlag(p,SHFLG_TestingMode) ){ + eputf(".%s unavailable without --unsafe-testing\n", + "imposter"); + rc = 1; + goto meta_command_exit; + } if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ - utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" - " .imposter off\n"); + eputz("Usage: .imposter INDEX IMPOSTER\n" + " .imposter off\n"); /* Also allowed, but not documented: ** ** .imposter TABLE IMPOSTER @@ -19731,7 +27280,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); + eputf("no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -19746,36 +27295,49 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); + eputf("Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - utf8_printf(stdout, "%s;\n", zSql); - raw_printf(stdout, - "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n", - azArg[1], isWO ? "table" : "index" - ); + sputf(stdout, "%s;\n", zSql); + sputf(stdout, "WARNING: writing to an imposter table will corrupt" + " the \"%s\" %s!\n", azArg[1], isWO ? "table" : "index"); } }else{ - raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + eputf("SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + rc = 1; + } + sqlite3_free(zSql); + }else +#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ + i64 iArg = 0; + if( nArg==2 ){ + iArg = integerValue(azArg[1]); + if( iArg==0 ) iArg = -1; + } + if( (nArg!=1 && nArg!=2) || iArg<0 ){ + eputf("%s","Usage: .intck STEPS_PER_UNLOCK\n"); rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); + open_db(p, 0); + rc = intckDatabaseCmd(p, iArg); }else -#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ #ifdef SQLITE_ENABLE_IOTRACE - if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); if( iotrace && iotrace!=stdout ) fclose(iotrace); iotrace = 0; if( nArg<2 ){ sqlite3IoTrace = 0; - }else if( strcmp(azArg[1], "-")==0 ){ + }else if( cli_strcmp(azArg[1], "-")==0 ){ sqlite3IoTrace = iotracePrintf; iotrace = stdout; }else{ iotrace = fopen(azArg[1], "w"); if( iotrace==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -19785,7 +27347,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif - if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ + if( c=='l' && n>=5 && cli_strncmp(azArg[0], "limits", n)==0 ){ static const struct { const char *zLimitName; /* Name of a limit */ int limitCode; /* Integer code for that limit */ @@ -19807,11 +27369,11 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; idb, aLimit[i].limitCode, -1)); + sputf(stdout, "%20s %d\n", aLimit[i].zLimitName, + sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ - raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n"); + eputz("Usage: .limit NAME ?NEW-VALUE?\n"); rc = 1; goto meta_command_exit; }else{ @@ -19822,16 +27384,16 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - utf8_printf(stderr, "ambiguous limit: \"%s\"\n", azArg[1]); + eputf("ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - utf8_printf(stderr, "unknown limit: \"%s\"\n" - "enter \".limits\" with no arguments for a list.\n", - azArg[1]); + eputf("unknown limit: \"%s\"\n" + "enter \".limits\" with no arguments for a list.\n", + azArg[1]); rc = 1; goto meta_command_exit; } @@ -19839,23 +27401,24 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); } - printf("%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); + sputf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else - if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + if( c=='l' && n>2 && cli_strncmp(azArg[0], "lint", n)==0 ){ open_db(p, 0); lintDotCommand(p, azArg, nArg); }else -#ifndef SQLITE_OMIT_LOAD_EXTENSION - if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) + if( c=='l' && cli_strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; char *zErrMsg = 0; failIfSafeMode(p, "cannot run .load in safe mode"); - if( nArg<2 ){ - raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); + if( nArg<2 || azArg[1][0]==0 ){ + /* Must have a non-empty FILE. (Will not load self.) */ + eputz("Usage: .load FILE ?ENTRYPOINT?\n"); rc = 1; goto meta_command_exit; } @@ -19864,91 +27427,162 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); sqlite3_free(zErrMsg); rc = 1; } }else #endif - if( c=='l' && strncmp(azArg[0], "log", n)==0 ){ - failIfSafeMode(p, "cannot run .log in safe mode"); + if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .log FILENAME\n"); + eputz("Usage: .log FILENAME\n"); rc = 1; }else{ const char *zFile = azArg[1]; + if( p->bSafeMode + && cli_strcmp(zFile,"on")!=0 + && cli_strcmp(zFile,"off")!=0 + ){ + sputz(stdout, "cannot set .log to anything other" + " than \"on\" or \"off\"\n"); + zFile = "off"; + } output_file_close(p->pLog); + if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; p->pLog = output_file_open(zFile, 0); } }else - if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){ - const char *zMode = nArg>=2 ? azArg[1] : ""; - int n2 = strlen30(zMode); - int c2 = zMode[0]; - if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ + if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ + const char *zMode = 0; + const char *zTabname = 0; + int i, n2; + ColModeOpts cmOpts = ColModeOpts_default; + for(i=1; imode==MODE_Column + || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + ){ + oputf("current output mode: %s --wrap %d --wordwrap %s --%squote\n", + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no"); + }else{ + oputf("current output mode: %s\n", modeDescr[p->mode]); + } + zMode = modeDescr[p->mode]; + } + n2 = strlen30(zMode); + if( cli_strncmp(zMode,"lines",n2)==0 ){ p->mode = MODE_Line; sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ + }else if( cli_strncmp(zMode,"columns",n2)==0 ){ p->mode = MODE_Column; if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ p->showHeader = 1; } sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"list",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ + }else if( cli_strncmp(zMode,"html",n2)==0 ){ p->mode = MODE_Html; - }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ + }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ + }else if( cli_strncmp(zMode,"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){ + }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( c2=='i' && strncmp(azArg[1],"insert",n2)==0 ){ + }else if( cli_strncmp(zMode,"insert",n2)==0 ){ p->mode = MODE_Insert; - set_table_name(p, nArg>=3 ? azArg[2] : "table"); - }else if( c2=='q' && strncmp(azArg[1],"quote",n2)==0 ){ + set_table_name(p, zTabname ? zTabname : "table"); + }else if( cli_strncmp(zMode,"quote",n2)==0 ){ p->mode = MODE_Quote; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( c2=='a' && strncmp(azArg[1],"ascii",n2)==0 ){ + }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ p->mode = MODE_Ascii; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( c2=='m' && strncmp(azArg[1],"markdown",n2)==0 ){ + }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ p->mode = MODE_Markdown; - }else if( c2=='t' && strncmp(azArg[1],"table",n2)==0 ){ + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"table",n2)==0 ){ p->mode = MODE_Table; - }else if( c2=='b' && strncmp(azArg[1],"box",n2)==0 ){ + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"box",n2)==0 ){ p->mode = MODE_Box; - }else if( c2=='j' && strncmp(azArg[1],"json",n2)==0 ){ + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"count",n2)==0 ){ + p->mode = MODE_Count; + }else if( cli_strncmp(zMode,"off",n2)==0 ){ + p->mode = MODE_Off; + }else if( cli_strncmp(zMode,"json",n2)==0 ){ p->mode = MODE_Json; - }else if( nArg==1 ){ - raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]); }else{ - raw_printf(stderr, "Error: mode should be one of: " - "ascii box column csv html insert json line list markdown " - "quote table tabs tcl\n"); + eputz("Error: mode should be one of: " + "ascii box column csv html insert json line list markdown " + "qbox quote table tabs tcl\n"); rc = 1; } p->cMode = p->mode; }else - if( c=='n' && strcmp(azArg[0], "nonce")==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){ if( nArg!=2 ){ - raw_printf(stderr, "Usage: .nonce NONCE\n"); + eputz("Usage: .nonce NONCE\n"); rc = 1; - }else if( p->zNonce==0 || strcmp(azArg[1],p->zNonce)!=0 ){ - raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); + }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ + eputf("line %d: incorrect nonce: \"%s\"\n", + p->lineno, azArg[1]); exit(1); }else{ p->bSafeMode = 0; @@ -19956,46 +27590,20 @@ static int do_meta_command(char *zLine, ShellState *p){ ** at the end of this procedure */ } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 ){ + if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); }else{ - raw_printf(stderr, "Usage: .nullvalue STRING\n"); + eputz("Usage: .nullvalue STRING\n"); rc = 1; } }else -#ifdef SQLITE_DEBUG - if( c=='o' && strcmp(azArg[0],"oom")==0 ){ - int i; - for(i=1; iout, "missing argument on \"%s\"\n", azArg[i]); - rc = 1; - }else{ - oomRepeat = (int)integerValue(azArg[++i]); - } - }else if( IsDigit(z[0]) ){ - oomCounter = (int)integerValue(azArg[i]); - }else{ - raw_printf(p->out, "unknown argument: \"%s\"\n", azArg[i]); - raw_printf(p->out, "Usage: .oom [--repeat N] [M]\n"); - rc = 1; - } - } - if( rc==0 ){ - raw_printf(p->out, "oomCounter = %d\n", oomCounter); - raw_printf(p->out, "oomRepeat = %d\n", oomRepeat); - } - }else -#endif /* SQLITE_DEBUG */ - - if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ + if( c=='o' && cli_strncmp(azArg[0], "open", n)==0 && n>=2 ){ + const char *zFN = 0; /* Pointer to constant filename */ char *zNewFilename = 0; /* Name of the database file to open */ int iName = 1; /* Index in azArg[] of the filename */ int newFlag = 0; /* True to delete file before opening */ @@ -20004,6 +27612,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* Check for command-line arguments */ for(iName=1; iNameszMax = integerValue(azArg[++iName]); #endif /* SQLITE_OMIT_DESERIALIZE */ - }else if( z[0]=='-' ){ - utf8_printf(stderr, "unknown option: %s\n", z); + }else +#endif /* !SQLITE_SHELL_FIDDLE */ + if( z[0]=='-' ){ + eputf("unknown option: %s\n", z); rc = 1; goto meta_command_exit; - }else if( zNewFilename ){ - utf8_printf(stderr, "extra argument: \"%s\"\n", z); + }else if( zFN ){ + eputf("extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ - zNewFilename = sqlite3_mprintf("%s", z); + zFN = z; } } @@ -20049,19 +27660,29 @@ static int do_meta_command(char *zLine, ShellState *p){ p->szMax = 0; /* If a filename is specified, try to open it first */ - if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){ - if( newFlag && !p->bSafeMode ) shellDeleteFile(zNewFilename); + if( zFN || p->openMode==SHELL_OPEN_HEXDB ){ + if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN); +#ifndef SQLITE_SHELL_FIDDLE if( p->bSafeMode && p->openMode!=SHELL_OPEN_HEXDB - && zNewFilename - && strcmp(zNewFilename,":memory:")!=0 + && zFN + && cli_strcmp(zFN,":memory:")!=0 ){ failIfSafeMode(p, "cannot open disk-based database files in safe mode"); } +#else + /* WASM mode has its own sandboxed pseudo-filesystem. */ +#endif + if( zFN ){ + zNewFilename = sqlite3_mprintf("%s", zFN); + shell_check_oom(zNewFilename); + }else{ + zNewFilename = 0; + } p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + eputf("Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -20074,57 +27695,60 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else +#ifndef SQLITE_SHELL_FIDDLE if( (c=='o' - && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) - || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) + && (cli_strncmp(azArg[0], "output", n)==0 + || cli_strncmp(azArg[0], "once", n)==0)) + || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) ){ char *zFile = 0; int bTxtMode = 0; int i; int eMode = 0; - int bBOM = 0; - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ + static const char *zBomUtf8 = "\xef\xbb\xbf"; + const char *zBom = 0; failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); if( c=='e' ){ eMode = 'x'; bOnce = 2; - }else if( strncmp(azArg[0],"once",n)==0 ){ + }else if( cli_strncmp(azArg[0],"once",n)==0 ){ bOnce = 1; } for(i=1; iout, "ERROR: unknown option: \"%s\". Usage:\n", - azArg[i]); + oputf("ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } }else if( zFile==0 && eMode!='e' && eMode!='x' ){ zFile = sqlite3_mprintf("%s", z); - if( zFile[0]=='|' ){ + if( zFile && zFile[0]=='|' ){ while( i+1out,"ERROR: extra parameter: \"%s\". Usage:\n", - azArg[i]); + oputf("ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; sqlite3_free(zFile); goto meta_command_exit; } } - if( zFile==0 ) zFile = sqlite3_mprintf("stdout"); + if( zFile==0 ){ + zFile = sqlite3_mprintf("stdout"); + } if( bOnce ){ p->outCount = 2; }else{ @@ -20151,46 +27775,48 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = sqlite3_mprintf("%s", p->zTempFile); } #endif /* SQLITE_NOHAVE_SYSTEM */ + shell_check_oom(zFile); if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + eputz("Error: pipes are not supported in this OS\n"); rc = 1; - p->out = stdout; + output_redir(p, stdout); #else - p->out = popen(zFile + 1, "w"); - if( p->out==0 ){ - utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - p->out = stdout; + FILE *pfPipe = popen(zFile + 1, "w"); + if( pfPipe==0 ){ + eputf("Error: cannot open pipe \"%s\"\n", zFile + 1); rc = 1; }else{ - if( bBOM ) fprintf(p->out,"\357\273\277"); + output_redir(p, pfPipe); + if( zBom ) oputz(zBom); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } #endif }else{ - p->out = output_file_open(zFile, bTxtMode); - if( p->out==0 ){ - if( strcmp(zFile,"off")!=0 ){ - utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + FILE *pfFile = output_file_open(zFile, bTxtMode); + if( pfFile==0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ + eputf("Error: cannot write to \"%s\"\n", zFile); } - p->out = stdout; rc = 1; } else { - if( bBOM ) fprintf(p->out,"\357\273\277"); + output_redir(p, pfFile); + if( zBom ) oputz(zBom); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } } sqlite3_free(zFile); }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); if( nArg<=1 ) goto parameter_syntax_error; /* .parameter clear ** Clear all bind parameters by dropping the TEMP table that holds them. */ - if( nArg==2 && strcmp(azArg[1],"clear")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"clear")==0 ){ sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp.sqlite_parameters;", 0, 0, 0); }else @@ -20198,7 +27824,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .parameter list ** List all bind parameters. */ - if( nArg==2 && strcmp(azArg[1],"list")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"list")==0 ){ sqlite3_stmt *pStmt = 0; int rx; int len = 0; @@ -20216,8 +27842,8 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), - sqlite3_column_text(pStmt,1)); + oputf("%-*s %s\n", len, sqlite3_column_text(pStmt,0), + sqlite3_column_text(pStmt,1)); } sqlite3_finalize(pStmt); } @@ -20227,7 +27853,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Make sure the TEMP table used to hold bind parameters exists. ** Create it if necessary. */ - if( nArg==2 && strcmp(azArg[1],"init")==0 ){ + if( nArg==2 && cli_strcmp(azArg[1],"init")==0 ){ bind_table_init(p); }else @@ -20237,7 +27863,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** VALUE can be in either SQL literal notation, or if not it will be ** understood to be a text string. */ - if( nArg==4 && strcmp(azArg[1],"set")==0 ){ + if( nArg==4 && cli_strcmp(azArg[1],"set")==0 ){ int rx; char *zSql; sqlite3_stmt *pStmt; @@ -20247,7 +27873,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "REPLACE INTO temp.sqlite_parameters(key,value)" "VALUES(%Q,%s);", zKey, zValue); - if( zSql==0 ) shell_out_of_memory(); + shell_check_oom(zSql); pStmt = 0; rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); @@ -20257,11 +27883,11 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "REPLACE INTO temp.sqlite_parameters(key,value)" "VALUES(%Q,%Q);", zKey, zValue); - if( zSql==0 ) shell_out_of_memory(); + shell_check_oom(zSql); rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + oputf("Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -20275,10 +27901,10 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Remove the NAME binding from the parameter binding table, if it ** exists. */ - if( nArg==3 && strcmp(azArg[1],"unset")==0 ){ + if( nArg==3 && cli_strcmp(azArg[1],"unset")==0 ){ char *zSql = sqlite3_mprintf( "DELETE FROM temp.sqlite_parameters WHERE key=%Q", azArg[2]); - if( zSql==0 ) shell_out_of_memory(); + shell_check_oom(zSql); sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); }else @@ -20287,17 +27913,17 @@ static int do_meta_command(char *zLine, ShellState *p){ showHelp(p->out, "parameter"); }else - if( c=='p' && n>=3 && strncmp(azArg[0], "print", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i1 ) raw_printf(p->out, " "); - utf8_printf(p->out, "%s", azArg[i]); + if( i>1 ) oputz(" "); + oputz(azArg[i]); } - raw_printf(p->out, "\n"); + oputz("\n"); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( c=='p' && n>=3 && strncmp(azArg[0], "progress", n)==0 ){ + if( c=='p' && n>=3 && cli_strncmp(azArg[0], "progress", n)==0 ){ int i; int nn = 0; p->flgProgress = 0; @@ -20308,21 +27934,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + if( cli_strcmp(z,"quiet")==0 || cli_strcmp(z,"q")==0 ){ p->flgProgress |= SHELL_PROGRESS_QUIET; continue; } - if( strcmp(z,"reset")==0 ){ + if( cli_strcmp(z,"reset")==0 ){ p->flgProgress |= SHELL_PROGRESS_RESET; continue; } - if( strcmp(z,"once")==0 ){ + if( cli_strcmp(z,"once")==0 ){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } - if( strcmp(z,"limit")==0 ){ + if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ - utf8_printf(stderr, "Error: missing argument on --limit\n"); + eputz("Error: missing argument on --limit\n"); rc = 1; goto meta_command_exit; }else{ @@ -20330,7 +27956,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } continue; } - utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]); + eputf("Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -20342,37 +27968,40 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ - if( c=='p' && strncmp(azArg[0], "prompt", n)==0 ){ + if( c=='p' && cli_strncmp(azArg[0], "prompt", n)==0 ){ if( nArg >= 2) { - strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); + shell_strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); } if( nArg >= 3) { - strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1); + shell_strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1); } }else - if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='q' && cli_strncmp(azArg[0], "quit", n)==0 ){ rc = 2; }else +#endif - if( c=='r' && n>=3 && strncmp(azArg[0], "read", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){ FILE *inSaved = p->in; int savedLineno = p->lineno; failIfSafeMode(p, "cannot run .read in safe mode"); if( nArg!=2 ){ - raw_printf(stderr, "Usage: .read FILE\n"); + eputz("Usage: .read FILE\n"); rc = 1; goto meta_command_exit; } if( azArg[1][0]=='|' ){ #ifdef SQLITE_OMIT_POPEN - raw_printf(stderr, "Error: pipes are not supported in this OS\n"); + eputz("Error: pipes are not supported in this OS\n"); rc = 1; p->out = stdout; #else p->in = popen(azArg[1]+1, "r"); if( p->in==0 ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p); @@ -20380,7 +28009,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + eputf("Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p); @@ -20389,8 +28018,10 @@ static int do_meta_command(char *zLine, ShellState *p){ p->in = inSaved; p->lineno = savedLineno; }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='r' && n>=3 && strncmp(azArg[0], "restore", n)==0 ){ +#ifndef SQLITE_SHELL_FIDDLE + if( c=='r' && n>=3 && cli_strncmp(azArg[0], "restore", n)==0 ){ const char *zSrcFile; const char *zDb; sqlite3 *pSrc; @@ -20405,20 +28036,20 @@ static int do_meta_command(char *zLine, ShellState *p){ zSrcFile = azArg[2]; zDb = azArg[1]; }else{ - raw_printf(stderr, "Usage: .restore ?DB? FILE\n"); + eputz("Usage: .restore ?DB? FILE\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile); + eputf("Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } open_db(p, 0); pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); if( pBackup==0 ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); close_db(pSrc); return 1; } @@ -20433,28 +28064,44 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){ - raw_printf(stderr, "Error: source database is busy\n"); + eputz("Error: source database is busy\n"); rc = 1; }else{ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); rc = 1; } close_db(pSrc); }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ - p->scanstatsOn = (u8)booleanValue(azArg[1]); -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); + if( cli_strcmp(azArg[1], "vm")==0 ){ + p->scanstatsOn = 3; + }else + if( cli_strcmp(azArg[1], "est")==0 ){ + p->scanstatsOn = 2; + }else{ + p->scanstatsOn = (u8)booleanValue(azArg[1]); + } + open_db(p, 0); + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + ); +#if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + eputz("Warning: .scanstats not available in this build.\n"); +#elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) + if( p->scanstatsOn==3 ){ + eputz("Warning: \".scanstats vm\" not available in this build.\n"); + } #endif }else{ - raw_printf(stderr, "Usage: .scanstats on|off\n"); + eputz("Usage: .scanstats on|off|est\n"); rc = 1; } }else - if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ ShellText sSelect; ShellState data; char *zErrMsg = 0; @@ -20478,13 +28125,13 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - utf8_printf(stderr, "Unknown option: \"%s\"\n", azArg[ii]); + eputf("Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ zName = azArg[ii]; }else{ - raw_printf(stderr, "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); + eputz("Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; } @@ -20504,6 +28151,7 @@ static int do_meta_command(char *zLine, ShellState *p){ " rootpage integer,\n" " sql text\n" ")", zName); + shell_check_oom(new_argv[0]); new_argv[1] = 0; new_colv[0] = "sql"; new_colv[1] = 0; @@ -20516,7 +28164,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", -1, &pStmt, 0); if( rc ){ - utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); + eputf("Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); rc = 1; goto meta_command_exit; @@ -20555,8 +28203,10 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sSelect, ") WHERE ", 0); if( zName ){ char *zQarg = sqlite3_mprintf("%Q", zName); - int bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || - strchr(zName, '[') != 0; + int bGlob; + shell_check_oom(zQarg); + bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; if( strchr(zName, '.') ){ appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); }else{ @@ -20576,31 +28226,33 @@ static int do_meta_command(char *zLine, ShellState *p){ appendText(&sSelect, "sql IS NOT NULL" " ORDER BY snum, rowid", 0); if( bDebug ){ - utf8_printf(p->out, "SQL: %s;\n", sSelect.z); + oputf("SQL: %s;\n", sSelect.z); }else{ rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg); } freeText(&sSelect); } if( zErrMsg ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); sqlite3_free(zErrMsg); rc = 1; }else if( rc != SQLITE_OK ){ - raw_printf(stderr,"Error: querying schema information\n"); + eputz("Error: querying schema information\n"); rc = 1; }else{ rc = 0; } }else - if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ - unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + if( (c=='s' && n==11 && cli_strncmp(azArg[0], "selecttrace", n)==0) + || (c=='t' && n==9 && cli_strncmp(azArg[0], "treetrace", n)==0) + ){ + unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &x); }else #if defined(SQLITE_ENABLE_SESSION) - if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + if( c=='s' && cli_strncmp(azArg[0],"session",n)==0 && n>=3 ){ struct AuxDb *pAuxDb = p->pAuxDb; OpenSession *pSession = &pAuxDb->aSession[0]; char **azCmd = &azArg[1]; @@ -20611,7 +28263,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg>=3 ){ for(iSes=0; iSesnSession; iSes++){ - if( strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; + if( cli_strcmp(pAuxDb->aSession[iSes].zName, azArg[1])==0 ) break; } if( iSesnSession ){ pSession = &pAuxDb->aSession[iSes]; @@ -20627,15 +28279,15 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Invoke the sqlite3session_attach() interface to attach a particular ** table so that it is never filtered. */ - if( strcmp(azCmd[0],"attach")==0 ){ + if( cli_strcmp(azCmd[0],"attach")==0 ){ if( nCmd!=2 ) goto session_syntax_error; if( pSession->p==0 ){ session_not_open: - raw_printf(stderr, "ERROR: No sessions are open\n"); + eputz("ERROR: No sessions are open\n"); }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + eputf("ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } } @@ -20645,15 +28297,17 @@ static int do_meta_command(char *zLine, ShellState *p){ ** .session patchset FILE ** Write a changeset or patchset into a file. The file is overwritten. */ - if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + if( cli_strcmp(azCmd[0],"changeset")==0 + || cli_strcmp(azCmd[0],"patchset")==0 + ){ FILE *out = 0; failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]); if( nCmd!=2 ) goto session_syntax_error; if( pSession->p==0 ) goto session_not_open; out = fopen(azCmd[1], "wb"); if( out==0 ){ - utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", - azCmd[1]); + eputf("ERROR: cannot open \"%s\" for writing\n", + azCmd[1]); }else{ int szChng; void *pChng; @@ -20663,13 +28317,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - printf("Error: error code %d\n", rc); + sputf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", - szChng); + eputf("ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); fclose(out); @@ -20679,7 +28332,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session close ** Close the identified session */ - if( strcmp(azCmd[0], "close")==0 ){ + if( cli_strcmp(azCmd[0], "close")==0 ){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ session_close(pSession); @@ -20690,21 +28343,20 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session enable ?BOOLEAN? ** Query or set the enable flag */ - if( strcmp(azCmd[0], "enable")==0 ){ + if( cli_strcmp(azCmd[0], "enable")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - utf8_printf(p->out, "session %s enable flag = %d\n", - pSession->zName, ii); + oputf("session %s enable flag = %d\n", pSession->zName, ii); } }else /* .session filter GLOB .... ** Set a list of GLOB patterns of table names to be excluded. */ - if( strcmp(azCmd[0], "filter")==0 ){ + if( cli_strcmp(azCmd[0], "filter")==0 ){ int ii, nByte; if( nCmd<2 ) goto session_syntax_error; if( pAuxDb->nSession ){ @@ -20714,12 +28366,10 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(pSession->azFilter); nByte = sizeof(pSession->azFilter[0])*(nCmd-1); pSession->azFilter = sqlite3_malloc( nByte ); - if( pSession->azFilter==0 ){ - raw_printf(stderr, "Error: out or memory\n"); - exit(1); - } + shell_check_oom( pSession->azFilter ); for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + char *x = pSession->azFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + shell_check_oom(x); } pSession->nFilter = ii-1; } @@ -20728,36 +28378,34 @@ static int do_meta_command(char *zLine, ShellState *p){ /* .session indirect ?BOOLEAN? ** Query or set the indirect flag */ - if( strcmp(azCmd[0], "indirect")==0 ){ + if( cli_strcmp(azCmd[0], "indirect")==0 ){ int ii; if( nCmd>2 ) goto session_syntax_error; ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - utf8_printf(p->out, "session %s indirect flag = %d\n", - pSession->zName, ii); + oputf("session %s indirect flag = %d\n", pSession->zName, ii); } }else /* .session isempty ** Determine if the session is empty */ - if( strcmp(azCmd[0], "isempty")==0 ){ + if( cli_strcmp(azCmd[0], "isempty")==0 ){ int ii; if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - utf8_printf(p->out, "session %s isempty flag = %d\n", - pSession->zName, ii); + oputf("session %s isempty flag = %d\n", pSession->zName, ii); } }else /* .session list ** List all currently open sessions */ - if( strcmp(azCmd[0],"list")==0 ){ + if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; inSession; i++){ - utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + oputf("%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -20765,25 +28413,25 @@ static int do_meta_command(char *zLine, ShellState *p){ ** Open a new session called NAME on the attached database DB. ** DB is normally "main". */ - if( strcmp(azCmd[0],"open")==0 ){ + if( cli_strcmp(azCmd[0],"open")==0 ){ char *zName; if( nCmd!=3 ) goto session_syntax_error; zName = azCmd[2]; if( zName[0]==0 ) goto session_syntax_error; for(i=0; inSession; i++){ - if( strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ + eputf("Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); + eputf("Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + eputf("Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -20791,6 +28439,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3session_table_filter(pSession->p, session_filter, pSession); pAuxDb->nSession++; pSession->zName = sqlite3_mprintf("%s", zName); + shell_check_oom(pSession->zName); }else /* If no command name matches, show a syntax error */ session_syntax_error: @@ -20801,27 +28450,27 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change ** without notice. */ - if( c=='s' && n>=10 && strncmp(azArg[0], "selftest-", 9)==0 ){ - if( strncmp(azArg[0]+9, "boolean", n-9)==0 ){ + if( c=='s' && n>=10 && cli_strncmp(azArg[0], "selftest-", 9)==0 ){ + if( cli_strncmp(azArg[0]+9, "boolean", n-9)==0 ){ int i, v; for(i=1; iout, "%s: %d 0x%x\n", azArg[i], v, v); + oputf("%s: %d 0x%x\n", azArg[i], v, v); } } - if( strncmp(azArg[0]+9, "integer", n-9)==0 ){ + if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ int i; sqlite3_int64 v; for(i=1; iout, "%s", zBuf); + oputz(zBuf); } } }else #endif - if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"selftest",n)==0 ){ int bIsInit = 0; /* True to initialize the SELFTEST table */ int bVerbose = 0; /* Verbose output */ int bSelftestExists; /* True if SELFTEST already exists */ @@ -20835,16 +28484,15 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=1; i0 ){ - char *zQuote = sqlite3_mprintf("%q", zSql); - printf("%d: %s %s\n", tno, zOp, zSql); - sqlite3_free(zQuote); + sputf(stdout, "%d: %s %s\n", tno, zOp, zSql); } - if( strcmp(zOp,"memo")==0 ){ - utf8_printf(p->out, "%s\n", zSql); + if( cli_strcmp(zOp,"memo")==0 ){ + oputf("%s\n", zSql); }else - if( strcmp(zOp,"run")==0 ){ + if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; str.n = 0; str.z[0] = 0; rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - utf8_printf(p->out, "Result: %s\n", str.z); + oputf("Result: %s\n", str.z); } if( rc || zErrMsg ){ nErr++; rc = 1; - utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + oputf("%d: error-code-%d: %s\n", tno, rc, zErrMsg); sqlite3_free(zErrMsg); - }else if( strcmp(zAns,str.z)!=0 ){ + }else if( cli_strcmp(zAns,str.z)!=0 ){ nErr++; rc = 1; - utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); - utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + oputf("%d: Expected: [%s]\n", tno, zAns); + oputf("%d: Got: [%s]\n", tno, str.z); } - }else - { - utf8_printf(stderr, - "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + } + else{ + eputf("Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; } @@ -20924,12 +28572,12 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + oputf("%d errors out of %d tests\n", nErr, nTest); }else - if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ - raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); + eputz("Usage: .separator COL ?ROW?\n"); rc = 1; } if( nArg>=2 ){ @@ -20942,7 +28590,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + if( c=='s' && n>=4 && cli_strncmp(azArg[0],"sha3sum",n)==0 ){ const char *zLike = 0; /* Which table to checksum. 0 means everything */ int i; /* Loop counter */ int bSchema = 0; /* Also hash the schema */ @@ -20960,26 +28608,25 @@ static int do_meta_command(char *zLine, ShellState *p){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; - if( strcmp(z,"schema")==0 ){ + if( cli_strcmp(z,"schema")==0 ){ bSchema = 1; }else - if( strcmp(z,"sha3-224")==0 || strcmp(z,"sha3-256")==0 - || strcmp(z,"sha3-384")==0 || strcmp(z,"sha3-512")==0 + if( cli_strcmp(z,"sha3-224")==0 || cli_strcmp(z,"sha3-256")==0 + || cli_strcmp(z,"sha3-384")==0 || cli_strcmp(z,"sha3-512")==0 ){ iSize = atoi(&z[5]); }else - if( strcmp(z,"debug")==0 ){ + if( cli_strcmp(z,"debug")==0 ){ bDebug = 1; }else { - utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", - azArg[i], azArg[0]); + eputf("Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } }else if( zLike ){ - raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); + eputz("Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; }else{ @@ -20989,12 +28636,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bSchema ){ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " UNION ALL SELECT 'sqlite_schema'" " ORDER BY 1 collate nocase"; }else{ - zSql = "SELECT lower(name) FROM sqlite_schema" + zSql = "SELECT lower(name) as tname FROM sqlite_schema" " WHERE type='table' AND coalesce(rootpage,0)>1" " AND name NOT LIKE 'sqlite_%'" " ORDER BY 1 collate nocase"; @@ -21006,21 +28653,22 @@ static int do_meta_command(char *zLine, ShellState *p){ zSep = "VALUES("; while( SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zTab==0 ) continue; if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; - if( strncmp(zTab, "sqlite_",7)!=0 ){ + if( cli_strncmp(zTab, "sqlite_",7)!=0 ){ appendText(&sQuery,"SELECT * FROM ", 0); appendText(&sQuery,zTab,'"'); appendText(&sQuery," NOT INDEXED;", 0); - }else if( strcmp(zTab, "sqlite_schema")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_schema")==0 ){ appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_schema" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_sequence")==0 ){ appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" " ORDER BY name;", 0); - }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat1")==0 ){ appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" " ORDER BY tbl,idx;", 0); - }else if( strcmp(zTab, "sqlite_stat4")==0 ){ + }else if( cli_strcmp(zTab, "sqlite_stat4")==0 ){ appendText(&sQuery, "SELECT * FROM ", 0); appendText(&sQuery, zTab, 0); appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); @@ -21046,87 +28694,160 @@ static int do_meta_command(char *zLine, ShellState *p){ " FROM [sha3sum$query]", sSql.z, iSize); } + shell_check_oom(zSql); freeText(&sQuery); freeText(&sSql); if( bDebug ){ - utf8_printf(p->out, "%s\n", zSql); + oputf("%s\n", zSql); }else{ shell_exec(p, zSql, 0); } +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE) + { + int lrc; + char *zRevText = /* Query for reversible to-blob-to-text check */ + "SELECT lower(name) as tname FROM sqlite_schema\n" + "WHERE type='table' AND coalesce(rootpage,0)>1\n" + "AND name NOT LIKE 'sqlite_%%'%s\n" + "ORDER BY 1 collate nocase"; + zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with tabcols as materialized(\n" + "select tname, cname\n" + "from (" + " select printf('\"%%w\"',ss.tname) as tname," + " printf('\"%%w\"',ti.name) as cname\n" + " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" + "select 'SELECT total(bad_text_count) AS bad_text_count\n" + "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" + " from (select 'SELECT COUNT(*) AS bad_text_count\n" + "FROM '||tname||' WHERE '\n" + "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n" + "|| ' AND typeof('||cname||')=''text'' ',\n" + "' OR ') as query, tname from tabcols group by tname)" + , zRevText); + shell_check_oom(zRevText); + if( bDebug ) oputf("%s\n", zRevText); + lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + if( lrc!=SQLITE_OK ){ + /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the + ** user does cruel and unnatural things like ".limit expr_depth 0". */ + rc = 1; + }else{ + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + if( lrc ){ + const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) oputf("%s\n", zGenQuery); + if( lrc!=SQLITE_OK ){ + rc = 1; + }else{ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int sz = (int)(countIrreversible + 0.5); + eputf("Digest includes %d invalidly encoded text field%s.\n", + sz, (sz>1)? "s": ""); + } + } + sqlite3_finalize(pCheckStmt); + } + sqlite3_finalize(pStmt); + } + } + if( rc ) eputz(".sha3sum failed.\n"); + sqlite3_free(zRevText); + } +#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ sqlite3_free(zSql); }else -#ifndef SQLITE_NOHAVE_SYSTEM +#if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) if( c=='s' - && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) + && (cli_strncmp(azArg[0], "shell", n)==0 + || cli_strncmp(azArg[0],"system",n)==0) ){ char *zCmd; int i, x; failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); if( nArg<2 ){ - raw_printf(stderr, "Usage: .system COMMAND\n"); + eputz("Usage: .system COMMAND\n"); rc = 1; goto meta_command_exit; } zCmd = sqlite3_mprintf(strchr(azArg[1],' ')==0?"%s":"\"%s\"", azArg[1]); - for(i=2; iout, "%12.12s: %s\n","echo", - azBool[ShellHasFlag(p, SHFLG_Echo)]); - utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); - utf8_printf(p->out, "%12.12s: %s\n","explain", - p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); - utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); - utf8_printf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->nullValue); - raw_printf(p->out, "\n"); - utf8_printf(p->out,"%12.12s: %s\n","output", - strlen30(p->outfile) ? p->outfile : "stdout"); - utf8_printf(p->out,"%12.12s: ", "colseparator"); - output_c_string(p->out, p->colSeparator); - raw_printf(p->out, "\n"); - utf8_printf(p->out,"%12.12s: ", "rowseparator"); - output_c_string(p->out, p->rowSeparator); - raw_printf(p->out, "\n"); + oputf("%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + oputf("%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); + oputf("%12.12s: %s\n","explain", + p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); + oputf("%12.12s: %s\n","headers", azBool[p->showHeader!=0]); + if( p->mode==MODE_Column + || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + ){ + oputf("%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no"); + }else{ + oputf("%12.12s: %s\n","mode", modeDescr[p->mode]); + } + oputf("%12.12s: ", "nullvalue"); + output_c_string(p->nullValue); + oputz("\n"); + oputf("%12.12s: %s\n","output", + strlen30(p->outfile) ? p->outfile : "stdout"); + oputf("%12.12s: ", "colseparator"); + output_c_string(p->colSeparator); + oputz("\n"); + oputf("%12.12s: ", "rowseparator"); + output_c_string(p->rowSeparator); + oputz("\n"); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - utf8_printf(p->out, "%12.12s: %s\n","stats", zOut); - utf8_printf(p->out, "%12.12s: ", "width"); + oputf("%12.12s: %s\n","stats", zOut); + oputf("%12.12s: ", "width"); for (i=0;inWidth;i++) { - raw_printf(p->out, "%d ", p->colWidth[i]); + oputf("%d ", p->colWidth[i]); } - raw_printf(p->out, "\n"); - utf8_printf(p->out, "%12.12s: %s\n", "filename", - p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); + oputz("\n"); + oputf("%12.12s: %s\n", "filename", + p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else - if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ + if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){ if( nArg==2 ){ - if( strcmp(azArg[1],"stmt")==0 ){ + if( cli_strcmp(azArg[1],"stmt")==0 ){ p->statsOn = 2; - }else if( strcmp(azArg[1],"vmstep")==0 ){ + }else if( cli_strcmp(azArg[1],"vmstep")==0 ){ p->statsOn = 3; }else{ p->statsOn = (u8)booleanValue(azArg[1]); @@ -21134,14 +28855,14 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( nArg==1 ){ display_stats(p->db, p, 0); }else{ - raw_printf(stderr, "Usage: .stats ?on|off|stmt|vmstep?\n"); + eputz("Usage: .stats ?on|off|stmt|vmstep?\n"); rc = 1; } }else - if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (strncmp(azArg[0], "indices", n)==0 - || strncmp(azArg[0], "indexes", n)==0) ) + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) ) ){ sqlite3_stmt *pStmt; char **azResult; @@ -21160,7 +28881,7 @@ static int do_meta_command(char *zLine, ShellState *p){ /* It is an historical accident that the .indexes command shows an error ** when called with the wrong number of arguments whereas the .tables ** command does not. */ - raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); + eputz("Usage: .indexes ?LIKE-PATTERN?\n"); rc = 1; sqlite3_finalize(pStmt); goto meta_command_exit; @@ -21209,12 +28930,12 @@ static int do_meta_command(char *zLine, ShellState *p){ char **azNew; int n2 = nAlloc*2 + 10; azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); - if( azNew==0 ) shell_out_of_memory(); + shell_check_oom(azNew); nAlloc = n2; azResult = azNew; } azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - if( 0==azResult[nRow] ) shell_out_of_memory(); + shell_check_oom(azResult[nRow]); nRow++; } if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ @@ -21236,10 +28957,9 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, - azResult[j] ? azResult[j]:""); + oputf("%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } - raw_printf(p->out, "\n"); + oputz("\n"); } } @@ -21247,12 +28967,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else +#ifndef SQLITE_SHELL_FIDDLE /* Begin redirecting output to the file "testcase-out.txt" */ - if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ + if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ output_reset(p); p->out = output_file_open("testcase-out.txt", 0); if( p->out==0 ){ - raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + eputz("Error: cannot open 'testcase-out.txt'\n"); } if( nArg>=2 ){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); @@ -21260,37 +28981,41 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE - if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 ){ + if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ - int unSafe; /* Not valid for --safe mode */ + int unSafe; /* Not valid unless --unsafe-testing */ const char *zUsage; /* Usage notes */ } aCtrl[] = { - { "always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, - { "assert", SQLITE_TESTCTRL_ASSERT, 1, "BOOLEAN" }, - /*{ "benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, "" },*/ - /*{ "bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ - { "byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, - { "extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, - /*{ "fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ - { "imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, - { "internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, - { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, - { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, - { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, + {"always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, + {"assert", SQLITE_TESTCTRL_ASSERT, 1, "BOOLEAN" }, + /*{"benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, "" },*/ + /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ + {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, + {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, + {"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..." }, + {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, + {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, + {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, + {"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" }, + {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, + {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, + {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, #ifdef YYCOVERAGE - { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, -#endif - { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET " }, - { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE,0, "" }, - { "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, 0, "" }, - { "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, 0, "SEED ?db?" }, - { "seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, - { "sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, - { "tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, + {"parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, +#endif + {"pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET " }, + {"prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE,0, "" }, + {"prng_save", SQLITE_TESTCTRL_PRNG_SAVE, 0, "" }, + {"prng_seed", SQLITE_TESTCTRL_PRNG_SEED, 0, "SEED ?db?" }, + {"seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, + {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, + {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, + {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, }; int testctrl = -1; int iCtrl = -1; @@ -21309,11 +29034,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* --help lists all test-controls */ - if( strcmp(zCmd,"help")==0 ){ - utf8_printf(p->out, "Available test-controls:\n"); + if( cli_strcmp(zCmd,"help")==0 ){ + oputz("Available test-controls:\n"); for(i=0; iout, " .testctrl %s %s\n", - aCtrl[i].zCtrlName, aCtrl[i].zUsage); + if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue; + oputf(" .testctrl %s %s\n", + aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; goto meta_command_exit; @@ -21323,31 +29049,28 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; ibSafeMode ){ - utf8_printf(stderr, - "line %d: \".testctrl %s\" may not be used in safe mode\n", - p->lineno, aCtrl[iCtrl].zCtrlName); - exit(1); + eputf("Error: unknown test-control: %s\n" + "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: + case SQLITE_TESTCTRL_FK_NO_ACTION: if( nArg==3 ){ unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); @@ -21379,9 +29102,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 || nArg==4 ){ int ii = (int)integerValue(azArg[2]); sqlite3 *db; - if( ii==0 && strcmp(azArg[2],"random")==0 ){ + if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - printf("-- random seed: %d\n", ii); + sputf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -21415,6 +29138,21 @@ static int do_meta_command(char *zLine, ShellState *p){ } break; + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_USELONGDOUBLE: { + int opt = -1; + if( nArg==3 ){ + if( cli_strcmp(azArg[2],"default")==0 ){ + opt = 2; + }else{ + opt = booleanValue(azArg[2]); + } + } + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 1; + break; + } + /* sqlite3_test_control(sqlite3*) */ case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: rc2 = sqlite3_test_control(testctrl, p->db); @@ -21434,7 +29172,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - utf8_printf(p->out, "%llu\n", x); + oputf("%llu\n", x); isOk = 3; break; } @@ -21465,11 +29203,11 @@ static int do_meta_command(char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) utf8_printf(p->out, " "); - utf8_printf(p->out, "%d: %d", id, val); + if( id>1 ) oputz(" "); + oputf("%d: %d", id, val); id++; } - if( id>1 ) utf8_printf(p->out, "\n"); + if( id>1 ) oputz("\n"); isOk = 3; } break; @@ -21482,39 +29220,119 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 3; } break; + case SQLITE_TESTCTRL_JSON_SELFCHECK: + if( nArg==2 ){ + rc2 = -1; + isOk = 1; + }else{ + rc2 = booleanValue(azArg[2]); + isOk = 3; + } + sqlite3_test_control(testctrl, &rc2); + break; + case SQLITE_TESTCTRL_FAULT_INSTALL: { + int kk; + int bShowHelp = nArg<=2; + isOk = 3; + for(kk=2; kk0 ) faultsim_state.eVerbose--; + }else if( cli_strcmp(z,"-id")==0 && kk+1=0 ){ - utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); + oputf("Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - raw_printf(p->out, "%d\n", rc2); + oputf("%d\n", rc2); }else if( isOk==2 ){ - raw_printf(p->out, "0x%08x\n", rc2); + oputf("0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ - if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ + if( c=='t' && n>4 && cli_strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); }else - if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ + if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); if( enableTimer && !HAS_TIMER ){ - raw_printf(stderr, "Error: timer not available on this system.\n"); + eputz("Error: timer not available on this system.\n"); enableTimer = 0; } }else{ - raw_printf(stderr, "Usage: .timer on|off\n"); + eputz("Usage: .timer on|off\n"); rc = 1; } }else #ifndef SQLITE_OMIT_TRACE - if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + if( c=='t' && cli_strncmp(azArg[0], "trace", n)==0 ){ int mType = 0; int jj; open_db(p, 0); @@ -21545,13 +29363,13 @@ static int do_meta_command(char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + eputf("Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ @@ -21564,12 +29382,12 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* !defined(SQLITE_OMIT_TRACE) */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_VIRTUALTABLE) - if( c=='u' && strncmp(azArg[0], "unmodule", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "unmodule", n)==0 ){ int ii; int lenOpt; char *zOpt; if( nArg<2 ){ - raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n"); + eputz("Usage: .unmodule [--allexcept] NAME ...\n"); rc = 1; goto meta_command_exit; } @@ -21577,7 +29395,7 @@ static int do_meta_command(char *zLine, ShellState *p){ zOpt = azArg[1]; if( zOpt[0]=='-' && zOpt[1]=='-' && zOpt[2]!=0 ) zOpt++; lenOpt = (int)strlen(zOpt); - if( lenOpt>=3 && strncmp(zOpt, "-allexcept",lenOpt)==0 ){ + if( lenOpt>=3 && cli_strncmp(zOpt, "-allexcept",lenOpt)==0 ){ assert( azArg[nArg]==0 ); sqlite3_drop_modules(p->db, nArg>2 ? (const char**)(azArg+2) : 0); }else{ @@ -21589,137 +29407,138 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif #if SQLITE_USER_AUTHENTICATION - if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){ if( nArg<2 ){ - raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n"); + eputz("Usage: .user SUBCOMMAND ...\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); - if( strcmp(azArg[1],"login")==0 ){ + if( cli_strcmp(azArg[1],"login")==0 ){ if( nArg!=4 ){ - raw_printf(stderr, "Usage: .user login USER PASSWORD\n"); + eputz("Usage: .user login USER PASSWORD\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], strlen30(azArg[3])); if( rc ){ - utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]); + eputf("Authentication failed for user %s\n", azArg[2]); rc = 1; } - }else if( strcmp(azArg[1],"add")==0 ){ + }else if( cli_strcmp(azArg[1],"add")==0 ){ if( nArg!=5 ){ - raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); + eputz("Usage: .user add USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]), booleanValue(azArg[4])); if( rc ){ - raw_printf(stderr, "User-Add failed: %d\n", rc); + eputf("User-Add failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"edit")==0 ){ + }else if( cli_strcmp(azArg[1],"edit")==0 ){ if( nArg!=5 ){ - raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); + eputz("Usage: .user edit USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]), booleanValue(azArg[4])); if( rc ){ - raw_printf(stderr, "User-Edit failed: %d\n", rc); + eputf("User-Edit failed: %d\n", rc); rc = 1; } - }else if( strcmp(azArg[1],"delete")==0 ){ + }else if( cli_strcmp(azArg[1],"delete")==0 ){ if( nArg!=3 ){ - raw_printf(stderr, "Usage: .user delete USER\n"); + eputz("Usage: .user delete USER\n"); rc = 1; goto meta_command_exit; } rc = sqlite3_user_delete(p->db, azArg[2]); if( rc ){ - raw_printf(stderr, "User-Delete failed: %d\n", rc); + eputf("User-Delete failed: %d\n", rc); rc = 1; } }else{ - raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); + eputz("Usage: .user login|add|edit|delete ...\n"); rc = 1; goto meta_command_exit; } }else #endif /* SQLITE_USER_AUTHENTICATION */ - if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ - utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, - sqlite3_libversion(), sqlite3_sourceid()); + if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ + char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; + oputf("SQLite %s %s\n" /*extra-version-info*/, + sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - utf8_printf(p->out, "zlib version %s\n", zlibVersion()); + oputf("zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." - CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__) "\n"); + oputf("clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n"); + oputf("msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - utf8_printf(p->out, "gcc-" __VERSION__ "\n"); + oputf("gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else - if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; sqlite3_vfs *pVfs = 0; if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + oputf("vfs.zName = \"%s\"\n", pVfs->zName); + oputf("vfs.iVersion = %d\n", pVfs->iVersion); + oputf("vfs.szOsFile = %d\n", pVfs->szOsFile); + oputf("vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else - if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfslist", n)==0 ){ sqlite3_vfs *pVfs; sqlite3_vfs *pCurrent = 0; if( p->db ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, - pVfs==pCurrent ? " <--- CURRENT" : ""); - raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + oputf("vfs.zName = \"%s\"%s\n", pVfs->zName, + pVfs==pCurrent ? " <--- CURRENT" : ""); + oputf("vfs.iVersion = %d\n", pVfs->iVersion); + oputf("vfs.szOsFile = %d\n", pVfs->szOsFile); + oputf("vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - raw_printf(p->out, "-----------------------------------\n"); + oputz("-----------------------------------\n"); } } }else - if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ + if( c=='v' && cli_strncmp(azArg[0], "vfsname", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; char *zVfsName = 0; if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - utf8_printf(p->out, "%s\n", zVfsName); + oputf("%s\n", zVfsName); sqlite3_free(zVfsName); } } }else - if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ - unsigned int x = nArg>=2 ? (unsigned int)integerValue(azArg[1]) : 0xffffffff; + if( c=='w' && cli_strncmp(azArg[0], "wheretrace", n)==0 ){ + unsigned int x = nArg>=2? (unsigned int)integerValue(azArg[1]) : 0xffffffff; sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &x); }else - if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; assert( nArg<=ArraySize(azArg) ); p->nWidth = nArg-1; @@ -21732,8 +29551,8 @@ static int do_meta_command(char *zLine, ShellState *p){ }else { - utf8_printf(stderr, "Error: unknown command or invalid arguments: " - " \"%s\". Enter \".help\" for help\n", azArg[0]); + eputf("Error: unknown command or invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } @@ -21767,7 +29586,8 @@ typedef enum { ** The scan is resumable for subsequent lines when prior ** return values are passed as the 2nd argument. */ -static QuickScanState quickscan(char *zLine, QuickScanState qss){ +static QuickScanState quickscan(char *zLine, QuickScanState qss, + SCAN_TRACKER_REFTYPE pst){ char cin; char cWait = (char)qss; /* intentional narrowing loss */ if( cWait==0 ){ @@ -21791,17 +29611,25 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ if( *zLine=='*' ){ ++zLine; cWait = '*'; + CONTINUE_PROMPT_AWAITS(pst, "/*"); qss = QSS_SETV(qss, cWait); goto TermScan; } break; case '[': cin = ']'; - /* fall thru */ + deliberate_fall_through; case '`': case '\'': case '"': cWait = cin; qss = QSS_HasDark | cWait; + CONTINUE_PROMPT_AWAITC(pst, cin); goto TermScan; + case '(': + CONTINUE_PAREN_INCR(pst, 1); + break; + case ')': + CONTINUE_PAREN_INCR(pst, -1); + break; default: break; } @@ -21817,19 +29645,22 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ continue; ++zLine; cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; case '`': case '\'': case '"': if(*zLine==cWait){ + /* Swallow doubled end-delimiter.*/ ++zLine; continue; } - /* fall thru */ + deliberate_fall_through; case ']': cWait = 0; + CONTINUE_PROMPT_AWAITC(pst, 0); qss = QSS_SETV(qss, 0); goto PlainScan; - default: assert(0); + default: assert(0); } } } @@ -21850,17 +29681,15 @@ static int line_is_command_terminator(char *zLine){ zLine += 2; /* SQL Server */ else return 0; - return quickscan(zLine,QSS_Start)==QSS_Start; + return quickscan(zLine, QSS_Start, 0)==QSS_Start; } /* -** We need a default sqlite3_complete() implementation to use in case -** the shell is compiled with SQLITE_OMIT_COMPLETE. The default assumes -** any arbitrary text is a complete SQL statement. This is not very -** user-friendly, but it does seem to work. +** The CLI needs a working sqlite3_complete() to work properly. So error +** out of the build if compiling with SQLITE_OMIT_COMPLETE. */ #ifdef SQLITE_OMIT_COMPLETE -#define sqlite3_complete(x) 1 +# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE. #endif /* @@ -21877,6 +29706,88 @@ static int line_is_complete(char *zSql, int nSql){ return rc; } +/* +** This function is called after processing each line of SQL in the +** runOneSqlLine() function. Its purpose is to detect scenarios where +** defensive mode should be automatically turned off. Specifically, when +** +** 1. The first line of input is "PRAGMA foreign_keys=OFF;", +** 2. The second line of input is "BEGIN TRANSACTION;", +** 3. The database is empty, and +** 4. The shell is not running in --safe mode. +** +** The implementation uses the ShellState.eRestoreState to maintain state: +** +** 0: Have not seen any SQL. +** 1: Have seen "PRAGMA foreign_keys=OFF;". +** 2-6: Currently running .dump transaction. If the "2" bit is set, +** disable DEFENSIVE when done. If "4" is set, disable DQS_DDL. +** 7: Nothing left to do. This function becomes a no-op. +*/ +static int doAutoDetectRestore(ShellState *p, const char *zSql){ + int rc = SQLITE_OK; + + if( p->eRestoreState<7 ){ + switch( p->eRestoreState ){ + case 0: { + const char *zExpect = "PRAGMA foreign_keys=OFF;"; + assert( strlen(zExpect)==24 ); + if( p->bSafeMode==0 && memcmp(zSql, zExpect, 25)==0 ){ + p->eRestoreState = 1; + }else{ + p->eRestoreState = 7; + } + break; + }; + + case 1: { + int bIsDump = 0; + const char *zExpect = "BEGIN TRANSACTION;"; + assert( strlen(zExpect)==18 ); + if( memcmp(zSql, zExpect, 19)==0 ){ + /* Now check if the database is empty. */ + const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1"; + sqlite3_stmt *pStmt = 0; + + bIsDump = 1; + shellPrepare(p->db, &rc, zQuery, &pStmt); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + bIsDump = 0; + } + shellFinalize(&rc, pStmt); + } + if( bIsDump && rc==SQLITE_OK ){ + int bDefense = 0; + int bDqsDdl = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, -1, &bDqsDdl); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 1, 0); + p->eRestoreState = (bDefense ? 2 : 0) + (bDqsDdl ? 4 : 0); + }else{ + p->eRestoreState = 7; + } + break; + } + + default: { + if( sqlite3_get_autocommit(p->db) ){ + if( (p->eRestoreState & 2) ){ + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); + } + if( (p->eRestoreState & 4) ){ + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 0, 0); + } + p->eRestoreState = 7; + } + break; + } + } + } + + return rc; +} + /* ** Run a single line of SQL. Return the number of errors. */ @@ -21892,30 +29803,79 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ END_TIMER; if( rc || zErrMsg ){ char zPrefix[100]; - if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, - "Error: near line %d:", startline); + const char *zErrorTail; + const char *zErrorType; + if( zErrMsg==0 ){ + zErrorType = "Error"; + zErrorTail = sqlite3_errmsg(p->db); + }else if( cli_strncmp(zErrMsg, "in prepare, ",12)==0 ){ + zErrorType = "Parse error"; + zErrorTail = &zErrMsg[12]; + }else if( cli_strncmp(zErrMsg, "stepping, ", 10)==0 ){ + zErrorType = "Runtime error"; + zErrorTail = &zErrMsg[10]; }else{ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); + zErrorType = "Error"; + zErrorTail = zErrMsg; } - if( zErrMsg!=0 ){ - utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg); - sqlite3_free(zErrMsg); - zErrMsg = 0; + if( in!=0 || !stdin_is_interactive ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d:", zErrorType, startline); }else{ - utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } + eputf("%s %s\n", zPrefix, zErrorTail); + sqlite3_free(zErrMsg); + zErrMsg = 0; return 1; }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ char zLineBuf[2000]; sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - raw_printf(p->out, "%s\n", zLineBuf); + oputf("%s\n", zLineBuf); } + + if( doAutoDetectRestore(p, zSql) ) return 1; return 0; } +static void echo_group_input(ShellState *p, const char *zDo){ + if( ShellHasFlag(p, SHFLG_Echo) ) oputf("%s\n", zDo); +} + +#ifdef SQLITE_SHELL_FIDDLE +/* +** Alternate one_input_line() impl for wasm mode. This is not in the primary +** impl because we need the global shellState and cannot access it from that +** function without moving lots of code around (creating a larger/messier diff). +*/ +static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ + /* Parse the next line from shellState.wasm.zInput. */ + const char *zBegin = shellState.wasm.zPos; + const char *z = zBegin; + char *zLine = 0; + i64 nZ = 0; + + UNUSED_PARAMETER(in); + UNUSED_PARAMETER(isContinuation); + if(!z || !*z){ + return 0; + } + while(*z && isspace(*z)) ++z; + zBegin = z; + for(; *z && '\n'!=*z; ++nZ, ++z){} + if(nZ>0 && '\r'==zBegin[nZ-1]){ + --nZ; + } + shellState.wasm.zPos = z; + zLine = realloc(zPrior, nZ+1); + shell_check_oom(zLine); + memcpy(zLine, zBegin, nZ); + zLine[nZ] = 0; + return zLine; +} +#endif /* SQLITE_SHELL_FIDDLE */ /* ** Read input from *in and process it. If *in==0 then input @@ -21929,21 +29889,29 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ static int process_input(ShellState *p){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ - int nLine; /* Length of current line */ - int nSql = 0; /* Bytes of zSql[] used */ - int nAlloc = 0; /* Allocated zSql[] space */ + i64 nLine; /* Length of current line */ + i64 nSql = 0; /* Bytes of zSql[] used */ + i64 nAlloc = 0; /* Allocated zSql[] space */ int rc; /* Error code */ int errCnt = 0; /* Number of errors seen */ - int startline = 0; /* Line number for start of current input */ + i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ + if( p->inputNesting==MAX_INPUT_NESTING ){ + /* This will be more informative in a later version. */ + eputf("Input nesting limit (%d) reached at line %d." + " Check recursion.\n", MAX_INPUT_NESTING, p->lineno); + return 1; + } + ++p->inputNesting; p->lineno = 0; + CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) printf("\n"); + if( p->in==0 && stdin_is_interactive ) oputz("\n"); break; } if( seenInterrupt ){ @@ -21956,16 +29924,16 @@ static int process_input(ShellState *p){ && line_is_complete(zSql, nSql) ){ memcpy(zLine,";",2); } - qss = quickscan(zLine, qss); + qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE); if( QSS_PLAINWHITE(qss) && nSql==0 ){ - if( ShellHasFlag(p, SHFLG_Echo) ) - printf("%s\n", zLine); /* Just swallow single-line whitespace */ + echo_group_input(p, zLine); qss = QSS_Start; continue; } if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){ - if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); + CONTINUE_PROMPT_RESET; + echo_group_input(p, zLine); if( zLine[0]=='.' ){ rc = do_meta_command(zLine, p); if( rc==2 ){ /* exit requested */ @@ -21978,15 +29946,15 @@ static int process_input(ShellState *p){ continue; } /* No single-line dispositions remain; accumulate line(s). */ - nLine = strlen30(zLine); + nLine = strlen(zLine); if( nSql+nLine+2>=nAlloc ){ /* Grow buffer by half-again increments when big. */ nAlloc = nSql+(nSql>>1)+nLine+100; zSql = realloc(zSql, nAlloc); - if( zSql==0 ) shell_out_of_memory(); + shell_check_oom(zSql); } if( nSql==0 ){ - int i; + i64 i; for(i=0; zLine[i] && IsSpace(zLine[i]); i++){} assert( nAlloc>0 && zSql!=0 ); memcpy(zSql, zLine+i, nLine+1-i); @@ -21998,7 +29966,9 @@ static int process_input(ShellState *p){ nSql += nLine; } if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ + echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET; nSql = 0; if( p->outCount ){ output_reset(p); @@ -22009,16 +29979,20 @@ static int process_input(ShellState *p){ p->bSafeMode = p->bSafeModePersist; qss = QSS_Start; }else if( nSql && QSS_PLAINWHITE(qss) ){ - if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); + echo_group_input(p, zSql); nSql = 0; qss = QSS_Start; } } - if( nSql && QSS_PLAINDARK(qss) ){ + if( nSql ){ + /* This may be incomplete. Let the SQL parser deal with that. */ + echo_group_input(p, zSql); errCnt += runOneSqlLine(p, zSql, p->in, startline); + CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); + --p->inputNesting; return errCnt>0; } @@ -22036,7 +30010,7 @@ static char *find_home_dir(int clearFlag){ if( home_dir ) return home_dir; #if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ - && !defined(__RTP__) && !defined(_WRS_KERNEL) + && !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) { struct passwd *pwent; uid_t uid = getuid(); @@ -22082,7 +30056,7 @@ static char *find_home_dir(int clearFlag){ #endif /* !_WIN32_WCE */ if( home_dir ){ - int n = strlen30(home_dir) + 1; + i64 n = strlen(home_dir) + 1; char *z = malloc( n ); if( z ) memcpy(z, home_dir, n); home_dir = z; @@ -22091,9 +30065,43 @@ static char *find_home_dir(int clearFlag){ return home_dir; } +/* +** On non-Windows platforms, look for $XDG_CONFIG_HOME. +** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return +** the path to it, else return 0. The result is cached for +** subsequent calls. +*/ +static const char *find_xdg_config(void){ +#if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ + || defined(__RTP__) || defined(_WRS_KERNEL) + return 0; +#else + static int alreadyTried = 0; + static char *zConfig = 0; + const char *zXdgHome; + + if( alreadyTried!=0 ){ + return zConfig; + } + alreadyTried = 1; + zXdgHome = getenv("XDG_CONFIG_HOME"); + if( zXdgHome==0 ){ + return 0; + } + zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); + shell_check_oom(zConfig); + if( access(zConfig,0)!=0 ){ + sqlite3_free(zConfig); + zConfig = 0; + } + return zConfig; +#endif +} + /* ** Read input from the file given by sqliterc_override. Or if that -** parameter is NULL, take input from ~/.sqliterc +** parameter is NULL, take input from the first of find_xdg_config() +** or ~/.sqliterc which is found. ** ** Returns the number of errors. */ @@ -22107,25 +30115,29 @@ static void process_sqliterc( FILE *inSaved = p->in; int savedLineno = p->lineno; - if (sqliterc == NULL) { + if( sqliterc == NULL ){ + sqliterc = find_xdg_config(); + } + if( sqliterc == NULL ){ home_dir = find_home_dir(0); if( home_dir==0 ){ - raw_printf(stderr, "-- warning: cannot find home directory;" - " cannot read ~/.sqliterc\n"); + eputz("-- warning: cannot find home directory;" + " cannot read ~/.sqliterc\n"); return; } zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir); + shell_check_oom(zBuf); sqliterc = zBuf; } p->in = fopen(sqliterc,"rb"); if( p->in ){ if( stdin_is_interactive ){ - utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc); + eputf("-- Loading resources from %s\n", sqliterc); } if( process_input(p) && bail_on_error ) exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + eputf("cannot open: \"%s\"\n", sqliterc); if( bail_on_error ) exit(1); } p->in = inSaved; @@ -22137,6 +30149,7 @@ static void process_sqliterc( ** Show available command line options */ static const char zOptions[] = + " -- treat no subsequent arguments as options\n" #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) " -A ARGS... run \".archive ARGS\" and exit\n" #endif @@ -22151,7 +30164,7 @@ static const char zOptions[] = #if !defined(SQLITE_OMIT_DESERIALIZE) " -deserialize open the database using sqlite3_deserialize()\n" #endif - " -echo print commands before execution\n" + " -echo print inputs before execution\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) @@ -22176,8 +30189,10 @@ static const char zOptions[] = " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" + " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -pcachetrace trace all page cache operations\n" " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" @@ -22188,6 +30203,7 @@ static const char zOptions[] = " -stats print memory stats before each finalize\n" " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" + " -unsafe-testing allow unsafe commands and modes for testing\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -22198,16 +30214,15 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - utf8_printf(stderr, - "Usage: %s [OPTIONS] FILENAME [SQL]\n" - "FILENAME is the name of an SQLite database. A new database is created\n" - "if the file does not previously exist.\n", Argv0); + eputf("Usage: %s [OPTIONS] [FILENAME [SQL]]\n" + "FILENAME is the name of an SQLite database. A new database is created\n" + "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); + eputf("OPTIONS include:\n%s", zOptions); }else{ - raw_printf(stderr, "Use the -help option for additional information\n"); + eputz("Use the -help option for additional information\n"); } - exit(1); + exit(0); } /* @@ -22216,8 +30231,8 @@ static void usage(int showDetail){ */ static void verify_uninitialized(void){ if( sqlite3_config(-1)==SQLITE_MISUSE ){ - utf8_printf(stdout, "WARNING: attempt to configure SQLite after" - " initialization.\n"); + sputz(stdout, "WARNING: attempt to configure SQLite after" + " initialization.\n"); } } @@ -22233,9 +30248,11 @@ static void main_init(ShellState *data) { memcpy(data->rowSeparator,SEP_Row, 2); data->showHeader = 0; data->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +#if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); +#endif sqlite3_config(SQLITE_CONFIG_URI, 1); - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); @@ -22244,7 +30261,7 @@ static void main_init(ShellState *data) { /* ** Output text to the console in a font that attracts extra attention. */ -#ifdef _WIN32 +#if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ #if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); @@ -22254,14 +30271,14 @@ static void printBold(const char *zText){ FOREGROUND_RED|FOREGROUND_INTENSITY ); #endif - printf("%s", zText); + sputz(stdout, zText); #if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); #endif } #else static void printBold(const char *zText){ - printf("\033[1m%s\033[0m", zText); + sputf(stdout, "\033[1m%s\033[0m", zText); } #endif @@ -22271,13 +30288,16 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - utf8_printf(stderr, "%s: Error: missing argument to %s\n", - argv[0], argv[argc-1]); + eputf("%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); exit(1); } return argv[i]; } +static void sayAbnormalExit(void){ + if( seenInterrupt ) eputz("Program interrupted.\n"); +} + #ifndef SQLITE_SHELL_IS_UTF8 # if (defined(_WIN32) || defined(WIN32)) \ && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) @@ -22287,42 +30307,60 @@ static char *cmdline_option_value(int argc, char **argv, int i){ # endif #endif +#ifdef SQLITE_SHELL_FIDDLE +# define main fiddle_main +#endif + #if SQLITE_SHELL_IS_UTF8 int SQLITE_CDECL main(int argc, char **argv){ #else int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; +#endif +#ifdef SQLITE_DEBUG + sqlite3_int64 mem_main_enter = 0; #endif char *zErrMsg = 0; +#ifdef SQLITE_SHELL_FIDDLE +# define data shellState +#else ShellState data; + StreamsAreConsole consStreams = SAC_NoConsole; +#endif const char *zInitFile = 0; int i; int rc = 0; int warnInmemoryDb = 0; int readStdin = 1; int nCmd = 0; + int nOptsEnd = argc; char **azCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ #if !SQLITE_SHELL_IS_UTF8 char **argvToFree = 0; int argcToFree = 0; #endif - - setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ - stdin_is_interactive = isatty(0); - stdout_is_console = isatty(1); +#ifdef SQLITE_SHELL_FIDDLE + stdin_is_interactive = 0; + stdout_is_console = 1; + data.wasm.zDefaultDbName = "/fiddle.sqlite3"; +#else + consStreams = consoleClassifySetup(stdin, stdout, stderr); + stdin_is_interactive = (consStreams & SAC_InConsole)!=0; + stdout_is_console = (consStreams & SAC_OutConsole)!=0; + atexit(consoleRestore); +#endif + atexit(sayAbnormalExit); #ifdef SQLITE_DEBUG - registerOomSimulator(); + mem_main_enter = sqlite3_memory_used(); #endif - #if !defined(_WIN32_WCE) if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ - fprintf(stderr, - "attach debugger to process %d and press any key to continue.\n", - GETPID()); + eputf("attach debugger to process %d and press any key to continue.\n", + GETPID()); fgetc(stdin); }else{ #if defined(_WIN32) || defined(WIN32) @@ -22337,11 +30375,19 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } } #endif + /* Register a valid signal handler early, before much else is done. */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){ + eputz("No ^C handler.\n"); + } +#endif #if USE_SYSTEM_SQLITE+0!=1 - if( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", - sqlite3_sourceid(), SQLITE_SOURCE_ID); + if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ + eputf("SQLite header and source version mismatch\n%s\n%s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); } #endif @@ -22356,16 +30402,16 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if !SQLITE_SHELL_IS_UTF8 sqlite3_initialize(); argvToFree = malloc(sizeof(argv[0])*argc*2); + shell_check_oom(argvToFree); argcToFree = argc; argv = argvToFree + argc; - if( argv==0 ) shell_out_of_memory(); for(i=0; i=1 && argv && argv[0] ); Argv0 = argv[0]; - /* Make sure we have a valid signal handler early, before anything - ** else is done. - */ -#ifdef SIGINT - signal(SIGINT, interrupt_handler); -#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) - SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); -#endif - #ifdef SQLITE_SHELL_DBNAME_PROC { /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name @@ -22399,42 +30436,55 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, - ** the size of the alternative malloc heap, - ** and the first command to execute. + ** the size of the alternative malloc heap, options affecting commands + ** or SQL run from the command line, and the first command to execute. */ +#ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); +#endif for(i=1; inOptsEnd ){ if( data.aAuxDb->zDbFilename==0 ){ data.aAuxDb->zDbFilename = z; }else{ - /* Excesss arguments are interpreted as SQL (or dot-commands) and + /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); - if( azCmd==0 ) shell_out_of_memory(); + shell_check_oom(azCmd); azCmd[nCmd-1] = z; } + continue; } if( z[1]=='-' ) z++; - if( strcmp(z,"-separator")==0 - || strcmp(z,"-nullvalue")==0 - || strcmp(z,"-newline")==0 - || strcmp(z,"-cmd")==0 + if( cli_strcmp(z, "-")==0 ){ + nOptsEnd = i; + continue; + }else if( cli_strcmp(z,"-separator")==0 + || cli_strcmp(z,"-nullvalue")==0 + || cli_strcmp(z,"-newline")==0 + || cli_strcmp(z,"-cmd")==0 ){ (void)cmdline_option_value(argc, argv, ++i); - }else if( strcmp(z,"-init")==0 ){ + }else if( cli_strcmp(z,"-init")==0 ){ zInitFile = cmdline_option_value(argc, argv, ++i); - }else if( strcmp(z,"-batch")==0 ){ + }else if( cli_strcmp(z,"-interactive")==0 ){ + }else if( cli_strcmp(z,"-batch")==0 ){ /* Need to check for batch mode here to so we can avoid printing ** informational messages (like from process_sqliterc) before ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; - }else if( strcmp(z,"-heap")==0 ){ + }else if( cli_strcmp(z,"-utf8")==0 ){ + }else if( cli_strcmp(z,"-no-utf8")==0 ){ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + int val = 0; + sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val); + assert( val==0 ); + }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; sqlite3_int64 szHeap; @@ -22442,11 +30492,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ zSize = cmdline_option_value(argc, argv, ++i); szHeap = integerValue(zSize); if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #else (void)cmdline_option_value(argc, argv, ++i); #endif - }else if( strcmp(z,"-pagecache")==0 ){ + }else if( cli_strcmp(z,"-pagecache")==0 ){ sqlite3_int64 n, sz; sz = integerValue(cmdline_option_value(argc,argv,++i)); if( sz>70000 ) sz = 70000; @@ -22455,27 +30506,30 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( sz>0 && n>0 && 0xffffffffffffLL/sz0 && sz>0) ? malloc(n*sz) : 0, sz, n); data.shellFlgs |= SHFLG_Pagecache; - }else if( strcmp(z,"-lookaside")==0 ){ + }else if( cli_strcmp(z,"-lookaside")==0 ){ int n, sz; sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz<0 ) sz = 0; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n<0 ) n = 0; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; - }else if( strcmp(z,"-threadsafe")==0 ){ + }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); switch( n ){ case 0: sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); break; case 2: sqlite3_config(SQLITE_CONFIG_MULTITHREAD); break; default: sqlite3_config(SQLITE_CONFIG_SERIALIZED); break; } #ifdef SQLITE_ENABLE_VFSTRACE - }else if( strcmp(z,"-vfstrace")==0 ){ + }else if( cli_strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( const char *zTraceName, const char *zOldVfsName, @@ -22486,54 +30540,62 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); #endif #ifdef SQLITE_ENABLE_MULTIPLEX - }else if( strcmp(z,"-multiplex")==0 ){ - extern int sqlite3_multiple_initialize(const char*,int); + }else if( cli_strcmp(z,"-multiplex")==0 ){ + extern int sqlite3_multiplex_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif - }else if( strcmp(z,"-mmap")==0 ){ + }else if( cli_strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); -#ifdef SQLITE_ENABLE_SORTER_REFERENCES - }else if( strcmp(z,"-sorterref")==0 ){ +#if defined(SQLITE_ENABLE_SORTER_REFERENCES) + }else if( cli_strcmp(z,"-sorterref")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); #endif - }else if( strcmp(z,"-vfs")==0 ){ + }else if( cli_strcmp(z,"-vfs")==0 ){ zVfs = cmdline_option_value(argc, argv, ++i); #ifdef SQLITE_HAVE_ZLIB - }else if( strcmp(z,"-zip")==0 ){ + }else if( cli_strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( strcmp(z,"-append")==0 ){ + }else if( cli_strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; #ifndef SQLITE_OMIT_DESERIALIZE - }else if( strcmp(z,"-deserialize")==0 ){ + }else if( cli_strcmp(z,"-deserialize")==0 ){ data.openMode = SHELL_OPEN_DESERIALIZE; - }else if( strcmp(z,"-maxsize")==0 && i+1zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0); + eputf("%s: Error: no database filename specified\n", Argv0); return 1; #endif } data.out = stdout; +#ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); +#endif /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -22595,129 +30659,141 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=1; i=nOptsEnd ) continue; if( z[1]=='-' ){ z++; } - if( strcmp(z,"-init")==0 ){ + if( cli_strcmp(z,"-init")==0 ){ i++; - }else if( strcmp(z,"-html")==0 ){ + }else if( cli_strcmp(z,"-html")==0 ){ data.mode = MODE_Html; - }else if( strcmp(z,"-list")==0 ){ + }else if( cli_strcmp(z,"-list")==0 ){ data.mode = MODE_List; - }else if( strcmp(z,"-quote")==0 ){ + }else if( cli_strcmp(z,"-quote")==0 ){ data.mode = MODE_Quote; sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma); sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); - }else if( strcmp(z,"-line")==0 ){ + }else if( cli_strcmp(z,"-line")==0 ){ data.mode = MODE_Line; - }else if( strcmp(z,"-column")==0 ){ + }else if( cli_strcmp(z,"-column")==0 ){ data.mode = MODE_Column; - }else if( strcmp(z,"-json")==0 ){ + }else if( cli_strcmp(z,"-json")==0 ){ data.mode = MODE_Json; - }else if( strcmp(z,"-markdown")==0 ){ + }else if( cli_strcmp(z,"-markdown")==0 ){ data.mode = MODE_Markdown; - }else if( strcmp(z,"-table")==0 ){ + }else if( cli_strcmp(z,"-table")==0 ){ data.mode = MODE_Table; - }else if( strcmp(z,"-box")==0 ){ + }else if( cli_strcmp(z,"-box")==0 ){ data.mode = MODE_Box; - }else if( strcmp(z,"-csv")==0 ){ + }else if( cli_strcmp(z,"-csv")==0 ){ data.mode = MODE_Csv; memcpy(data.colSeparator,",",2); #ifdef SQLITE_HAVE_ZLIB - }else if( strcmp(z,"-zip")==0 ){ + }else if( cli_strcmp(z,"-zip")==0 ){ data.openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( strcmp(z,"-append")==0 ){ + }else if( cli_strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; #ifndef SQLITE_OMIT_DESERIALIZE - }else if( strcmp(z,"-deserialize")==0 ){ + }else if( cli_strcmp(z,"-deserialize")==0 ){ data.openMode = SHELL_OPEN_DESERIALIZE; - }else if( strcmp(z,"-maxsize")==0 && i+10 ){ - utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands" - " with \"%s\"\n", z); + eputf("Error: cannot mix regular SQL or dot-commands" + " with \"%s\"\n", z); return 1; } open_db(&data, OPEN_DB_ZIPFILE); @@ -22755,11 +30831,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ readStdin = 0; break; #endif - }else if( strcmp(z,"-safe")==0 ){ + }else if( cli_strcmp(z,"-safe")==0 ){ data.bSafeMode = data.bSafeModePersist = 1; + }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ + /* Acted upon in first pass. */ }else{ - utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); - raw_printf(stderr,"Use -help for a list of options.\n"); + eputf("%s: Error: unknown option: %s\n", Argv0, z); + eputz("Use -help for a list of options.\n"); return 1; } data.cMode = data.mode; @@ -22779,12 +30857,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ open_db(&data, 0); + echo_group_input(&data, azCmd[i]); rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); + eputf("Error: %s\n", zErrMsg); }else{ - utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]); + eputf("Error: unable to process SQL: %s\n", azCmd[i]); } sqlite3_free(zErrMsg); free(azCmd); @@ -22799,16 +30878,19 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *zHome; char *zHistory; int nHistory; - printf( - "SQLite version %s %.19s\n" /*extra-version-info*/ - "Enter \".help\" for usage hints.\n", - sqlite3_libversion(), sqlite3_sourceid() - ); +#if CIO_WIN_WC_XLATE +# define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "") +#else +# define SHELL_CIO_CHAR_SET "" +#endif + sputf(stdout, "SQLite version %s %.19s%s\n" /*extra-version-info*/ + "Enter \".help\" for usage hints.\n", + sqlite3_libversion(), sqlite3_sourceid(), SHELL_CIO_CHAR_SET); if( warnInmemoryDb ){ - printf("Connected to a "); + sputz(stdout, "Connected to a "); printBold("transient in-memory database"); - printf(".\nUse \".open FILENAME\" to reopen on a " - "persistent database.\n"); + sputz(stdout, ".\nUse \".open FILENAME\" to reopen on a" + " persistent database.\n"); } zHistory = getenv("SQLITE_HISTORY"); if( zHistory ){ @@ -22837,6 +30919,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } +#ifndef SQLITE_SHELL_FIDDLE + /* In WASM mode we have to leave the db state in place so that + ** client code can "push" SQL into it after this call returns. */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( data.expert.pExpert ){ + expertFinish(&data, 1, 0); + } +#endif free(azCmd); set_table_name(&data, 0); if( data.db ){ @@ -22863,5 +30953,147 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); +#ifdef SQLITE_DEBUG + if( sqlite3_memory_used()>mem_main_enter ){ + eputf("Memory leaked: %u bytes\n", + (unsigned int)(sqlite3_memory_used()-mem_main_enter)); + } +#endif +#endif /* !SQLITE_SHELL_FIDDLE */ + return rc; +} + + +#ifdef SQLITE_SHELL_FIDDLE +/* Only for emcc experimentation purposes. */ +int fiddle_experiment(int a,int b){ + return a + b; +} + +/* +** Returns a pointer to the current DB handle. +*/ +sqlite3 * fiddle_db_handle(){ + return globalDb; +} + +/* +** Returns a pointer to the given DB name's VFS. If zDbName is 0 then +** "main" is assumed. Returns 0 if no db with the given name is +** open. +*/ +sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ + sqlite3_vfs * pVfs = 0; + if(globalDb){ + sqlite3_file_control(globalDb, zDbName ? zDbName : "main", + SQLITE_FCNTL_VFS_POINTER, &pVfs); + } + return pVfs; +} + +/* Only for emcc experimentation purposes. */ +sqlite3 * fiddle_db_arg(sqlite3 *arg){ + oputf("fiddle_db_arg(%p)\n", (const void*)arg); + return arg; +} + +/* +** Intended to be called via a SharedWorker() while a separate +** SharedWorker() (which manages the wasm module) is performing work +** which should be interrupted. Unfortunately, SharedWorker is not +** portable enough to make real use of. +*/ +void fiddle_interrupt(void){ + if( globalDb ) sqlite3_interrupt(globalDb); +} + +/* +** Returns the filename of the given db name, assuming "main" if +** zDbName is NULL. Returns NULL if globalDb is not opened. +*/ +const char * fiddle_db_filename(const char * zDbName){ + return globalDb + ? sqlite3_db_filename(globalDb, zDbName ? zDbName : "main") + : NULL; +} + +/* +** Completely wipes out the contents of the currently-opened database +** but leaves its storage intact for reuse. If any transactions are +** active, they are forcibly rolled back. +*/ +void fiddle_reset_db(void){ + if( globalDb ){ + int rc; + while( sqlite3_txn_state(globalDb,0)>0 ){ + /* + ** Resolve problem reported in + ** https://sqlite.org/forum/forumpost/0b41a25d65 + */ + oputz("Rolling back in-progress transaction.\n"); + sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); + } + rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } +} + +/* +** Uses the current database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +*/ +int fiddle_export_db( int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = shellState.db + ? sqlite3_file_control(shellState.db, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size when + ** exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPospMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } return rc; } + +/* +** Trivial exportable function for emscripten. It processes zSql as if +** it were input to the sqlite3 shell and redirects all output to the +** wasm binding. fiddle_main() must have been called before this +** is called, or results are undefined. +*/ +void fiddle_exec(const char * zSql){ + if(zSql && *zSql){ + if('.'==*zSql) puts(zSql); + shellState.wasm.zInput = zSql; + shellState.wasm.zPos = zSql; + process_input(&shellState); + shellState.wasm.zInput = shellState.wasm.zPos = 0; + } +} +#endif /* SQLITE_SHELL_FIDDLE */ diff --git a/mock/sqlite/src/sqlite3.c b/mock/sqlite/src/sqlite3.c index 8df30ae3..582cc3e9 100644 --- a/mock/sqlite/src/sqlite3.c +++ b/mock/sqlite/src/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.37.2. By combining all the individual C code files into this +** version 3.46.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -16,10 +16,9 @@ ** if you want a wrapper to interface SQLite with your choice of programming ** language. The code for the "sqlite3" command-line shell is also in a ** separate file. This file contains only code for the core SQLite library. -*/ -/* -** 2019.09.02-Complete codec logic for encryption and decryption. -** Huawei Technologies Co, Ltd. +** +** The content in this amalgamation comes from Fossil check-in +** c9c2ab54ba1f5f46360f1b4f35d849cd3f08. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -54,11 +53,11 @@ ** used on lines of code that actually ** implement parts of coverage testing. ** -** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** OPTIMIZATION-IF-TRUE - This branch is allowed to always be false ** and the correct answer is still obtained, ** though perhaps more slowly. ** -** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** OPTIMIZATION-IF-FALSE - This branch is allowed to always be true ** and the correct answer is still obtained, ** though perhaps more slowly. ** @@ -127,6 +126,10 @@ #define SQLITE_4_BYTE_ALIGNED_MALLOC #endif /* defined(_MSC_VER) && !defined(_WIN64) */ +#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 +#define HAVE_LOG2 0 +#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ @@ -456,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.37.2" -#define SQLITE_VERSION_NUMBER 3037002 -#define SQLITE_SOURCE_ID "2022-01-06 13:25:41 872ba256cbf61d9290b571c0e6d82a20c224ca3ad82971edc46b29818d5d17a0" +#define SQLITE_VERSION "3.46.1" +#define SQLITE_VERSION_NUMBER 3046001 +#define SQLITE_SOURCE_ID "2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -730,6 +733,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ SQLITE_API int sqlite3_exec( @@ -783,6 +788,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ +#define SQLITE_META_RECOVERED 66 /* meta page recovered*/ /* end-of-error-codes */ /* @@ -838,6 +844,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -873,10 +880,12 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_WARNING_DUMP (SQLITE_WARNING | (2<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ /* ** CAPI3REF: Flags For File Open Operations @@ -980,13 +989,17 @@ SQLITE_API int sqlite3_exec( ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -1064,7 +1077,14 @@ struct sqlite3_file { **
  • [SQLITE_LOCK_PENDING], or **
  • [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +** If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, ** PENDING, or EXCLUSIVE lock on the file. It returns true @@ -1169,9 +1189,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1475,7 +1494,6 @@ struct sqlite3_io_methods { ** in wal mode after the client has finished copying pages from the wal ** file to the database file, but before the *-shm file is updated to ** record the fact that the pages have been checkpointed. -** ** **
  • [[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect @@ -1488,10 +1506,16 @@ struct sqlite3_io_methods { ** the database is not a wal-mode db, or if there is no such connection in any ** other process. This opcode cannot be used to detect transactions opened ** by clients within the current process, only within other processes. -** ** **
  • [[SQLITE_FCNTL_CKSM_FILE]] -** Used by the cksmvfs VFS module only. +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the +** [checksum VFS shim] only. +** +**
  • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control +** purges the contents of the in-memory page cache. If there is an open +** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1534,6 +1558,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKPT_START 39 #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1563,6 +1588,26 @@ typedef struct sqlite3_mutex sqlite3_mutex; */ typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
      +**
    • sqlite3_filename_database() +**
    • sqlite3_filename_journal() +**
    • sqlite3_filename_wal() +**
    • sqlite3_uri_parameter() +**
    • sqlite3_uri_boolean() +**
    • sqlite3_uri_int64() +**
    • sqlite3_uri_key() +**
    +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1741,7 +1786,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -1928,20 +1973,23 @@ SQLITE_API int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -2049,6 +2097,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
      +**
    • SQLITE_CONFIG_LOG +**
    • SQLITE_CONFIG_PCACHE_HDRSZ +**
    +** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -2379,7 +2444,7 @@ struct sqlite3_mem_methods { ** is stored in each sorted record and the required column values loaded ** from the database as records are returned in sorted order. The default ** value for this option is to never use this optimization. Specifying a -** negative value for this option restores the default behaviour. +** negative value for this option restores the default behavior. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** @@ -2393,30 +2458,61 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. +** +** [[SQLITE_CONFIG_CORRUPTION]]
    SQLITE_CONFIG_CORRUPTION
    +**
    The SQLITE_CONFIG_CORRUPTION option is used to configure the SQLite +** global [ corruption error]. +** (^The SQLITE_CONFIG_CORRUPTION option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,const void*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked to process each data corruption event. ^If the +** function pointer is NULL, no=op will do when corruption detect. +** ^The void pointer that is the second argument to SQLITE_CONFIG_CORRUPTION is +** passed through as the first parameter to the application-defined corruption +** function whenever that function is invoked. ^The second parameter to +** the corruption function is a corruption message after formatting via [sqlite3_snprintf()]. +** In a multi-threaded application, the application-defined corruption +** function must be threadsafe.
    ** */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ @@ -2424,6 +2520,9 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ +#define SQLITE_CONFIG_CORRUPTION 31 /* xCorruption */ +#define SQLITE_CONFIG_ENABLE_ICU 32 /* boolean */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2457,7 +2556,7 @@ struct sqlite3_mem_methods { ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words ** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^ @@ -2554,7 +2653,7 @@ struct sqlite3_mem_methods { ** database handle, SQLite checks if this will mean that there are now no ** connections at all to the database. If so, it performs a checkpoint ** operation before closing the connection. This option may be used to -** override this behaviour. The first parameter passed to this operation +** override this behavior. The first parameter passed to this operation ** is an integer - positive to disable checkpoints-on-close, or zero (the ** default) to enable them, and negative to leave the setting unchanged. ** The second parameter is a pointer to an integer @@ -2607,8 +2706,12 @@ struct sqlite3_mem_methods { **
  • sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); ** ** Because resetting a database is destructive and irreversible, the -** process requires the use of this obscure API and multiple steps to help -** ensure that it does not happen by accident. +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. ** ** [[SQLITE_DBCONFIG_DEFENSIVE]]
    SQLITE_DBCONFIG_DEFENSIVE
    **
    The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the @@ -2619,6 +2722,7 @@ struct sqlite3_mem_methods { **
      **
    • The [PRAGMA writable_schema=ON] statement. **
    • The [PRAGMA journal_mode=OFF] statement. +**
    • The [PRAGMA schema_version=N] statement. **
    • Writes to the [sqlite_dbpage] virtual table. **
    • Direct writes to [shadow tables]. **
    @@ -2646,7 +2750,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
    SQLITE_DBCONFIG_DQS_DML +**
    SQLITE_DBCONFIG_DQS_DML
    **
    The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2655,7 +2759,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
    SQLITE_DBCONFIG_DQS_DDL +**
    SQLITE_DBCONFIG_DQS_DDL
    **
    The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2664,7 +2768,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA
    **
    The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2684,7 +2788,7 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
    **
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2693,7 +2797,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2702,8 +2806,40 @@ struct sqlite3_mem_methods { ** the [VACUUM] command will fail with an obscure error when attempting to ** process a table with generated columns and a descending index. This is ** not considered a bug since SQLite versions 3.3.0 and earlier do not support -** either generated columns or decending indexes. +** either generated columns or descending indexes. **
    +** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
    SQLITE_DBCONFIG_STMT_SCANSTATUS
    +**
    The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
    +** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
    SQLITE_DBCONFIG_REVERSE_SCANORDER
    +**
    The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
    +** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2724,11 +2860,15 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ + #ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION #define SQLITE_DBCONFIG_SET_SHAREDBLOCK 2004 #define SQLITE_DBCONFIG_USE_SHAREDBLOCK 2005 -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + /* ** CAPI3REF: Enable Or Disable Extended Result Codes ** METHOD: sqlite3 @@ -2949,8 +3089,13 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); ** ^A call to sqlite3_interrupt(D) that occurs when there are no running ** SQL statements is a no-op and has no effect on SQL statements ** that are started after the sqlite3_interrupt() call returns. +** +** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether +** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API int sqlite3_is_interrupted(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -3155,6 +3300,17 @@ SQLITE_API int sqlite3_get_table( ); SQLITE_API void sqlite3_free_table(char **result); +// hw export the symbols +#ifdef SQLITE_EXPORT_SYMBOLS +#if defined(__GNUC__) +# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) +#elif defined(_MSC_VER) +# define EXPORT_SYMBOLS __declspec(dllexport) +#else +# define EXPORT_SYMBOLS +#endif +#endif // SQLITE_EXPORT_SYMBOLS + /* ** CAPI3REF: Formatted String Printing Functions ** @@ -3497,8 +3653,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -3568,8 +3724,8 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, **
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same ** information as is provided by the [sqlite3_profile()] callback. ** ^The P argument is a pointer to the [prepared statement] and the -** X argument points to a 64-bit integer which is the estimated of -** the number of nanosecond that the prepared statement took to run. +** X argument points to a 64-bit integer which is approximately +** the number of nanoseconds that the prepared statement took to run. ** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. ** ** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    @@ -3601,8 +3757,10 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, ** M argument should be the bitwise OR-ed combination of ** zero or more [SQLITE_TRACE] constants. ** -** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides -** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P) +** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or +** sqlite3_trace_v2(D,M,X,P) for the [database connection] D. Each +** database connection may have at most one trace callback. ** ** ^The X callback is invoked whenever any of the events identified by ** mask M occur. ^The integer return value from the callback is currently @@ -3632,7 +3790,7 @@ SQLITE_API int sqlite3_trace_v2( ** ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback ** function X to be invoked periodically during long running calls to -** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** [sqlite3_step()] and [sqlite3_prepare()] and similar for ** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** @@ -3657,6 +3815,13 @@ SQLITE_API int sqlite3_trace_v2( ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** +** The progress handler callback would originally only be invoked from the +** bytecode engine. It still might be invoked during [sqlite3_prepare()] +** and similar because those routines might force a reparse of the schema +** which involves running the bytecode engine. However, beginning with +** SQLite version 3.41.0, the progress handler callback might also be +** invoked directly from [sqlite3_prepare()] while analyzing and generating +** code for complex queries. */ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); @@ -3693,13 +3858,18 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** **
    ** ^(
    [SQLITE_OPEN_READONLY]
    -**
    The database is opened in read-only mode. If the database does not -** already exist, an error is returned.
    )^ +**
    The database is opened in read-only mode. If the database does +** not already exist, an error is returned.
    )^ ** ** ^(
    [SQLITE_OPEN_READWRITE]
    -**
    The database is opened for reading and writing if possible, or reading -** only if the file is write protected by the operating system. In either -** case the database must already exist, otherwise an error is returned.
    )^ +**
    The database is opened for reading and writing if possible, or +** reading only if the file is write protected by the operating +** system. In either case the database must already exist, otherwise +** an error is returned. For historical reasons, if opening in +** read-write mode fails due to OS-level permissions, an attempt is +** made to open it in read-only mode. [sqlite3_db_readonly()] can be +** used to determine whether the database is actually +** read-write.
    )^ ** ** ^(
    [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
    **
    The database is opened for reading and writing, and is created if @@ -3737,6 +3907,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
    The database is opened [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    **
    The database is opened [shared cache] disabled, overriding @@ -3752,7 +3925,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** to return an extended result code.
    ** ** [[OPEN_NOFOLLOW]] ^(
    [SQLITE_OPEN_NOFOLLOW]
    -**
    The database filename is not allowed to be a symbolic link
    +**
    The database filename is not allowed to contain a symbolic link
    **
    )^ ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the @@ -3956,7 +4129,7 @@ SQLITE_API int sqlite3_open_v2( ** as F) must be one of: **
      **
    • A database filename pointer created by the SQLite core and -** passed into the xOpen() method of a VFS implemention, or +** passed into the xOpen() method of a VFS implementation, or **
    • A filename obtained from [sqlite3_db_filename()], or **
    • A new filename constructed using [sqlite3_create_filename()]. **
    @@ -4011,10 +4184,10 @@ SQLITE_API int sqlite3_open_v2( ** ** See the [URI filename] documentation for additional information. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); -SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); -SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); -SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); /* ** CAPI3REF: Translate filenames @@ -4043,9 +4216,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); ** return value from [sqlite3_db_filename()], then the result is ** undefined and is likely a memory access violation. */ -SQLITE_API const char *sqlite3_filename_database(const char*); -SQLITE_API const char *sqlite3_filename_journal(const char*); -SQLITE_API const char *sqlite3_filename_wal(const char*); +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); /* ** CAPI3REF: Database File Corresponding To A Journal @@ -4069,7 +4242,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); /* ** CAPI3REF: Create and Destroy VFS Filenames ** -** These interfces are provided for use by [VFS shim] implementations and +** These interfaces are provided for use by [VFS shim] implementations and ** are not useful outside of that context. ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of @@ -4111,14 +4284,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** then the corresponding [sqlite3_module.xClose() method should also be ** invoked prior to calling sqlite3_free_filename(Y). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API sqlite3_filename sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, int nParam, const char **azParam ); -SQLITE_API void sqlite3_free_filename(char*); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages @@ -4137,27 +4310,38 @@ SQLITE_API void sqlite3_free_filename(char*); ** sqlite3_extended_errcode() might change with each API call. ** Except, there are some interfaces that are guaranteed to never ** change the value of the error code. The error-code preserving -** interfaces are: +** interfaces include the following: ** **
      **
    • sqlite3_errcode() **
    • sqlite3_extended_errcode() **
    • sqlite3_errmsg() **
    • sqlite3_errmsg16() +**
    • sqlite3_error_offset() **
    ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language -** text that describes the error, as either UTF-8 or UTF-16 respectively. +** text that describes the error, as either UTF-8 or UTF-16 respectively, +** or NULL if no error message is available. +** (See how SQLite handles [invalid UTF] for exceptions to this rule.) ** ^(Memory to hold the error message string is managed internally. ** The application does not need to worry about freeing the result. ** However, the error string might be overwritten or deallocated by ** subsequent calls to other SQLite interface functions.)^ ** -** ^The sqlite3_errstr() interface returns the English-language text -** that describes the [result code], as UTF-8. +** ^The sqlite3_errstr(E) interface returns the English-language text +** that describes the [result code] E, as UTF-8, or NULL if E is not an +** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. ** +** ^If the most recent error references a specific token in the input +** SQL, the sqlite3_error_offset() interface returns the byte offset +** of the start of that token. ^The byte offset returned by +** sqlite3_error_offset() assumes that the input SQL is UTF8. +** ^If the most recent error does not reference a specific token in the input +** SQL, then the sqlite3_error_offset() function returns -1. +** ** When the serialized [threading mode] is in use, it might be the ** case that a second error occurs on a separate thread in between ** the time of the first error and the call to these interfaces. @@ -4177,6 +4361,7 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); SQLITE_API const char *sqlite3_errmsg(sqlite3*); SQLITE_API const void *sqlite3_errmsg16(sqlite3*); SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int sqlite3_error_offset(sqlite3 *db); /* ** CAPI3REF: Prepared Statement Object @@ -4588,6 +4773,10 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); ** be false. ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a ** read-only no-op if the table already exists, but ** sqlite3_stmt_readonly() still returns false for such a statement. +** +** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN] +** statement, then sqlite3_stmt_readonly(X) returns the same value as +** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted. */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); @@ -4603,6 +4792,41 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for [prepared statement] S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if +** its SQL text began with "[EXPLAIN QUERY PLAN]". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using [sqlite3_prepare()] instead of +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4656,6 +4880,8 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); ** ** ^The sqlite3_value objects that are passed as parameters into the ** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()] +** are protected. ** ^The sqlite3_value object returned by ** [sqlite3_column_value()] is unprotected. ** Unprotected sqlite3_value objects may only be used as arguments @@ -4764,7 +4990,7 @@ typedef struct sqlite3_context sqlite3_context; ** with it may be passed. ^It is called to dispose of the BLOB or string even ** if the call to the bind API fails, except the destructor is not called if ** the third parameter is a NULL pointer or the fourth parameter is negative. -** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that +** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that ** the application remains responsible for disposing of the object. ^In this ** case, the object and the provided pointer to it must remain valid until ** either the prepared statement is finalized or the same SQL parameter is @@ -5127,7 +5353,8 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK SQLITE_API int sqlite3_set_droptable_handle(sqlite3*, void (*xFunc)(sqlite3*,const char*,const char*)); -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + /* ** CAPI3REF: Number of columns in a result set ** METHOD: sqlite3_stmt @@ -5280,6 +5507,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** even empty strings, are always zero-terminated. ^The return ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** +** ^Strings returned by sqlite3_column_text16() always have the endianness +** which is native to the platform, regardless of the text encoding set +** for the database. +** ** Warning: ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. In a multithreaded environment, ** an unprotected sqlite3_value object may only be used safely with @@ -5293,7 +5524,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** [application-defined SQL functions] or [virtual tables], not within ** top-level application code. ** -** The these routines may attempt to convert the datatype of the result. +** These routines may attempt to convert the datatype of the result. ** ^For example, if the internal representation is FLOAT and a text result ** is requested, [sqlite3_snprintf()] is used internally to perform the ** conversion automatically. ^(The following table details the conversions @@ -5318,7 +5549,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); **
    • TEXT BLOB No change **
    BLOB INTEGER [CAST] to INTEGER **
    BLOB FLOAT [CAST] to REAL -**
    BLOB TEXT Add a zero terminator if needed +**
    BLOB TEXT [CAST] to TEXT, ensure zero terminator **
    ** )^ ** @@ -5442,20 +5673,33 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S ** back to the beginning of its program. ** -** ^If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. ** ** ^If the most recent call to [sqlite3_step(S)] for the ** [prepared statement] S indicated an error, then ** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. ** ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} @@ -5661,10 +5905,21 @@ SQLITE_API int sqlite3_create_window_function( ** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in ** schema structures such as [CHECK constraints], [DEFAULT clauses], ** [expression indexes], [partial indexes], or [generated columns]. -** The SQLITE_DIRECTONLY flags is a security feature which is recommended -** for all [application-defined SQL functions], and especially for functions -** that have side-effects or that could potentially leak sensitive -** information. +**

    +** The SQLITE_DIRECTONLY flag is recommended for any +** [application-defined SQL function] +** that has side-effects or that could potentially leak sensitive information. +** This will prevent attacks in which an application is tricked +** into using a database file that has had its schema surreptitiously +** modified to invoke the application-defined function in ways that are +** harmful. +**

    +** Some people say it is good practice to set SQLITE_DIRECTONLY on all +** [application-defined SQL functions], regardless of whether or not they +** are security sensitive, as doing so prevents those functions from being used +** inside of the database schema, and thus ensures that the database +** can be inspected and modified using generic tools (such as the [CLI]) +** that do not have access to the application-defined functions. **

    • ** ** [[SQLITE_INNOCUOUS]]
    SQLITE_INNOCUOUS
    @@ -5691,13 +5946,27 @@ SQLITE_API int sqlite3_create_window_function( **
    ** ** [[SQLITE_SUBTYPE]]
    SQLITE_SUBTYPE
    -** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. -** Specifying this flag makes no difference for scalar or aggregate user -** functions. However, if it is not specified for a user-defined window -** function, then any sub-types belonging to arguments passed to the window -** function may be discarded before the window function is called (i.e. -** sqlite3_value_subtype() will always return 0). +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** SQL functions that invokes [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
    SQLITE_RESULT_SUBTYPE
    +** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. **
    ** */ @@ -5705,6 +5974,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_DIRECTONLY 0x000080000 #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 /* ** CAPI3REF: Deprecated Functions @@ -5870,6 +6140,28 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); SQLITE_API int sqlite3_value_frombind(sqlite3_value*); +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); + /* ** CAPI3REF: Finding The Subtype Of SQL Values ** METHOD: sqlite3_value @@ -5879,6 +6171,12 @@ SQLITE_API int sqlite3_value_frombind(sqlite3_value*); ** information can be used to pass a limited amount of context from ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. +** +** Every [application-defined SQL function] that invoke this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. */ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); @@ -5890,7 +6188,8 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); ** object D and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a -** memory allocation fails. +** memory allocation fails. ^If V is a [pointer value], then the result +** of sqlite3_value_dup(V) is a NULL value. ** ** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object ** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer @@ -5921,7 +6220,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the @@ -5976,48 +6275,56 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to -** associate metadata with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. An example -** of where this might be useful is in a regular-expression matching -** function. The compiled version of the regular expression can be stored as -** metadata associated with the pattern string. +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. ** Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument ** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no metadata +** function argument. ^If there is no auxiliary data ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface ** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th -** argument of the application-defined function. ^Subsequent +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent ** calls to sqlite3_get_auxdata(C,N) return P from the most recent -** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or -** NULL if the metadata has been discarded. +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, ** SQLite will invoke the destructor function X with parameter P exactly -** once, when the metadata is discarded. -** SQLite is free to discard the metadata at any time, including:
      +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
        **
      • ^(when the corresponding function parameter changes)^, or **
      • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the ** SQL statement)^, or **
      • ^(when sqlite3_set_auxdata() is invoked again on the same ** parameter)^, or **
      • ^(during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.)^
      +** allocation error occurs.)^ +**
    • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^
    ** -** Note the last bullet in particular. The destructor X in +** Note the last two bullets in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the ** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() ** should be called near the end of the function implementation and the ** function implementation should not make any use of P after -** sqlite3_set_auxdata() has been called. -** -** ^(In practice, metadata is preserved between function calls for +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. +** +** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal ** values and [parameters] and expressions composed from the same.)^ ** @@ -6027,10 +6334,67 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** ** These routines must be called from the same thread in which ** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
      +**
    • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
    • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
    • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
    +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior @@ -6126,9 +6490,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -6231,6 +6596,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); ** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. */ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); @@ -6412,7 +6791,7 @@ SQLITE_API int sqlite3_rekey_v2( const void *pKey, int nKey /* The new key */ ); -#endif +#endif /* SQLITE_HAS_CODEC */ #ifdef SQLITE_ENABLE_CEROD /* @@ -6440,6 +6819,13 @@ SQLITE_API void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ SQLITE_API int sqlite3_sleep(int); @@ -6610,6 +6996,28 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*); */ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +/* +** CAPI3REF: Return The Schema Name For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name +** for the N-th database on database connection D, or a NULL pointer of N is +** out of range. An N value of 0 means the main database file. An N of 1 is +** the "temp" schema. Larger values of N correspond to various ATTACH-ed +** databases. +** +** Space to hold the string that is returned by sqlite3_db_name() is managed +** by SQLite itself. The string might be deallocated by any operation that +** changes the schema, including [ATTACH] or [DETACH] or calls to +** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that +** occur on a different thread. Applications that need to +** remember the string long-term should make their own copy. Applications that +** are accessing the same database connection simultaneously on multiple +** threads should mutex-protect calls to this API and should make their own +** private copy of the result prior to releasing the mutex. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); + /* ** CAPI3REF: Return The Filename For A Database Connection ** METHOD: sqlite3 @@ -6640,7 +7048,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); **
  • [sqlite3_filename_wal()] ** */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -6671,7 +7079,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); /* -** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** CAPI3REF: Allowed return values from sqlite3_txn_state() ** KEYWORDS: {transaction state} ** ** These constants define the current transaction state of a database file. @@ -6777,7 +7185,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** function C that is invoked prior to each autovacuum of the database ** file. ^The callback is passed a copy of the generic data pointer (P), ** the schema-name of the attached database that is being autovacuumed, -** the the size of the database file in pages, the number of free pages, +** the size of the database file in pages, the number of free pages, ** and the number of bytes per page, respectively. The callback should ** return the number of free pages that should be removed by the ** autovacuum. ^If the callback returns zero, then no autovacuum happens. @@ -6803,7 +7211,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all ** previous invocations for that database connection. ^If the callback ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, -** then the autovacuum steps callback is cancelled. The return value +** then the autovacuum steps callback is canceled. The return value ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might ** be some other error code if something goes wrong. The current ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other @@ -6869,6 +7277,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -6898,6 +7312,11 @@ SQLITE_API void *sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). ** In prior versions of SQLite, @@ -6996,7 +7415,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** ^The soft heap limit may not be greater than the hard heap limit. ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) ** is invoked with a value of N that is greater than the hard heap limit, -** the the soft heap limit is set to the value of the hard heap limit. +** the soft heap limit is set to the value of the hard heap limit. ** ^The soft heap limit is automatically enabled whenever the hard heap ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and ** the soft heap limit is outside the range of 1..N, then the soft heap @@ -7257,15 +7676,6 @@ SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); */ SQLITE_API void sqlite3_reset_auto_extension(void); -/* -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** Structures used by the virtual table interface */ @@ -7326,6 +7736,10 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); + /* The methods above are in versions 1 through 3 of the sqlite_module object. + ** Those below are for version 4 and greater. */ + int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema, + const char *zTabName, int mFlags, char **pzErr); }; /* @@ -7384,10 +7798,10 @@ struct sqlite3_module { ** when the omit flag is true there is no guarantee that the constraint will ** not be checked again using byte code.)^ ** -** ^The idxNum and idxPtr values are recorded and passed into the +** ^The idxNum and idxStr values are recorded and passed into the ** [xFilter] method. -** ^[sqlite3_free()] is used to free idxPtr if and only if -** needToFreeIdxPtr is true. +** ^[sqlite3_free()] is used to free idxStr if and only if +** needToFreeIdxStr is true. ** ** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in ** the correct order to satisfy the ORDER BY clause so that no separate @@ -7476,24 +7890,56 @@ struct sqlite3_index_info { ** ** These macros define the allowed values for the ** [sqlite3_index_info].aConstraint[].op field. Each value represents -** an operator that is part of a constraint term in the wHERE clause of +** an operator that is part of a constraint term in the WHERE clause of ** a query that uses a [virtual table]. -*/ -#define SQLITE_INDEX_CONSTRAINT_EQ 2 -#define SQLITE_INDEX_CONSTRAINT_GT 4 -#define SQLITE_INDEX_CONSTRAINT_LE 8 -#define SQLITE_INDEX_CONSTRAINT_LT 16 -#define SQLITE_INDEX_CONSTRAINT_GE 32 -#define SQLITE_INDEX_CONSTRAINT_MATCH 64 -#define SQLITE_INDEX_CONSTRAINT_LIKE 65 -#define SQLITE_INDEX_CONSTRAINT_GLOB 66 -#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 -#define SQLITE_INDEX_CONSTRAINT_NE 68 -#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 -#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 -#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 -#define SQLITE_INDEX_CONSTRAINT_IS 72 -#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 +** +** ^The left-hand operand of the operator is given by the corresponding +** aConstraint[].iColumn field. ^An iColumn of -1 indicates the left-hand +** operand is the rowid. +** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET +** operators have no left-hand operand, and so for those operators the +** corresponding aConstraint[].iColumn is meaningless and should not be +** used. +** +** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through +** value 255 are reserved to represent functions that are overloaded +** by the [xFindFunction|xFindFunction method] of the virtual table +** implementation. +** +** The right-hand operands for each constraint might be accessible using +** the [sqlite3_vtab_rhs_value()] interface. Usually the right-hand +** operand is only available if it appears as a single constant literal +** in the input SQL. If the right-hand operand is another column or an +** expression (even a constant expression) or a parameter, then the +** sqlite3_vtab_rhs_value() probably will not be able to extract it. +** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and +** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand +** and hence calls to sqlite3_vtab_rhs_value() for those operators will +** always return SQLITE_NOTFOUND. +** +** The collating sequence to be used for comparison can be found using +** the [sqlite3_vtab_collation()] interface. For most real-world virtual +** tables, the collating sequence of constraints does not matter (for example +** because the constraints are numeric) and so the sqlite3_vtab_collation() +** interface is not commonly needed. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 +#define SQLITE_INDEX_CONSTRAINT_NE 68 +#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 +#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 +#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 +#define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_LIMIT 73 +#define SQLITE_INDEX_CONSTRAINT_OFFSET 74 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -7522,7 +7968,7 @@ struct sqlite3_index_info { ** destructor. ** ** ^If the third parameter (the pointer to the sqlite3_module object) is -** NULL then no new module is create and any existing modules with the +** NULL then no new module is created and any existing modules with the ** same name are dropped. ** ** See also: [sqlite3_drop_modules()] @@ -7634,16 +8080,6 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); */ SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); -/* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stabilizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -*/ - /* ** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} @@ -7791,7 +8227,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); ** code is returned and the transaction rolled back. ** ** Calling this function with an argument that is not a NULL pointer or an -** open blob handle results in undefined behaviour. ^Calling this routine +** open blob handle results in undefined behavior. ^Calling this routine ** with a null pointer (such as would be returned by a failed call to ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function ** is passed a valid open blob handle, the values returned by the @@ -8018,18 +8454,20 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable -** behavior.)^ +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was ** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ @@ -8271,6 +8709,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 #define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -8278,6 +8717,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ @@ -8298,7 +8738,9 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SEEK_COUNT 30 #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 -#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_LOGEST 33 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -8311,7 +8753,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -8821,6 +9263,16 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** The counter is incremented on the first [sqlite3_step()] call of each ** cycle. ** +** [[SQLITE_STMTSTATUS_FILTER_MISS]] +** [[SQLITE_STMTSTATUS_FILTER HIT]] +**
    SQLITE_STMTSTATUS_FILTER_HIT
    +** SQLITE_STMTSTATUS_FILTER_MISS
    +**
    ^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join +** step was bypassed because a Bloom filter returned not-found. The +** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of +** times that the Bloom filter returned a find, and thus the join step +** had to be processed as normal. +** ** [[SQLITE_STMTSTATUS_MEMUSED]]
    SQLITE_STMTSTATUS_MEMUSED
    **
    ^This is the approximate number of bytes of heap memory ** used to store the prepared statement. ^This value is not actually @@ -8835,6 +9287,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); #define SQLITE_STMTSTATUS_VM_STEP 4 #define SQLITE_STMTSTATUS_REPREPARE 5 #define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_FILTER_MISS 7 +#define SQLITE_STMTSTATUS_FILTER_HIT 8 #define SQLITE_STMTSTATUS_MEMUSED 99 /* @@ -9246,7 +9700,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database @@ -9674,7 +10128,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -9742,7 +10196,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_DIRECTONLY]]
    SQLITE_VTAB_DIRECTONLY
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** prohibits that virtual table from being used from within triggers and ** views. **
    @@ -9750,18 +10204,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
    SQLITE_VTAB_INNOCUOUS
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
    +** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
    SQLITE_VTAB_USES_ALL_SCHEMAS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
    ** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -9794,7 +10258,7 @@ struct Sqlite3SharedBlockMethods { int (*xPutOther)(void *pCtx, int addedRows, int column); /* Additional methods may be added in future releases */ }; -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ /* ** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE @@ -9824,19 +10288,296 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); /* ** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** METHOD: sqlite3_index_info ** ** This function may only be called from within a call to the [xBestIndex] -** method of a [virtual table]. +** method of a [virtual table]. This function returns a pointer to a string +** that is the name of the appropriate collation sequence to use for text +** comparisons on the constraint identified by its arguments. +** +** The first argument must be the pointer to the [sqlite3_index_info] object +** that is the first parameter to the xBestIndex() method. The second argument +** must be an index into the aConstraint[] array belonging to the +** sqlite3_index_info structure passed to xBestIndex. ** -** The first argument must be the sqlite3_index_info object that is the -** first parameter to the xBestIndex() method. The second argument must be -** an index into the aConstraint[] array belonging to the sqlite3_index_info -** structure passed to xBestIndex. This function returns a pointer to a buffer -** containing the name of the collation sequence for the corresponding -** constraint. +** Important: +** The first parameter must be the same pointer that is passed into the +** xBestMethod() method. The first parameter may not be a pointer to a +** different [sqlite3_index_info] object, even an exact copy. +** +** The return value is computed as follows: +** +**
      +**

    1. If the constraint comes from a WHERE clause expression that contains +** a [COLLATE operator], then the name of the collation specified by +** that COLLATE operator is returned. +**

    2. If there is no COLLATE operator, but the column that is the subject +** of the constraint specifies an alternative collating sequence via +** a [COLLATE clause] on the column definition within the CREATE TABLE +** statement that was passed into [sqlite3_declare_vtab()], then the +** name of that alternative collating sequence is returned. +**

    3. Otherwise, "BINARY" is returned. +**

    */ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); +/* +** CAPI3REF: Determine if a virtual table query is DISTINCT +** METHOD: sqlite3_index_info +** +** This API may only be used from within an [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this +** interface from outside of xBestIndex() is undefined and probably harmful. +** +** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and +** 3. The integer returned by sqlite3_vtab_distinct() +** gives the virtual table additional information about how the query +** planner wants the output to be ordered. As long as the virtual table +** can meet the ordering requirements of the query planner, it may set +** the "orderByConsumed" flag. +** +**

    1. +** ^If the sqlite3_vtab_distinct() interface returns 0, that means +** that the query planner needs the virtual table to return all rows in the +** sort order defined by the "nOrderBy" and "aOrderBy" fields of the +** [sqlite3_index_info] object. This is the default expectation. If the +** virtual table outputs all rows in sorted order, then it is always safe for +** the xBestIndex method to set the "orderByConsumed" flag, regardless of +** the return value from sqlite3_vtab_distinct(). +**

    2. +** ^(If the sqlite3_vtab_distinct() interface returns 1, that means +** that the query planner does not need the rows to be returned in sorted order +** as long as all rows with the same values in all columns identified by the +** "aOrderBy" field are adjacent.)^ This mode is used when the query planner +** is doing a GROUP BY. +**

    3. +** ^(If the sqlite3_vtab_distinct() interface returns 2, that means +** that the query planner does not need the rows returned in any particular +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for a DISTINCT query. +**

    4. +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries +** that have both DISTINCT and ORDER BY clauses. +**

    +** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** +** ^For the purposes of comparing virtual table output values to see if the +** values are same value for sorting purposes, two NULL values are considered +** to be the same. In other words, the comparison operator is "IS" +** (or "IS NOT DISTINCT FROM") and not "==". +** +** If a virtual table implementation is unable to meet the requirements +** specified above, then it must not set the "orderByConsumed" flag in the +** [sqlite3_index_info] object or an incorrect answer may result. +** +** ^A virtual table implementation is always free to return rows in any order +** it wants, as long as the "orderByConsumed" flag is not set. ^When the +** the "orderByConsumed" flag is unset, the query planner will add extra +** [bytecode] to ensure that the final results returned by the SQL query are +** ordered correctly. The use of the "orderByConsumed" flag and the +** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful +** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed" +** flag might help queries against a virtual table to run faster. Being +** overly aggressive and setting the "orderByConsumed" flag when it is not +** valid to do so, on the other hand, might cause SQLite to return incorrect +** results. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); + +/* +** CAPI3REF: Identify and handle IN constraints in xBestIndex +** +** This interface may only be used from within an +** [xBestIndex|xBestIndex() method] of a [virtual table] implementation. +** The result of invoking this interface from any other context is +** undefined and probably harmful. +** +** ^(A constraint on a virtual table of the form +** "[IN operator|column IN (...)]" is +** communicated to the xBestIndex method as a +** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use +** this constraint, it must set the corresponding +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under +** the usual mode of handling IN operators, SQLite generates [bytecode] +** that invokes the [xFilter|xFilter() method] once for each value +** on the right-hand side of the IN operator.)^ Thus the virtual table +** only sees a single value from the right-hand side of the IN operator +** at a time. +** +** In some cases, however, it would be advantageous for the virtual +** table to see all values on the right-hand of the IN operator all at +** once. The sqlite3_vtab_in() interfaces facilitates this in two ways: +** +**

      +**

    1. +** ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero) +** if and only if the [sqlite3_index_info|P->aConstraint][N] constraint +** is an [IN operator] that can be processed all at once. ^In other words, +** sqlite3_vtab_in() with -1 in the third argument is a mechanism +** by which the virtual table can ask SQLite if all-at-once processing +** of the IN operator is even possible. +** +**

    2. +** ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates +** to SQLite that the virtual table does or does not want to process +** the IN operator all-at-once, respectively. ^Thus when the third +** parameter (F) is non-negative, this interface is the mechanism by +** which the virtual table tells SQLite how it wants to process the +** IN operator. +**

    +** +** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times +** within the same xBestIndex method call. ^For any given P,N pair, +** the return value from sqlite3_vtab_in(P,N,F) will always be the same +** within the same xBestIndex call. ^If the interface returns true +** (non-zero), that means that the constraint is an IN operator +** that can be processed all-at-once. ^If the constraint is not an IN +** operator or cannot be processed all-at-once, then the interface returns +** false. +** +** ^(All-at-once processing of the IN operator is selected if both of the +** following conditions are met: +** +**
      +**

    1. The P->aConstraintUsage[N].argvIndex value is set to a positive +** integer. This is how the virtual table tells SQLite that it wants to +** use the N-th constraint. +** +**

    2. The last call to sqlite3_vtab_in(P,N,F) for which F was +** non-negative had F>=1. +**

    )^ +** +** ^If either or both of the conditions above are false, then SQLite uses +** the traditional one-at-a-time processing strategy for the IN constraint. +** ^If both conditions are true, then the argvIndex-th parameter to the +** xFilter method will be an [sqlite3_value] that appears to be NULL, +** but which can be passed to [sqlite3_vtab_in_first()] and +** [sqlite3_vtab_in_next()] to find all values on the right-hand side +** of the IN constraint. +*/ +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); + +/* +** CAPI3REF: Find all elements on the right-hand side of an IN constraint. +** +** These interfaces are only useful from within the +** [xFilter|xFilter() method] of a [virtual table] implementation. +** The result of invoking these interfaces from any other context +** is undefined and probably harmful. +** +** The X parameter in a call to sqlite3_vtab_in_first(X,P) or +** sqlite3_vtab_in_next(X,P) should be one of the parameters to the +** xFilter method which invokes these routines, and specifically +** a parameter that was previously selected for all-at-once IN constraint +** processing use the [sqlite3_vtab_in()] interface in the +** [xBestIndex|xBestIndex method]. ^(If the X parameter is not +** an xFilter argument that was selected for all-at-once IN constraint +** processing, then these routines return [SQLITE_ERROR].)^ +** +** ^(Use these routines to access all values on the right-hand side +** of the IN constraint using code like the following: +** +** 
    +**    for(rc=sqlite3_vtab_in_first(pList, &pVal);
+**        rc==SQLITE_OK && pVal;
+**        rc=sqlite3_vtab_in_next(pList, &pVal)
+**    ){
+**      // do something with pVal
+**    }
+**    if( rc!=SQLITE_OK ){
+**      // an error has occurred
+**    }
+**
    )^ +** +** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P) +** routines return SQLITE_OK and set *P to point to the first or next value +** on the RHS of the IN constraint. ^If there are no more values on the +** right hand side of the IN constraint, then *P is set to NULL and these +** routines return [SQLITE_DONE]. ^The return value might be +** some other value, such as SQLITE_NOMEM, in the event of a malfunction. +** +** The *ppOut values returned by these routines are only valid until the +** next call to either of these routines or until the end of the xFilter +** method from which these routines were called. If the virtual table +** implementation needs to retain the *ppOut values for longer, it must make +** copies. The *ppOut values are [protected sqlite3_value|protected]. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut); +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); + +/* +** CAPI3REF: Constraint values in xBestIndex() +** METHOD: sqlite3_index_info +** +** This API may only be used from within the [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this interface +** from outside of an xBestIndex method are undefined and probably harmful. +** +** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within +** the [xBestIndex] method of a [virtual table] implementation, with P being +** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and +** J being a 0-based index into P->aConstraint[], then this routine +** attempts to set *V to the value of the right-hand operand of +** that constraint if the right-hand operand is known. ^If the +** right-hand operand is not known, then *V is set to a NULL pointer. +** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if +** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) +** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th +** constraint is not available. ^The sqlite3_vtab_rhs_value() interface +** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** something goes wrong. +** +** The sqlite3_vtab_rhs_value() interface is usually only successful if +** the right-hand operand of a constraint is a literal value in the original +** SQL statement. If the right-hand operand is an expression or a reference +** to some other column or a [host parameter], then sqlite3_vtab_rhs_value() +** will probably return [SQLITE_NOTFOUND]. +** +** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and +** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand. For such +** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^ +** +** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value +** and remains valid for the duration of the xBestIndex method call. +** ^When xBestIndex returns, the sqlite3_value object returned by +** sqlite3_vtab_rhs_value() is automatically deallocated. +** +** The "_rhs_" in the name of this routine is an abbreviation for +** "Right-Hand Side". +*/ +SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); + /* ** CAPI3REF: Conflict resolution modes ** KEYWORDS: {conflict resolution mode} @@ -9867,6 +10608,10 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    **
    ^The [sqlite3_int64] variable pointed to by the V parameter will be @@ -9894,12 +10639,24 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECTID
    **
    ^The "int" variable pointed to by the V parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_PARENTID]]
    SQLITE_SCANSTAT_PARENTID
    +**
    The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
    SQLITE_SCANSTAT_NCYCLE
    +**
    The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. **
    */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -9908,12 +10665,14 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. @@ -9924,19 +10683,25 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. -** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. +** +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ @@ -9946,6 +10711,19 @@ SQLITE_API int sqlite3_stmt_scanstatus( int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ ); +SQLITE_API int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 /* ** CAPI3REF: Zero Scan-Status Counters @@ -10036,6 +10814,10 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** function is not defined for operations on WITHOUT ROWID tables, or for ** DELETE operations on rowid tables. ** +** ^The sqlite3_preupdate_hook(D,C,P) function returns the P argument from +** the previous call on the same [database connection] D, or NULL for +** the first call on D. +** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces ** provide additional information about a preupdate event. These routines @@ -10075,7 +10857,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** When the [sqlite3_blob_write()] API is used to update a blob column, ** the pre-update hook is invoked with SQLITE_DELETE. This is because the ** in this case the new values are not available. In this case, when a -** callback made with op==SQLITE_DELETE is actuall a write using the +** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the ** pre-update hook is being invoked for some other reason, including a @@ -10336,6 +11118,13 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy ** of the database exists. ** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory ** allocation error occurs. @@ -10384,6 +11173,9 @@ SQLITE_API unsigned char *sqlite3_serialize( ** SQLite will try to increase the buffer size using sqlite3_realloc64() ** if writes on the database cause it to grow larger than M bytes. ** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the ** database is currently in a read transaction or is involved in a backup ** operation. @@ -10392,6 +11184,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** ** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then ** [sqlite3_free()] is invoked on argument P prior to returning. @@ -10441,6 +11240,19 @@ SQLITE_API int sqlite3_deserialize( # undef double #endif +#if defined(__wasi__) +# undef SQLITE_WASI +# define SQLITE_WASI 1 +# undef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ +# ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION +# endif +# ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +# endif +#endif + #if 0 } /* End of the 'extern "C"' block */ #endif @@ -10647,16 +11459,20 @@ SQLITE_API int sqlite3session_create( SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** **
    SQLITE_SESSION_OBJCONFIG_SIZE
    @@ -10672,12 +11488,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +**
    SQLITE_SESSION_OBJCONFIG_ROWID
    +** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -11438,6 +12263,18 @@ SQLITE_API int sqlite3changeset_concat( ); +/* +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + /* ** CAPI3REF: Changegroup Handle ** @@ -11484,6 +12321,38 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; */ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); +/* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
      +**
    • The name identified by the changeset, and +**
    • at least as many columns as recorded in the changeset, and +**
    • the primary key columns in the same position as recorded in +** the changeset. +**
    +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + /* ** CAPI3REF: Add A Changeset To A Changegroup ** METHOD: sqlite3_changegroup @@ -11552,16 +12421,45 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); ** If the new changeset contains changes to a table that is already present ** in the changegroup, then the number of columns and the position of the ** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the state -** of the final contents of the changegroup is undefined. +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. ** -** If no error occurs, SQLITE_OK is returned. +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. +** +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -11810,9 +12708,30 @@ SQLITE_API int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
    SQLITE_CHANGESETAPPLY_IGNORENOOP
    +** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
      +**
    • a delete change if the row being deleted cannot be found, +**
    • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
    • an insert change if all fields of the conflicting row match +** the row being inserted. +**
    +** +**
    SQLITE_CHANGESETAPPLY_FKNOACTION
    +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -12345,8 +13264,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -12378,8 +13297,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -12389,8 +13311,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -12406,12 +13330,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -12437,6 +13362,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -12551,6 +13480,39 @@ struct Fts5PhraseIter { ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ @@ -12588,6 +13550,13 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); }; /* @@ -12782,8 +13751,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -12831,7 +13800,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -12840,7 +13809,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -12848,7 +13817,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); @@ -12869,12 +13838,17 @@ struct fts5_api { /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* +** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory. +*/ +#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1 + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "config.h" +#include "sqlite_cfg.h" #define SQLITECONFIG_H 1 #endif @@ -12954,7 +13928,7 @@ struct fts5_api { ** level of recursion for each term. A stack overflow can result ** if the number of terms is too large. In practice, most SQL ** never has more than 3 or 4 terms. Use a value of 0 to disable -** any limit on the number of terms in a compount SELECT. +** any limit on the number of terms in a compound SELECT. */ #ifndef SQLITE_MAX_COMPOUND_SELECT # define SQLITE_MAX_COMPOUND_SELECT 500 @@ -13069,7 +14043,7 @@ struct fts5_api { ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT -# define SQLITE_MAX_PAGE_COUNT 1073741823 +# define SQLITE_MAX_PAGE_COUNT 0xfffffffe /* 4294967294 */ #endif /* @@ -13104,8 +14078,8 @@ struct fts5_api { #endif /* -** WAL mode depends on atomic aligned 32-bit loads and stores in a few -** places. The following macros try to make this explicit. +** A few places in the code require atomic load/store of aligned +** integer values. */ #ifndef __has_extension # define __has_extension(x) 0 /* compatibility with non-clang compilers */ @@ -13161,15 +14135,22 @@ struct fts5_api { #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) +# undef SQLITE_INLINE +# define SQLITE_INLINE #endif /* @@ -13191,6 +14172,29 @@ struct fts5_api { # endif #endif +/* +** Enable SQLITE_USE_SEH by default on MSVC builds. Only omit +** SEH support if the -DSQLITE_OMIT_SEH option is given. +*/ +#if defined(_MSC_VER) && !defined(SQLITE_OMIT_SEH) +# define SQLITE_USE_SEH 1 +#else +# undef SQLITE_USE_SEH +#endif + +/* +** Enable SQLITE_DIRECT_OVERFLOW_READ, unless the build explicitly +** disables it using -DSQLITE_DIRECT_OVERFLOW_READ=0 +*/ +#if defined(SQLITE_DIRECT_OVERFLOW_READ) && SQLITE_DIRECT_OVERFLOW_READ+1==1 + /* Disable if -DSQLITE_DIRECT_OVERFLOW_READ=0 */ +# undef SQLITE_DIRECT_OVERFLOW_READ +#else + /* In all other cases, enable */ +# define SQLITE_DIRECT_OVERFLOW_READ 1 +#endif + + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -13459,6 +14463,8 @@ struct fts5_api { # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -13690,78 +14696,80 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_SLASH 109 #define TK_REM 110 #define TK_CONCAT 111 -#define TK_COLLATE 112 -#define TK_BITNOT 113 -#define TK_ON 114 -#define TK_INDEXED 115 -#define TK_STRING 116 -#define TK_JOIN_KW 117 -#define TK_CONSTRAINT 118 -#define TK_DEFAULT 119 -#define TK_NULL 120 -#define TK_PRIMARY 121 -#define TK_UNIQUE 122 -#define TK_CHECK 123 -#define TK_REFERENCES 124 -#define TK_AUTOINCR 125 -#define TK_INSERT 126 -#define TK_DELETE 127 -#define TK_UPDATE 128 -#define TK_SET 129 -#define TK_DEFERRABLE 130 -#define TK_FOREIGN 131 -#define TK_DROP 132 -#define TK_UNION 133 -#define TK_ALL 134 -#define TK_EXCEPT 135 -#define TK_INTERSECT 136 -#define TK_SELECT 137 -#define TK_VALUES 138 -#define TK_DISTINCT 139 -#define TK_DOT 140 -#define TK_FROM 141 -#define TK_JOIN 142 -#define TK_USING 143 -#define TK_ORDER 144 -#define TK_GROUP 145 -#define TK_HAVING 146 -#define TK_LIMIT 147 -#define TK_WHERE 148 -#define TK_RETURNING 149 -#define TK_INTO 150 -#define TK_NOTHING 151 -#define TK_FLOAT 152 -#define TK_BLOB 153 -#define TK_INTEGER 154 -#define TK_VARIABLE 155 -#define TK_CASE 156 -#define TK_WHEN 157 -#define TK_THEN 158 -#define TK_ELSE 159 -#define TK_INDEX 160 -#define TK_ALTER 161 -#define TK_ADD 162 -#define TK_WINDOW 163 -#define TK_OVER 164 -#define TK_FILTER 165 -#define TK_COLUMN 166 -#define TK_AGG_FUNCTION 167 -#define TK_AGG_COLUMN 168 -#define TK_TRUEFALSE 169 -#define TK_ISNOT 170 -#define TK_FUNCTION 171 -#define TK_UMINUS 172 +#define TK_PTR 112 +#define TK_COLLATE 113 +#define TK_BITNOT 114 +#define TK_ON 115 +#define TK_INDEXED 116 +#define TK_STRING 117 +#define TK_JOIN_KW 118 +#define TK_CONSTRAINT 119 +#define TK_DEFAULT 120 +#define TK_NULL 121 +#define TK_PRIMARY 122 +#define TK_UNIQUE 123 +#define TK_CHECK 124 +#define TK_REFERENCES 125 +#define TK_AUTOINCR 126 +#define TK_INSERT 127 +#define TK_DELETE 128 +#define TK_UPDATE 129 +#define TK_SET 130 +#define TK_DEFERRABLE 131 +#define TK_FOREIGN 132 +#define TK_DROP 133 +#define TK_UNION 134 +#define TK_ALL 135 +#define TK_EXCEPT 136 +#define TK_INTERSECT 137 +#define TK_SELECT 138 +#define TK_VALUES 139 +#define TK_DISTINCT 140 +#define TK_DOT 141 +#define TK_FROM 142 +#define TK_JOIN 143 +#define TK_USING 144 +#define TK_ORDER 145 +#define TK_GROUP 146 +#define TK_HAVING 147 +#define TK_LIMIT 148 +#define TK_WHERE 149 +#define TK_RETURNING 150 +#define TK_INTO 151 +#define TK_NOTHING 152 +#define TK_FLOAT 153 +#define TK_BLOB 154 +#define TK_INTEGER 155 +#define TK_VARIABLE 156 +#define TK_CASE 157 +#define TK_WHEN 158 +#define TK_THEN 159 +#define TK_ELSE 160 +#define TK_INDEX 161 +#define TK_ALTER 162 +#define TK_ADD 163 +#define TK_WINDOW 164 +#define TK_OVER 165 +#define TK_FILTER 166 +#define TK_COLUMN 167 +#define TK_AGG_FUNCTION 168 +#define TK_AGG_COLUMN 169 +#define TK_TRUEFALSE 170 +#define TK_ISNOT 171 +#define TK_FUNCTION 172 #define TK_UPLUS 173 -#define TK_TRUTH 174 -#define TK_REGISTER 175 -#define TK_VECTOR 176 -#define TK_SELECT_COLUMN 177 -#define TK_IF_NULL_ROW 178 -#define TK_ASTERISK 179 -#define TK_SPAN 180 -#define TK_ERROR 181 -#define TK_SPACE 182 -#define TK_ILLEGAL 183 +#define TK_UMINUS 174 +#define TK_TRUTH 175 +#define TK_REGISTER 176 +#define TK_VECTOR 177 +#define TK_SELECT_COLUMN 178 +#define TK_IF_NULL_ROW 179 +#define TK_ASTERISK 180 +#define TK_SPAN 181 +#define TK_ERROR 182 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -13986,15 +14994,9 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ /* ** The datatype used to store estimates of the number of rows in a -** table or index. This is an unsigned integer type. For 99.9% of -** the world, a 32-bit integer is sufficient. But a 64-bit integer -** can be used at compile-time if desired. +** table or index. */ -#ifdef SQLITE_64BIT_STATS - typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ -#else - typedef u32 tRowcnt; /* 32-bit is the default */ -#endif +typedef u64 tRowcnt; /* ** Estimated quantities used for query planning are stored as 16-bit @@ -14029,7 +15031,7 @@ typedef INT16_TYPE LogEst; # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ - (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else @@ -14055,8 +15057,31 @@ typedef INT16_TYPE LogEst; ** the end of buffer S. This macro returns true if P points to something ** contained within the buffer S. */ -#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +/* +** P is one byte past the end of a large buffer. Return true if a span of bytes +** between S..E crosses the end of that buffer. In other words, return true +** if the sub-buffer S..E-1 overflows the buffer whose last byte is P-1. +** +** S is the start of the span. E is one byte past the end of end of span. +** +** P +** |-----------------| FALSE +** |-------| +** S E +** +** P +** |-----------------| +** |-------| TRUE +** S E +** +** P +** |-----------------| +** |-------| FALSE +** S E +*/ +#define SQLITE_OVERFLOW(P,S,E) (((uptr)(S)<(uptr)(P))&&((uptr)(E)>(uptr)(P))) /* ** Macros to determine whether the machine is big or little endian, @@ -14066,16 +15091,33 @@ typedef INT16_TYPE LogEst; ** using C-preprocessor macros. If that is unsuccessful, or if ** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined ** at run-time. +** +** If you are building SQLite on some obscure platform for which the +** following ifdef magic does not work, you can always include either: +** +** -DSQLITE_BYTEORDER=1234 +** +** or +** +** -DSQLITE_BYTEORDER=4321 +** +** to cause the build to work for little-endian or big-endian processors, +** respectively. */ -#ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) -# define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) || \ - defined(__ARMEB__) || defined(__AARCH64EB__) -# define SQLITE_BYTEORDER 4321 +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0 # endif @@ -14111,8 +15153,19 @@ typedef INT16_TYPE LogEst; /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. +** +** ROUND8() always does the rounding, for any argument. +** +** ROUND8P() assumes that the argument is already an integer number of +** pointers in size, and so it is a no-op on systems where the pointer +** size is 8. */ #define ROUND8(x) (((x)+7)&~7) +#if SQLITE_PTRSIZE==8 +# define ROUND8P(x) (x) +#else +# define ROUND8P(x) (((x)+7)&~7) +#endif /* ** Round down to the nearest multiple of 8 @@ -14129,9 +15182,9 @@ typedef INT16_TYPE LogEst; ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) #else -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif /* @@ -14175,24 +15228,49 @@ typedef INT16_TYPE LogEst; #endif /* -** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not -** the Select query generator tracing logic is turned on. +** TREETRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Abstract Syntax Tree tracing logic is turned on. */ #if !defined(SQLITE_AMALGAMATION) -SQLITE_PRIVATE u32 sqlite3SelectTrace; +SQLITE_PRIVATE u32 sqlite3TreeTrace; #endif #if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE)) -# define SELECTTRACE_ENABLED 1 -# define SELECTTRACE(K,P,S,X) \ - if(sqlite3SelectTrace&(K)) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ + || defined(SQLITE_ENABLE_TREETRACE)) +# define TREETRACE_ENABLED 1 +# define TREETRACE(K,P,S,X) \ + if(sqlite3TreeTrace&(K)) \ sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else -# define SELECTTRACE(K,P,S,X) -# define SELECTTRACE_ENABLED 0 +# define TREETRACE(K,P,S,X) +# define TREETRACE_ENABLED 0 #endif +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction +*/ + /* ** Macros for "wheretrace" */ @@ -14205,6 +15283,36 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; # define WHERETRACE(K,X) #endif +/* +** Bits for the sqlite3WhereTrace mask: +** +** (---any--) Top-level block structure +** 0x-------F High-level debug messages +** 0x----FFF- More detail +** 0xFFFF---- Low-level debug messages +** +** 0x00000001 Code generation +** 0x00000002 Solver +** 0x00000004 Solver costs +** 0x00000008 WhereLoop inserts +** +** 0x00000010 Display sqlite3_index_info xBestIndex calls +** 0x00000020 Range an equality scan metrics +** 0x00000040 IN operator decisions +** 0x00000080 WhereLoop cost adjustments +** 0x00000100 +** 0x00000200 Covering index decisions +** 0x00000400 OR optimization +** 0x00000800 Index scanner +** 0x00001000 More details associated with code generation +** 0x00002000 +** 0x00004000 Show all WHERE terms at key points +** 0x00008000 Show the full SELECT statement at key places +** +** 0x00010000 Show more detail when printing WHERE terms +** 0x00020000 Show WHERE terms returned from whereScanNext() +*/ + /* ** An instance of the following structure is used to store the busy-handler @@ -14225,7 +15333,7 @@ struct BusyHandler { /* ** Name of table that holds the database schema. ** -** The PREFERRED names are used whereever possible. But LEGACY is also +** The PREFERRED names are used wherever possible. But LEGACY is also ** used for backwards compatibility. ** ** 1. Queries can use either the PREFERRED or the LEGACY names @@ -14275,7 +15383,7 @@ struct BusyHandler { ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomFault) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomClear) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -14334,16 +15442,19 @@ typedef struct Column Column; typedef struct Cte Cte; typedef struct CteUse CteUse; typedef struct Db Db; +typedef struct DbClientData DbClientData; typedef struct DbFixer DbFixer; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct FKey FKey; +typedef struct FpDecode FpDecode; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -14351,10 +15462,12 @@ typedef struct Lookaside Lookaside; typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; +typedef struct OnOrUsing OnOrUsing; typedef struct Parse Parse; typedef struct ParseCleanup ParseCleanup; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RCStr RCStr; typedef struct RenameToken RenameToken; typedef struct Returning Returning; typedef struct RowSet RowSet; @@ -14403,10 +15516,12 @@ typedef struct With With; /* ** A bit in a Bitmask */ -#define MASKBIT(n) (((Bitmask)1)<<(n)) -#define MASKBIT64(n) (((u64)1)<<(n)) -#define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT64(n) (((u64)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0) +#define ALLBITS ((Bitmask)-1) +#define TOPBIT (((Bitmask)1)<<(BMS-1)) /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer @@ -14421,6 +15536,330 @@ typedef int VList; ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of +** +** SQLITE_OS_KV +** SQLITE_OS_OTHER +** SQLITE_OS_UNIX +** SQLITE_OS_WIN +** +** will defined to either 1 or 0. One of them will be 1. The others will be 0. +** If none of the macros are initially defined, then select either +** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform. +** +** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application +** must provide its own VFS implementation together with sqlite3_os_init() +** and sqlite3_os_end() routines. +*/ +#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ + !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +#endif +#if SQLITE_OS_OTHER+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_KV+1>1 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# define SQLITE_OMIT_LOAD_EXTENSION 1 +# define SQLITE_OMIT_WAL 1 +# define SQLITE_OMIT_DEPRECATED 1 +# undef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */ +# define SQLITE_DQS 0 +# define SQLITE_OMIT_SHARED_CACHE 1 +# define SQLITE_OMIT_AUTOINIT 1 +#endif +#if SQLITE_OS_UNIX+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_WIN+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +#endif + +#endif /* SQLITE_OS_SETUP_H */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h +*/ +#ifndef SQLITE_MAX_PATHLEN +# define SQLITE_MAX_PATHLEN FILENAME_MAX +#endif + +/* Maximum number of symlinks that will be resolved while trying to +** expand a filename in xFullPathname() in the VFS. +*/ +#ifndef SQLITE_MAX_SYMLINK +# define SQLITE_MAX_SYMLINK 200 +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ + +/************** End of os.h **************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pager.h in the middle of sqliteInt.h *****************/ /************** Begin file pager.h *******************************************/ /* @@ -14468,14 +15907,15 @@ typedef struct Pager Pager; typedef struct PgHdr DbPage; /* -** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is +** Page number PAGER_SJ_PGNO is never used in an SQLite database (it is ** reserved for working around a windows/posix incompatibility). It is ** used in the journal to signify that the remainder of the journal file ** is devoted to storing a super-journal name - there are no more pages to ** roll back. See comments for function writeSuperJournal() in pager.c ** for details. */ -#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) +#define PAGER_SJ_PGNO_COMPUTED(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) +#define PAGER_SJ_PGNO(x) ((x)->lckPgno) /* ** Allowed values for the flags parameter to sqlite3PagerOpen(). @@ -14555,7 +15995,7 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); #ifdef SQLITE_HAS_CODEC SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*, Pager*); -#endif +#endif /* SQLITE_HAS_CODEC */ SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager*, Pgno); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); @@ -14643,7 +16083,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); -SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, u64*); SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); @@ -14654,7 +16094,7 @@ SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); -#endif +#endif /* defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) */ /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) @@ -14671,6 +16111,10 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); # define enable_simulated_io_errors() #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager*); +#endif + #endif /* SQLITE_PAGER_H */ /************** End of pager.h ***********************************************/ @@ -14862,7 +16306,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); ** reduce network bandwidth. ** ** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use +** standard SQLite. The other hints are provided for extensions that use ** the SQLite parser and code generator but substitute their own storage ** engine. */ @@ -15000,15 +16444,22 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); -#endif SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,Pgno*aRoot,int nRoot,int,int*); +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ +); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); @@ -15047,6 +16498,8 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree*); + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the @@ -15159,19 +16612,18 @@ struct VdbeOp { #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif - int (*xAdvance)(BtCursor *, int); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS char *zComment; /* Comment to improve readability */ #endif -#ifdef VDBE_PROFILE - u32 cnt; /* Number of times this instruction was executed */ - u64 cycles; /* Total time spent executing this instruction */ -#endif #ifdef SQLITE_VDBE_COVERAGE u32 iSrcLine; /* Source-code line that generated this opcode ** with flags in the upper 8 bits */ #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; +#endif }; typedef struct VdbeOp VdbeOp; @@ -15210,21 +16662,20 @@ typedef struct VdbeOpList VdbeOpList; #define P4_COLLSEQ (-2) /* P4 is a pointer to a CollSeq structure */ #define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ #define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ -#define P4_ADVANCE (-5) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_TABLE (-6) /* P4 is a pointer to a Table structure */ +#define P4_TABLE (-5) /* P4 is a pointer to a Table structure */ /* Above do not own any resources. Must free those below */ -#define P4_FREE_IF_LE (-7) -#define P4_DYNAMIC (-7) /* Pointer to memory from sqliteMalloc() */ -#define P4_FUNCDEF (-8) /* P4 is a pointer to a FuncDef structure */ -#define P4_KEYINFO (-9) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ -#define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ -#define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ -#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ -#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ -#define P4_DYNBLOB (-17) /* Pointer to memory from sqliteMalloc() */ +#define P4_FREE_IF_LE (-6) +#define P4_DYNAMIC (-6) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-7) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-8) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-9) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-10) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-11) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -15269,53 +16720,53 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Savepoint 0 #define OP_AutoCommit 1 #define OP_Transaction 2 -#define OP_SorterNext 3 /* jump */ -#define OP_Prev 4 /* jump */ -#define OP_Next 5 /* jump */ -#define OP_Checkpoint 6 -#define OP_JournalMode 7 -#define OP_Vacuum 8 -#define OP_VFilter 9 /* jump, synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 10 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 11 /* jump */ -#define OP_Gosub 12 /* jump */ -#define OP_InitCoroutine 13 /* jump */ -#define OP_Yield 14 /* jump */ -#define OP_MustBeInt 15 /* jump */ -#define OP_Jump 16 /* jump */ -#define OP_Once 17 /* jump */ -#define OP_If 18 /* jump */ +#define OP_Checkpoint 3 +#define OP_JournalMode 4 +#define OP_Vacuum 5 +#define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ +#define OP_Init 8 /* jump0, synopsis: Start at P2 */ +#define OP_Goto 9 /* jump */ +#define OP_Gosub 10 /* jump */ +#define OP_InitCoroutine 11 /* jump0 */ +#define OP_Yield 12 /* jump0 */ +#define OP_MustBeInt 13 /* jump0 */ +#define OP_Jump 14 /* jump */ +#define OP_Once 15 /* jump */ +#define OP_If 16 /* jump */ +#define OP_IfNot 17 /* jump */ +#define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_IfNot 20 /* jump */ -#define OP_IsNullOrType 21 /* jump, synopsis: if typeof(r[P1]) IN (P3,5) goto P2 */ -#define OP_IfNullRow 22 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 24 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 25 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IfNotOpen 27 /* jump, synopsis: if( !csr[P1] ) goto P2 */ -#define OP_IfNoHope 28 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NoConflict 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NotFound 30 /* jump, synopsis: key=r[P3@P4] */ -#define OP_Found 31 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 32 /* jump, synopsis: intkey=r[P3] */ -#define OP_NotExists 33 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 34 /* jump */ -#define OP_IfSmaller 35 /* jump */ -#define OP_SorterSort 36 /* jump */ -#define OP_Sort 37 /* jump */ -#define OP_Rewind 38 /* jump */ -#define OP_IdxLE 39 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGE 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 21 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ +#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 32 /* jump0 */ +#define OP_IfSizeBetween 33 /* jump */ +#define OP_SorterSort 34 /* jump */ +#define OP_Sort 35 /* jump */ +#define OP_Rewind 36 /* jump0 */ +#define OP_SorterNext 37 /* jump */ +#define OP_Prev 38 /* jump */ +#define OP_Next 39 /* jump */ +#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_RowSetRead 45 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 46 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 47 /* jump */ -#define OP_FkIfZero 48 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 49 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 48 /* jump0 */ +#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -15325,49 +16776,49 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ #define OP_ElseEq 58 /* jump, same as TK_ESCAPE */ -#define OP_IfNotZero 59 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 60 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 61 /* jump */ -#define OP_VNext 62 /* jump */ -#define OP_Init 63 /* jump, synopsis: Start at P2 */ -#define OP_PureFunc 64 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Function 65 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Return 66 -#define OP_EndCoroutine 67 -#define OP_HaltIfNull 68 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 69 -#define OP_Integer 70 /* synopsis: r[P2]=P1 */ -#define OP_Int64 71 /* synopsis: r[P2]=P4 */ -#define OP_String 72 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 73 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 74 /* synopsis: r[P1]=NULL */ -#define OP_Blob 75 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 76 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 77 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 78 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 79 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 80 /* synopsis: r[P2]=r[P1] */ -#define OP_ChngCntRow 81 /* synopsis: output=r[P1] */ -#define OP_ResultRow 82 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 83 -#define OP_AddImm 84 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 85 -#define OP_Cast 86 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 87 -#define OP_Compare 88 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_IsTrue 89 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_ZeroOrNull 90 /* synopsis: r[P2] = 0 OR NULL */ -#define OP_Offset 91 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 92 /* synopsis: r[P3]=PX */ -#define OP_TypeCheck 93 /* synopsis: typecheck(r[P1@P2]) */ -#define OP_Affinity 94 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 95 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 96 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 97 -#define OP_SetCookie 98 -#define OP_ReopenIdx 99 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 100 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 101 /* synopsis: root=P2 iDb=P3 */ +#define OP_IfPos 59 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 62 /* jump */ +#define OP_VNext 63 /* jump */ +#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 67 +#define OP_EndCoroutine 68 +#define OP_HaltIfNull 69 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 70 +#define OP_Integer 71 /* synopsis: r[P2]=P1 */ +#define OP_Int64 72 /* synopsis: r[P2]=P4 */ +#define OP_String 73 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 74 /* synopsis: r[P2]=NULL */ +#define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ +#define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 82 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 83 +#define OP_ResultRow 84 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 85 +#define OP_AddImm 86 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 87 +#define OP_Cast 88 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 89 +#define OP_Compare 90 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 91 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 92 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 93 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 94 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 95 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 96 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 97 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 98 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 99 +#define OP_SetCookie 100 +#define OP_ReopenIdx 101 /* synopsis: root=P2 iDb=P3 */ #define OP_BitAnd 102 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 103 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 104 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 159 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 160 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 161 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 162 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 163 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 164 -#define OP_CursorLock 165 -#define OP_CursorUnlock 166 -#define OP_TableLock 167 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 168 -#define OP_VCreate 169 -#define OP_VDestroy 170 -#define OP_VOpen 171 -#define OP_VColumn 172 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 173 -#define OP_Pagecount 174 -#define OP_MaxPgcnt 175 -#define OP_Trace 176 -#define OP_CursorHint 177 -#define OP_ReleaseReg 178 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 179 -#define OP_Explain 180 -#define OP_Abortable 181 +#define OP_OpenRead 112 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 113 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitNot 114 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenDup 115 +#define OP_OpenAutoindex 116 /* synopsis: nColumn=P2 */ +#define OP_String8 117 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_OpenEphemeral 118 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 119 +#define OP_SequenceTest 120 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 121 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 122 +#define OP_ColumnsUsed 123 +#define OP_SeekScan 124 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 125 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 126 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 127 /* synopsis: r[P2]=rowid */ +#define OP_Insert 128 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 129 +#define OP_Delete 130 +#define OP_ResetCount 131 +#define OP_SorterCompare 132 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 133 /* synopsis: r[P2]=data */ +#define OP_RowData 134 /* synopsis: r[P2]=data */ +#define OP_Rowid 135 /* synopsis: r[P2]=PX rowid of P1 */ +#define OP_NullRow 136 +#define OP_SeekEnd 137 +#define OP_IdxInsert 138 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 139 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 140 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 141 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 142 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 143 +#define OP_Destroy 144 +#define OP_Clear 145 +#define OP_ResetSorter 146 +#define OP_CreateBtree 147 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 148 +#define OP_ParseSchema 149 +#define OP_LoadAnalysis 150 +#define OP_DropTable 151 +#define OP_DropIndex 152 +#define OP_Real 153 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_DropTrigger 154 +#define OP_IntegrityCk 155 +#define OP_RowSetAdd 156 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 157 +#define OP_FkCounter 158 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 159 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 160 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 161 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 162 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 164 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 165 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 166 +#define OP_CursorLock 167 +#define OP_CursorUnlock 168 +#define OP_TableLock 169 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 170 +#define OP_VCreate 171 +#define OP_VDestroy 172 +#define OP_VOpen 173 +#define OP_VCheck 174 +#define OP_VInitIn 175 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 176 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 177 +#define OP_Pagecount 178 +#define OP_MaxPgcnt 179 +#define OP_ClrSubtype 180 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 181 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 182 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 183 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 184 +#define OP_CursorHint 185 +#define OP_ReleaseReg 186 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 187 +#define OP_Explain 188 +#define OP_Abortable 189 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -15459,30 +16918,33 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_IN3 0x08 /* in3: P3 is an input */ #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ +#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ -/* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ -/* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x03, 0x01, 0x09,\ -/* 24 */ 0x09, 0x09, 0x09, 0x01, 0x09, 0x09, 0x09, 0x09,\ -/* 32 */ 0x09, 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x23, 0x0b, 0x01,\ -/* 48 */ 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x01, 0x01,\ -/* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ -/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ -/* 80 */ 0x10, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00,\ -/* 88 */ 0x00, 0x12, 0x1e, 0x20, 0x00, 0x00, 0x00, 0x00,\ -/* 96 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x26, 0x26,\ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ +/* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ +/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ +/* 48 */ 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ +/* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ +/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ +/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ +/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40, 0x00,\ +/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x00, 0x12, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ -/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,\ -/* 136 */ 0x04, 0x04, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00,\ -/* 144 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x10, 0x00, 0x06, 0x10, 0x00, 0x04, 0x1a, 0x00,\ -/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ -/* 176 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 112 */ 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40, 0x00,\ +/* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ +/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x50,\ +/* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ +/* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ +/* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x50,\ +/* 176 */ 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12, 0x00,\ +/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -15490,7 +16952,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 63 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 64 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -15528,19 +16990,27 @@ SQLITE_PRIVATE void sqlite3VdbeVerifyNoResultRow(Vdbe *p); #endif #if defined(SQLITE_DEBUG) SQLITE_PRIVATE void sqlite3VdbeVerifyAbortable(Vdbe *p, int); +SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn(Vdbe*,int,int,int); #else # define sqlite3VdbeVerifyAbortable(A,B) +# define sqlite3VdbeNoJumpsOutsideSubrtn(A,B,C,D) #endif SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); #ifndef SQLITE_OMIT_EXPLAIN -SQLITE_PRIVATE void sqlite3VdbeExplain(Parse*,u8,const char*,...); +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse*,u8,const char*,...); SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse*); SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse*); # define ExplainQueryPlan(P) sqlite3VdbeExplain P +# ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define ExplainQueryPlan2(V,P) (V = sqlite3VdbeExplain P) +# else +# define ExplainQueryPlan2(V,P) ExplainQueryPlan(P) +# endif # define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) # define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) #else # define ExplainQueryPlan(P) +# define ExplainQueryPlan2(V,P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ @@ -15556,6 +17026,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe*, int); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); @@ -15570,11 +17041,11 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); @@ -15616,6 +17087,8 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); #ifdef SQLITE_ENABLE_BYTECODE_VTAB SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); @@ -15648,7 +17121,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** The VdbeCoverage macros are used to set a coverage testing point ** for VDBE branch instructions. The coverage testing points are line ** numbers in the sqlite3.c source file. VDBE branch coverage testing -** only works with an amalagmation build. That's ok since a VDBE branch +** only works with an amalgamation build. That's ok since a VDBE branch ** coverage build designed for testing the test suite only. No application ** should ever ship with VDBE branch coverage measuring turned on. ** @@ -15666,7 +17139,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** // NULL option is not possible ** ** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested -** // in distingishing equal and not-equal. +** // in distinguishing equal and not-equal. ** ** Every VDBE branch operation must be tagged with one of the macros above. ** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and @@ -15676,7 +17149,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** During testing, the test application will invoke ** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback ** routine that is invoked as each bytecode branch is taken. The callback -** contains the sqlite3.c source line number ov the VdbeCoverage macro and +** contains the sqlite3.c source line number of the VdbeCoverage macro and ** flags to indicate whether or not the branch was taken. The test application ** is responsible for keeping track of this and reporting byte-code branches ** that are never taken. @@ -15712,14 +17185,22 @@ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); #else -# define sqlite3VdbeScanStatus(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e,f) +# define sqlite3VdbeScanStatusRange(a,b,c,d) +# define sqlite3VdbeScanStatusCounters(a,b,c,d) #endif #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + #endif /* SQLITE_VDBE_H */ /************** End of vdbe.h ************************************************/ @@ -15768,7 +17249,7 @@ struct PgHdr { ** private to pcache.c and should not be accessed by other modules. ** pCache is grouped with the public elements for efficiency. */ - i16 nRef; /* Number of users of this page */ + i64 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ /* NB: pDirtyNext and pDirtyPrev are undefined if the @@ -15849,12 +17330,12 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); /* Return the total number of outstanding page references */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache*); /* Increment the reference count of an existing page */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr*); /* Return the total number of pages stored in the cache */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); @@ -15919,290 +17400,6 @@ SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ -/************** Include os.h in the middle of sqliteInt.h ********************/ -/************** Begin file os.h **********************************************/ -/* -** 2001 September 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file (together with is companion C source-code file -** "os.c") attempt to abstract the underlying operating system so that -** the SQLite library will work on both POSIX and windows systems. -** -** This header file is #include-ed by sqliteInt.h and thus ends up -** being included by every source file. -*/ -#ifndef _SQLITE_OS_H_ -#define _SQLITE_OS_H_ - -/* -** Attempt to automatically detect the operating system and setup the -** necessary pre-processor macros for it. -*/ -/************** Include os_setup.h in the middle of os.h *********************/ -/************** Begin file os_setup.h ****************************************/ -/* -** 2013 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains pre-processor directives related to operating system -** detection and/or setup. -*/ -#ifndef SQLITE_OS_SETUP_H -#define SQLITE_OS_SETUP_H - -/* -** Figure out if we are dealing with Unix, Windows, or some other operating -** system. -** -** After the following block of preprocess macros, all of SQLITE_OS_UNIX, -** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the three will be 1. The other two will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ - defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#endif /* SQLITE_OS_SETUP_H */ - -/************** End of os_setup.h ********************************************/ -/************** Continuing where we left off in os.h *************************/ - -/* If the SET_FULLSYNC macro is not defined above, then make it -** a no-op -*/ -#ifndef SET_FULLSYNC -# define SET_FULLSYNC(x,y) -#endif - -/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h -*/ -#ifndef SQLITE_MAX_PATHLEN -# define SQLITE_MAX_PATHLEN FILENAME_MAX -#endif - -/* -** The default size of a disk sector -*/ -#ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 4096 -#endif - -/* -** Temporary files are named starting with this prefix followed by 16 random -** alphanumeric characters, and no file extension. They are stored in the -** OS's standard temporary file directory, and are deleted prior to exit. -** If sqlite is being embedded in another program, you may wish to change the -** prefix to reflect your program's name, so that if your program exits -** prematurely, old temporary files can be easily identified. This can be done -** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. -** -** 2006-10-31: The default prefix used to be "sqlite_". But then -** Mcafee started using SQLite in their anti-virus product and it -** started putting files with the "sqlite" name in the c:/temp folder. -** This annoyed many windows users. Those users would then do a -** Google search for "sqlite", find the telephone numbers of the -** developers and call to wake them up at night and complain. -** For this reason, the default name prefix is changed to be "sqlite" -** spelled backwards. So the temp files are still identified, but -** anybody smart enough to figure out the code is also likely smart -** enough to know that calling the developer will not help get rid -** of the file. -*/ -#ifndef SQLITE_TEMP_FILE_PREFIX -# define SQLITE_TEMP_FILE_PREFIX "etilqs_" -#endif - -/* -** The following values may be passed as the second argument to -** sqlite3OsLock(). The various locks exhibit the following semantics: -** -** SHARED: Any number of processes may hold a SHARED lock simultaneously. -** RESERVED: A single process may hold a RESERVED lock on a file at -** any time. Other processes may hold and obtain new SHARED locks. -** PENDING: A single process may hold a PENDING lock on a file at -** any one time. Existing SHARED locks may persist, but no new -** SHARED locks may be obtained by other processes. -** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. -** -** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a -** process that requests an EXCLUSIVE lock may actually obtain a PENDING -** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to -** sqlite3OsLock(). -*/ -#define NO_LOCK 0 -#define SHARED_LOCK 1 -#define RESERVED_LOCK 2 -#define PENDING_LOCK 3 -#define EXCLUSIVE_LOCK 4 - -/* -** File Locking Notes: (Mostly about windows but also some info for Unix) -** -** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because -** those functions are not available. So we use only LockFile() and -** UnlockFile(). -** -** LockFile() prevents not just writing but also reading by other processes. -** A SHARED_LOCK is obtained by locking a single randomly-chosen -** byte out of a specific range of bytes. The lock byte is obtained at -** random so two separate readers can probably access the file at the -** same time, unless they are unlucky and choose the same lock byte. -** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. -** There can only be one writer. A RESERVED_LOCK is obtained by locking -** a single byte of the file that is designated as the reserved lock byte. -** A PENDING_LOCK is obtained by locking a designated byte different from -** the RESERVED_LOCK byte. -** -** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, -** which means we can use reader/writer locks. When reader/writer locks -** are used, the lock is placed on the same range of bytes that is used -** for probabilistic locking in Win95/98/ME. Hence, the locking scheme -** will support two or more Win95 readers or two or more WinNT readers. -** But a single Win95 reader will lock out all WinNT readers and a single -** WinNT reader will lock out all other Win95 readers. -** -** The following #defines specify the range of bytes used for locking. -** SHARED_SIZE is the number of bytes available in the pool from which -** a random byte is selected for a shared lock. The pool of bytes for -** shared locks begins at SHARED_FIRST. -** -** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possibility of having -** clients on win95, winNT, and unix all talking to the same shared file -** and all locking correctly. To do so would require that samba (or whatever -** tool is being used for file sharing) implements locks correctly between -** windows and unix. I'm guessing that isn't likely to happen, but by -** using the same locking range we are at least open to the possibility. -** -** Locking in windows is manditory. For this reason, we cannot store -** actual data in the bytes used for locking. The pager never allocates -** the pages involved in locking therefore. SHARED_SIZE is selected so -** that all locks will fit on a single page even at the minimum page size. -** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE -** is set high so that we don't have to allocate an unused page except -** for very large databases. But one should test the page skipping logic -** by setting PENDING_BYTE low and running the entire regression suite. -** -** Changing the value of PENDING_BYTE results in a subtly incompatible -** file format. Depending on how it is changed, you might not notice -** the incompatibility right away, even running a full regression test. -** The default location of PENDING_BYTE is the first byte past the -** 1GB boundary. -** -*/ -#ifdef SQLITE_OMIT_WSD -# define PENDING_BYTE (0x40000000) -#else -# define PENDING_BYTE sqlite3PendingByte -#endif -#define RESERVED_BYTE (PENDING_BYTE+1) -#define SHARED_FIRST (PENDING_BYTE+2) -#define SHARED_SIZE 510 - -/* -** Wrapper around OS specific sqlite3_os_init() function. -*/ -SQLITE_PRIVATE int sqlite3OsInit(void); - -/* -** Functions for accessing sqlite3_file methods -*/ -SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); -SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); -SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); -SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); -SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); -#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 -SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -#ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); -SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); -SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); -#endif /* SQLITE_OMIT_WAL */ -SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); - - -/* -** Functions for accessing sqlite3_vfs methods -*/ -SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); -SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); -SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); -#ifndef SQLITE_OMIT_LOAD_EXTENSION -SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); -SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); -SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ -SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); -SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); - -/* -** Convenience functions for opening and closing files using -** sqlite3_malloc() to obtain space for the file-handle structure. -*/ -SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); - -#endif /* _SQLITE_OS_H_ */ - -/************** End of os.h **************************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include mutex.h in the middle of sqliteInt.h *****************/ /************** Begin file mutex.h *******************************************/ /* @@ -16291,7 +17488,7 @@ SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); /* ** Default synchronous levels. ** -** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** Note that (for historical reasons) the PAGER_SYNCHRONOUS_* macros differ ** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. ** ** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS @@ -16330,7 +17527,7 @@ struct Db { ** An instance of the following structure stores a database schema. ** ** Most Schema objects are associated with a Btree. The exception is -** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** the Schema for the TEMP database (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. ** @@ -16441,13 +17638,14 @@ struct Lookaside { LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE - LookasideSlot *pSmallInit; /* List of small buffers not prediously used */ + LookasideSlot *pSmallInit; /* List of small buffers not previously used */ LookasideSlot *pSmallFree; /* List of available small buffers */ void *pMiddle; /* First byte past end of full-size buffers and ** the first byte of LOOKASIDE_SMALL buffers */ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ + void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */ }; struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ @@ -16457,7 +17655,7 @@ struct LookasideSlot { #define EnableLookaside db->lookaside.bDisable--;\ db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue -/* Size of the smaller allocations in two-size lookside */ +/* Size of the smaller allocations in two-size lookaside */ #ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE # define LOOKASIDE_SMALL 0 #else @@ -16478,6 +17676,10 @@ struct FuncDefHash { }; #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) +#if defined(SQLITE_USER_AUTHENTICATION) +# warning "The SQLITE_USER_AUTHENTICATION extension is deprecated. \ + See ext/userauth/user-auth.txt for details." +#endif #ifdef SQLITE_USER_AUTHENTICATION /* ** Information held in the "sqlite3" database connection object and used @@ -16534,7 +17736,20 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); */ #define SQLITE_MAX_DB (SQLITE_MAX_ATTACHED+2) +#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK typedef void (*sqlite3_xDropTableHandle)(sqlite3*, const char*, const char*); +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ + +#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) +typedef struct CodecParameter { + int kdfIter; + int pageSize; + u8 cipher; + u8 hmacAlgo; + u8 kdfAlgo; + u8 reserved; +} CodecParameter; +#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ /* ** Each database connection is an instance of the following structure. @@ -16553,6 +17768,7 @@ struct sqlite3 { u32 nSchemaLock; /* Do not reset the schema when non-zero */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ + int errByteOffset; /* Byte offset of error in SQL statement */ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ u32 dbOptFlags; /* Flags to enable/disable optimizations */ @@ -16658,6 +17874,7 @@ struct sqlite3 { i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ + DbClientData *pDbData; /* sqlite3_set_clientdata() content */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -16683,7 +17900,10 @@ struct sqlite3 { char *mDropTableName; char *mDropSchemaName; sqlite3_xDropTableHandle xDropTableHandle; /* User drop table callback */ -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ +#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) + CodecParameter codecParm; +#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ }; /* @@ -16719,7 +17939,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -16745,6 +17965,8 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_FkNoAction HI(0x00008) /* Treat all FK as NO ACTION */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -16785,7 +18007,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -16793,6 +18015,16 @@ struct sqlite3 { #define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */ #define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */ /* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */ +#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */ +#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */ +#define SQLITE_BalancedMerge 0x00200000 /* Balance multi-way merges */ +#define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ +#define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ + /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ +#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ +#define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -16875,10 +18107,17 @@ struct FuncDestructor { ** SQLITE_FUNC_ANYORDER == NC_OrderAgg == SF_OrderByReqd ** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG +** SQLITE_FUNC_BYTELEN == OPFLAG_BYTELENARG ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API ** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API -** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS +** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS -- opposite meanings!!! ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API +** +** Note that even though SQLITE_FUNC_UNSAFE and SQLITE_INNOCUOUS have the +** same bit value, their meanings are inverted. SQLITE_FUNC_UNSAFE is +** used internally and if set means that the function has side effects. +** SQLITE_INNOCUOUS is used by application code and means "not unsafe". +** See multiple instances of tag-20230109-1. */ #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ #define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ @@ -16887,6 +18126,7 @@ struct FuncDestructor { #define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ #define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ #define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_BYTELEN 0x00c0 /* Built-in octet_length() function */ #define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ /* 0x0200 -- available for reuse */ #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ @@ -16895,14 +18135,15 @@ struct FuncDestructor { #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a ** single query - might change over time */ #define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ -#define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ +#define SQLITE_FUNC_RUNONLY 0x8000 /* Cannot be used by valueFromFunction */ #define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ #define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ -#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ +/* SQLITE_SUBTYPE 0x00100000 // Consumer of subtypes */ #define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ #define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ #define SQLITE_FUNC_BUILTIN 0x00800000 /* This is a built-in function */ +/* SQLITE_RESULT_SUBTYPE 0x01000000 // Generator of subtypes */ #define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */ /* Identifier numbers for each in-line function */ @@ -16912,6 +18153,7 @@ struct FuncDestructor { #define INLINEFUNC_expr_compare 3 #define INLINEFUNC_affinity 4 #define INLINEFUNC_iif 5 +#define INLINEFUNC_sqlite_offset 6 #define INLINEFUNC_unlikely 99 /* Default case */ /* @@ -16966,7 +18208,7 @@ struct FuncDestructor { ** are interpreted in the same way as the first 4 parameters to ** FUNCTION(). ** -** WFUNCTION(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) +** WAGGREGATE(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters ** are interpreted in the same way as the first 4 parameters to @@ -16993,6 +18235,11 @@ struct FuncDestructor { #define MFUNCTION(zName, nArg, xPtr, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ xPtr, 0, xFunc, 0, 0, 0, #zName, {0} } +#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\ + SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\ + ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \ + SQLITE_INT_TO_PTR(iArg|((bJsonB)*JSON_BLOB)),0,xFunc,0, 0, 0, #zName, {0} } #define INLINE_FUNC(zName, nArg, iArg, mFlags) \ {nArg, SQLITE_FUNC_BUILTIN|\ SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ @@ -17134,6 +18381,7 @@ struct Column { #define COLFLAG_NOTAVAIL 0x0080 /* STORED column not yet calculated */ #define COLFLAG_BUSY 0x0100 /* Blocks recursion on GENERATED columns */ #define COLFLAG_HASCOLL 0x0200 /* Has collating sequence name in zCnName */ +#define COLFLAG_NOEXPAND 0x0400 /* Omit this column when expanding "*" */ #define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ #define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ @@ -17181,6 +18429,7 @@ struct CollSeq { #define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ +#define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -17251,6 +18500,7 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ @@ -17330,8 +18580,7 @@ struct Table { #define TF_HasStored 0x00000040 /* Has one or more STORED columns */ #define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ #define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ -#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by - ** Index.aiRowLogEst[] values */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ @@ -17359,7 +18608,7 @@ struct Table { #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) # define ExprIsVtab(X) \ - ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB) + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else # define IsVirtual(X) 0 # define ExprIsVtab(X) 0 @@ -17387,6 +18636,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -17458,7 +18716,7 @@ struct FKey { ** foreign key. ** ** The OE_Default value is a place holder that means to use whatever -** conflict resolution algorthm is required from context. +** conflict resolution algorithm is required from context. ** ** The following symbolic values are used to record which type ** of conflict resolution action to take. @@ -17540,6 +18798,11 @@ struct KeyInfo { struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ Mem *aMem; /* Values */ + union { + char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ + i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ + } u; + int n; /* Cache of aMem[0].n used by vdbeRecordCompareString() */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ @@ -17571,10 +18834,22 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution -** algorithm to employ whenever an attempt is made to insert a non-unique +** and the value of Index.onError indicates which conflict resolution +** algorithm to employ when an attempt is made to insert a non-unique ** element. ** +** The colNotIdxed bitmask is used in combination with SrcItem.colUsed +** for a fast test to see if an index can serve as a covering index. +** colNotIdxed has a 1 bit for every column of the original table that +** is *not* available in the index. Thus the expression +** "colUsed & colNotIdxed" will be non-zero if the index is not a +** covering index. The most significant bit of of colNotIdxed will always +** be true (note-20221022-a). If a column beyond the 63rd column of the +** table is used, the "colUsed & colNotIdxed" test will always be non-zero +** and we have to assume either that the index is not covering, or use +** an alternative (slower) algorithm to determine whether or not +** the index is covering. +** ** While parsing a CREATE TABLE or CREATE INDEX statement in order to ** generate VDBE code (as opposed to parsing one read from an sqlite_schema ** table as part of parsing an existing database schema), transient instances @@ -17607,18 +18882,22 @@ struct Index { unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ + int mxSample; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif - Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ + Bitmask colNotIdxed; /* Unindexed columns in pTab */ }; /* @@ -17693,16 +18972,15 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iMem; /* Memory location that acts as accumulator */ i16 iColumn; /* Column number within the source table */ i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; @@ -17713,14 +18991,31 @@ struct AggInfo { struct AggInfo_func { /* For each aggregate function */ Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ + int iOBTab; /* Ephemeral table to implement ORDER BY */ + u8 bOBPayload; /* iOBTab has payload columns separate from key */ + u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ + u8 bUseSubtype; /* Transfer subtype info through sorter */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ +#ifdef SQLITE_DEBUG + Select *pSelect; /* SELECT statement that this AggInfo supports */ +#endif }; +/* +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. +*/ +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater @@ -17840,14 +19135,17 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood - ** TK_IN: ephemerial table holding RHS + ** TK_IN: ephemeral table holding RHS ** TK_SELECT_COLUMN: Number of columns on the LHS ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). ** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ - int iRightJoinTable; /* If EP_FromJoin, the right table of the join */ + union { + int iJoin; /* If EP_OuterON or EP_InnerON, the right table */ + int iOfst; /* else: start of token from start of statement */ + } w; AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ union { Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL @@ -17866,29 +19164,29 @@ struct Expr { ** EP_Agg == NC_HasAgg == SF_HasAgg ** EP_Win == NC_HasWin */ -#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ -#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ -#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ -#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ +#define EP_OuterON 0x000001 /* Originates in ON/USING clause of outer join */ +#define EP_InnerON 0x000002 /* Originates in ON/USING of an inner join */ +#define EP_Distinct 0x000004 /* Aggregate function with DISTINCT keyword */ +#define EP_HasFunc 0x000008 /* Contains one or more functions of any kind */ #define EP_Agg 0x000010 /* Contains one or more aggregate functions */ -#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ -#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ -#define EP_Commuted 0x000200 /* Comparison operator has been commuted */ -#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ -#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ -#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_FixedCol 0x000020 /* TK_Column with a known fixed value */ +#define EP_VarSelect 0x000040 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000080 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000100 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000200 /* Tree contains a TK_COLLATE operator */ +#define EP_Commuted 0x000400 /* Comparison operator has been commuted */ +#define EP_IntValue 0x000800 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x001000 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x002000 /* Operator does not contribute to affinity */ +#define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_Win 0x008000 /* Contains window functions */ -#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ -#define EP_IfNullRow 0x020000 /* The TK_IF_NULL_ROW opcode */ -#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ -#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ -#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ - /* 0x400000 // Available */ +#define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_FullSize 0x020000 /* Expr structure must remain full sized */ +#define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ +#define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ +#define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x200000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x400000 /* Tree contains a TK_SELECT operator */ #define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ #define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ #define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ @@ -17911,14 +19209,17 @@ struct Expr { #define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) #define ExprSetProperty(E,P) (E)->flags|=(P) #define ExprClearProperty(E,P) (E)->flags&=~(P) -#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) -#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse) +#define ExprIsFullSize(E) (((E)->flags&(EP_Reduced|EP_TokenOnly))==0) /* Macros used to ensure that the correct members of unions are accessed ** in Expr. */ #define ExprUseUToken(E) (((E)->flags&EP_IntValue)==0) #define ExprUseUValue(E) (((E)->flags&EP_IntValue)!=0) +#define ExprUseWOfst(E) (((E)->flags&(EP_InnerON|EP_OuterON))==0) +#define ExprUseWJoin(E) (((E)->flags&(EP_InnerON|EP_OuterON))!=0) #define ExprUseXList(E) (((E)->flags&EP_xIsSelect)==0) #define ExprUseXSelect(E) (((E)->flags&EP_xIsSelect)!=0) #define ExprUseYTab(E) (((E)->flags&(EP_WinFunc|EP_Subrtn))==0) @@ -17999,12 +19300,18 @@ struct ExprList { struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The parse tree for this expression */ char *zEName; /* Token associated with this expression */ - u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ - unsigned eEName :2; /* Meaning of zEName */ - unsigned done :1; /* A flag to indicate when processing is finished */ - unsigned reusable :1; /* Constant expression is reusable */ - unsigned bSorterRef :1; /* Defer evaluation until after sorting */ - unsigned bNulls: 1; /* True if explicit "NULLS FIRST/LAST" */ + struct { + u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ + unsigned eEName :2; /* Meaning of zEName */ + unsigned done :1; /* Indicates when processing is finished */ + unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ + unsigned bNulls :1; /* True if explicit "NULLS FIRST/LAST" */ + unsigned bUsed :1; /* This column used in a SF_NestedFrom subquery */ + unsigned bUsingTerm:1; /* Term from the USING clause of a NestedFrom */ + unsigned bNoExpand: 1; /* Term is an auxiliary in NestedFrom and should + ** not be expanded by "*" in parent queries */ + } fg; union { struct { /* Used by any ExprList other than Parse.pConsExpr */ u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -18022,6 +19329,7 @@ struct ExprList { #define ENAME_NAME 0 /* The AS clause of a result set */ #define ENAME_SPAN 1 /* Complete text of the result set expression */ #define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ +#define ENAME_ROWID 3 /* "DB.TABLE._rowid_" for * expansion of rowid */ /* ** An instance of this structure can hold a simple list of identifiers, @@ -18039,23 +19347,45 @@ struct ExprList { ** If "a" is the k-th column of table "t", then IdList.a[0].idx==k. */ struct IdList { + int nId; /* Number of identifiers on the list */ + u8 eU4; /* Which element of a.u4 is valid */ struct IdList_item { char *zName; /* Name of the identifier */ - int idx; /* Index in some Table.aCol[] of a column named zName */ - } *a; - int nId; /* Number of identifiers on the list */ + union { + int idx; /* Index in some Table.aCol[] of a column named zName */ + Expr *pExpr; /* Expr to implement a USING variable -- NOT USED */ + } u4; + } a[1]; }; +/* +** Allowed values for IdList.eType, which determines which value of the a.u4 +** is valid. +*/ +#define EU4_NONE 0 /* Does not use IdList.a.u4 */ +#define EU4_IDX 1 /* Uses IdList.a.u4.idx */ +#define EU4_EXPR 2 /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */ + /* ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. ** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +** ** Union member validity: ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc +** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy && !fg.isCte +** u2.pCteUse fg.isCte && !fg.isIndexedBy */ struct SrcItem { Schema *pSchema; /* Schema to which this item is fixed */ @@ -18073,44 +19403,50 @@ struct SrcItem { unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ unsigned isTabFunc :1; /* True if table-valued-function syntax */ unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned isMaterialized:1; /* This is a materialized view */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ unsigned isRecursive :1; /* True for recursive reference in WITH */ unsigned fromDDL :1; /* Comes from sqlite_schema */ unsigned isCte :1; /* This is a CTE */ unsigned notCte :1; /* This item may not match a CTE */ + unsigned isUsing :1; /* u3.pUsing is valid */ + unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ + unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ + unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ - Expr *pOn; /* The ON clause of a join */ - IdList *pUsing; /* The USING clause of a join */ - Bitmask colUsed; /* Bit N (1< The ON clause of a join */ + IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ + } u3; + Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ - CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */ + CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; }; /* -** The following structure describes the FROM clause of a SELECT statement. -** Each table or subquery in the FROM clause is a separate element of -** the SrcList.a[] array. -** -** With the addition of multiple database support, the following structure -** can also be used to describe a particular table such as the table that -** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, -** such a table must be a simple name: ID. But in SQLite, the table can -** now be identified by a database name, a dot, then the table name: ID.ID. -** -** The jointype starts out showing the join type between the current table -** and the next table on the list. The parser builds the list this way. -** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each -** jointype expresses the join between the table and the previous table. +** The OnOrUsing object represents either an ON clause or a USING clause. +** It can never be both at the same time, but it can be neither. +*/ +struct OnOrUsing { + Expr *pOn; /* The ON clause of a join */ + IdList *pUsing; /* The USING clause of a join */ +}; + +/* +** This object represents one or more tables that are the source of +** content for an SQL statement. For example, a single SrcList object +** is used to hold the FROM clause of a SELECT statement. SrcList also +** represents the target tables for DELETE, INSERT, and UPDATE statements. ** -** In the colUsed field, the high-order bit (bit 63) is set if the table -** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { int nSrc; /* Number of tables or subqueries in the FROM clause */ @@ -18121,14 +19457,15 @@ struct SrcList { /* ** Permitted values of the SrcList.a.jointype field */ -#define JT_INNER 0x0001 /* Any kind of inner or cross join */ -#define JT_CROSS 0x0002 /* Explicit use of the CROSS keyword */ -#define JT_NATURAL 0x0004 /* True for a "natural" join */ -#define JT_LEFT 0x0008 /* Left outer join */ -#define JT_RIGHT 0x0010 /* Right outer join */ -#define JT_OUTER 0x0020 /* The "OUTER" keyword is present */ -#define JT_ERROR 0x0040 /* unknown or unsupported join type */ - +#define JT_INNER 0x01 /* Any kind of inner or cross join */ +#define JT_CROSS 0x02 /* Explicit use of the CROSS keyword */ +#define JT_NATURAL 0x04 /* True for a "natural" join */ +#define JT_LEFT 0x08 /* Left outer join */ +#define JT_RIGHT 0x10 /* Right outer join */ +#define JT_OUTER 0x20 /* The "OUTER" keyword is present */ +#define JT_LTORJ 0x40 /* One of the LEFT operands of a RIGHT JOIN + ** Mnemonic: Left Table Of Right Join */ +#define JT_ERROR 0x80 /* unknown or unsupported join type */ /* ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() @@ -18151,8 +19488,8 @@ struct SrcList { #define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ - /* 0x1000 not currently used */ - /* 0x2000 not currently used */ +#define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -18197,6 +19534,7 @@ struct NameContext { int nRef; /* Number of names resolved by this context */ int nNcErr; /* Number of errors encountered while resolving names */ int ncFlags; /* Zero or more NC_* flags defined below */ + u32 nNestedSelect; /* Number of nested selects using this NC */ Select *pWinSelect; /* SELECT statement for any window functions */ }; @@ -18217,7 +19555,7 @@ struct NameContext { #define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ #define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ #define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ -#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ #define NC_UEList 0x000080 /* True if uNC.pEList is used */ #define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ @@ -18230,6 +19568,7 @@ struct NameContext { #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ #define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ #define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ /* @@ -18253,6 +19592,7 @@ struct Upsert { Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ /* Above this point is the parse tree for the ON CONFLICT clauses. ** The next group of fields stores intermediate data. */ void *pToFree; /* Free memory when deleting the Upsert object */ @@ -18342,10 +19682,15 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ +#define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ + +/* True if S exists and has SF_NestedFrom */ +#define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) /* ** The results of a SELECT can be distributed in several ways, as defined @@ -18451,7 +19796,7 @@ struct SelectDest { int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ - char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + char *zAffSdst; /* Affinity used for SRT_Set */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; @@ -18510,11 +19855,34 @@ struct TriggerPrg { #else typedef unsigned int yDbMask; # define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) -# define DbMaskZero(M) (M)=0 -# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) -# define DbMaskAllZero(M) (M)==0 -# define DbMaskNonZero(M) (M)!=0 +# define DbMaskZero(M) ((M)=0) +# define DbMaskSet(M,I) ((M)|=(((yDbMask)1)<<(I))) +# define DbMaskAllZero(M) ((M)==0) +# define DbMaskNonZero(M) ((M)!=0) +#endif + +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + u8 aff; /* Affinity of the pExpr expression */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ #endif +}; /* ** An instance of the ParseCleanup object specifies an operation that @@ -18557,9 +19925,14 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 disableVtab; /* Disable all virtual tables for this parse */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ + u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasWith; /* True if statement contains WITH */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ #endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -18573,6 +19946,8 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ @@ -18580,6 +19955,9 @@ struct Parse { int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ +#endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -18593,9 +19971,9 @@ struct Parse { int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ Returning *pReturning; /* The RETURNING clause */ } u1; - u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ @@ -18609,6 +19987,7 @@ struct Parse { **************************************************************************/ int aTempReg[8]; /* Holding area for temporary registers */ + Parse *pOuterParse; /* Outer Parse object when nested */ Token sNameToken; /* Token with unqualified schema object name */ /************************************************************************ @@ -18659,7 +20038,8 @@ struct Parse { /* ** Sizes and pointers of various parts of the Parse object. */ -#define PARSE_HDR_SZ offsetof(Parse,aTempReg) /* Recursive part w/o aColCache*/ +#define PARSE_HDR(X) (((char*)(X))+offsetof(Parse,zErrMsg)) +#define PARSE_HDR_SZ (offsetof(Parse,aTempReg)-offsetof(Parse,zErrMsg)) /* Recursive part w/o aColCache*/ #define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ #define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ #define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ @@ -18717,6 +20097,7 @@ struct AuthContext { #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BYTELENARG 0xc0 /* OP_Column only for octet_length() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ #define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ @@ -18728,20 +20109,20 @@ struct AuthContext { #define OPFLAG_PREFORMAT 0x80 /* OP_Insert uses preformatted cell */ /* - * Each trigger present in the database schema is stored as an instance of - * struct Trigger. - * - * Pointers to instances of struct Trigger are stored in two ways. - * 1. In the "trigHash" hash table (part of the sqlite3* that represents the - * database). This allows Trigger structures to be retrieved by name. - * 2. All triggers associated with a single table form a linked list, using the - * pNext member of struct Trigger. A pointer to the first element of the - * linked list is stored as the "pTrigger" member of the associated - * struct Table. - * - * The "step_list" member points to the first element of a linked list - * containing the SQL statements specified as the trigger program. - */ +** Each trigger present in the database schema is stored as an instance of +** struct Trigger. +** +** Pointers to instances of struct Trigger are stored in two ways. +** 1. In the "trigHash" hash table (part of the sqlite3* that represents the +** database). This allows Trigger structures to be retrieved by name. +** 2. All triggers associated with a single table form a linked list, using the +** pNext member of struct Trigger. A pointer to the first element of the +** linked list is stored as the "pTrigger" member of the associated +** struct Table. +** +** The "step_list" member points to the first element of a linked list +** containing the SQL statements specified as the trigger program. +*/ struct Trigger { char *zName; /* The name of the trigger */ char *table; /* The table or view to which the trigger applies */ @@ -18768,43 +20149,48 @@ struct Trigger { #define TRIGGER_AFTER 2 /* - * An instance of struct TriggerStep is used to store a single SQL statement - * that is a part of a trigger-program. - * - * Instances of struct TriggerStep are stored in a singly linked list (linked - * using the "pNext" member) referenced by the "step_list" member of the - * associated struct Trigger instance. The first element of the linked list is - * the first step of the trigger-program. - * - * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or - * "SELECT" statement. The meanings of the other members is determined by the - * value of "op" as follows: - * - * (op == TK_INSERT) - * orconf -> stores the ON CONFLICT algorithm - * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then - * this stores a pointer to the SELECT statement. Otherwise NULL. - * zTarget -> Dequoted name of the table to insert into. - * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then - * this stores values to be inserted. Otherwise NULL. - * pIdList -> If this is an INSERT INTO ... () VALUES ... - * statement, then this stores the column-names to be - * inserted into. - * - * (op == TK_DELETE) - * zTarget -> Dequoted name of the table to delete from. - * pWhere -> The WHERE clause of the DELETE statement if one is specified. - * Otherwise NULL. - * - * (op == TK_UPDATE) - * zTarget -> Dequoted name of the table to update. - * pWhere -> The WHERE clause of the UPDATE statement if one is specified. - * Otherwise NULL. - * pExprList -> A list of the columns to update and the expressions to update - * them to. See sqlite3Update() documentation of "pChanges" - * argument. - * - */ +** An instance of struct TriggerStep is used to store a single SQL statement +** that is a part of a trigger-program. +** +** Instances of struct TriggerStep are stored in a singly linked list (linked +** using the "pNext" member) referenced by the "step_list" member of the +** associated struct Trigger instance. The first element of the linked list is +** the first step of the trigger-program. +** +** The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or +** "SELECT" statement. The meanings of the other members is determined by the +** value of "op" as follows: +** +** (op == TK_INSERT) +** orconf -> stores the ON CONFLICT algorithm +** pSelect -> The content to be inserted - either a SELECT statement or +** a VALUES clause. +** zTarget -> Dequoted name of the table to insert into. +** pIdList -> If this is an INSERT INTO ... () VALUES ... +** statement, then this stores the column-names to be +** inserted into. +** pUpsert -> The ON CONFLICT clauses for an Upsert +** +** (op == TK_DELETE) +** zTarget -> Dequoted name of the table to delete from. +** pWhere -> The WHERE clause of the DELETE statement if one is specified. +** Otherwise NULL. +** +** (op == TK_UPDATE) +** zTarget -> Dequoted name of the table to update. +** pWhere -> The WHERE clause of the UPDATE statement if one is specified. +** Otherwise NULL. +** pExprList -> A list of the columns to update and the expressions to update +** them to. See sqlite3Update() documentation of "pChanges" +** argument. +** +** (op == TK_SELECT) +** pSelect -> The SELECT statement +** +** (op == TK_RETURNING) +** pExprList -> The list of expressions that follow the RETURNING keyword. +** +*/ struct TriggerStep { u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT, ** or TK_RETURNING */ @@ -18833,6 +20219,7 @@ struct Returning { int iRetCur; /* Transient table holding RETURNING results */ int nRetCol; /* Number of in pReturnEL after expansion */ int iRetReg; /* Register array for holding a row of RETURNING */ + char zName[40]; /* Name of trigger: "sqlite_returning_%p" */ }; /* @@ -18854,6 +20241,28 @@ struct sqlite3_str { #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) +/* +** The following object is the header for an "RCStr" or "reference-counted +** string". An RCStr is passed around and used like any other char* +** that has been dynamically allocated. The important interface +** differences: +** +** 1. RCStr strings are reference counted. They are deallocated +** when the reference count reaches zero. +** +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than +** sqlite3_free() +** +** 3. Make a (read-only) copy of a read-only RCStr string using +** sqlite3RCStrRef(). +** +** "String" is in the name, but an RCStr object can also be used to hold +** binary data. +*/ +struct RCStr { + u64 nRCRef; /* Number of references */ + /* Total structure size should be a multiple of 8 bytes for alignment */ +}; /* ** A pointer to this structure is used to communicate information @@ -18880,7 +20289,7 @@ typedef struct { /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning ** parameters are for temporary use during development, to help find -** optimial values for parameters in the query planner. The should not +** optimal values for parameters in the query planner. The should not ** be used on trunk check-ins. They are a temporary mechanism available ** for transient development builds only. ** @@ -18906,6 +20315,10 @@ struct Sqlite3Config { u8 bUseCis; /* Use covering indices for full-scans */ u8 bSmallMalloc; /* Avoid large memory allocations if true */ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ + u8 bUseLongDouble; /* Make use of long double */ +#ifdef SQLITE_DEBUG + u8 bJsonSelfcheck; /* Double-check JSON parsing */ +#endif int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -18952,11 +20365,19 @@ struct Sqlite3Config { #endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ + int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ + void (*xCorruption)(void *, const void *); + void *pCorruptionArg; /* vvvv--- must be last ---vvv */ #ifdef SQLITE_DEBUG sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */ @@ -18991,6 +20412,7 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ int walkerDepth; /* Number of subqueries */ u16 eCode; /* A small processing code */ + u16 mWFlags; /* Use-dependent flags */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int n; /* A counter */ @@ -19000,15 +20422,16 @@ struct Walker { struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ int *aiCol; /* array of column indexes */ struct IdxCover *pIdxCover; /* Check for index coverage */ - struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ struct WindowRewrite *pRewrite; /* Window rewrite context */ struct WhereConst *pConst; /* WHERE clause constants */ struct RenameCtx *pRename; /* RENAME COLUMN context */ struct Table *pTab; /* Table of generated column */ + struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ - DbFixer *pFix; + DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; @@ -19029,6 +20452,7 @@ struct DbFixer { /* Forward declarations */ SQLITE_PRIVATE int sqlite3WalkExpr(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3WalkExprNN(Walker*, Expr*); SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); @@ -19109,6 +20533,16 @@ struct CteUse { }; +/* Client data associated with sqlite3_set_clientdata() and +** sqlite3_get_clientdata(). +*/ +struct DbClientData { + DbClientData *pNext; /* Next in a linked list */ + void *pData; /* The data */ + void (*xDestructor)(void*); /* Destructor. Might be NULL */ + char zName[1]; /* Name of this client data. MUST BE LAST */ +}; + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of @@ -19158,7 +20592,7 @@ struct Window { Window **ppThis; /* Pointer to this object in Select.pWin list */ Window *pNextWin; /* Next window function belonging to this SELECT */ Expr *pFilter; /* The FILTER expression */ - FuncDef *pFunc; /* The function */ + FuncDef *pWFunc; /* The function */ int iEphCsr; /* Partition buffer or Peer buffer */ int regAccum; /* Accumulator */ int regResult; /* Interim result */ @@ -19175,6 +20609,9 @@ struct Window { ** due to the SQLITE_SUBTYPE flag */ }; +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); @@ -19208,6 +20645,57 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprLis } \ } +#define SQLITE_PRINT_CORRUPT_SIZE (SQLITE_PRINT_BUF_SIZE * 2) + +#define SQLITE_CORRUPT_CONTEXT(N,P,T,O,S,M,R) \ + { .nPage=(N), .pgno=(P), .type=(T), \ + .zoneRange={(O),(S)}, .zMsg=(M), .reservedArgs=(R) } + +typedef struct { + int offset; + size_t size; +}sqlite3CorruptRange; + +typedef enum { + CORRUPT_TYPE_PAGE_BTREE_LEAF, + CORRUPT_TYPE_PAGE_BTREE_INTERIOR, + CORRUPT_TYPE_PAGE_INDEX_LEAF, + CORRUPT_TYPE_PAGE_INDEX_INTERIOR, + CORRUPT_TYPE_PAGE_OVERFLOW, + CORRUPT_TYPE_PAGE_PTR_MAP, + CORRUPT_TYPE_PAGE_FREE_LIST, + CORRUPT_TYPE_FRAME_WAL, + CORRUPT_TYPE_ENTRY_JOURNAL, + CORRUPT_TYPE_VDBE, + CORRUPT_TYPE_FILE_HEADER, + CORRUPT_TYPE_UNKOWN, +} CorruptType; + +typedef struct { + size_t nPage; /* Number of pages */ + unsigned int pgno; /* Page number for corrupted page */ + CorruptType type; + sqlite3CorruptRange zoneRange; + const char *zMsg; + void *reservedArgs; +}sqlite3CorruptContext; + +// Encode buffer with base16, return size after encode +static size_t sqlite3base16Encode(const unsigned char *buffer, size_t bufSize, char *encodeBuf, size_t encodeBufSize) +{ + if (buffer == NULL || bufSize == 0 || encodeBuf == NULL || encodeBufSize == 0) { + return 0; + } + static const char base16Code[] = "0123456789ABCDEF"; + size_t i = 0; + for (; i < bufSize && (i * 2 < encodeBufSize - 1); i++) { + *encodeBuf++ = base16Code[(buffer[i] >> 4) & 0x0F]; + *encodeBuf++ = base16Code[buffer[i] & 0x0F]; + } + *encodeBuf = '\0'; + return i * 2; +} + /* ** The SQLITE_*_BKPT macros are substitutes for the error codes with ** the same name but without the _BKPT suffix. These macros invoke @@ -19216,10 +20704,11 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprLis ** to set a debugger breakpoint. */ SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType); -SQLITE_PRIVATE int sqlite3CorruptError(int); +SQLITE_PRIVATE int sqlite3CorruptError(int lineno, sqlite3CorruptContext *context); SQLITE_PRIVATE int sqlite3MisuseError(int); SQLITE_PRIVATE int sqlite3CantopenError(int); -#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) +#define SQLITE_CORRUPT_REPORT(context) sqlite3CorruptError(__LINE__,(context)) +#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__,NULL) #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) #ifdef SQLITE_DEBUG @@ -19231,12 +20720,13 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define SQLITE_NOMEM_BKPT SQLITE_NOMEM # define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM #endif -#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +#if (defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO)) SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); -# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) +# define SQLITE_CORRUPT_PGNO(P,context) sqlite3CorruptPgnoError(__LINE__,(P)) #else -# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) +# define SQLITE_CORRUPT_PGNO(P,context) sqlite3CorruptError(__LINE__,(context)) #endif +# define SQLITE_CORRUPT_REPORT_PGNO(context) SQLITE_CORRUPT_PGNO((context)->pgno,(context)) /* ** FTS3 and FTS4 both require virtual table support @@ -19278,6 +20768,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -19287,6 +20779,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif SQLITE_PRIVATE int sqlite3IsIdChar(u8); @@ -19314,6 +20808,7 @@ SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3*, void*); SQLITE_PRIVATE int sqlite3MallocSize(const void*); SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); @@ -19334,12 +20829,14 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); */ #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) -# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) +# define sqlite3StackAllocRawNN(D,N) alloca(N) # define sqlite3StackFree(D,P) +# define sqlite3StackFreeNN(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) -# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) +# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P) #endif /* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they @@ -19387,10 +20884,13 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) SQLITE_PRIVATE int sqlite3IsNaN(double); +SQLITE_PRIVATE int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -19403,6 +20903,20 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; +/* +** An instance of this object receives the decoding of a floating point +** value into an approximate decimal representation. +*/ +struct FpDecode { + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[24]; /* Storage for significant digits */ +}; + +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode*,double,int,int); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) @@ -19413,34 +20927,75 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif #if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView*, const char *zFormat, ...); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewBareIdList(TreeView*, const IdList*, const char*); +SQLITE_PRIVATE void sqlite3TreeViewIdList(TreeView*, const IdList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewColumnList(TreeView*, const Column*, int, u8); SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); +SQLITE_PRIVATE void sqlite3TreeViewUpsert(TreeView*, const Upsert*, u8); +#if TREETRACE_ENABLED +SQLITE_PRIVATE void sqlite3TreeViewDelete(const With*, const SrcList*, const Expr*, + const ExprList*,const Expr*, const Trigger*); +SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*, + const IdList*, const Select*, const ExprList*, + int, const Upsert*, const Trigger*); +SQLITE_PRIVATE void sqlite3TreeViewUpdate(const With*, const SrcList*, const ExprList*, + const Expr*, int, const ExprList*, const Expr*, + const Upsert*, const Trigger*); +#endif +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3TreeViewTriggerStep(TreeView*, const TriggerStep*, u8, u8); +SQLITE_PRIVATE void sqlite3TreeViewTrigger(TreeView*, const Trigger*, u8, u8); +#endif #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView*, const Window*, u8); SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); #endif +SQLITE_PRIVATE void sqlite3ShowExpr(const Expr*); +SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList*); +SQLITE_PRIVATE void sqlite3ShowIdList(const IdList*); +SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList*); +SQLITE_PRIVATE void sqlite3ShowSelect(const Select*); +SQLITE_PRIVATE void sqlite3ShowWith(const With*); +SQLITE_PRIVATE void sqlite3ShowUpsert(const Upsert*); +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3ShowTriggerStep(const TriggerStep*); +SQLITE_PRIVATE void sqlite3ShowTriggerStepList(const TriggerStep*); +SQLITE_PRIVATE void sqlite3ShowTrigger(const Trigger*); +SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger*); +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3ShowWindow(const Window*); +SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window*); +#endif #endif - SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); -SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); +SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*); SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); SQLITE_PRIVATE int sqlite3GetTempReg(Parse*); SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +SQLITE_PRIVATE void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); #endif @@ -19452,10 +21007,13 @@ SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int); +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy(Parse*,Expr*,ExprList*); +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse*,Expr*); SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3*,void*); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); @@ -19464,6 +21022,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE void sqlite3ExprListDeleteGeneric(sqlite3*,void*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); @@ -19484,7 +21043,7 @@ SQLITE_PRIVATE const char *sqlite3ColumnColl(Column*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); SQLITE_PRIVATE void sqlite3GenerateColumnNames(Parse *pParse, Select *pSelect); SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); @@ -19513,11 +21072,14 @@ SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u32,Select*); SQLITE_PRIVATE void sqlite3AddReturning(Parse*,ExprList*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); -#ifdef SQLITE_HAS_CODEC -SQLITE_PRIVATE int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); -#else -# define sqlite3CodecQueryParameters(A,B,C) 0 -#endif +#ifndef SQLITE_HAS_CODEC +#define sqlite3CodecQueryParameters(A,B,C) 0 +#else +SQLITE_PRIVATE int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); +#endif /* !SQLITE_HAS_CODEC */ +#if defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) +SQLITE_PRIVATE void sqlite3CodecResetParameters(CodecParameter *p); +#endif /* defined(SQLITE_HAS_CODEC) && defined(SQLITE_CODEC_ATTACH_CHANGED) */ SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); #ifdef SQLITE_UNTESTABLE @@ -19558,6 +21120,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3DeleteTableGeneric(sqlite3*, void*); SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3*, Index*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); @@ -19577,13 +21140,14 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, - Token*, Select*, Expr*, IdList*); + Token*, Select*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, SrcItem *); -SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(Parse*,SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); +SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, @@ -19593,16 +21157,19 @@ SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); +SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); #endif +SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe*,int,const char*); SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*); SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*, Upsert*); -SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*, + ExprList*,Select*,u16,int); SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); @@ -19654,7 +21221,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(const Parse*,const Expr*,const Expr*, int) SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*,Expr*,int); SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList*,const ExprList*, int); SQLITE_PRIVATE int sqlite3ExprImpliesExpr(const Parse*,const Expr*,const Expr*, int); -SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int); +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int,int); SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); @@ -19676,11 +21243,10 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -19688,6 +21254,7 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab); SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); @@ -19717,9 +21284,14 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,const Select*,int); SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum*,sqlite3_value*); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void); SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) +SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3*); +#endif SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); @@ -19773,7 +21345,8 @@ SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol); -SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int); +SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem*,int); +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int,u32); SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); #ifndef SQLITE_OMIT_AUTHORIZATION @@ -19796,8 +21369,10 @@ SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); + SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); -SQLITE_PRIVATE void sqlite3Int64ToText(i64,char*); +SQLITE_PRIVATE i64 sqlite3RealToI64(double); +SQLITE_PRIVATE int sqlite3Int64ToText(i64,char*); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); @@ -19807,16 +21382,11 @@ SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); #endif SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8**); +SQLITE_PRIVATE int sqlite3Utf8ReadLimited(const u8*, int, u32*); SQLITE_PRIVATE LogEst sqlite3LogEst(u64); SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); -#ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); -#endif -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); -#endif SQLITE_PRIVATE VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int); SQLITE_PRIVATE const char *sqlite3VListNumToName(VList*,int); SQLITE_PRIVATE int sqlite3VListNameToNum(VList*,const char*,int); @@ -19848,18 +21418,22 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table*,int); SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -19869,6 +21443,9 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int); #ifndef SQLITE_OMIT_DESERIALIZE SQLITE_PRIVATE int sqlite3MemdbInit(void); +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs*); +#else +# define sqlite3IsMemdb(X) 0 #endif SQLITE_PRIVATE const char *sqlite3ErrStr(int); @@ -19900,6 +21477,7 @@ SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -19919,7 +21497,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[]; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[]; -SQLITE_PRIVATE const char sqlite3StdTypeMap[]; SQLITE_PRIVATE const char *sqlite3StdType[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char *sqlite3aLTb; @@ -19952,7 +21529,8 @@ SQLITE_PRIVATE int sqlite3MatchEName( const struct ExprList_item*, const char*, const char*, - const char* + const char*, + int* ); SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); @@ -20003,17 +21581,25 @@ SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, FuncDestructor *pDestructor ); SQLITE_PRIVATE void sqlite3NoopDestructor(void*); -SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); +SQLITE_PRIVATE void *sqlite3OomFault(sqlite3*); SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); +SQLITE_PRIVATE char *sqlite3RCStrRef(char*); +SQLITE_PRIVATE void sqlite3RCStrUnref(void*); +SQLITE_PRIVATE char *sqlite3RCStrNew(u64); +SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); + SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest*,int,int); SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); +SQLITE_PRIVATE void sqlite3RecordErrorByteOffset(sqlite3*,const char*); +SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3*,const Expr*); SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); @@ -20118,11 +21704,14 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3*, int, const char *, char **); SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse*, Table*); SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); + SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse*); SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); -SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +SQLITE_PRIVATE void sqlite3ParseObjectInit(Parse*,sqlite3*); +SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse*); SQLITE_PRIVATE void *sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); #ifdef SQLITE_ENABLE_NORMALIZE SQLITE_PRIVATE char *sqlite3Normalize(Vdbe*, const char*); @@ -20142,6 +21731,7 @@ SQLITE_PRIVATE Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,u8); SQLITE_PRIVATE void sqlite3CteDelete(sqlite3*,Cte*); SQLITE_PRIVATE With *sqlite3WithAdd(Parse*,With*,Cte*); SQLITE_PRIVATE void sqlite3WithDelete(sqlite3*,With*); +SQLITE_PRIVATE void sqlite3WithDeleteGeneric(sqlite3*,void*); SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); #else # define sqlite3CteNew(P,T,E,S) ((void*)0) @@ -20154,7 +21744,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*); SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); -SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*); @@ -20253,6 +21843,7 @@ SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr*,int); SQLITE_PRIVATE u32 sqlite3Get4byte(const u8*); SQLITE_PRIVATE void sqlite3Put4byte(u8*, u32); @@ -20354,6 +21945,22 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*); SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); #endif +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void); +#endif + +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +SQLITE_PRIVATE sqlite3_uint64 sqlite3Hwtime(void); +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + #endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ @@ -20395,101 +22002,6 @@ SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); */ #ifdef SQLITE_PERFORMANCE_TRACE -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 and x86_64 class CPUs. -*/ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - /* - ** asm() is needed for hardware timing support. Without asm(), - ** disable the sqlite3Hwtime() routine. - ** - ** sqlite3Hwtime() is only used for some obscure debugging - ** and analysis configurations, not in any deliverable, so this - ** should not be a great loss. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(SQLITE_HWTIME_H) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ - static sqlite_uint64 g_start; static sqlite_uint64 g_elapsed; #define TIMER_START g_start=sqlite3Hwtime() @@ -20551,10 +22063,22 @@ SQLITE_API extern int sqlite3_open_file_count; #define OpenCounter(X) #endif /* defined(SQLITE_TEST) */ +#ifdef FDSAN_ENABLE +#define SQLITE_FDSAN_TAG 5351 +#endif /* FDSAN_ENABLE */ + #endif /* !defined(_OS_COMMON_H_) */ /************** End of os_common.h *******************************************/ /************** Begin file ctime.c *******************************************/ +/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkctimec.tcl. +** +** To modify this header, edit any of the various lists in that script +** which specify categories of generated conditionals in this file. +*/ + /* ** 2010 February 23 ** @@ -20577,7 +22101,7 @@ SQLITE_API extern int sqlite3_open_file_count; ** autoconf-based build */ #if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -/* #include "config.h" */ +/* #include "sqlite_cfg.h" */ #define SQLITECONFIG_H 1 #endif @@ -20603,23 +22127,20 @@ SQLITE_API extern int sqlite3_open_file_count; */ static const char * const sqlite3azCompileOpt[] = { -/* -** BEGIN CODE GENERATED BY tool/mkctime.tcl -*/ #ifdef SQLITE_32BIT_ROWID "32BIT_ROWID", #endif #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC "4_BYTE_ALIGNED_MALLOC", #endif -#ifdef SQLITE_64BIT_STATS - "64BIT_STATS", -#endif #ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN # if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif @@ -20745,6 +22266,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT "DISABLE_SKIPAHEAD_DISTINCT", #endif +#ifdef SQLITE_DQS + "DQS=" CTIMEOPT_VAL(SQLITE_DQS), +#endif #ifdef SQLITE_ENABLE_8_3_NAMES "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif @@ -20814,9 +22338,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", #endif -#ifdef SQLITE_ENABLE_JSON1 - "ENABLE_JSON1", -#endif #ifdef SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif @@ -20862,9 +22383,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif -#ifdef SQLITE_ENABLE_SELECTTRACE - "ENABLE_SELECTTRACE", -#endif #ifdef SQLITE_ENABLE_SESSION "ENABLE_SESSION", #endif @@ -20886,6 +22404,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_STMT_SCANSTATUS "ENABLE_STMT_SCANSTATUS", #endif +#ifdef SQLITE_ENABLE_TREETRACE + "ENABLE_TREETRACE", +#endif #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION "ENABLE_UNKNOWN_SQL_FUNCTION", #endif @@ -20910,6 +22431,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS "EXPLAIN_ESTIMATED_ROWS", #endif +#ifdef SQLITE_EXTRA_AUTOEXT + "EXTRA_AUTOEXT=" CTIMEOPT_VAL(SQLITE_EXTRA_AUTOEXT), +#endif #ifdef SQLITE_EXTRA_IFNULLROW "EXTRA_IFNULLROW", #endif @@ -20940,7 +22464,7 @@ static const char * const sqlite3azCompileOpt[] = { # endif #endif #if SQLITE_SHARED_BLOCK_OPTIMIZATION - "SHARED_BLOCK_OPTIMIZATION", + "SHARED_BLOCK_OPTIMIZATION", #endif #ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS "IGNORE_AFP_LOCK_ERRORS", @@ -20957,6 +22481,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), #endif +#ifdef SQLITE_LEGACY_JSON_VALID + "LEGACY_JSON_VALID", +#endif #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS "LIKE_DOESNT_MATCH_BLOBS", #endif @@ -21146,6 +22673,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_INTROSPECTION_PRAGMAS "OMIT_INTROSPECTION_PRAGMAS", #endif +#ifdef SQLITE_OMIT_JSON + "OMIT_JSON", +#endif #ifdef SQLITE_OMIT_LIKE_OPTIMIZATION "OMIT_LIKE_OPTIMIZATION", #endif @@ -21191,6 +22721,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS "OMIT_SCHEMA_VERSION_PRAGMAS", #endif +#ifdef SQLITE_OMIT_SEH + "OMIT_SEH", +#endif #ifdef SQLITE_OMIT_SHARED_CACHE "OMIT_SHARED_CACHE", #endif @@ -21241,9 +22774,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - "PCACHE_SEPARATE_HEADER", -#endif #ifdef SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif @@ -21334,10 +22864,8 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ZERO_MALLOC "ZERO_MALLOC", #endif -/* -** END CODE GENERATED BY tool/mkctime.tcl -*/ -}; + +} ; SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt){ *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); @@ -21447,7 +22975,7 @@ SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 -** SQLite identifier character 0x40 +** SQLite identifier character 0x40 $, _, or non-ascii ** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper @@ -21518,11 +23046,11 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** enabled. */ #ifndef SQLITE_USE_URI -# ifdef SQLITE_HAS_CODEC -# define SQLITE_USE_URI 1 -# else -# define SQLITE_USE_URI 0 -# endif +#ifndef SQLITE_HAS_CODEC +#define SQLITE_USE_URI 0 +#else +#define SQLITE_USE_URI 1 +#endif /* !SQLITE_HAS_CODEC */ #endif /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the @@ -21600,6 +23128,10 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0, /* bSmallMalloc */ 1, /* bExtraSchemaChecks */ + sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ +#ifdef SQLITE_DEBUG + 0, /* bJsonSelfcheck */ +#endif 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ @@ -21641,11 +23173,20 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ #endif 0, /* bLocaltimeFault */ + 0, /* xAltLocaltime */ 0x7ffffffe, /* iOnceResetThreshold */ SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ 0, /* iPrngSeed */ + 0, /* xCorruption */ + 0, /* pCorruptionArg */ +#ifdef SQLITE_DEBUG + {0,0,0,0,0,0}, /* aTune */ +#endif }; /* @@ -21700,7 +23241,7 @@ SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; /* ** Tracing flags set by SQLITE_TESTCTRL_TRACEFLAGS. */ -SQLITE_PRIVATE u32 sqlite3SelectTrace = 0; +SQLITE_PRIVATE u32 sqlite3TreeTrace = 0; SQLITE_PRIVATE u32 sqlite3WhereTrace = 0; /* #include "opcodes.h" */ @@ -21728,10 +23269,6 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; ** ** sqlite3StdTypeAffinity[] The affinity associated with each entry ** in sqlite3StdType[]. -** -** sqlite3StdTypeMap[] The type value (as returned from -** sqlite3_column_type() or sqlite3_value_type()) -** for each entry in sqlite3StdType[]. */ SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { @@ -21742,14 +23279,6 @@ SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { SQLITE_AFF_REAL, SQLITE_AFF_TEXT }; -SQLITE_PRIVATE const char sqlite3StdTypeMap[] = { - 0, - SQLITE_BLOB, - SQLITE_INTEGER, - SQLITE_INTEGER, - SQLITE_FLOAT, - SQLITE_TEXT -}; SQLITE_PRIVATE const char *sqlite3StdType[] = { "ANY", "BLOB", @@ -21837,6 +23366,9 @@ typedef struct VdbeSorter VdbeSorter; /* Elements of the linked list at Vdbe.pAuxData */ typedef struct AuxData AuxData; +/* A cache of large TEXT or BLOB values in a VdbeCursor */ +typedef struct VdbeTxtBlbCache VdbeTxtBlbCache; + /* Types of VDBE cursors */ #define CURTYPE_BTREE 0 #define CURTYPE_SORTER 1 @@ -21856,7 +23388,7 @@ typedef struct AuxData AuxData; typedef struct VdbeCursor VdbeCursor; struct VdbeCursor { u8 eCurType; /* One of the CURTYPE_* values above */ - i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ + i8 iDb; /* Index of cursor database in db->aDb[] */ u8 nullRow; /* True if pointing to a row with no data */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ u8 isTable; /* True for rowid tables. False for indexes */ @@ -21867,11 +23399,14 @@ struct VdbeCursor { Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ - Bool hasBeenDuped:1; /* This cursor was source or target of OP_OpenDup */ + Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ - Btree *pBtx; /* Separate file holding temporary table */ + union { /* pBtx for isEphermeral. pAltMap otherwise */ + Btree *pBtx; /* Separate file holding temporary table */ + u32 *aAltMap; /* Mapping from table to index column numbers */ + } ub; i64 seqCount; /* Sequence counter */ - u32 *aAltMap; /* Mapping from table to index column numbers */ /* Cached OP_Column parse information is only valid if cacheStatus matches ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of @@ -21906,6 +23441,7 @@ struct VdbeCursor { #ifdef SQLITE_ENABLE_COLUMN_USED_MASK u64 maskUsed; /* Mask of columns used by this cursor */ #endif + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for @@ -21913,12 +23449,30 @@ struct VdbeCursor { u32 aType[1]; /* Type values record decode. MUST BE LAST */ }; +/* Return true if P is a null-only cursor +*/ +#define IsNullCursor(P) \ + ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) /* ** A value for VdbeCursor.cacheStatus that means the cache is always invalid. */ #define CACHE_STALE 0 +/* +** Large TEXT or BLOB values can be slow to load, so we want to avoid +** loading them more than once. For that reason, large TEXT and BLOB values +** can be stored in a cache defined by this object, and attached to the +** VdbeCursor using the pCache field. +*/ +struct VdbeTxtBlbCache { + char *pCValue; /* A RCStr buffer to hold the value */ + i64 iOffset; /* File offset of the row being cached */ + int iCol; /* Column for which the cache is valid */ + u32 cacheStatus; /* Vdbe.cacheCtr value */ + u32 colCacheCtr; /* Column cache counter */ +}; + /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as @@ -21945,7 +23499,6 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u8 *aOnce; /* Bitmask used by OP_Once */ @@ -21987,16 +23540,16 @@ struct sqlite3_value { const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ } u; + char *z; /* String or BLOB value */ + int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ u8 eSubtype; /* Subtype for this value */ - int n; /* Number of characters in string value, excluding '\0' */ - char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ - char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + sqlite3 *db; /* The associated database connection */ int szMalloc; /* Size of the zMalloc allocation */ u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ - sqlite3 *db; /* The associated database connection */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ @@ -22008,11 +23561,43 @@ struct sqlite3_value { ** Size of struct Mem not including the Mem.zMalloc member or anything that ** follows. */ -#define MEMCELLSIZE offsetof(Mem,zMalloc) +#define MEMCELLSIZE offsetof(Mem,db) -/* One or more of the following flags are set to indicate the validOK +/* One or more of the following flags are set to indicate the ** representations of the value stored in the Mem struct. ** +** * MEM_Null An SQL NULL value +** +** * MEM_Null|MEM_Zero An SQL NULL with the virtual table +** UPDATE no-change flag set +** +** * MEM_Null|MEM_Term| An SQL NULL, but also contains a +** MEM_Subtype pointer accessible using +** sqlite3_value_pointer(). +** +** * MEM_Null|MEM_Cleared Special SQL NULL that compares non-equal +** to other NULLs even using the IS operator. +** +** * MEM_Str A string, stored in Mem.z with +** length Mem.n. Zero-terminated if +** MEM_Term is set. This flag is +** incompatible with MEM_Blob and +** MEM_Null, but can appear with MEM_Int, +** MEM_Real, and MEM_IntReal. +** +** * MEM_Blob A blob, stored in Mem.z length Mem.n. +** Incompatible with MEM_Str, MEM_Null, +** MEM_Int, MEM_Real, and MEM_IntReal. +** +** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus +** MEM.u.i extra 0x00 bytes at the end. +** +** * MEM_Int Integer stored in Mem.u.i. +** +** * MEM_Real Real stored in Mem.u.r. +** +** * MEM_IntReal Real stored as an integer in Mem.u.i. +** ** If the MEM_Null flag is set, then the value is an SQL NULL value. ** For a pointer type created using sqlite3_bind_pointer() or ** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set. @@ -22023,6 +23608,7 @@ struct sqlite3_value { ** set, then the string is nul terminated. The MEM_Int and MEM_Real ** flags may coexist with the MEM_Str flag. */ +#define MEM_Undefined 0x0000 /* Value is undefined */ #define MEM_Null 0x0001 /* Value is NULL (or a pointer) */ #define MEM_Str 0x0002 /* Value is a string */ #define MEM_Int 0x0004 /* Value is an integer */ @@ -22030,28 +23616,24 @@ struct sqlite3_value { #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ #define MEM_AffMask 0x003f /* Mask of affinity bits */ + +/* Extra bits that modify the meanings of the core datatypes above +*/ #define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ -#define MEM_Undefined 0x0080 /* Value is undefined */ + /* 0x0080 // Available */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0xc1bf /* Mask of type bits */ - +#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */ +#define MEM_Zero 0x0400 /* Mem.i contains count of 0s appended to blob */ +#define MEM_Subtype 0x0800 /* Mem.eSubtype is valid */ +#define MEM_TypeMask 0x0dbf /* Mask of type bits */ -/* Whenever Mem contains a valid string or blob representation, one of -** the following flags must be set to determine the memory management -** policy for Mem.z. The MEM_Term flag tells us whether or not the -** string is \000 or \u0000 terminated +/* Bits that determine the storage for Mem.z for a string or blob or +** aggregate accumulator. */ -#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */ -#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ -#define MEM_Static 0x0800 /* Mem.z points to a static string */ -#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ -#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ -#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ -#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ -#ifdef SQLITE_OMIT_INCRBLOB - #undef MEM_Zero - #define MEM_Zero 0x0000 -#endif +#define MEM_Dyn 0x1000 /* Need to call Mem.xDel() on Mem.z */ +#define MEM_Static 0x2000 /* Mem.z points to a static string */ +#define MEM_Ephem 0x4000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x8000 /* Mem.z points to an agg function context */ /* Return TRUE if Mem X contains dynamically allocated content - anything ** that needs to be deallocated to avoid a leak. @@ -22073,11 +23655,15 @@ struct sqlite3_value { && (X)->n==0 && (X)->u.nZero==0) /* -** Return true if a memory cell is not marked as invalid. This macro +** Return true if a memory cell has been initialized and is valid. ** is for use inside assert() statements only. +** +** A Memory cell is initialized if at least one of the +** MEM_Null, MEM_Str, MEM_Int, MEM_Real, MEM_Blob, or MEM_IntReal bits +** is set. It is "undefined" if all those bits are zero. */ #ifdef SQLITE_DEBUG -#define memIsValid(M) ((M)->flags & MEM_Undefined)==0 +#define memIsValid(M) ((M)->flags & MEM_AffMask)!=0 #endif /* @@ -22115,6 +23701,7 @@ struct sqlite3_context { Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ + u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ u8 argc; /* Number of arguments */ sqlite3_value *argv[1]; /* Argument set */ @@ -22127,10 +23714,19 @@ typedef unsigned bft; /* Bit Field Type */ /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. +** +** aAddrRange[]: +** This array is used by ScanStatus elements associated with EQP +** notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is +** an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[] +** values should be summed to calculate the NCYCLE value. Each pair of +** integer addresses is a start and end address (both inclusive) for a range +** instructions. A start value of 0 indicates an empty range. */ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ + int aAddrRange[6]; int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ @@ -22160,10 +23756,9 @@ struct DblquoteStr { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Vdbe **ppVPrev,*pVNext; /* Linked list of VDBEs with the same Vdbe.db */ Parse *pParse; /* Parsing context used to create this Vdbe */ ynVar nVar; /* Number of entries in aVar[] */ - u32 iVdbeMagic; /* Magic number defining state of the SQL statement */ int nMem; /* Number of memory locations currently allocated */ int nCursor; /* Number of slots in apCsr[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ @@ -22187,7 +23782,7 @@ struct Vdbe { int nOp; /* Number of instructions in the program */ int nOpAlloc; /* Slots allocated for aOp[] */ Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ + Mem *pResultRow; /* Current output row */ char *zErrMsg; /* Error message written here */ VList *pVList; /* Name of variables */ #ifndef SQLITE_OMIT_TRACE @@ -22198,20 +23793,21 @@ struct Vdbe { u32 nWrite; /* Number of write operations that have occurred */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nResAlloc; /* Column slots allocated to aColName[] */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ - u8 doingRerun; /* True if rerunning after an auto-reprepare */ + u8 eVdbeState; /* On of the VDBE_*_STATE values */ bft expired:2; /* 1: recompile VM immediately 2: when convenient */ - bft explain:2; /* True if EXPLAIN present on SQL command */ + bft explain:2; /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */ bft changeCntOn:1; /* True to update the change-counter */ - bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ + bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ - u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ + u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ #ifdef SQLITE_ENABLE_NORMALIZE char *zNormSql; /* Normalization of the associated SQL statement */ @@ -22225,7 +23821,6 @@ struct Vdbe { SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif @@ -22235,17 +23830,16 @@ struct Vdbe { int blockFull; int startPos; int addedRows; -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ }; /* -** The following are allowed values for Vdbe.magic +** The following are allowed values for Vdbe.eVdbeState */ -#define VDBE_MAGIC_INIT 0x16bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0x2df20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x319c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_RESET 0x48fa9f76 /* Reset and ready to run again */ -#define VDBE_MAGIC_DEAD 0x5606c3c8 /* The VDBE has been deallocated */ +#define VDBE_INIT_STATE 0 /* Prepared statement under construction */ +#define VDBE_READY_STATE 1 /* Ready to run but not yet started */ +#define VDBE_RUN_STATE 2 /* Run in progress */ +#define VDBE_HALT_STATE 3 /* Finished. Need reset() or finalize() */ /* ** Structure used to store the context required by the @@ -22264,23 +23858,54 @@ struct PreUpdate { i64 iKey1; /* First key value passed to hook */ i64 iKey2; /* Second key value passed to hook */ Mem *aNew; /* Array of new.* values */ - Table *pTab; /* Schema object being upated */ + Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ }; +/* +** An instance of this object is used to pass an vector of values into +** OP_VFilter, the xFilter method of a virtual table. The vector is the +** set of values on the right-hand side of an IN constraint. +** +** The value as passed into xFilter is an sqlite3_value with a "pointer" +** type, such as is generated by sqlite3_result_pointer() and read by +** sqlite3_value_pointer. Such values have MEM_Term|MEM_Subtype|MEM_Null +** and a subtype of 'p'. The sqlite3_vtab_in_first() and _next() interfaces +** know how to use this object to step through all the values in the +** right operand of the IN constraint. +*/ +typedef struct ValueList ValueList; +struct ValueList { + BtCursor *pCsr; /* An ephemeral table holding all values */ + sqlite3_value *pOut; /* Register to hold each decoded output value */ +}; + +/* Size of content associated with serial types that fit into a +** single-byte varint. +*/ +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE const u8 sqlite3SmallTypeSizes[]; +#endif + /* ** Function prototypes */ SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe*,VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeHandleMovedCursor(VdbeCursor *p); SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor*); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, u32*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +SQLITE_PRIVATE u64 sqlite3FloatSwap(u64 in); +# define swapMixedEndianFloat(X) X = sqlite3FloatSwap(X) +#else +# define swapMixedEndianFloat(X) +#endif +SQLITE_PRIVATE void sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); @@ -22323,10 +23948,11 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetZeroBlob(Mem*,int); SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); #endif SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem*); SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); @@ -22337,6 +23963,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset(BtCursor*,u32,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE void sqlite3VdbeMemReleaseMalloc(Mem*p); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); @@ -22368,6 +23995,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void*); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe*, VdbeCursor*); SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe*); @@ -22696,6 +24325,8 @@ SQLITE_API int sqlite3_db_status( sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( ALWAYS(pSchema!=0) ){ @@ -22721,6 +24352,7 @@ SQLITE_API int sqlite3_db_status( } } db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); *pHighwater = 0; @@ -22738,10 +24370,12 @@ SQLITE_API int sqlite3_db_status( int nByte = 0; /* Used to accumulate return value */ db->pnBytesFreed = &nByte; - for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ - sqlite3VdbeClearObject(db, pVdbe); - sqlite3DbFree(db, pVdbe); + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pVNext){ + sqlite3VdbeDelete(pVdbe); } + db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; *pHighwater = 0; /* IMP: R-64479-57858 */ @@ -22762,7 +24396,7 @@ SQLITE_API int sqlite3_db_status( case SQLITE_DBSTATUS_CACHE_MISS: case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; - int nRet = 0; + u64 nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); @@ -22775,7 +24409,7 @@ SQLITE_API int sqlite3_db_status( *pHighwater = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = nRet; + *pCurrent = (int)nRet & 0x7fffffff; break; } @@ -22872,12 +24506,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -22910,8 +24546,8 @@ struct DateTime { */ static int getDigits(const char *zDate, const char *zFormat, ...){ /* The aMx[] array translates the 3rd character of each format - ** spec into a max size: a b c d e f */ - static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 14712 }; va_list ap; int cnt = 0; char nextC; @@ -22975,6 +24611,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -22987,7 +24625,6 @@ static int parseTimezone(const char *zDate, DateTime *p){ p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -23031,7 +24668,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -23077,16 +24713,41 @@ static void computeJD(DateTime *p){ p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); - if( p->validTZ ){ + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -23125,12 +24786,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -23140,6 +24805,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -23192,6 +24860,11 @@ static int parseDateOrTime( }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; + }else if( (sqlite3StrICmp(zDate,"subsec")==0 + || sqlite3StrICmp(zDate,"subsecond")==0) + && sqlite3NotPureFunc(context) ){ + p->useSubsec = 1; + return setDateTimeToCurrent(context, p); } return 1; } @@ -23247,17 +24920,14 @@ static void computeYMD(DateTime *p){ ** Compute the Hour, Minute, and Seconds from the julian day number. */ static void computeHMS(DateTime *p){ - int s; + int day_ms, day_min; /* milliseconds, minutes into the day */ if( p->validHMS ) return; computeJD(p); - s = (int)((p->iJD + 43200000) % 86400000); - p->s = s/1000.0; - s = (int)p->s; - p->s -= s; - p->h = s/3600; - s -= p->h*3600; - p->m = s/60; - p->s += s - p->m*60; + day_ms = (int)((p->iJD + 43200000) % 86400000); + p->s = (day_ms % 60000)/1000.0; + day_min = day_ms/60000; + p->m = day_min % 60; + p->h = day_min / 60; p->rawS = 0; p->validHMS = 1; } @@ -23276,7 +24946,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -23304,8 +24974,10 @@ static void clearYMD_HMS_TZ(DateTime *p){ ** is available. This routine returns 0 on success and ** non-zero on any kind of error. ** -** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this -** routine will always fail. +** If the sqlite3GlobalConfig.bLocaltimeFault variable is non-zero then this +** routine will always fail. If bLocaltimeFault is nonzero and +** sqlite3GlobalConfig.xAltLocaltime is not NULL, then xAltLocaltime() is +** invoked in place of the OS-defined localtime() function. ** ** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C ** library function localtime_r() is used to assist in the calculation of @@ -23321,14 +24993,30 @@ static int osLocaltime(time_t *t, struct tm *pTm){ sqlite3_mutex_enter(mutex); pX = localtime(t); #ifndef SQLITE_UNTESTABLE - if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; + if( sqlite3GlobalConfig.bLocaltimeFault ){ + if( sqlite3GlobalConfig.xAltLocaltime!=0 + && 0==sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm) + ){ + pX = pTm; + }else{ + pX = 0; + } + } #endif if( pX ) *pTm = *pX; +#if SQLITE_THREADSAFE>0 sqlite3_mutex_leave(mutex); +#endif rc = pX==0; #else #ifndef SQLITE_UNTESTABLE - if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; + if( sqlite3GlobalConfig.bLocaltimeFault ){ + if( sqlite3GlobalConfig.xAltLocaltime!=0 ){ + return sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm); + }else{ + return 1; + } + } #endif #if HAVE_LOCALTIME_R rc = localtime_r(t, pTm)==0; @@ -23343,67 +25031,56 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in milliseconds) between localtime and UTC -** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, -** return this value and set *pRc to SQLITE_OK. -** -** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value -** is undefined in this case. +** Assuming the input DateTime is UTC, move it to its localtime equivalent. */ -static sqlite3_int64 localtimeOffset( - DateTime *p, /* Date at which to calculate offset */ - sqlite3_context *pCtx, /* Write error here if one occurs */ - int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +static int toLocaltime( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx /* Write error here if one occurs */ ){ - DateTime x, y; time_t t; struct tm sLocal; + int iYearDiff; /* Initialize the contents of sLocal to avoid a compiler warning. */ memset(&sLocal, 0, sizeof(sLocal)); - x = *p; - computeYMD_HMS(&x); - if( x.Y<1971 || x.Y>=2038 ){ + computeJD(p); + if( p->iJD<2108667600*(i64)100000 /* 1970-01-01 */ + || p->iJD>2130141456*(i64)100000 /* 2038-01-18 */ + ){ /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only ** works for years between 1970 and 2037. For dates outside this range, ** SQLite attempts to map the year into an equivalent year within this ** range, do the calculation, then map the year back. */ - x.Y = 2000; - x.M = 1; - x.D = 1; - x.h = 0; - x.m = 0; - x.s = 0.0; - } else { - int s = (int)(x.s + 0.5); - x.s = s; + DateTime x = *p; + computeYMD_HMS(&x); + iYearDiff = (2000 + x.Y%4) - x.Y; + x.Y += iYearDiff; + x.validJD = 0; + computeJD(&x); + t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); + }else{ + iYearDiff = 0; + t = (time_t)(p->iJD/1000 - 21086676*(i64)10000); } - x.tz = 0; - x.validJD = 0; - computeJD(&x); - t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); if( osLocaltime(&t, &sLocal) ){ sqlite3_result_error(pCtx, "local time unavailable", -1); - *pRc = SQLITE_ERROR; - return 0; + return SQLITE_ERROR; } - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - y.validYMD = 1; - y.validHMS = 1; - y.validJD = 0; - y.rawS = 0; - y.validTZ = 0; - y.isError = 0; - computeJD(&y); - *pRc = SQLITE_OK; - return y.iJD - x.iJD; + p->Y = sLocal.tm_year + 1900 - iYearDiff; + p->M = sLocal.tm_mon + 1; + p->D = sLocal.tm_mday; + p->h = sLocal.tm_hour; + p->m = sLocal.tm_min; + p->s = sLocal.tm_sec + (p->iJD%1000)*0.001; + p->validYMD = 1; + p->validHMS = 1; + p->validJD = 0; + p->rawS = 0; + p->tz = 0; + p->isError = 0; + return SQLITE_OK; } #endif /* SQLITE_OMIT_LOCALTIME */ @@ -23416,20 +25093,38 @@ static sqlite3_int64 localtimeOffset( ** of several units of time. */ static const struct { - u8 eType; /* Transformation type code */ - u8 nName; /* Length of th name */ - char *zName; /* Name of the transformation */ - double rLimit; /* Maximum NNN value for this transform */ - double rXform; /* Constant used for this transform */ + u8 nName; /* Length of the name */ + char zName[7]; /* Name of the transformation */ + float rLimit; /* Maximum NNN value for this transform */ + float rXform; /* Constant used for this transform */ } aXformType[] = { - { 0, 6, "second", 464269060800.0, 1000.0 }, - { 0, 6, "minute", 7737817680.0, 60000.0 }, - { 0, 4, "hour", 128963628.0, 3600000.0 }, - { 0, 3, "day", 5373485.0, 86400000.0 }, - { 1, 5, "month", 176546.0, 2592000000.0 }, - { 2, 4, "year", 14713.0, 31536000000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 30.0*86400.0 }, + /* 5 */ { 4, "year", 14713.0, 365.0*86400.0 }, }; +/* +** If the DateTime p is raw number, try to figure out if it is +** a julian day number of a unix timestamp. Set the p value +** appropriately. +*/ +static void autoAdjustDate(DateTime *p){ + if( !p->rawS || p->validJD ){ + p->rawS = 0; + }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ + && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ + ){ + double r = p->s*1000.0 + 210866760000000.0; + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + } +} + /* ** Process a modifier to a date-time stamp. The modifiers are ** as follows: @@ -23440,14 +25135,20 @@ static const struct { ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -23458,11 +25159,75 @@ static int parseModifier( sqlite3_context *pCtx, /* Function context */ const char *z, /* The text of the modifier */ int n, /* Length of zMod in bytes */ - DateTime *p /* The date/time value to be modified */ + DateTime *p, /* The date/time value to be modified */ + int idx /* Parameter index of the modifier */ ){ int rc = 1; double r; switch(sqlite3UpperToLower[(u8)z[0]] ){ + case 'a': { + /* + ** auto + ** + ** If rawS is available, then interpret as a julian day number, or + ** a unix timestamp, depending on its magnitude. + */ + if( sqlite3_stricmp(z, "auto")==0 ){ + if( idx>1 ) return 1; /* IMP: R-33611-57934 */ + autoAdjustDate(p); + rc = 0; + } + break; + } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } + case 'j': { + /* + ** julianday + ** + ** Always interpret the prior number as a julian-day value. If this + ** is not the first modifier, or if the prior argument is not a numeric + ** value in the allowed range of julian day numbers understood by + ** SQLite (0..5373484.5) then the result will be NULL. + */ + if( sqlite3_stricmp(z, "julianday")==0 ){ + if( idx>1 ) return 1; /* IMP: R-31176-64601 */ + if( p->validJD && p->rawS ){ + rc = 0; + p->rawS = 0; + } + } + break; + } #ifndef SQLITE_OMIT_LOCALTIME case 'l': { /* localtime @@ -23471,9 +25236,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - computeJD(p); - p->iJD += localtimeOffset(p, pCtx, &rc); - clearYMD_HMS_TZ(p); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -23486,6 +25251,7 @@ static int parseModifier( ** seconds since 1970. Convert to a real julian day number. */ if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){ + if( idx>1 ) return 1; /* IMP: R-49255-55373 */ r = p->s*1000.0 + 210866760000000.0; if( r>=0.0 && r<464269060800000.0 ){ clearYMD_HMS_TZ(p); @@ -23497,19 +25263,33 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ - sqlite3_int64 c1; + if( p->isUtc==0 ){ + i64 iOrigJD; /* Original localtime */ + i64 iGuess; /* Guess at the corresponding utc time */ + int cnt = 0; /* Safety to prevent infinite loop */ + i64 iErr; /* Guess is off by this much */ + computeJD(p); - c1 = localtimeOffset(p, pCtx, &rc); - if( rc==SQLITE_OK ){ - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p, pCtx, &rc); - } - p->tzSet = 1; - }else{ - rc = SQLITE_OK; + iGuess = iOrigJD = p->iJD; + iErr = 0; + do{ + DateTime new; + memset(&new, 0, sizeof(new)); + iGuess -= iErr; + new.iJD = iGuess; + new.validJD = 1; + rc = toLocaltime(&new, pCtx); + if( rc ) return rc; + computeJD(&new); + iErr = new.iJD - iOrigJD; + }while( iErr && cnt++<3 ); + memset(p, 0, sizeof(*p)); + p->iJD = iGuess; + p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; } + rc = SQLITE_OK; } #endif break; @@ -23524,10 +25304,10 @@ static int parseModifier( */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -23544,8 +25324,22 @@ static int parseModifier( ** ** Move the date backwards to the beginning of the current day, ** or month or year. + ** + ** subsecond + ** subsec + ** + ** Show subsecond precision in the output of datetime() and + ** unixepoch() and strftime('%s'). */ - if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ){ + if( sqlite3_stricmp(z, "subsec")==0 + || sqlite3_stricmp(z, "subsecond")==0 + ){ + p->useSubsec = 1; + rc = 0; + } + break; + } if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); @@ -23553,7 +25347,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -23581,18 +25375,74 @@ static int parseModifier( case '9': { double rRounder; int i; - for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + int Y,M,D,h,m,x; + const char *z2 = z; + char z0 = z[0]; + for(n=1; z[n]; n++){ + if( z[n]==':' ) break; + if( sqlite3Isspace(z[n]) ) break; + if( z[n]=='-' ){ + if( n==5 && getDigits(&z[1], "40f", &Y)==1 ) break; + if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; + } + } if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ - rc = 1; + assert( rc==1 ); break; } - if( z[n]==':' ){ + if( z[n]=='-' ){ + /* A modifier of the form (+|-)YYYY-MM-DD adds or subtracts the + ** specified number of years, months, and days. MM is limited to + ** the range 0-11 and DD is limited to 0-30. + */ + if( z0!='+' && z0!='-' ) break; /* Must start with +/- */ + if( n==5 ){ + if( getDigits(&z[1], "40f-20a-20d", &Y, &M, &D)!=3 ) break; + }else{ + assert( n==6 ); + if( getDigits(&z[1], "50f-20a-20d", &Y, &M, &D)!=3 ) break; + z++; + } + if( M>=12 ) break; /* M range 0..11 */ + if( D>=31 ) break; /* D range 0..30 */ + computeYMD_HMS(p); + p->validJD = 0; + if( z0=='-' ){ + p->Y -= Y; + p->M -= M; + D = -D; + }else{ + p->Y += Y; + p->M += M; + } + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + computeFloor(p); + computeJD(p); + p->validHMS = 0; + p->validYMD = 0; + p->iJD += (i64)D*86400000; + if( z[11]==0 ){ + rc = 0; + break; + } + if( sqlite3Isspace(z[11]) + && getDigits(&z[12], "20c:20e", &h, &m)==2 + ){ + z2 = &z[12]; + n = 2; + }else{ + break; + } + } + if( z2[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be ** omitted. */ - const char *z2 = z; + DateTime tx; sqlite3_int64 day; if( !sqlite3Isdigit(*z2) ) z2++; @@ -23602,7 +25452,7 @@ static int parseModifier( tx.iJD -= 43200000; day = tx.iJD/86400000; tx.iJD -= day*86400000; - if( z[0]=='-' ) tx.iJD = -tx.iJD; + if( z0=='-' ) tx.iJD = -tx.iJD; computeJD(p); clearYMD_HMS_TZ(p); p->iJD += tx.iJD; @@ -23615,39 +25465,44 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); - rc = 1; + assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; i-aXformType[i].rLimit && rM += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } - case 2: { /* Special processing to add years */ + case 5: { /* Special processing to add years */ int y = (int)r; + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } } computeJD(p); - p->iJD += (sqlite3_int64)(r*aXformType[i].rXform + rRounder); + p->iJD += (sqlite3_int64)(r*1000.0*aXformType[i].rXform + rRounder); rc = 0; break; } @@ -23697,10 +25552,16 @@ static int isDate( for(i=1; iisError || !validJulianDay(p->iJD) ) return 1; + if( argc==1 && p->validYMD && p->D>28 ){ + /* Make sure a YYYY-MM-DD is normalized. + ** Example: 2023-02-31 -> 2023-03-03 */ + assert( p->validJD ); + p->validYMD = 0; + } return 0; } @@ -23727,6 +25588,28 @@ static void juliandayFunc( } } +/* +** unixepoch( TIMESTRING, MOD, MOD, ...) +** +** Return the number of seconds (including fractional seconds) since +** the unix epoch of 1970-01-01 00:00:00 GMT. +*/ +static void unixepochFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + computeJD(&x); + if( x.useSubsec ){ + sqlite3_result_double(context, (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + } + } +} + /* ** datetime( TIMESTRING, MOD, MOD, ...) ** @@ -23739,11 +25622,51 @@ static void datetimeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int Y, s, n; + char zBuf[32]; computeYMD_HMS(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d %02d:%02d:%02d", - x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + Y = x.Y; + if( Y<0 ) Y = -Y; + zBuf[1] = '0' + (Y/1000)%10; + zBuf[2] = '0' + (Y/100)%10; + zBuf[3] = '0' + (Y/10)%10; + zBuf[4] = '0' + (Y)%10; + zBuf[5] = '-'; + zBuf[6] = '0' + (x.M/10)%10; + zBuf[7] = '0' + (x.M)%10; + zBuf[8] = '-'; + zBuf[9] = '0' + (x.D/10)%10; + zBuf[10] = '0' + (x.D)%10; + zBuf[11] = ' '; + zBuf[12] = '0' + (x.h/10)%10; + zBuf[13] = '0' + (x.h)%10; + zBuf[14] = ':'; + zBuf[15] = '0' + (x.m/10)%10; + zBuf[16] = '0' + (x.m)%10; + zBuf[17] = ':'; + if( x.useSubsec ){ + s = (int)(1000.0*x.s + 0.5); + zBuf[18] = '0' + (s/10000)%10; + zBuf[19] = '0' + (s/1000)%10; + zBuf[20] = '.'; + zBuf[21] = '0' + (s/100)%10; + zBuf[22] = '0' + (s/10)%10; + zBuf[23] = '0' + (s)%10; + zBuf[24] = 0; + n = 24; + }else{ + s = (int)x.s; + zBuf[18] = '0' + (s/10)%10; + zBuf[19] = '0' + (s)%10; + zBuf[20] = 0; + n = 20; + } + if( x.Y<0 ){ + zBuf[0] = '-'; + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(context, &zBuf[1], n-1, SQLITE_TRANSIENT); + } } } @@ -23759,10 +25682,33 @@ static void timeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int s, n; + char zBuf[16]; computeHMS(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%02d:%02d:%02d", x.h, x.m, (int)x.s); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + zBuf[0] = '0' + (x.h/10)%10; + zBuf[1] = '0' + (x.h)%10; + zBuf[2] = ':'; + zBuf[3] = '0' + (x.m/10)%10; + zBuf[4] = '0' + (x.m)%10; + zBuf[5] = ':'; + if( x.useSubsec ){ + s = (int)(1000.0*x.s + 0.5); + zBuf[6] = '0' + (s/10000)%10; + zBuf[7] = '0' + (s/1000)%10; + zBuf[8] = '.'; + zBuf[9] = '0' + (s/100)%10; + zBuf[10] = '0' + (s/10)%10; + zBuf[11] = '0' + (s)%10; + zBuf[12] = 0; + n = 12; + }else{ + s = (int)x.s; + zBuf[6] = '0' + (s/10)%10; + zBuf[7] = '0' + (s)%10; + zBuf[8] = 0; + n = 8; + } + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); } } @@ -23778,29 +25724,108 @@ static void dateFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int Y; + char zBuf[16]; computeYMD(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d", x.Y, x.M, x.D); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + Y = x.Y; + if( Y<0 ) Y = -Y; + zBuf[1] = '0' + (Y/1000)%10; + zBuf[2] = '0' + (Y/100)%10; + zBuf[3] = '0' + (Y/10)%10; + zBuf[4] = '0' + (Y)%10; + zBuf[5] = '-'; + zBuf[6] = '0' + (x.M/10)%10; + zBuf[7] = '0' + (x.M)%10; + zBuf[8] = '-'; + zBuf[9] = '0' + (x.D/10)%10; + zBuf[10] = '0' + (x.D)%10; + zBuf[11] = 0; + if( x.Y<0 ){ + zBuf[0] = '-'; + sqlite3_result_text(context, zBuf, 11, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(context, &zBuf[1], 10, SQLITE_TRANSIENT); + } } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "am" or "pm" +** %P "AM" or "PM" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -23825,44 +25850,61 @@ static void strftimeFunc( computeJD(&x); computeYMD_HMS(&x); for(i=j=0; zFmt[i]; i++){ + char cf; if( zFmt[i]!='%' ) continue; if( j59.999 ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); break; } - case 'H': { - sqlite3_str_appendf(&sRes, "%02d", x.h); + case 'F': { + sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ + case 'G': /* Fall thru */ + case 'g': { DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( zFmt[i]=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); + sqlite3_str_appendf(&sRes, "%04d", y.Y); } break; } - case 'J': { + case 'H': + case 'k': { + sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); + break; + } + case 'I': /* Fall thru */ + case 'l': { + int h = x.h; + if( h>12 ) h -= 12; + if( h==0 ) h = 12; + sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); + break; + } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); + break; + } + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -23874,18 +25916,62 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d",x.m); break; } + case 'p': /* Fall thru */ + case 'P': { + if( x.h>=12 ){ + sqlite3_str_append(&sRes, cf=='p' ? "PM" : "pm", 2); + }else{ + sqlite3_str_append(&sRes, cf=='p' ? "AM" : "am", 2); + } + break; + } + case 'R': { + sqlite3_str_appendf(&sRes, "%02d:%02d", x.h, x.m); + break; + } case 's': { - i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); - sqlite3_str_appendf(&sRes,"%lld",iS); + if( x.useSubsec ){ + sqlite3_str_appendf(&sRes,"%.3f", + (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3_str_appendf(&sRes,"%lld",iS); + } break; } case 'S': { sqlite3_str_appendf(&sRes,"%02d",(int)x.s); break; } - case 'w': { - sqlite3_str_appendchar(&sRes, 1, - (char)(((x.iJD+129600000)/86400000) % 7) + '0'); + case 'T': { + sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); + break; + } + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; + if( c=='0' && cf=='u' ) c = '7'; + sqlite3_str_appendchar(&sRes, 1, c); + break; + } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); break; } case 'Y': { @@ -23934,6 +26020,115 @@ static void cdateFunc( dateFunc(context, 0, 0); } +/* +** timediff(DATE1, DATE2) +** +** Return the amount of time that must be added to DATE2 in order to +** convert it into DATE2. The time difference format is: +** +** +YYYY-MM-DD HH:MM:SS.SSS +** +** The initial "+" becomes "-" if DATE1 occurs before DATE2. For +** date/time values A and B, the following invariant should hold: +** +** datetime(A) == (datetime(B, timediff(A,B)) +** +** Both DATE arguments must be either a julian day number, or an +** ISO-8601 string. The unix timestamps are not supported by this +** routine. +*/ +static void timediffFunc( + sqlite3_context *context, + int NotUsed1, + sqlite3_value **argv +){ + char sign; + int Y, M; + DateTime d1, d2; + sqlite3_str sRes; + UNUSED_PARAMETER(NotUsed1); + if( isDate(context, 1, &argv[0], &d1) ) return; + if( isDate(context, 1, &argv[1], &d2) ) return; + computeYMD_HMS(&d1); + computeYMD_HMS(&d2); + if( d1.iJD>=d2.iJD ){ + sign = '+'; + Y = d1.Y - d2.Y; + if( Y ){ + d2.Y = d1.Y; + d2.validJD = 0; + computeJD(&d2); + } + M = d1.M - d2.M; + if( M<0 ){ + Y--; + M += 12; + } + if( M!=0 ){ + d2.M = d1.M; + d2.validJD = 0; + computeJD(&d2); + } + while( d1.iJDd2.iJD ){ + M--; + if( M<0 ){ + M = 11; + Y--; + } + d2.M++; + if( d2.M>12 ){ + d2.M = 1; + d2.Y++; + } + d2.validJD = 0; + computeJD(&d2); + } + d1.iJD = d2.iJD - d1.iJD; + d1.iJD += (u64)1486995408 * (u64)100000; + } + clearYMD_HMS_TZ(&d1); + computeYMD_HMS(&d1); + sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); + sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", + sign, Y, M, d1.D-1, d1.h, d1.m, d1.s); + sqlite3ResultStrAccum(context, &sRes); +} + + /* ** current_timestamp() ** @@ -23994,6 +26189,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -24003,10 +26228,15 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS PURE_DATE(julianday, -1, 0, 0, juliandayFunc ), + PURE_DATE(unixepoch, -1, 0, 0, unixepochFunc ), PURE_DATE(date, -1, 0, 0, dateFunc ), PURE_DATE(time, -1, 0, 0, timeFunc ), PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -24129,9 +26359,11 @@ SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ } SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); + assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE ); return id->pMethods->xLock(id, lockType); } SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED ); return id->pMethods->xUnlock(id, lockType); } SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ @@ -24158,7 +26390,7 @@ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite ** is using a regular VFS, it is called after the corresponding ** transaction has been committed. Injecting a fault at this point - ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** confuses the test scripts - the COMMIT command returns SQLITE_NOMEM ** but the transaction is committed anyway. ** ** The core must call OsFileControl() though, not OsFileControlHint(), @@ -24246,6 +26478,7 @@ SQLITE_PRIVATE int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ + assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) ); rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; @@ -24778,7 +27011,7 @@ static void *sqlite3MemMalloc(int nByte){ ** or sqlite3MemRealloc(). ** ** For this low-level routine, we already know that pPrior!=0 since -** cases where pPrior==0 will have been intecepted and dealt with +** cases where pPrior==0 will have been intercepted and dealt with ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ @@ -24866,7 +27099,7 @@ static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + /* One usually wants to use hw.activecpu for MT decisions, but not here */ sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); if( cpuCount>1 ){ /* defer MT decisions to system malloc */ @@ -26557,8 +28790,17 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ */ static int memsys5Roundup(int n){ int iFullSz; - if( n > 0x40000000 ) return 0; - for(iFullSz=mem5.szAtom; iFullSz0x10000000 ){ + if( n>0x40000000 ) return 0; + if( n>0x20000000 ) return 0x40000000; + return 0x20000000; + } + for(iFullSz=mem5.szAtom*8; iFullSz=(i64)n ) return iFullSz/2; return iFullSz; } @@ -26849,7 +29091,7 @@ static void checkMutexFree(sqlite3_mutex *p){ assert( SQLITE_MUTEX_FAST<2 ); assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( ((CheckMutex*)p)->iType<2 ) #endif { @@ -27324,7 +29566,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ /* ** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields -** are necessary under two condidtions: (1) Debug builds and (2) using +** are necessary under two conditions: (1) Debug builds and (2) using ** home-grown mutexes. Encapsulate these conditions into a single #define. */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) @@ -27521,7 +29763,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) #endif { @@ -27825,7 +30067,7 @@ struct sqlite3_mutex { CRITICAL_SECTION mutex; /* Mutex controlling the lock */ int id; /* Mutex type */ #ifdef SQLITE_DEBUG - volatile int nRef; /* Number of enterances */ + volatile int nRef; /* Number of entrances */ volatile DWORD owner; /* Thread holding this mutex */ volatile LONG trace; /* True to trace changes */ #endif @@ -27874,7 +30116,7 @@ SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif MSVC_VERSION>=1300 +#elif MSVC_VERSION>=1400 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); @@ -28410,6 +30652,24 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -28436,6 +30696,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -28459,18 +30720,34 @@ static void mallocWithAlarm(int n, void **pp){ *pp = p; } +/* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Paranoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + /* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ void *p; - if( n==0 || n>=0x7fffff00 ){ - /* A memory allocation of a number of bytes which is near the maximum - ** signed integer value might cause an integer overflow inside of the - ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving - ** 255 bytes of overhead. SQLite itself will never use anything near - ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + if( n==0 || n>SQLITE_MAX_ALLOCATION_SIZE ){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); @@ -28506,7 +30783,7 @@ SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, const void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd); } #else #define isLookaside(A,B) 0 @@ -28530,18 +30807,16 @@ static int lookasideMallocSize(sqlite3 *db, const void *p){ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){ assert( p!=0 ); #ifdef SQLITE_DEBUG - if( db==0 || !isLookaside(db,p) ){ - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - }else{ - assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - } + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else if( !isLookaside(db,p) ){ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } #endif if( db ){ - if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ + if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ assert( sqlite3_mutex_held(db->mutex) ); @@ -28597,14 +30872,11 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( p!=0 ); if( db ){ - if( db->pnBytesFreed ){ - measureAllocationSize(db, p); - return; - } if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif @@ -28615,6 +30887,7 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); #ifdef SQLITE_DEBUG memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ #endif @@ -28623,6 +30896,10 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ return; } } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -28630,6 +30907,43 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); } +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); if( p ) sqlite3DbFreeNN(db, p); @@ -28671,6 +30985,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } @@ -28929,9 +31244,14 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ */ SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]0) && sqlite3Isspace(zStart[n-1]) ) n--; + while( sqlite3Isspace(zStart[n-1]) ) n--; return sqlite3DbStrNDup(db, zStart, n); } @@ -28939,8 +31259,9 @@ SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const cha ** Free any prior content in *pz and replace it with a copy of zNew. */ SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ + char *z = sqlite3DbStrDup(db, zNew); sqlite3DbFree(db, *pz); - *pz = sqlite3DbStrDup(db, zNew); + *pz = z; } /* @@ -28948,8 +31269,15 @@ SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ ** has happened. This routine will set db->mallocFailed, and also ** temporarily disable the lookaside memory allocator and interrupt ** any running VDBEs. +** +** Always return a NULL pointer so that this routine can be invoked using +** +** return sqlite3OomFault(db); +** +** and thereby avoid unnecessary stack frame allocations for the overwhelmingly +** common case where no OOM occurs. */ -SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ +SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){ if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ db->mallocFailed = 1; if( db->nVdbeExec>0 ){ @@ -28957,9 +31285,16 @@ SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ } DisableLookaside; if( db->pParse ){ + Parse *pParse; + sqlite3ErrorMsg(db->pParse, "out of memory"); db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } } + return 0; } /* @@ -29012,7 +31347,7 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ if( db->mallocFailed || rc ){ return apiHandleError(db, rc); } - return rc & db->errMask; + return 0; } /************** End of malloc.c **********************************************/ @@ -29124,43 +31459,6 @@ static const et_info fmtinfo[] = { ** %!S Like %S but prefer the zName over the zAlias */ -/* Floating point constants used for rounding */ -static const double arRound[] = { - 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, - 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, -}; - -/* -** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point -** conversions will work. -*/ -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a double such that 0.1 <= *val < 10.0 -** Return the ascii code for the leading digit of *val, then -** multiply "*val" by 10.0 to renormalize. -** -** Example: -** input: *val = 3.14159 -** output: *val = 1.4159 function return = '3' -** -** The counter *cnt is incremented each time. After counter exceeds -** 16 (the number of significant digits in a 64-bit float) '0' is -** always returned. -*/ -static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ - int digit; - LONGDOUBLE_TYPE d; - if( (*cnt)<=0 ) return '0'; - (*cnt)--; - digit = (int)*val; - d = digit; - digit += '0'; - *val = (*val - d)*10.0; - return (char)digit; -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ - /* ** Set the StrAccum object to an error mode. */ @@ -29252,18 +31550,15 @@ SQLITE_API void sqlite3_str_vappendf( u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ - LONGDOUBLE_TYPE realvalue; /* Value for real types */ + double realvalue; /* Value for real types */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ char *zExtra = 0; /* Malloced memory used by some conversion */ -#ifndef SQLITE_OMIT_FLOATING_POINT - int exp, e2; /* exponent of real numbers */ - int nsd; /* Number of significant digits returned */ - double rounder; /* Used for rounding floating point values */ + int exp, e2; /* exponent of real numbers */ etByte flag_dp; /* True if decimal point should be shown */ etByte flag_rtz; /* True if trailing zeros should be removed */ -#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ @@ -29538,74 +31833,69 @@ SQLITE_API void sqlite3_str_vappendf( break; case etFLOAT: case etEXP: - case etGENERIC: + case etGENERIC: { + FpDecode s; + int iRound; + int j; + if( bArgList ){ realvalue = getDoubleArg(pArgList); }else{ realvalue = va_arg(ap,double); } -#ifdef SQLITE_OMIT_FLOATING_POINT - length = 0; -#else if( precision<0 ) precision = 6; /* Set default precision */ #ifdef SQLITE_FP_PRECISION_LIMIT if( precision>SQLITE_FP_PRECISION_LIMIT ){ precision = SQLITE_FP_PRECISION_LIMIT; } #endif - if( realvalue<0.0 ){ - realvalue = -realvalue; - prefix = '-'; - }else{ - prefix = flag_prefix; - } - if( xtype==etGENERIC && precision>0 ) precision--; - testcase( precision>0xfff ); - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - exp = 0; - if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; - break; + iRound = -precision; + }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; + iRound = precision; + }else{ + iRound = precision+1; } - if( realvalue>0.0 ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + if( s.isSpecial ){ + if( s.isSpecial==2 ){ + bufpt = flag_zeropad ? "null" : "NaN"; + length = sqlite3Strlen30(bufpt); + break; + }else if( flag_zeropad ){ + s.z[0] = '9'; + s.iDP = 1000; + s.n = 1; + }else{ + memcpy(buf, "-Inf", 5); bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( s.sign=='-' ){ + /* no-op */ + }else if( flag_prefix ){ + buf[0] = flag_prefix; + }else{ + bufpt++; + } + length = sqlite3Strlen30(bufpt); break; } } - bufpt = buf; + if( s.sign=='-' ){ + prefix = '-'; + }else{ + prefix = flag_prefix; + } + + exp = s.iDP-1; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -29619,29 +31909,32 @@ SQLITE_API void sqlite3_str_vappendf( if( xtype==etEXP ){ e2 = 0; }else{ - e2 = exp; + e2 = s.iDP - 1; } + bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; if( szBufNeeded > etBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); if( bufpt==0 ) return; } } zOut = bufpt; - nsd = 16 + flag_altform2*10; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ *(bufpt++) = prefix; } /* Digits prior to the decimal point */ + j = 0; if( e2<0 ){ *(bufpt++) = '0'; }else{ for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = j1 ) *(bufpt++) = ','; } } /* The decimal point */ @@ -29650,13 +31943,12 @@ SQLITE_API void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0; precision--, e2++){ - assert( precision>0 ); + for(e2++; e2<0 && precision>0; precision--, e2++){ *(bufpt++) = '0'; } /* Significant digits after the decimal point */ while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = jcharset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -29705,8 +31998,8 @@ SQLITE_API void sqlite3_str_vappendf( while( nPad-- ) bufpt[i++] = '0'; length = width; } -#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ break; + } case etSIZE: if( !bArgList ){ *(va_arg(ap,int*)) = pAccum->nChar; @@ -29755,13 +32048,26 @@ SQLITE_API void sqlite3_str_vappendf( } } if( precision>1 ){ + i64 nPrior = 1; width -= precision-1; if( width>1 && !flag_leftjustify ){ sqlite3_str_appendchar(pAccum, width-1, ' '); width = 0; } - while( precision-- > 1 ){ - sqlite3_str_append(pAccum, buf, length); + sqlite3_str_append(pAccum, buf, length); + precision--; + while( precision > 1 ){ + i64 nCopyBytes; + if( nPrior > precision-1 ) nPrior = precision - 1; + nCopyBytes = length*nPrior; + if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ + sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + } + if( pAccum->accError ) break; + sqlite3_str_append(pAccum, + &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); + precision -= nPrior; + nPrior *= 2; } } bufpt = buf; @@ -29822,8 +32128,8 @@ SQLITE_API void sqlite3_str_vappendf( case etSQLESCAPE: /* %q: Escape ' characters */ case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ case etSQLESCAPE3: { /* %w: Escape " characters */ - int i, j, k, n, isnull; - int needQuote; + i64 i, j, k, n; + int needQuote, isnull; char ch; char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; @@ -29868,12 +32174,22 @@ SQLITE_API void sqlite3_str_vappendf( goto adjust_width_for_utf8; } case etTOKEN: { - Token *pToken; if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; - pToken = va_arg(ap, Token*); - assert( bArgList==0 ); - if( pToken && pToken->n ){ - sqlite3_str_append(pAccum, (const char*)pToken->z, pToken->n); + if( flag_alternateform ){ + /* %#T means an Expr pointer that uses Expr.u.zToken */ + Expr *pExpr = va_arg(ap,Expr*); + if( ALWAYS(pExpr) && ALWAYS(!ExprHasProperty(pExpr,EP_IntValue)) ){ + sqlite3_str_appendall(pAccum, (const char*)pExpr->u.zToken); + sqlite3RecordErrorOffsetOfExpr(pAccum->db, pExpr); + } + }else{ + /* %T means a Token pointer */ + Token *pToken = va_arg(ap, Token*); + assert( bArgList==0 ); + if( pToken && pToken->n ){ + sqlite3_str_append(pAccum, (const char*)pToken->z, pToken->n); + sqlite3RecordErrorByteOffset(pAccum->db, pToken->z); + } } length = width = 0; break; @@ -29893,8 +32209,18 @@ SQLITE_API void sqlite3_str_vappendf( sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else if( ALWAYS(pItem->pSelect) ){ - sqlite3_str_appendf(pAccum, "SUBQUERY %u", pItem->pSelect->selId); + }else{ + Select *pSel = pItem->pSelect; + assert( pSel!=0 ); /* Because of tag-20240424-1 */ + if( pSel->selFlags & SF_NestedFrom ){ + sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); + }else{ + sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); + } } length = width = 0; break; @@ -29928,6 +32254,44 @@ SQLITE_API void sqlite3_str_vappendf( }/* End for loop over the format string */ } /* End of function */ + +/* +** The z string points to the first character of a token that is +** associated with an error. If db does not already have an error +** byte offset recorded, try to compute the error byte offset for +** z and set the error byte offset in db. +*/ +SQLITE_PRIVATE void sqlite3RecordErrorByteOffset(sqlite3 *db, const char *z){ + const Parse *pParse; + const char *zText; + const char *zEnd; + assert( z!=0 ); + if( NEVER(db==0) ) return; + if( db->errByteOffset!=(-2) ) return; + pParse = db->pParse; + if( NEVER(pParse==0) ) return; + zText =pParse->zTail; + if( NEVER(zText==0) ) return; + zEnd = &zText[strlen(zText)]; + if( SQLITE_WITHIN(z,zText,zEnd) ){ + db->errByteOffset = (int)(z-zText); + } +} + +/* +** If pExpr has a byte offset for the start of a token, record that as +** as the error offset. +*/ +SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExpr){ + while( pExpr + && (ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) || pExpr->w.iOfst<=0) + ){ + pExpr = pExpr->pLeft; + } + if( pExpr==0 ) return; + db->errByteOffset = pExpr->w.iOfst; +} + /* ** Enlarge the memory allocation on a StrAccum object so that it is ** able to accept at least N more bytes of text. @@ -29935,9 +32299,9 @@ SQLITE_API void sqlite3_str_vappendf( ** Return the number of bytes of text that StrAccum is able to accept ** after the attempted enlargement. The value returned might be zero. */ -static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ char *zNew; - assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==SQLITE_TOOBIG); testcase(p->accError==SQLITE_NOMEM); @@ -29948,8 +32312,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; - i64 szNew = p->nChar; - szNew += (sqlite3_int64)N + 1; + i64 szNew = p->nChar + N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ @@ -29979,7 +32342,8 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return 0; } } - return N; + assert( N>=0 && N<=0x7fffffff ); + return (int)N; } /* @@ -30270,12 +32634,22 @@ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_li return zBuf; } SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; + StrAccum acc; va_list ap; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); va_start(ap,zFormat); - z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - return z; + zBuf[acc.nChar] = 0; + return zBuf; } /* @@ -30294,7 +32668,11 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ +#ifndef LOG_DUMP char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ +#else +char zMsg[SQLITE_PRINT_BUF_SIZE*10]; /* Complete log message */ +#endif /* !LOG_DUMP */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3_str_vappendf(&acc, zFormat, ap); @@ -30353,6 +32731,75 @@ SQLITE_API void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ va_end(ap); } + +/***************************************************************************** +** Reference counted string/blob storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +SQLITE_PRIVATE char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3RCStrUnref(void *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + sqlite3_free(p); + } +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +SQLITE_PRIVATE char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); + if( p==0 ) return 0; + p->nRCRef = 1; + return (char*)&p[1]; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +SQLITE_PRIVATE char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} + /************** End of printf.c **********************************************/ /************** Begin file treeview.c ****************************************/ /* @@ -30381,40 +32828,44 @@ SQLITE_API void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ ** Add a new subitem to the tree. The moreToFollow flag indicates that this ** is not the last item in the tree. */ -static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ +static void sqlite3TreeViewPush(TreeView **pp, u8 moreToFollow){ + TreeView *p = *pp; if( p==0 ){ - p = sqlite3_malloc64( sizeof(*p) ); - if( p==0 ) return 0; + *pp = p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return; memset(p, 0, sizeof(*p)); }else{ p->iLevel++; } assert( moreToFollow==0 || moreToFollow==1 ); - if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; - return p; + if( p->iLevel<(int)sizeof(p->bLine) ) p->bLine[p->iLevel] = moreToFollow; } /* ** Finished with one layer of the tree */ -static void sqlite3TreeViewPop(TreeView *p){ +static void sqlite3TreeViewPop(TreeView **pp){ + TreeView *p = *pp; if( p==0 ) return; p->iLevel--; - if( p->iLevel<0 ) sqlite3_free(p); + if( p->iLevel<0 ){ + sqlite3_free(p); + *pp = 0; + } } /* ** Generate a single line of output for the tree, with a prefix that contains ** all the appropriate tree lines */ -static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; - char zBuf[500]; + char zBuf[1000]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); if( p ){ - for(i=0; iiLevel && ibLine)-1; i++){ + for(i=0; iiLevel && i<(int)sizeof(p->bLine)-1; i++){ sqlite3_str_append(&acc, p->bLine[i] ? "| " : " ", 4); } sqlite3_str_append(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); @@ -30435,10 +32886,57 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ ** Shorthand for starting a new tree item that consists of a single label */ static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ - p = sqlite3TreeViewPush(p, moreFollows); + sqlite3TreeViewPush(&p, moreFollows); sqlite3TreeViewLine(p, "%s", zLabel); } +/* +** Show a list of Column objects in tree format. +*/ +SQLITE_PRIVATE void sqlite3TreeViewColumnList( + TreeView *pView, + const Column *aCol, + int nCol, + u8 moreToFollow +){ + int i; + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewLine(pView, "COLUMNS"); + for(i=0; inCte>0 ){ - pView = sqlite3TreeViewPush(pView, 1); + sqlite3TreeViewPush(&pView, moreToFollow); for(i=0; inCte; i++){ StrAccum x; char zLine[1000]; @@ -30468,6 +32966,10 @@ SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 m } sqlite3_str_appendf(&x, ")"); } + if( pCte->eM10d!=M10d_Any ){ + sqlite3_str_appendf(&x, " %sMATERIALIZED", + pCte->eM10d==M10d_No ? "NOT " : ""); + } if( pCte->pUse ){ sqlite3_str_appendf(&x, " (pUse=0x%p, nUse=%d)", pCte->pUse, pCte->pUse->nUse); @@ -30475,9 +32977,9 @@ SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 m sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inCte-1); sqlite3TreeViewSelect(pView, pCte->pSelect, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } } @@ -30486,37 +32988,72 @@ SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 m */ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ int i; + if( pSrc==0 ) return; for(i=0; inSrc; i++){ const SrcItem *pItem = &pSrc->a[i]; StrAccum x; - char zLine[100]; + int n = 0; + char zLine[1000]; sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); - } - if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); + } + if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ + sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); + }else if( pItem->fg.jointype & JT_LEFT ){ sqlite3_str_appendf(&x, " LEFT-JOIN"); + }else if( pItem->fg.jointype & JT_RIGHT ){ + sqlite3_str_appendf(&x, " RIGHT-JOIN"); }else if( pItem->fg.jointype & JT_CROSS ){ sqlite3_str_appendf(&x, " CROSS-JOIN"); } + if( pItem->fg.jointype & JT_LTORJ ){ + sqlite3_str_appendf(&x, " LTORJ"); + } if( pItem->fg.fromDDL ){ sqlite3_str_appendf(&x, " DDL"); } if( pItem->fg.isCte ){ sqlite3_str_appendf(&x, " CteUse=0x%p", pItem->u2.pCteUse); } + if( pItem->fg.isOn || (pItem->fg.isUsing==0 && pItem->u3.pOn!=0) ){ + sqlite3_str_appendf(&x, " ON"); + } + if( pItem->fg.isTabFunc ) sqlite3_str_appendf(&x, " isTabFunc"); + if( pItem->fg.isCorrelated ) sqlite3_str_appendf(&x, " isCorrelated"); + if( pItem->fg.isMaterialized ) sqlite3_str_appendf(&x, " isMaterialized"); + if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); + if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); + if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); + n = 0; + if( pItem->pSelect ) n++; + if( pItem->fg.isTabFunc ) n++; + if( pItem->fg.isUsing ) n++; + if( pItem->fg.isUsing ){ + sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); + } if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + sqlite3TreeViewPush(&pView, i+1nSrc); + if( pItem->pTab ){ + Table *pTab = pItem->pTab; + sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); + } + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + sqlite3TreeViewPop(&pView); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } } @@ -30530,11 +33067,11 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewLine(pView, "nil-SELECT"); return; } - pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewPush(&pView, moreToFollow); if( p->pWith ){ sqlite3TreeViewWith(pView, p->pWith, 1); cnt = 1; - sqlite3TreeViewPush(pView, 1); + sqlite3TreeViewPush(&pView, 1); } do{ if( p->selFlags & SF_WhereBegin ){ @@ -30548,7 +33085,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m (int)p->nSelectRow ); } - if( cnt++ ) sqlite3TreeViewPop(pView); + if( cnt++ ) sqlite3TreeViewPop(&pView); if( p->pPrior ){ n = 1000; }else{ @@ -30571,24 +33108,24 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m #ifndef SQLITE_OMIT_WINDOWFUNC if( p->pWin ){ Window *pX; - pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewPush(&pView, (n--)>0); sqlite3TreeViewLine(pView, "window-functions"); for(pX=p->pWin; pX; pX=pX->pNextWin){ sqlite3TreeViewWinFunc(pView, pX, pX->pNextWin!=0); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #endif if( p->pSrc && p->pSrc->nSrc ){ - pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewPush(&pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); sqlite3TreeViewSrcList(pView, p->pSrc); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( p->pWhere ){ sqlite3TreeViewItem(pView, "WHERE", (n--)>0); sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( p->pGroupBy ){ sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); @@ -30596,7 +33133,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m if( p->pHaving ){ sqlite3TreeViewItem(pView, "HAVING", (n--)>0); sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #ifndef SQLITE_OMIT_WINDOWFUNC if( p->pWinDefn ){ @@ -30605,7 +33142,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m for(pX=p->pWinDefn; pX; pX=pX->pNextWin){ sqlite3TreeViewWindow(pView, pX, pX->pNextWin!=0); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #endif if( p->pOrderBy ){ @@ -30615,11 +33152,11 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); if( p->pLimit->pRight ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewItem(pView, "OFFSET", 0); sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( p->pPrior ){ const char *zOp = "UNION"; @@ -30632,7 +33169,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m } p = p->pPrior; }while( p!=0 ); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #ifndef SQLITE_OMIT_WINDOWFUNC @@ -30648,24 +33185,24 @@ SQLITE_PRIVATE void sqlite3TreeViewBound( switch( eBound ){ case TK_UNBOUNDED: { sqlite3TreeViewItem(pView, "UNBOUNDED", moreToFollow); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); break; } case TK_CURRENT: { sqlite3TreeViewItem(pView, "CURRENT", moreToFollow); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); break; } case TK_PRECEDING: { sqlite3TreeViewItem(pView, "PRECEDING", moreToFollow); sqlite3TreeViewExpr(pView, pExpr, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); break; } case TK_FOLLOWING: { sqlite3TreeViewItem(pView, "FOLLOWING", moreToFollow); sqlite3TreeViewExpr(pView, pExpr, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); break; } } @@ -30678,12 +33215,14 @@ SQLITE_PRIVATE void sqlite3TreeViewBound( */ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ int nElement = 0; + if( pWin==0 ) return; if( pWin->pFilter ){ sqlite3TreeViewItem(pView, "FILTER", 1); sqlite3TreeViewExpr(pView, pWin->pFilter, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); + if( pWin->eFrmType==TK_FILTER ) return; } - pView = sqlite3TreeViewPush(pView, more); + sqlite3TreeViewPush(&pView, more); if( pWin->zName ){ sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); }else{ @@ -30691,12 +33230,12 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u } if( pWin->zBase ) nElement++; if( pWin->pOrderBy ) nElement++; - if( pWin->eFrmType ) nElement++; + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ) nElement++; if( pWin->eExclude ) nElement++; if( pWin->zBase ){ - sqlite3TreeViewPush(pView, (--nElement)>0); + sqlite3TreeViewPush(&pView, (--nElement)>0); sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( pWin->pPartition ){ sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); @@ -30704,7 +33243,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u if( pWin->pOrderBy ){ sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); } - if( pWin->eFrmType ){ + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ){ char zBuf[30]; const char *zFrmType = "ROWS"; if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; @@ -30714,7 +33253,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( pWin->eExclude ){ char zBuf[30]; @@ -30729,11 +33268,11 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u zExclude = zBuf; break; } - sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewPush(&pView, 0); sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #endif /* SQLITE_OMIT_WINDOWFUNC */ @@ -30742,11 +33281,12 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u ** Generate a human-readable explanation for a Window Function object */ SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView *pView, const Window *pWin, u8 more){ - pView = sqlite3TreeViewPush(pView, more); + if( pWin==0 ) return; + sqlite3TreeViewPush(&pView, more); sqlite3TreeViewLine(pView, "WINFUNC %s(%d)", - pWin->pFunc->zName, pWin->pFunc->nArg); + pWin->pWFunc->zName, pWin->pWFunc->nArg); sqlite3TreeViewWindow(pView, pWin, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } #endif /* SQLITE_OMIT_WINDOWFUNC */ @@ -30757,19 +33297,22 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ char zFlgs[200]; - pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewPush(&pView, moreToFollow); if( pExpr==0 ){ sqlite3TreeViewLine(pView, "nil"); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - sqlite3_str_appendf(&x, " iRJT=%d", pExpr->iRightJoinTable); + if( ExprHasProperty(pExpr, EP_OuterON) ){ + sqlite3_str_appendf(&x, " outer.iJoin=%d", pExpr->w.iJoin); + } + if( ExprHasProperty(pExpr, EP_InnerON) ){ + sqlite3_str_appendf(&x, " inner.iJoin=%d", pExpr->w.iJoin); } if( ExprHasProperty(pExpr, EP_FromDDL) ){ sqlite3_str_appendf(&x, " DDL"); @@ -30777,6 +33320,9 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; @@ -30906,7 +33452,8 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m }; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); - assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); + assert( sqlite3ExprSkipCollateAndLikely(pExpr->pRight)->op + == TK_TRUEFALSE ); x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break; @@ -30944,7 +33491,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m assert( ExprUseXList(pExpr) ); pFarg = pExpr->x.pList; #ifndef SQLITE_OMIT_WINDOWFUNC - pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0; + pWin = IsWindowFunc(pExpr) ? pExpr->y.pWin : 0; #else pWin = 0; #endif @@ -30970,7 +33517,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); } if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); + sqlite3TreeViewExprList(pView, pFarg, pWin!=0 || pExpr->pLeft, 0); + if( pExpr->pLeft ){ + Expr *pOB = pExpr->pLeft; + assert( pOB->op==TK_ORDER ); + assert( ExprUseXList(pOB) ); + sqlite3TreeViewExprList(pView, pOB->x.pList, pWin!=0, "ORDERBY"); + } } #ifndef SQLITE_OMIT_WINDOWFUNC if( pWin ){ @@ -30979,6 +33532,10 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m #endif break; } + case TK_ORDER: { + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, "ORDERBY"); + break; + } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: { assert( ExprUseXSelect(pExpr) ); @@ -30993,7 +33550,17 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m break; } case TK_IN: { - sqlite3TreeViewLine(pView, "IN flags=0x%x", pExpr->flags); + sqlite3_str *pStr = sqlite3_str_new(0); + char *z; + sqlite3_str_appendf(pStr, "IN flags=0x%x", pExpr->flags); + if( pExpr->iTable ) sqlite3_str_appendf(pStr, " iTable=%d",pExpr->iTable); + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + sqlite3_str_appendf(pStr, " subrtn(%d,%d)", + pExpr->y.sub.regReturn, pExpr->y.sub.iAddr); + } + z = sqlite3_str_finish(pStr); + sqlite3TreeViewLine(pView, z); + sqlite3_free(z); sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); if( ExprUseXSelect(pExpr) ){ sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); @@ -31022,7 +33589,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m assert( pExpr->x.pList->nExpr==2 ); pY = pExpr->x.pList->a[0].pExpr; pZ = pExpr->x.pList->a[1].pExpr; - sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewLine(pView, "BETWEEN%s", zFlgs); sqlite3TreeViewExpr(pView, pX, 1); sqlite3TreeViewExpr(pView, pY, 1); sqlite3TreeViewExpr(pView, pZ, 0); @@ -31117,7 +33684,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } @@ -31139,13 +33706,25 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList( int j = pList->a[i].u.x.iOrderByCol; char *zName = pList->a[i].zEName; int moreToFollow = inExpr - 1; - if( pList->a[i].eEName!=ENAME_NAME ) zName = 0; if( j || zName ){ - sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewPush(&pView, moreToFollow); moreToFollow = 0; sqlite3TreeViewLine(pView, 0); if( zName ){ - fprintf(stdout, "AS %s ", zName); + switch( pList->a[i].fg.eEName ){ + default: + fprintf(stdout, "AS %s ", zName); + break; + case ENAME_TAB: + fprintf(stdout, "TABLE-ALIAS-NAME(\"%s\") ", zName); + if( pList->a[i].fg.bUsed ) fprintf(stdout, "(used) "); + if( pList->a[i].fg.bUsingTerm ) fprintf(stdout, "(USING-term) "); + if( pList->a[i].fg.bNoExpand ) fprintf(stdout, "(NoExpand) "); + break; + case ENAME_SPAN: + fprintf(stdout, "SPAN(\"%s\") ", zName); + break; + } } if( j ){ fprintf(stdout, "iOrderByCol=%d", j); @@ -31155,7 +33734,7 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList( } sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow); if( j || zName ){ - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } } } @@ -31166,10 +33745,377 @@ SQLITE_PRIVATE void sqlite3TreeViewExprList( u8 moreToFollow, const char *zLabel ){ - pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewPush(&pView, moreToFollow); sqlite3TreeViewBareExprList(pView, pList, zLabel); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); +} + +/* +** Generate a human-readable explanation of an id-list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewBareIdList( + TreeView *pView, + const IdList *pList, + const char *zLabel +){ + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + int i; + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inId; i++){ + char *zName = pList->a[i].zName; + int moreToFollow = inId - 1; + if( zName==0 ) zName = "(null)"; + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewLine(pView, 0); + if( pList->eU4==EU4_NONE ){ + fprintf(stdout, "%s\n", zName); + }else if( pList->eU4==EU4_IDX ){ + fprintf(stdout, "%s (%d)\n", zName, pList->a[i].u4.idx); + }else{ + assert( pList->eU4==EU4_EXPR ); + if( pList->a[i].u4.pExpr==0 ){ + fprintf(stdout, "%s (pExpr=NULL)\n", zName); + }else{ + fprintf(stdout, "%s\n", zName); + sqlite3TreeViewPush(&pView, inId-1); + sqlite3TreeViewExpr(pView, pList->a[i].u4.pExpr, 0); + sqlite3TreeViewPop(&pView); + } + } + sqlite3TreeViewPop(&pView); + } + } +} +SQLITE_PRIVATE void sqlite3TreeViewIdList( + TreeView *pView, + const IdList *pList, + u8 moreToFollow, + const char *zLabel +){ + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewBareIdList(pView, pList, zLabel); + sqlite3TreeViewPop(&pView); +} + +/* +** Generate a human-readable explanation of a list of Upsert objects +*/ +SQLITE_PRIVATE void sqlite3TreeViewUpsert( + TreeView *pView, + const Upsert *pUpsert, + u8 moreToFollow +){ + if( pUpsert==0 ) return; + sqlite3TreeViewPush(&pView, moreToFollow); + while( pUpsert ){ + int n; + sqlite3TreeViewPush(&pView, pUpsert->pNextUpsert!=0 || moreToFollow); + sqlite3TreeViewLine(pView, "ON CONFLICT DO %s", + pUpsert->isDoUpdate ? "UPDATE" : "NOTHING"); + n = (pUpsert->pUpsertSet!=0) + (pUpsert->pUpsertWhere!=0); + sqlite3TreeViewExprList(pView, pUpsert->pUpsertTarget, (n--)>0, "TARGET"); + sqlite3TreeViewExprList(pView, pUpsert->pUpsertSet, (n--)>0, "SET"); + if( pUpsert->pUpsertWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, pUpsert->pUpsertWhere, 0); + sqlite3TreeViewPop(&pView); + } + sqlite3TreeViewPop(&pView); + pUpsert = pUpsert->pNextUpsert; + } + sqlite3TreeViewPop(&pView); +} + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an DELETE statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewDelete( + const With *pWith, + const SrcList *pTabList, + const Expr *pWhere, + const ExprList *pOrderBy, + const Expr *pLimit, + const Trigger *pTrigger +){ + int n = 0; + TreeView *pView = 0; + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "DELETE"); + if( pWith ) n++; + if( pTabList ) n++; + if( pWhere ) n++; + if( pOrderBy ) n++; + if( pLimit ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pWhere ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "WHERE"); + sqlite3TreeViewExpr(pView, pWhere, 0); + sqlite3TreeViewPop(&pView); + } + if( pOrderBy ){ + sqlite3TreeViewExprList(pView, pOrderBy, (--n)>0, "ORDER-BY"); + } + if( pLimit ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "LIMIT"); + sqlite3TreeViewExpr(pView, pLimit, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); +} +#endif /* TREETRACE_ENABLED */ + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an INSERT statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewInsert( + const With *pWith, + const SrcList *pTabList, + const IdList *pColumnList, + const Select *pSelect, + const ExprList *pExprList, + int onError, + const Upsert *pUpsert, + const Trigger *pTrigger +){ + TreeView *pView = 0; + int n = 0; + const char *zLabel = "INSERT"; + switch( onError ){ + case OE_Replace: zLabel = "REPLACE"; break; + case OE_Ignore: zLabel = "INSERT OR IGNORE"; break; + case OE_Rollback: zLabel = "INSERT OR ROLLBACK"; break; + case OE_Abort: zLabel = "INSERT OR ABORT"; break; + case OE_Fail: zLabel = "INSERT OR FAIL"; break; + } + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, zLabel); + if( pWith ) n++; + if( pTabList ) n++; + if( pColumnList ) n++; + if( pSelect ) n++; + if( pExprList ) n++; + if( pUpsert ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "INTO"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pColumnList ){ + sqlite3TreeViewIdList(pView, pColumnList, (--n)>0, "COLUMNS"); + } + if( pSelect ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "DATA-SOURCE"); + sqlite3TreeViewSelect(pView, pSelect, 0); + sqlite3TreeViewPop(&pView); + } + if( pExprList ){ + sqlite3TreeViewExprList(pView, pExprList, (--n)>0, "VALUES"); + } + if( pUpsert ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "UPSERT"); + sqlite3TreeViewUpsert(pView, pUpsert, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); } +#endif /* TREETRACE_ENABLED */ + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an UPDATE statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewUpdate( + const With *pWith, + const SrcList *pTabList, + const ExprList *pChanges, + const Expr *pWhere, + int onError, + const ExprList *pOrderBy, + const Expr *pLimit, + const Upsert *pUpsert, + const Trigger *pTrigger +){ + int n = 0; + TreeView *pView = 0; + const char *zLabel = "UPDATE"; + switch( onError ){ + case OE_Replace: zLabel = "UPDATE OR REPLACE"; break; + case OE_Ignore: zLabel = "UPDATE OR IGNORE"; break; + case OE_Rollback: zLabel = "UPDATE OR ROLLBACK"; break; + case OE_Abort: zLabel = "UPDATE OR ABORT"; break; + case OE_Fail: zLabel = "UPDATE OR FAIL"; break; + } + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, zLabel); + if( pWith ) n++; + if( pTabList ) n++; + if( pChanges ) n++; + if( pWhere ) n++; + if( pOrderBy ) n++; + if( pLimit ) n++; + if( pUpsert ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pChanges ){ + sqlite3TreeViewExprList(pView, pChanges, (--n)>0, "SET"); + } + if( pWhere ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "WHERE"); + sqlite3TreeViewExpr(pView, pWhere, 0); + sqlite3TreeViewPop(&pView); + } + if( pOrderBy ){ + sqlite3TreeViewExprList(pView, pOrderBy, (--n)>0, "ORDER-BY"); + } + if( pLimit ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "LIMIT"); + sqlite3TreeViewExpr(pView, pLimit, 0); + sqlite3TreeViewPop(&pView); + } + if( pUpsert ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "UPSERT"); + sqlite3TreeViewUpsert(pView, pUpsert, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); +} +#endif /* TREETRACE_ENABLED */ + +#ifndef SQLITE_OMIT_TRIGGER +/* +** Show a human-readable graph of a TriggerStep +*/ +SQLITE_PRIVATE void sqlite3TreeViewTriggerStep( + TreeView *pView, + const TriggerStep *pStep, + u8 moreToFollow, + u8 showFullList +){ + int cnt = 0; + if( pStep==0 ) return; + sqlite3TreeViewPush(&pView, + moreToFollow || (showFullList && pStep->pNext!=0)); + do{ + if( cnt++ && pStep->pNext==0 ){ + sqlite3TreeViewPop(&pView); + sqlite3TreeViewPush(&pView, 0); + } + sqlite3TreeViewLine(pView, "%s", pStep->zSpan ? pStep->zSpan : "RETURNING"); + }while( showFullList && (pStep = pStep->pNext)!=0 ); + sqlite3TreeViewPop(&pView); +} + +/* +** Show a human-readable graph of a Trigger +*/ +SQLITE_PRIVATE void sqlite3TreeViewTrigger( + TreeView *pView, + const Trigger *pTrigger, + u8 moreToFollow, + u8 showFullList +){ + int cnt = 0; + if( pTrigger==0 ) return; + sqlite3TreeViewPush(&pView, + moreToFollow || (showFullList && pTrigger->pNext!=0)); + do{ + if( cnt++ && pTrigger->pNext==0 ){ + sqlite3TreeViewPop(&pView); + sqlite3TreeViewPush(&pView, 0); + } + sqlite3TreeViewLine(pView, "TRIGGER %s", pTrigger->zName); + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewTriggerStep(pView, pTrigger->step_list, 0, 1); + sqlite3TreeViewPop(&pView); + }while( showFullList && (pTrigger = pTrigger->pNext)!=0 ); + sqlite3TreeViewPop(&pView); +} +#endif /* SQLITE_OMIT_TRIGGER */ + + +/* +** These simplified versions of the tree-view routines omit unnecessary +** parameters. These variants are intended to be used from a symbolic +** debugger, such as "gdb", during interactive debugging sessions. +** +** This routines are given external linkage so that they will always be +** accessible to the debugging, and to avoid warnings about unused +** functions. But these routines only exist in debugging builds, so they +** do not contaminate the interface. +*/ +SQLITE_PRIVATE void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} +SQLITE_PRIVATE void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } +SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } +SQLITE_PRIVATE void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3ShowTriggerStep(const TriggerStep *p){ + sqlite3TreeViewTriggerStep(0,p,0,0); +} +SQLITE_PRIVATE void sqlite3ShowTriggerStepList(const TriggerStep *p){ + sqlite3TreeViewTriggerStep(0,p,0,1); +} +SQLITE_PRIVATE void sqlite3ShowTrigger(const Trigger *p){ sqlite3TreeViewTrigger(0,p,0,0); } +SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger *p){ sqlite3TreeViewTrigger(0,p,0,1);} +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3ShowWindow(const Window *p){ sqlite3TreeViewWindow(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window *p){ sqlite3TreeViewWinFunc(0,p,0); } +#endif #endif /* SQLITE_DEBUG */ @@ -31199,16 +34145,41 @@ SQLITE_PRIVATE void sqlite3TreeViewExprList( ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ - unsigned char t; unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -31238,53 +34209,46 @@ SQLITE_API void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ - if( !wsdPrng.isInit ){ + if( wsdPrng.s[0]==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); if( NEVER(pVfs==0) ){ - memset(k, 0, sizeof(k)); + memset(&wsdPrng.s[4], 0, 44); }else{ - sqlite3OsRandomness(pVfs, 256, k); + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; - } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[15] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } @@ -31760,7 +34724,38 @@ SQLITE_PRIVATE u32 sqlite3Utf8Read( return c; } - +/* +** Read a single UTF8 character out of buffer z[], but reading no +** more than n characters from the buffer. z[] is not zero-terminated. +** +** Return the number of bytes used to construct the character. +** +** Invalid UTF8 might generate a strange result. No effort is made +** to detect invalid UTF8. +** +** At most 4 bytes will be read out of z[]. The return value will always +** be between 1 and 4. +*/ +SQLITE_PRIVATE int sqlite3Utf8ReadLimited( + const u8 *z, + int n, + u32 *piOut +){ + u32 c; + int i = 1; + assert( n>0 ); + c = z[0]; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + if( n>4 ) n = 4; + while( ierrCode = err_code; - if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); + if( err_code || db->pErr ){ + sqlite3ErrorFinish(db, err_code); + }else{ + db->errByteOffset = -1; + } } /* @@ -32261,6 +35273,7 @@ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3 *db){ assert( db!=0 ); db->errCode = SQLITE_OK; + db->errByteOffset = -1; if( db->pErr ) sqlite3ValueSetNull(db->pErr); } @@ -32270,6 +35283,23 @@ SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3 *db){ */ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ if( rc==SQLITE_IOERR_NOMEM ) return; +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) + if( rc==SQLITE_IOERR_IN_PAGE ){ + int ii; + int iErr; + sqlite3BtreeEnterAll(db); + for(ii=0; iinDb; ii++){ + if( db->aDb[ii].pBt ){ + iErr = sqlite3PagerWalSystemErrno(sqlite3BtreePager(db->aDb[ii].pBt)); + if( iErr ){ + db->iSysErrno = iErr; + } + } + } + sqlite3BtreeLeaveAll(db); + return; + } +#endif rc &= 0xff; if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ db->iSysErrno = sqlite3OsGetLastError(db->pVfs); @@ -32281,17 +35311,8 @@ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ ** handle "db". The error code is set to "err_code". ** ** If it is not NULL, string zFormat specifies the format of the -** error string in the style of the printf functions: The following -** format characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList -** -** zFormat and any string tokens that follow it are assumed to be -** encoded in UTF-8. +** error string. zFormat and any string tokens that follow it are +** assumed to be encoded in UTF-8. ** ** To clear the most recent error for sqlite handle "db", sqlite3Error ** should be called with err_code set to SQLITE_OK and zFormat set @@ -32313,15 +35334,32 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *z } } +/* +** Check for interrupts and invoke progress callback. +*/ +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse *p){ + sqlite3 *db = p->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + if( p->rc==SQLITE_INTERRUPT ){ + p->nProgressSteps = 0; + }else if( (++p->nProgressSteps)>=db->nProgressOps ){ + if( db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } + p->nProgressSteps = 0; + } + } +#endif +} + /* ** Add an error message to pParse->zErrMsg and increment pParse->nErr. -** The following formatting characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList ** ** This function should be used to report any error that occurs while ** compiling an SQL statement (i.e. within sqlite3_prepare()). The @@ -32334,11 +35372,19 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ char *zMsg; va_list ap; sqlite3 *db = pParse->db; + assert( db!=0 ); + assert( db->pParse==pParse || db->pParse->pToplevel==pParse ); + db->errByteOffset = -2; va_start(ap, zFormat); zMsg = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); + if( db->errByteOffset<-1 ) db->errByteOffset = -1; if( db->suppressErr ){ sqlite3DbFree(db, zMsg); + if( db->mallocFailed ){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } }else{ pParse->nErr++; sqlite3DbFree(db, pParse->zErrMsg); @@ -32407,6 +35453,44 @@ SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ sqlite3Dequote(p->u.zToken); } +/* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + /* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: @@ -32503,43 +35587,40 @@ SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ return h; } -/* -** Compute 10 to the E-th power. Examples: E==1 results in 10. -** E==2 results in 100. E==50 results in 1.0e50. +/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) ** -** This routine only works for values of E between 1 and 341. +** Reference: +** T. J. Dekker, "A Floating-Point Technique for Extending the +** Available Precision". 1971-07-26. */ -static LONGDOUBLE_TYPE sqlite3Pow10(int E){ -#if defined(_MSC_VER) - static const LONGDOUBLE_TYPE x[] = { - 1.0e+001L, - 1.0e+002L, - 1.0e+004L, - 1.0e+008L, - 1.0e+016L, - 1.0e+032L, - 1.0e+064L, - 1.0e+128L, - 1.0e+256L - }; - LONGDOUBLE_TYPE r = 1.0; - int i; - assert( E>=0 && E<=307 ); - for(i=0; E!=0; i++, E >>=1){ - if( E & 1 ) r *= x[i]; - } - return r; -#else - LONGDOUBLE_TYPE x = 10.0; - LONGDOUBLE_TYPE r = 1.0; - while(1){ - if( E & 1 ) r *= x; - E >>= 1; - if( E==0 ) break; - x *= x; - } - return r; -#endif +static void dekkerMul2(volatile double *x, double y, double yy){ + /* + ** The "volatile" keywords on parameter x[] and on local variables + ** below are needed force intermediate results to be truncated to + ** binary64 rather than be carried around in an extended-precision + ** format. The truncation is necessary for the Dekker algorithm to + ** work. Intel x86 floating point might omit the truncation without + ** the use of volatile. + */ + volatile double tx, ty, p, q, c, cc; + double hx, hy; + u64 m; + memcpy(&m, (void*)&x[0], 8); + m &= 0xfffffffffc000000LL; + memcpy(&hx, &m, 8); + tx = x[0] - hx; + memcpy(&m, &y, 8); + m &= 0xfffffffffc000000LL; + memcpy(&hy, &m, 8); + ty = y - hy; + p = hx*hy; + q = hx*ty + tx*hy; + c = p+q; + cc = p - c + q + tx*ty; + cc = x[0]*yy + x[1]*y + cc; + x[0] = c + cc; + x[1] = c - x[0]; + x[1] += cc; } /* @@ -32580,12 +35661,11 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ + u64 s = 0; /* significand */ int d = 0; /* adjust exponent for shifting decimal point */ int esign = 1; /* sign of exponent */ int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ - double result; int nDigit = 0; /* Number of digits processed */ int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ @@ -32625,7 +35705,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en while( z=((LARGEST_INT64-9)/10) ){ + if( s>=((LARGEST_UINT64-9)/10) ){ /* skip non-significant significand digits ** (increase exponent by d to shift decimal left) */ while( z0 ){ /*OPTIMIZATION-IF-TRUE*/ - if( esign>0 ){ - if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ - s *= 10; - }else{ - if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ - s /= 10; - } - e--; - } + /* adjust exponent by d, and update sign */ + e = (e*esign) + d; - /* adjust the sign of significand */ - s = sign<0 ? -s : s; + /* Try to adjust the exponent to make it smaller */ + while( e>0 && s<(LARGEST_UINT64/10) ){ + s *= 10; + e--; + } + while( e<0 && (s%10)==0 ){ + s /= 10; + e++; + } - if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ - result = (double)s; + if( e==0 ){ + *pResult = s; + }else if( sqlite3Config.bUseLongDouble ){ + LONGDOUBLE_TYPE r = (LONGDOUBLE_TYPE)s; + if( e>0 ){ + while( e>=100 ){ e-=100; r *= 1.0e+100L; } + while( e>=10 ){ e-=10; r *= 1.0e+10L; } + while( e>=1 ){ e-=1; r *= 1.0e+01L; } }else{ - /* attempt to handle extremely small/large numbers better */ - if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ - if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308); - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else{ assert( e>=342 ); - if( esign<0 ){ - result = 0.0*s; - }else{ + while( e<=-100 ){ e+=100; r *= 1.0e-100L; } + while( e<=-10 ){ e+=10; r *= 1.0e-10L; } + while( e<=-1 ){ e+=1; r *= 1.0e-01L; } + } + assert( r>=0.0 ); + if( r>+1.7976931348623157081452742373e+308L ){ #ifdef INFINITY - result = INFINITY*s; + *pResult = +INFINITY; #else - result = 1e308*1e308*s; /* Infinity */ + *pResult = 1.0e308*10.0; #endif - } - } - }else{ - LONGDOUBLE_TYPE scale = sqlite3Pow10(e); - if( esign<0 ){ - result = s / scale; - }else{ - result = s * scale; - } + }else{ + *pResult = (double)r; + } + }else{ + double rr[2]; + u64 s2; + rr[0] = (double)s; + s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif + rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); + if( e>0 ){ + while( e>=100 ){ + e -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( e>=10 ){ + e -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( e>=1 ){ + e -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + }else{ + while( e<=-100 ){ + e += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( e<=-10 ){ + e += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( e<=-1 ){ + e += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); } } + *pResult = rr[0]+rr[1]; + if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; } + if( sign<0 ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); - /* store the result */ - *pResult = result; - - /* return true if number and no extra non-whitespace chracters after */ +atof_return: + /* return true if number and no extra non-whitespace characters after */ if( z==zEnd && nDigit>0 && eValid && eType>0 ){ return eType; }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ @@ -32769,11 +35862,14 @@ do_atof_calc: #endif /* -** Render an signed 64-bit integer as text. Store the result in zOut[]. +** Render an signed 64-bit integer as text. Store the result in zOut[] and +** return the length of the string that was stored, in bytes. The value +** returned does not include the zero terminator at the end of the output +** string. ** ** The caller must ensure that zOut[] is at least 21 bytes in size. */ -SQLITE_PRIVATE void sqlite3Int64ToText(i64 v, char *zOut){ +SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; char zTemp[22]; @@ -32784,12 +35880,15 @@ SQLITE_PRIVATE void sqlite3Int64ToText(i64 v, char *zOut){ } i = sizeof(zTemp)-2; zTemp[sizeof(zTemp)-1] = 0; - do{ - zTemp[i--] = (x%10) + '0'; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; x = x/10; - }while( x ); - if( v<0 ) zTemp[i--] = '-'; - memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* @@ -32882,7 +35981,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc /* This test and assignment is needed only to suppress UB warnings ** from clang and -fsanitize=undefined. This test and assignment make ** the code a little larger and slower, and no harm comes from omitting - ** them, but we must appaise the undefined-behavior pharisees. */ + ** them, but we must appease the undefined-behavior pharisees. */ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; }else if( neg ){ *pNum = -(i64)u; @@ -32954,11 +36053,15 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 2; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); + if( z[n] ) n++; + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); } } @@ -32990,7 +36093,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ @@ -33037,6 +36140,153 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ return x; } +/* +** Decode a floating-point value into an approximate decimal +** representation. +** +** Round the decimal representation to n significant digits if +** n is positive. Or round to -n signficant digits after the +** decimal point if n is negative. No rounding is performed if +** n is zero. +** +** The significant digits of the decimal representation are +** stored in p->z[] which is a often (but not always) a pointer +** into the middle of p->zBuf[]. There are p->n significant digits. +** The p->z[] array is *not* zero-terminated. +*/ +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ + int i; + u64 v; + int e, exp = 0; + p->isSpecial = 0; + p->z = p->zBuf; + + /* Convert negative numbers to positive. Deal with Infinity, 0.0, and + ** NaN. */ + if( r<0.0 ){ + p->sign = '-'; + r = -r; + }else if( r==0.0 ){ + p->sign = '+'; + p->n = 1; + p->iDP = 1; + p->z = "0"; + return; + }else{ + p->sign = '+'; + } + memcpy(&v,&r,8); + e = v>>52; + if( (e&0x7ff)==0x7ff ){ + p->isSpecial = 1 + (v!=0x7ff0000000000000LL); + p->n = 0; + p->iDP = 0; + return; + } + + /* Multiply r by powers of ten until it lands somewhere in between + ** 1.0e+19 and 1.0e+17. + */ + if( sqlite3Config.bUseLongDouble ){ + LONGDOUBLE_TYPE rr = r; + if( rr>=1.0e+19 ){ + while( rr>=1.0e+119L ){ exp+=100; rr *= 1.0e-100L; } + while( rr>=1.0e+29L ){ exp+=10; rr *= 1.0e-10L; } + while( rr>=1.0e+19L ){ exp++; rr *= 1.0e-1L; } + }else{ + while( rr<1.0e-97L ){ exp-=100; rr *= 1.0e+100L; } + while( rr<1.0e+07L ){ exp-=10; rr *= 1.0e+10L; } + while( rr<1.0e+17L ){ exp--; rr *= 1.0e+1L; } + } + v = (u64)rr; + }else{ + /* If high-precision floating point is not available using "long double", + ** then use Dekker-style double-double computation to increase the + ** precision. + ** + ** The error terms on constants like 1.0e+100 computed using the + ** decimal extension, for example as follows: + ** + ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); + */ + double rr[2]; + rr[0] = r; + rr[1] = 0.0; + if( rr[0]>9.223372036854774784e+18 ){ + while( rr[0]>9.223372036854774784e+118 ){ + exp += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( rr[0]>9.223372036854774784e+28 ){ + exp += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( rr[0]>9.223372036854774784e+18 ){ + exp += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); + } + }else{ + while( rr[0]<9.223372036854774784e-83 ){ + exp -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( rr[0]<9.223372036854774784e+07 ){ + exp -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( rr[0]<9.22337203685477478e+17 ){ + exp -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + } + v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + } + + + /* Extract significant digits. */ + i = sizeof(p->zBuf)-1; + assert( v>0 ); + while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } + assert( i>=0 && izBuf)-1 ); + p->n = sizeof(p->zBuf) - 1 - i; + assert( p->n>0 ); + assert( p->nzBuf) ); + p->iDP = p->n + exp; + if( iRound<=0 ){ + iRound = p->iDP - iRound; + if( iRound==0 && p->zBuf[i+1]>='5' ){ + iRound = 1; + p->zBuf[i--] = '0'; + p->n++; + p->iDP++; + } + } + if( iRound>0 && (iRoundn || p->n>mxRound) ){ + char *z = &p->zBuf[i+1]; + if( iRound>mxRound ) iRound = mxRound; + p->n = iRound; + if( z[iRound]>='5' ){ + int j = iRound-1; + while( 1 /*exit-by-break*/ ){ + z[j]++; + if( z[j]<='9' ) break; + z[j] = '0'; + if( j==0 ){ + p->z[i--] = '1'; + p->n++; + p->iDP++; + break; + }else{ + j--; + } + } + } + } + p->z = &p->zBuf[i+1]; + assert( i+p->n < sizeof(p->zBuf) ); + while( ALWAYS(p->n>0) && p->z[p->n-1]=='0' ){ p->n--; } +} + /* ** Try to convert z into an unsigned 32-bit integer. Return true on ** success and false if there is an error. @@ -33300,121 +36550,32 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ ** this function assumes the single-byte case has already been handled. */ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ - u32 a,b; + u64 v64; + u8 n; - /* The 1-byte case. Overwhelmingly the most common. Handled inline - ** by the getVarin32() macro */ - a = *p; - /* a: p0 (unmasked) */ -#ifndef getVarint32 - if (!(a&0x80)) - { - /* Values between 0 and 127 */ - *v = a; - return 1; - } -#endif + /* Assume that the single-byte case has already been handled by + ** the getVarint32() macro */ + assert( (p[0] & 0x80)!=0 ); - /* The 2-byte case */ - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 128 and 16383 */ - a &= 0x7f; - a = a<<7; - *v = a | b; + if( (p[1] & 0x80)==0 ){ + /* This is the two-byte case */ + *v = ((p[0]&0x7f)<<7) | p[1]; return 2; } - - /* The 3-byte case */ - p++; - a = a<<14; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 16384 and 2097151 */ - a &= (0x7f<<14)|(0x7f); - b &= 0x7f; - b = b<<7; - *v = a | b; + if( (p[2] & 0x80)==0 ){ + /* This is the three-byte case */ + *v = ((p[0]&0x7f)<<14) | ((p[1]&0x7f)<<7) | p[2]; return 3; } - - /* A 32-bit varint is used to store size information in btrees. - ** Objects are rarely larger than 2MiB limit of a 3-byte varint. - ** A 3-byte varint is sufficient, for example, to record the size - ** of a 1048569-byte BLOB or string. - ** - ** We only unroll the first 1-, 2-, and 3- byte cases. The very - ** rare larger cases can be handled by the slower 64-bit varint - ** routine. - */ -#if 1 - { - u64 v64; - u8 n; - - n = sqlite3GetVarint(p-2, &v64); - assert( n>3 && n<=9 ); - if( (v64 & SQLITE_MAX_U32)!=v64 ){ - *v = 0xffffffff; - }else{ - *v = (u32)v64; - } - return n; - } - -#else - /* For following code (kept for historical record only) shows an - ** unrolling for the 3- and 4-byte varint cases. This code is - ** slightly faster, but it is also larger and much harder to test. - */ - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 2097152 and 268435455 */ - b &= (0x7f<<14)|(0x7f); - a &= (0x7f<<14)|(0x7f); - a = a<<7; - *v = a | b; - return 4; - } - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 268435456 and 34359738367 */ - a &= SLOT_4_2_0; - b &= SLOT_4_2_0; - b = b<<7; - *v = a | b; - return 5; - } - - /* We can only reach this point when reading a corrupt database - ** file. In that case we are not in any hurry. Use the (relatively - ** slow) general-purpose sqlite3GetVarint() routine to extract the - ** value. */ - { - u64 v64; - u8 n; - - p -= 4; - n = sqlite3GetVarint(p, &v64); - assert( n>5 && n<=9 ); + /* four or more bytes */ + n = sqlite3GetVarint(p, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ *v = (u32)v64; - return n; } -#endif + return n; } /* @@ -33505,7 +36666,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ } return zBlob; } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ +#endif /* !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) */ /* ** Log an error that is an API call on a connection pointer that should @@ -33565,7 +36726,7 @@ SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ } /* -** Attempt to add, substract, or multiply the 64-bit signed value iB against +** Attempt to add, subtract, or multiply the 64-bit signed value iB against ** the other 64-bit signed integer at *pA and store the result in *pA. ** Return 0 on success. Or if the operation would have resulted in an ** overflow, leave *pA unchanged and return 1. @@ -33720,7 +36881,6 @@ SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ return a[x&7] + y - 10; } -#ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Convert a double into a LogEst ** In other words, compute an approximation for 10*log2(x). @@ -33735,16 +36895,9 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ e = (a>>52) - 1022; return e*10; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. -** -** Note that this routine is only used when one or more of various -** non-standard compile-time options is enabled. */ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ u64 n; @@ -33752,17 +36905,9 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) if( x>60 ) return (u64)LARGEST_INT64; -#else - /* If only SQLITE_ENABLE_STAT4 is on, then the largest input - ** possible to this routine is 310, resulting in a maximum x of 31 */ - assert( x<=60 ); -#endif return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); } -#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ /* ** Add a new name/number pair to a VList. This might require that the @@ -33867,6 +37012,104 @@ SQLITE_PRIVATE int sqlite3VListNameToNum(VList *pIn, const char *zName, int nNam return 0; } +/* +** High-resolution hardware timer used for debugging and testing only. +*/ +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/************** Include hwtime.h in the middle of util.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H + +/* +** The following routine only works on Pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in util.c ***********************/ +#endif + /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ /* @@ -33968,7 +37211,7 @@ static void insertElement( } -/* Resize the hash table so that it cantains "new_size" buckets. +/* Resize the hash table so that it contains "new_size" buckets. ** ** The hash table might fail to resize if sqlite3_malloc() fails or ** if the new size is the same as the prior size. @@ -34037,12 +37280,13 @@ static HashElem *findElementWithHash( count = pH->count; } if( pHash ) *pHash = h; - while( count-- ){ + while( count ){ assert( elem!=0 ); if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; + count--; } return &nullElement; } @@ -34156,53 +37400,53 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 0 */ "Savepoint" OpHelp(""), /* 1 */ "AutoCommit" OpHelp(""), /* 2 */ "Transaction" OpHelp(""), - /* 3 */ "SorterNext" OpHelp(""), - /* 4 */ "Prev" OpHelp(""), - /* 5 */ "Next" OpHelp(""), - /* 6 */ "Checkpoint" OpHelp(""), - /* 7 */ "JournalMode" OpHelp(""), - /* 8 */ "Vacuum" OpHelp(""), - /* 9 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 10 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 11 */ "Goto" OpHelp(""), - /* 12 */ "Gosub" OpHelp(""), - /* 13 */ "InitCoroutine" OpHelp(""), - /* 14 */ "Yield" OpHelp(""), - /* 15 */ "MustBeInt" OpHelp(""), - /* 16 */ "Jump" OpHelp(""), - /* 17 */ "Once" OpHelp(""), - /* 18 */ "If" OpHelp(""), + /* 3 */ "Checkpoint" OpHelp(""), + /* 4 */ "JournalMode" OpHelp(""), + /* 5 */ "Vacuum" OpHelp(""), + /* 6 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 7 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 8 */ "Init" OpHelp("Start at P2"), + /* 9 */ "Goto" OpHelp(""), + /* 10 */ "Gosub" OpHelp(""), + /* 11 */ "InitCoroutine" OpHelp(""), + /* 12 */ "Yield" OpHelp(""), + /* 13 */ "MustBeInt" OpHelp(""), + /* 14 */ "Jump" OpHelp(""), + /* 15 */ "Once" OpHelp(""), + /* 16 */ "If" OpHelp(""), + /* 17 */ "IfNot" OpHelp(""), + /* 18 */ "IsType" OpHelp("if typeof(P1.P3) in P5 goto P2"), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "IfNot" OpHelp(""), - /* 21 */ "IsNullOrType" OpHelp("if typeof(r[P1]) IN (P3,5) goto P2"), - /* 22 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), - /* 23 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 24 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 25 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 26 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 27 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), - /* 28 */ "IfNoHope" OpHelp("key=r[P3@P4]"), - /* 29 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 30 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 31 */ "Found" OpHelp("key=r[P3@P4]"), - /* 32 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 33 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 34 */ "Last" OpHelp(""), - /* 35 */ "IfSmaller" OpHelp(""), - /* 36 */ "SorterSort" OpHelp(""), - /* 37 */ "Sort" OpHelp(""), - /* 38 */ "Rewind" OpHelp(""), - /* 39 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 40 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 20 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 21 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 29 */ "Found" OpHelp("key=r[P3@P4]"), + /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 32 */ "Last" OpHelp(""), + /* 33 */ "IfSizeBetween" OpHelp(""), + /* 34 */ "SorterSort" OpHelp(""), + /* 35 */ "Sort" OpHelp(""), + /* 36 */ "Rewind" OpHelp(""), + /* 37 */ "SorterNext" OpHelp(""), + /* 38 */ "Prev" OpHelp(""), + /* 39 */ "Next" OpHelp(""), + /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 46 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 47 */ "Program" OpHelp(""), - /* 48 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 49 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 48 */ "Program" OpHelp(""), + /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -34212,49 +37456,49 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), /* 58 */ "ElseEq" OpHelp(""), - /* 59 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 60 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 61 */ "IncrVacuum" OpHelp(""), - /* 62 */ "VNext" OpHelp(""), - /* 63 */ "Init" OpHelp("Start at P2"), - /* 64 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), - /* 65 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), - /* 66 */ "Return" OpHelp(""), - /* 67 */ "EndCoroutine" OpHelp(""), - /* 68 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 69 */ "Halt" OpHelp(""), - /* 70 */ "Integer" OpHelp("r[P2]=P1"), - /* 71 */ "Int64" OpHelp("r[P2]=P4"), - /* 72 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 73 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 74 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 75 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 76 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 77 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 78 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 79 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 80 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 81 */ "ChngCntRow" OpHelp("output=r[P1]"), - /* 82 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 83 */ "CollSeq" OpHelp(""), - /* 84 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 85 */ "RealAffinity" OpHelp(""), - /* 86 */ "Cast" OpHelp("affinity(r[P1])"), - /* 87 */ "Permutation" OpHelp(""), - /* 88 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 89 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 90 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), - /* 91 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 92 */ "Column" OpHelp("r[P3]=PX"), - /* 93 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), - /* 94 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 95 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 96 */ "Count" OpHelp("r[P2]=count()"), - /* 97 */ "ReadCookie" OpHelp(""), - /* 98 */ "SetCookie" OpHelp(""), - /* 99 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 100 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 101 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 59 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 62 */ "IncrVacuum" OpHelp(""), + /* 63 */ "VNext" OpHelp(""), + /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 67 */ "Return" OpHelp(""), + /* 68 */ "EndCoroutine" OpHelp(""), + /* 69 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 70 */ "Halt" OpHelp(""), + /* 71 */ "Integer" OpHelp("r[P2]=P1"), + /* 72 */ "Int64" OpHelp("r[P2]=P4"), + /* 73 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 74 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 82 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 83 */ "FkCheck" OpHelp(""), + /* 84 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 85 */ "CollSeq" OpHelp(""), + /* 86 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 87 */ "RealAffinity" OpHelp(""), + /* 88 */ "Cast" OpHelp("affinity(r[P1])"), + /* 89 */ "Permutation" OpHelp(""), + /* 90 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 91 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 92 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 93 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 94 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 95 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 96 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 97 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 98 */ "Count" OpHelp("r[P2]=count()"), + /* 99 */ "ReadCookie" OpHelp(""), + /* 100 */ "SetCookie" OpHelp(""), + /* 101 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), /* 102 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 103 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 104 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 159 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 160 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 161 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 162 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 163 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 164 */ "Expire" OpHelp(""), - /* 165 */ "CursorLock" OpHelp(""), - /* 166 */ "CursorUnlock" OpHelp(""), - /* 167 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 168 */ "VBegin" OpHelp(""), - /* 169 */ "VCreate" OpHelp(""), - /* 170 */ "VDestroy" OpHelp(""), - /* 171 */ "VOpen" OpHelp(""), - /* 172 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 173 */ "VRename" OpHelp(""), - /* 174 */ "Pagecount" OpHelp(""), - /* 175 */ "MaxPgcnt" OpHelp(""), - /* 176 */ "Trace" OpHelp(""), - /* 177 */ "CursorHint" OpHelp(""), - /* 178 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 179 */ "Noop" OpHelp(""), - /* 180 */ "Explain" OpHelp(""), - /* 181 */ "Abortable" OpHelp(""), + /* 112 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 113 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 114 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 115 */ "OpenDup" OpHelp(""), + /* 116 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 117 */ "String8" OpHelp("r[P2]='P4'"), + /* 118 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 119 */ "SorterOpen" OpHelp(""), + /* 120 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 121 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 122 */ "Close" OpHelp(""), + /* 123 */ "ColumnsUsed" OpHelp(""), + /* 124 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 125 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 126 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 127 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 128 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 129 */ "RowCell" OpHelp(""), + /* 130 */ "Delete" OpHelp(""), + /* 131 */ "ResetCount" OpHelp(""), + /* 132 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 133 */ "SorterData" OpHelp("r[P2]=data"), + /* 134 */ "RowData" OpHelp("r[P2]=data"), + /* 135 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), + /* 136 */ "NullRow" OpHelp(""), + /* 137 */ "SeekEnd" OpHelp(""), + /* 138 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 139 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 140 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 141 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 142 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 143 */ "FinishSeek" OpHelp(""), + /* 144 */ "Destroy" OpHelp(""), + /* 145 */ "Clear" OpHelp(""), + /* 146 */ "ResetSorter" OpHelp(""), + /* 147 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 148 */ "SqlExec" OpHelp(""), + /* 149 */ "ParseSchema" OpHelp(""), + /* 150 */ "LoadAnalysis" OpHelp(""), + /* 151 */ "DropTable" OpHelp(""), + /* 152 */ "DropIndex" OpHelp(""), + /* 153 */ "Real" OpHelp("r[P2]=P4"), + /* 154 */ "DropTrigger" OpHelp(""), + /* 155 */ "IntegrityCk" OpHelp(""), + /* 156 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 157 */ "Param" OpHelp(""), + /* 158 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 159 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 160 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 161 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 162 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 163 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 164 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 165 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 166 */ "Expire" OpHelp(""), + /* 167 */ "CursorLock" OpHelp(""), + /* 168 */ "CursorUnlock" OpHelp(""), + /* 169 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 170 */ "VBegin" OpHelp(""), + /* 171 */ "VCreate" OpHelp(""), + /* 172 */ "VDestroy" OpHelp(""), + /* 173 */ "VOpen" OpHelp(""), + /* 174 */ "VCheck" OpHelp(""), + /* 175 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 176 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 177 */ "VRename" OpHelp(""), + /* 178 */ "Pagecount" OpHelp(""), + /* 179 */ "MaxPgcnt" OpHelp(""), + /* 180 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 181 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 182 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 183 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 184 */ "Trace" OpHelp(""), + /* 185 */ "CursorHint" OpHelp(""), + /* 186 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 187 */ "Noop" OpHelp(""), + /* 188 */ "Explain" OpHelp(""), + /* 189 */ "Abortable" OpHelp(""), }; return azName[i]; } #endif /************** End of opcodes.c *********************************************/ +/************** Begin file os_kv.c *******************************************/ +/* +** 2022-09-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an experimental VFS layer that operates on a +** Key/Value storage engine where both keys and values must be pure +** text. +*/ +/* #include */ +#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)) + +/***************************************************************************** +** Debugging logic +*/ + +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +#if 0 +#define SQLITE_KV_TRACE(X) printf X +#else +#define SQLITE_KV_TRACE(X) +#endif + +/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ +#if 0 +#define SQLITE_KV_LOG(X) printf X +#else +#define SQLITE_KV_LOG(X) +#endif + + +/* +** Forward declaration of objects used by this VFS implementation +*/ +typedef struct KVVfsFile KVVfsFile; + +/* A single open file. There are only two files represented by this +** VFS - the database and the rollback journal. +*/ +struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + char *aData; /* Buffer to hold page data */ +}; +#define SQLITE_KVOS_SZ 133073 + +/* +** Methods for KVVfsFile +*/ +static int kvvfsClose(sqlite3_file*); +static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); +static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); +static int kvvfsSyncDb(sqlite3_file*, int flags); +static int kvvfsSyncJrnl(sqlite3_file*, int flags); +static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsLock(sqlite3_file*, int); +static int kvvfsUnlock(sqlite3_file*, int); +static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg); +static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg); +static int kvvfsSectorSize(sqlite3_file*); +static int kvvfsDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for sqlite3_vfs +*/ +static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); +static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int kvvfsSleep(sqlite3_vfs*, int microseconds); +static int kvvfsCurrentTime(sqlite3_vfs*, double*); +static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs sqlite3OsKvvfsObject = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +/* Methods for sqlite3_file objects referencing a database file +*/ +static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlDb, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* Methods for sqlite3_file objects referencing a rollback journal +*/ +static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlJrnl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/****** Storage subsystem **************************************************/ +#include +#include +#include + +/* Forward declarations for the low-level storage engine +*/ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 + +/* Expand the key name with an appropriate prefix and put the result +** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVSTORAGE_KEY_SZ bytes. +*/ +static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut +){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); +} + +/* Write content into a key. zClass is the particular namespace of the +** underlying key/value store to use - either "local" or "session". +** +** Both zKey and zData are zero-terminated pure text strings. +** +** Return the number of errors. +*/ +static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData +){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } +} + +/* Delete a key (with its corresponding data) from the key/value +** namespace given by zClass. If the key does not previously exist, +** this routine is a no-op. +*/ +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; +} + +/* Read the value associated with a zKey from the key/value namespace given +** by zClass and put the text data associated with that key in the first +** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large +** enough to hold it all. The value put into zBuf must always be zero +** terminated, even if it gets truncated because nBuf is not large enough. +** +** Return the total number of bytes in the data, without truncation, and +** not counting the final zero terminator. Return -1 if the key does +** not exist. +** +** If nBuf<=0 then this routine simply returns the size of the data without +** actually reading it. +*/ +static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf +){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } +} + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; + +/****** Utility subroutines ************************************************/ + +/* +** Encode binary into the text encoded used to persist on disk. +** The output text is stored in aOut[], which must be at least +** nData+1 bytes in length. +** +** Return the actual length of the encoded text, not counting the +** zero terminator at the end. +** +** Encoding format +** --------------- +** +** * Non-zero bytes are encoded as upper-case hexadecimal +** +** * A sequence of one or more zero-bytes that are not at the +** beginning of the buffer are encoded as a little-endian +** base-26 number using a..z. "a" means 0. "b" means 1, +** "z" means 25. "ab" means 26. "ac" means 52. And so forth. +** +** * Because there is no overlap between the encoding characters +** of hexadecimal and base-26 numbers, it is always clear where +** one stops and the next begins. +*/ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; +} + +static const signed char kvvfsHexValue[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* +** Decode the text encoding back to binary. The binary content is +** written into pOut, which must be at least nOut bytes in length. +** +** The return value is the number of bytes actually written into aOut[]. +*/ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ + int i, j; + int c; + const unsigned char *aIn = (const unsigned char*)a; + i = 0; + j = 0; + while( 1 ){ + c = kvvfsHexValue[aIn[i]]; + if( c<0 ){ + int n = 0; + int mult = 1; + c = aIn[i]; + if( c==0 ) break; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + memset(&aOut[j], 0, n); + j += n; + if( c==0 || mult==1 ) break; /* progress stalled if mult==1 */ + }else{ + aOut[j] = c<<4; + c = kvvfsHexValue[aIn[++i]]; + if( c<0 ) break; + aOut[j++] += c; + i++; + } + } + return j; +} + +/* +** Decode a complete journal file. Allocate space in pFile->aJrnl +** and store the decoding there. Or leave pFile->aJrnl set to NULL +** if an error is encountered. +** +** The first few characters of the text encoding will be a little-endian +** base-26 number (digits a..z) that is the total number of bytes +** in the decoded journal file image. This base-26 number is followed +** by a single space, then the encoding of the journal. The space +** separator is required to act as a terminator for the base-26 number. +*/ +static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ +){ + unsigned int n = 0; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( nnJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } +} + +/* +** Read or write the "sz" element, containing the database file size. +*/ +static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); +} +static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); +} + +/****** sqlite3_io_methods methods ******************************************/ + +/* +** Close an kvvfs-file. +*/ +static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + sqlite3_free(pFile->aData); + return SQLITE_OK; +} + +/* +** Read from the -journal file. +*/ +static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Read from the database file. +*/ +static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char *aData = pFile->aData; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + int k = iOfst+iAmt; + aData[k*2] = 0; + n = kvvfsDecode(aData, &aData[2000], SQLITE_KVOS_SZ-2000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[2000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( nzClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; +} + +/* +** Write into the database file. +*/ +static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char *aData = pFile->aData; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + assert( pFile->szPage<0 || pFile->szPage==iAmt ); + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; + } + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; +} + +/* +** Truncate an kvvfs-file. +*/ +static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; +} +static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK; + } + return SQLITE_IOERR; +} + +/* +** Sync an kvvfs-file. +*/ +static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return i ? SQLITE_IOERR : SQLITE_OK; +} +static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an kvvfs-file. +*/ +static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; +} +static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Lock an kvvfs-file. +*/ +static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Unlock an kvvfs-file. +*/ +static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an kvvfs-file. +*/ +static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an kvvfs-file. +*/ +static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op)); + return SQLITE_NOTFOUND; +} +static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on database\n", op)); + if( op==SQLITE_FCNTL_SYNC ){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + int rc = SQLITE_OK; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){ + rc = SQLITE_IOERR; + } + return rc; + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector-size in bytes for an kvvfs-file. +*/ +static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; +} + +/* +** Return the device characteristic flags supported by an kvvfs-file. +*/ +static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; +} + +/****** sqlite3_vfs methods *************************************************/ + +/* +** Open an kvvfs file handle. +*/ +static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + if( zName==0 ) zName = ""; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); + if( pFile->aData==0 ){ + return SQLITE_NOMEM; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("session", "jrnl"); + } + return SQLITE_OK; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut +){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + size_t nPath; +#ifdef SQLITE_OS_KV_ALWAYS_LOCAL + zPath = "local"; +#endif + nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut +static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ + +#if SQLITE_OS_KV +/* +** This routine is called initialize the KV-vfs as the default VFS. +*/ +SQLITE_API int sqlite3_os_init(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1); +} +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV */ + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0); +} +#endif + +/************** End of os_kv.c ***********************************************/ +/************** Define dump function *****************************************/ +#if SQLITE_OS_UNIX +#define DB_LOCK_NUM 3 +#define WAL_LOCK_NUM 9 +// 8 wal lock, 1 SHM_DMS lock, 1 TRX lock +#define MAX_LOCK_NUM (WAL_LOCK_NUM + 1) +#define SHM_DMS_IDX (WAL_LOCK_NUM - 1) +#define TRX_LOCK_IDX WAL_LOCK_NUM +#define LOCK_BY_PROCESS 1 +#define LOCK_BY_THREAD 0 +#define DUMP_BUF_MAX_LEN 180 +#define DUMP_MAX_STR_LEN 21 + +typedef struct LocalLockStatus { + u8 busyLockIdx; + u8 busyLockType; + u8 lockByProcess; + u8 reserved; + u32 lockLen; + u32 busyLine; + int curTid; + u8 lockStatus[MAX_LOCK_NUM]; // last index is trx lock +} LocalLockStatus; +__thread LocalLockStatus g_lockStatus = {0}; + +#define MARK_LAST_BUSY_LINE(rc) {if ((rc)==SQLITE_BUSY || (rc) == SQLITE_LOCKED) g_lockStatus.busyLine = __LINE__;} +static void ResetLockStatus(void); +static void TryRecordTid(int *tidBuf, int ofs, int lockLen); +static void TryClearTid(int *tidBuf, int ofs, int lockLen); +static void MarkLockBusy(u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess); +static void MarkLockStatus(u32 lockIdx, u32 lockLen, u8 lockType); +static void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess); +#else +#define MARK_LAST_BUSY_LINE(rc) +#define ResetLockStatus(void) +#define MarkLockBusy(A, B, C, D) +#define MarkLockStatusByRc(A, B, C, D, E) +#endif +/************** End define dump function *************************************/ /************** Begin file os_unix.c *****************************************/ /* ** 2004 May 22 @@ -34366,7 +38639,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** This source file is organized into divisions where the logic for various ** subfunctions is contained within the appropriate division. PLEASE ** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed -** in the correct division and should be clearly labeled. +** in the correct division and should be clearly labelled. ** ** The layout of divisions is as follows: ** @@ -34416,7 +38689,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #endif /* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) # define HAVE_PREAD 1 # define HAVE_PWRITE 1 #endif @@ -34431,15 +38704,16 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* ** standard include files. */ -#include -#include +#include /* amalgamator: keep */ +#include /* amalgamator: keep */ #include #include -#include +#include /* amalgamator: keep */ /* #include */ -#include +#include /* amalgamator: keep */ #include -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) # include #endif @@ -34527,9 +38801,46 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define SQLITE_MAX_SYMLINKS 100 +/* +** Remove and stub certain info for WASI (WebAssembly System +** Interface) builds. +*/ +#ifdef SQLITE_WASI +# undef HAVE_FCHMOD +# undef HAVE_FCHOWN +# undef HAVE_MREMAP +# define HAVE_MREMAP 0 +# ifndef SQLITE_DEFAULT_UNIX_VFS +# define SQLITE_DEFAULT_UNIX_VFS "unix-dotfile" + /* ^^^ should SQLITE_DEFAULT_UNIX_VFS be "unix-none"? */ +# endif +# ifndef F_RDLCK +# define F_RDLCK 0 +# define F_WRLCK 1 +# define F_UNLCK 2 +# if __LONG_MAX == 0x7fffffffL +# define F_GETLK 12 +# define F_SETLK 13 +# define F_SETLKW 14 +# else +# define F_GETLK 5 +# define F_SETLK 6 +# define F_SETLKW 7 +# endif +# endif +#else /* !SQLITE_WASI */ +# ifndef HAVE_FCHMOD +# define HAVE_FCHMOD +# endif +#endif /* SQLITE_WASI */ + +#ifdef SQLITE_WASI +# define osGetpid(X) (pid_t)1 +#else /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ -#define osGetpid(X) (pid_t)getpid() +# define osGetpid(X) (pid_t)getpid() +#endif /* ** Only set the lastErrno if the error code is a real error and not @@ -34702,6 +39013,29 @@ static pid_t randomnessPid = 0; #define F2FS_FEATURE_ATOMIC_WRITE 0x0004 #endif /* __linux__ */ +#ifdef HARMONY_OS +#define HMFS_MONITOR_FL 0x00000002 +#define HMFS_IOCTL_HW_GET_FLAGS _IOR(0xf5, 70, unsigned int) +#define HMFS_IOCTL_HW_SET_FLAGS _IOR(0xf5, 71, unsigned int) + +static void enableDbFileDelMonitor(int32_t fd) +{ + unsigned int flags = 0; + int ret = ioctl(fd, HMFS_IOCTL_HW_GET_FLAGS, &flags); + if (ret < 0) { + return; + } + if (flags & HMFS_MONITOR_FL) { + return; + } + flags |= HMFS_MONITOR_FL; + ret = ioctl(fd, HMFS_IOCTL_HW_SET_FLAGS, &flags); + if (ret < 0) { + sqlite3_log(SQLITE_WARNING, "Fd %d enable del monitor go wrong, errno = %d", fd, errno); + } +} + +#endif /* HARMONY_OS */ /* ** Different Unix systems declare open() in different ways. Same use @@ -34712,7 +39046,29 @@ static pid_t randomnessPid = 0; ** which always has the same well-defined interface. */ static int posixOpen(const char *zFile, int flags, int mode){ - return open(zFile, flags, mode); + int fd = open(zFile, flags, mode); +#ifdef FDSAN_ENABLE + if( fd >= 0 ){ + fdsan_exchange_owner_tag(fd, 0, fdsan_create_owner_tag(FDSAN_OWNER_TYPE_FILE, SQLITE_FDSAN_TAG)); + } +#endif /* FDSAN_ENABLE */ +#ifdef HARMONY_OS + if( fd >= 0 ){ + enableDbFileDelMonitor(fd); + } +#endif /* HARMONY_OS */ + return fd; +} + +/* +** Change close to posixClose, use fdsan_close_with_tag when fdsan enable. +*/ +static int posixClose(int fd) { +#ifdef FDSAN_ENABLE + return fdsan_close_with_tag(fd, fdsan_create_owner_tag(FDSAN_OWNER_TYPE_FILE, SQLITE_FDSAN_TAG)); +#else + return close(fd); +#endif /* FDSAN_ENABLE */ } /* Forward reference */ @@ -34733,7 +39089,7 @@ static struct unix_syscall { { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, #define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) - { "close", (sqlite3_syscall_ptr)close, 0 }, + { "close", (sqlite3_syscall_ptr)posixClose, 0 }, #define osClose ((int(*)(int))aSyscall[1].pCurrent) { "access", (sqlite3_syscall_ptr)access, 0 }, @@ -34801,7 +39157,11 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) +#if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#else + { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -34837,14 +39197,16 @@ static struct unix_syscall { #endif #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #else { "mmap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #else { "munmap", (sqlite3_syscall_ptr)0, 0 }, @@ -34909,7 +39271,7 @@ static int robustFchown(int fd, uid_t uid, gid_t gid){ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** "unix" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -35030,6 +39392,9 @@ static int robust_open(const char *z, int f, mode_t m){ break; } if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + if( (f & (O_EXCL|O_CREAT))==(O_EXCL|O_CREAT) ){ + (void)osUnlink(z); + } osClose(fd); sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); @@ -35428,7 +39793,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** If you close a file descriptor that points to a file that has locks, ** all locks on that file that are owned by the current process are ** released. To work around this problem, each unixInodeInfo object -** maintains a count of the number of pending locks on tha inode. +** maintains a count of the number of pending locks on the inode. ** When an attempt is made to close an unixFile, if there are ** other unixFile open on the same inode that are holding locks, the call ** to close() the file descriptor is deferred until all of the locks clear. @@ -35442,7 +39807,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** not posix compliant. Under LinuxThreads, a lock created by thread ** A cannot be modified or overridden by a different thread B. ** Only thread A can modify the lock. Locking behavior is correct -** if the appliation uses the newer Native Posix Thread Library (NPTL) +** if the application uses the newer Native Posix Thread Library (NPTL) ** on linux - with NPTL a lock created by thread A can override locks ** in thread B. But there is no way to know at compile-time which ** threading library is being used. So there is no way to know at @@ -35592,8 +39957,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -35644,7 +40013,7 @@ static void storeLastErrno(unixFile *pFile, int error){ } /* -** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +** Close all file descriptors accumulated in the unixInodeInfo->pUnused list. */ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; @@ -35724,6 +40093,9 @@ static int findInodeInfo( #if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; #endif +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR, "findInodeInfo-osFstat, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); +#endif /* LOG_DUMP */ return SQLITE_IOERR; } @@ -35742,11 +40114,17 @@ static int findInodeInfo( do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); if( rc!=1 ){ storeLastErrno(pFile, errno); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR, "findInodeInfo-osWrite, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); +#endif /* LOG_DUMP */ return SQLITE_IOERR; } rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR, "findInodeInfo-msdos-osFstat, fd[%d], errno[%d], osFstat[%d]", fd, errno, rc); +#endif /* LOG_DUMP */ return SQLITE_IOERR; } } @@ -35992,7 +40370,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ ** ** UNLOCKED -> SHARED ** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE +** SHARED -> EXCLUSIVE ** RESERVED -> (PENDING) -> EXCLUSIVE ** PENDING -> EXCLUSIVE ** @@ -36007,7 +40385,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** - ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** Symbols defined in os.h identify the 'pending byte' and the 'reserved ** byte', each single bytes at well known offsets, and the 'shared byte ** range', a range of 510 bytes at a well known offset. ** @@ -36015,7 +40393,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** byte'. If this is successful, 'shared byte range' is read-locked ** and the lock on the 'pending byte' released. (Legacy note: When ** SQLite was first developed, Windows95 systems were still very common, - ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** and Windows95 lacks a shared-lock capability. So on Windows95, a ** single randomly selected by from the 'shared byte range' is locked. ** Windows95 is now pretty much extinct, but this work-around for the ** lack of shared-locks on Windows95 lives on, for backwards @@ -36025,19 +40403,20 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** A RESERVED lock is implemented by grabbing a write-lock on the ** 'reserved byte'. ** - ** A process may only obtain a PENDING lock after it has obtained a - ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock - ** on the 'pending byte'. This ensures that no new SHARED locks can be - ** obtained, but existing SHARED locks are allowed to persist. A process - ** does not have to obtain a RESERVED lock on the way to a PENDING lock. - ** This property is used by the algorithm for rolling back a journal file - ** after a crash. + ** An EXCLUSIVE lock may only be requested after either a SHARED or + ** RESERVED lock is held. An EXCLUSIVE lock is implemented by obtaining + ** a write-lock on the entire 'shared byte range'. Since all other locks + ** require a read-lock on one of the bytes within this range, this ensures + ** that no other locks are held on the database. ** - ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is - ** implemented by obtaining a write-lock on the entire 'shared byte - ** range'. Since all other locks require a read-lock on one of the bytes - ** within this range, this ensures that no other locks are held on the - ** database. + ** If a process that holds a RESERVED lock requests an EXCLUSIVE, then + ** a PENDING lock is obtained first. A PENDING lock is implemented by + ** obtaining a write-lock on the 'pending byte'. This ensures that no new + ** SHARED locks can be obtained, but existing SHARED locks are allowed to + ** persist. If the call to this function fails to obtain the EXCLUSIVE + ** lock in this case, it holds the PENDING lock instead. The client may + ** then re-attempt the EXCLUSIVE lock later on, after existing SHARED + ** locks have cleared. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -36058,12 +40437,13 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( pFile->eFileLock>=eFileLock ){ OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, azFileLock(eFileLock))); + MarkLockStatus(TRX_LOCK_IDX, 1, eFileLock); return SQLITE_OK; } /* Make sure the locking sequence is correct. ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -36082,6 +40462,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; + MarkLockBusy(TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_THREAD); goto end_lock; } @@ -36097,6 +40478,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ pFile->eFileLock = SHARED_LOCK; pInode->nShared++; pInode->nLock++; + MarkLockStatus(TRX_LOCK_IDX, 1, eFileLock); goto end_lock; } @@ -36108,7 +40490,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ lock.l_len = 1L; lock.l_whence = SEEK_SET; if( eFileLock==SHARED_LOCK - || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockeFileLock==RESERVED_LOCK) ){ lock.l_type = (eFileLock==SHARED_LOCK?F_RDLCK:F_WRLCK); lock.l_start = PENDING_BYTE; @@ -36118,11 +40500,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( rc!=SQLITE_BUSY ){ storeLastErrno(pFile, tErrno); } + MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); goto end_lock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; } } - /* If control gets to this point, then actually go ahead and make ** operating system calls for the specified lock. */ @@ -36138,7 +40523,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); } - + MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); /* Drop the temporary PENDING lock */ lock.l_start = PENDING_BYTE; lock.l_len = 1L; @@ -36163,6 +40548,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; + MarkLockBusy(TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_THREAD); }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file @@ -36187,6 +40573,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ storeLastErrno(pFile, tErrno); } } + MarkLockStatusByRc(rc, TRX_LOCK_IDX, 1, eFileLock, LOCK_BY_PROCESS); } @@ -36206,13 +40593,9 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } #endif - if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; pInode->eFileLock = eFileLock; - }else if( eFileLock==EXCLUSIVE_LOCK ){ - pFile->eFileLock = PENDING_LOCK; - pInode->eFileLock = PENDING_LOCK; } end_lock: @@ -36384,7 +40767,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ pFile->eFileLock = NO_LOCK; } } - /* Decrement the count of locks against this same file. When the ** count reaches zero, close any other file descriptors whose close ** was deferred because of outstanding locks. @@ -36908,6 +41290,9 @@ static int flockUnlock(sqlite3_file *id, int eFileLock) { #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS return SQLITE_OK; #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_UNLOCK, "IOERR_UNLOCK fd[%d], eFileLock[%d]", pFile->h, eFileLock); +#endif /* LOG_DUMP */ return SQLITE_IOERR_UNLOCK; }else{ pFile->eFileLock = NO_LOCK; @@ -37282,7 +41667,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -37398,7 +41783,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 0)) ){ int failed2 = SQLITE_OK; - /* now attemmpt to get the exclusive lock range */ + /* now attempt to get the exclusive lock range */ failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 1); if( failed && (failed2 = afpSetLock(context->dbPath, pFile, @@ -37447,9 +41832,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unixInodeInfo *pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; int skipShared = 0; -#ifdef SQLITE_TEST - int h = pFile->h; -#endif assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, @@ -37465,9 +41847,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); #ifdef SQLITE_DEBUG /* When reducing a lock such that other processes can start @@ -37516,9 +41895,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; pInode->nShared--; if( pInode->nShared==0 ){ - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); if( !skipShared ){ rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); } @@ -37619,12 +41995,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** in any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** ** To avoid stomping the errno value on a failed read the lastErrno value ** is set before returning. */ @@ -37699,7 +42069,7 @@ static int unixRead( #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -37735,8 +42105,14 @@ static int unixRead( #ifdef EDEVERR case EDEVERR: #endif +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_CORRUPTFS, "unixRead-EDEVERR, fd:[%d], amt[%d], got[%d], offset[%lld]", pFile->h, amt, got, offset); +#endif /* LOG_DUMP */ return SQLITE_IOERR_CORRUPTFS; } +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_READ, "unixRead-got<0, fd: [%d], amt[%d], got[%d], offset[%lld]", pFile->h, amt, got, offset); +#endif /* LOG_DUMP */ return SQLITE_IOERR_READ; }else{ storeLastErrno(pFile, 0); /* not a system error */ @@ -37851,7 +42227,7 @@ static int unixWrite( #endif #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -37878,9 +42254,19 @@ static int unixWrite( if( amt>wrote ){ if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_WRITE, + "unixWrite, lastErrno set by seekAndWrite, fd[%d], offset[%lld], wrote[%d], amt[%d], lastErrno[%d]", + pFile->h, offset, wrote, amt, pFile->lastErrno); +#endif /* LOG_DUMP */ return SQLITE_IOERR_WRITE; }else{ storeLastErrno(pFile, 0); /* not a system error */ +#ifdef LOG_DUMP + sqlite3_log(SQLITE_FULL, + "unixWrite, not a system error, fd[%d], offset[%lld], wrote[%d], amt[%d], lastErrno[%d]", + pFile->h, offset, wrote, amt, pFile->lastErrno); +#endif /* LOG_DUMP */ return SQLITE_FULL; } } @@ -37973,7 +42359,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op. But go ahead and call fstat() to validate the file ** descriptor as we need a method to provoke a failure during - ** coverate testing. + ** coverage testing. */ #ifdef SQLITE_NO_SYNC { @@ -38234,7 +42620,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ do{ err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); }while( err==EINTR ); - if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; + if( err && err!=EINVAL ) { +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_WRITE, + "fcntlSizeHint-fallocate, fd[%d], bufSize[%lld], nSize[%lld] err[%d]", + pFile->h, buf.st_size, nSize, err); +#endif /* LOG_DUMP */ + return SQLITE_IOERR_WRITE; + } #else /* If the OS does not have posix_fallocate(), fake it. Write a ** single byte to the last byte in each block that falls entirely @@ -38253,7 +42646,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; nWrite = seekAndWrite(pFile, iWrite, "", 1); - if( nWrite!=1 ) return SQLITE_IOERR_WRITE; + if( nWrite!=1 ) { +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_WRITE, + "fcntlSizeHint-seekAndWrite, fd[%d], nWrite[%d], nSize[%d], nBlk[%d], iWrite[%lld]", + pFile->h, nWrite, nSize, nBlk, iWrite); +#endif /* LOG_DUMP */ + return SQLITE_IOERR_WRITE; + } } #endif } @@ -38308,14 +42708,29 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); +#ifdef LOG_DUMP + if( rc ){ + sqlite3_log(SQLITE_IOERR_BEGIN_ATOMIC, "unixFileControl-begin, fd[%d], op[%d]", pFile->h, op); + } +#endif /* LOG_DUMP */ return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; } case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); +#ifdef LOG_DUMP + if( rc ){ + sqlite3_log(SQLITE_IOERR_COMMIT_ATOMIC, "unixFileControl-commit, fd[%d], op[%d]", pFile->h, op); + } +#endif /* LOG_DUMP */ return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; } case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); +#ifdef LOG_DUMP + if( rc ){ + sqlite3_log(SQLITE_IOERR_ROLLBACK_ATOMIC, "unixFileControl-rollback, fd[%d], op[%d]", pFile->h, op); + } +#endif /* LOG_DUMP */ return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; } #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ @@ -38366,7 +42781,13 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT case SQLITE_FCNTL_LOCK_TIMEOUT: { int iOld = pFile->iBusyTimeout; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 pFile->iBusyTimeout = *(int*)pArg; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = !!(*(int*)pArg); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif *(int*)pArg = iOld; return SQLITE_OK; } @@ -38619,6 +43040,25 @@ static int unixGetpagesize(void){ ** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. +** +** aLock[SQLITE_SHM_NLOCK]: +** This array records the various locks held by clients on each of the +** SQLITE_SHM_NLOCK slots. If the aLock[] entry is set to 0, then no +** locks are held by the process on this slot. If it is set to -1, then +** some client holds an EXCLUSIVE lock on the locking slot. If the aLock[] +** value is set to a positive value, then it is the number of shared +** locks currently held on the slot. +** +** aMutex[SQLITE_SHM_NLOCK]: +** Normally, when SQLITE_ENABLE_SETLK_TIMEOUT is not defined, mutex +** pShmMutex is used to protect the aLock[] array and the right to +** call fcntl() on unixShmNode.hShm to obtain or release locks. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined though, we use an array +** of mutexes - one for each locking slot. To read or write locking +** slot aLock[iSlot], the caller must hold the corresponding mutex +** aMutex[iSlot]. Similarly, to call fcntl() to obtain or release a +** lock corresponding to slot iSlot, mutex aMutex[iSlot] must be held. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ @@ -38632,10 +43072,12 @@ struct unixShmNode { char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK]; +#endif int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ + int aLockTid[SQLITE_SHM_NLOCK]; #ifdef SQLITE_DEBUG - u8 exclMask; /* Mask of exclusive locks held */ - u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif }; @@ -38718,16 +43160,35 @@ static int unixShmSystemLock( struct flock f; /* The posix advisory locking structure */ int rc = SQLITE_OK; /* Result code form fcntl() */ - /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); - assert( pShmNode->nRef>0 || unixMutexHeld() ); + + /* Assert that the parameters are within expected range and that the + ** correct mutex or mutexes are held. */ + assert( pShmNode->nRef>=0 ); + assert( (ofst==UNIX_SHM_DMS && n==1) + || (ofst>=UNIX_SHM_BASE && ofst+n<=(UNIX_SHM_BASE+SQLITE_SHM_NLOCK)) + ); + if( ofst==UNIX_SHM_DMS ){ + assert( pShmNode->nRef>0 || unixMutexHeld() ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int ii; + for(ii=ofst-UNIX_SHM_BASE; iiaMutex[ii]) ); + } +#else + assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 ); +#endif + } /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) ); if( pShmNode->hShm>=0 ){ int res; @@ -38738,7 +43199,7 @@ static int unixShmSystemLock( f.l_len = n; res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); if( res==-1 ){ -#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) && SQLITE_ENABLE_SETLK_TIMEOUT==1 rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); #else rc = SQLITE_BUSY; @@ -38746,39 +43207,28 @@ static int unixShmSystemLock( } } - /* Update the global lock state and do debug tracing */ + /* Do debug tracing */ #ifdef SQLITE_DEBUG - { u16 mask; OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("unlock %d..%d ok\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; + OSTRACE(("read-lock %d..%d ok\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("write-lock %d..%d ok\n", ofst, ofst+n-1)); } }else{ if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); + OSTRACE(("unlock %d..%d failed\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); + OSTRACE(("read-lock %d..%d failed\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); + OSTRACE(("write-lock %d..%d failed\n", ofst, ofst+n-1)); } } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); - } #endif return rc; @@ -38815,6 +43265,11 @@ static void unixShmPurge(unixFile *pFd){ int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->pShmMutex); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + for(i=0; iaMutex[i]); + } +#endif for(i=0; inRegion; i+=nShmPerMap){ if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); @@ -38874,7 +43329,22 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ pShmNode->isUnlocked = 1; rc = SQLITE_READONLY_CANTINIT; }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* Do not use a blocking lock here. If the lock cannot be obtained + ** immediately, it means some other connection is truncating the + ** *-shm file. And after it has done so, it will not release its + ** lock, but only downgrade it to a shared lock. So no point in + ** blocking here. The call below to obtain the shared DMS lock may + ** use a blocking lock. */ + int iSaveTimeout = pDbFd->iBusyTimeout; + pDbFd->iBusyTimeout = 0; +#endif rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pDbFd->iBusyTimeout = iSaveTimeout; +#endif + MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, EXCLUSIVE_LOCK, LOCK_BY_PROCESS); + MARK_LAST_BUSY_LINE(rc); /* The first connection to attach must truncate the -shm file. We ** truncate to 3 bytes (an arbitrary small number, less than the ** -shm header size) rather than 0 as a system debugging aid, to @@ -38886,11 +43356,15 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ } }else if( lock.l_type==F_WRLCK ){ rc = SQLITE_BUSY; + MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); + MARK_LAST_BUSY_LINE(rc); } if( rc==SQLITE_OK ){ assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + MarkLockStatusByRc(rc, SHM_DMS_IDX, 1, SHARED_LOCK, LOCK_BY_PROCESS); + MARK_LAST_BUSY_LINE(rc); } return rc; } @@ -38995,6 +43469,18 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { + int ii; + for(ii=0; iiaMutex[ii] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->aMutex[ii]==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + } + } +#endif } if( pInode->bProcessLock==0 ){ @@ -39216,9 +43702,11 @@ shmpage_out: */ #ifdef SQLITE_DEBUG static int assertLockingArrayOk(unixShmNode *pShmNode){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + return 1; +#else unixShm *pX; int aLock[SQLITE_SHM_NLOCK]; - assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); memset(aLock, 0, sizeof(aLock)); for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ @@ -39236,13 +43724,14 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){ assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) ); return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); +#endif } #endif /* ** Change the lock state for a shared-memory segment. ** -** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** Note that the relationship between SHARED and EXCLUSIVE locks is a little ** different here than in posix. In xShmLock(), one can go from unlocked ** to shared and back or from unlocked to exclusive and back. But one may ** not go from shared to exclusive or from exclusive to shared. @@ -39254,12 +43743,28 @@ static int unixShmLock( int flags /* What to do with the lock */ ){ unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ - unixShm *p = pDbFd->pShm; /* The shared memory being locked */ - unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + unixShm *p; /* The shared memory being locked */ + unixShmNode *pShmNode; /* The underlying file iNode */ int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ - int *aLock = pShmNode->aLock; + u16 mask = (1<<(ofst+n)) - (1<pShm; + if( p==0 ) { +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_SHMLOCK, "unixShmLock-pShm, fd[%d], ofst[%d], n[%d], flags[%d]", pDbFd->h, ofst, n, flags); +#endif + return SQLITE_IOERR_SHMLOCK; + } + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) { +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_SHMLOCK, "unixShmLock-pShmNode, fd[%d], ofst[%d], n[%d], flags[%d]", pDbFd->h, ofst, n, flags); +#endif + return SQLITE_IOERR_SHMLOCK; + } + aLock = pShmNode->aLock; + int *aLockTid = pShmNode->aLockTid; assert( pShmNode==pDbFd->pInode->pShmNode ); assert( pShmNode->pInode==pDbFd->pInode ); assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); @@ -39286,88 +43791,158 @@ static int unixShmLock( ** It is not permitted to block on the RECOVER lock. */ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ - && (ofst!=1 || (p->exclMask|p->sharedMask)==0) - && (ofst!=0 || (p->exclMask|p->sharedMask)<3) - && (ofst<3 || (p->exclMask|p->sharedMask)<(1<exclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2) /* not RECOVER */ + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<1 || mask==(1<pShmMutex); - assert( assertLockingArrayOk(pShmNode) ); - if( flags & SQLITE_SHM_UNLOCK ){ - if( (p->exclMask|p->sharedMask) & mask ){ - int ii; - int bUnlock = 1; + /* Check if there is any work to do. There are three cases: + ** + ** a) An unlock operation where there are locks to unlock, + ** b) An shared lock where the requested lock is not already held + ** c) An exclusive lock where the requested lock is not already held + ** + ** The SQLite core never requests an exclusive lock that it already holds. + ** This is assert()ed below. + */ + assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) + || 0==(p->exclMask & mask) + ); + u8 useProcessLock = LOCK_BY_THREAD; + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ - for(ii=ofst; ii((p->sharedMask & (1<aMutex[iMutex]); + if( rc!=SQLITE_OK ) goto leave_shmnode_mutexes; + }else{ + sqlite3_mutex_enter(pShmNode->aMutex[iMutex]); } + } +#else + sqlite3_mutex_enter(pShmNode->pShmMutex); +#endif - if( bUnlock ){ - rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); - if( rc==SQLITE_OK ){ - memset(&aLock[ofst], 0, sizeof(int)*n); + if( ALWAYS(rc==SQLITE_OK) ){ + if( flags & SQLITE_SHM_UNLOCK ){ + /* Case (a) - unlock. */ + int bUnlock = 1; + assert( (p->exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); + + /* If this is a SHARED lock being unlocked, it is possible that other + ** clients within this process are holding the same SHARED lock. In + ** this case, set bUnlock to 0 so that the posix lock is not removed + ** from the file-descriptor below. */ + if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( aLock[ofst]>=1 ); + if( aLock[ofst]>1 ){ + bUnlock = 0; + aLock[ofst]--; + p->sharedMask &= ~mask; + } } - }else if( ALWAYS(p->sharedMask & (1<1 ); - aLock[ofst]--; - } - - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - } - }else if( flags & SQLITE_SHM_SHARED ){ - assert( n==1 ); - assert( (p->exclMask & (1<sharedMask & mask)==0 ){ - if( aLock[ofst]<0 ){ - rc = SQLITE_BUSY; - }else if( aLock[ofst]==0 ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); - } + MarkLockStatusByRc(rc, ofst, n, NO_LOCK, useProcessLock); + if( bUnlock ){ + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ + memset(&aLock[ofst], 0, sizeof(int)*n); + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + } + useProcessLock = LOCK_BY_PROCESS; + TryClearTid(aLockTid, ofst, n); + } + }else if( flags & SQLITE_SHM_SHARED ){ + /* Case (b) - a shared lock. */ - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - aLock[ofst]++; - } - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. */ - int ii; - for(ii=ofst; iisharedMask & mask)==0 ); - if( ALWAYS((p->exclMask & (1<sharedMask |= mask; + aLock[ofst]++; + TryRecordTid(aLockTid, ofst, n); + } + }else{ + /* Case (c) - an exclusive lock. */ + int ii; - /* Get the exclusive locks at the system level. Then if successful - ** also update the in-memory values. */ - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n); - if( rc==SQLITE_OK ){ + assert( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) ); assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; + assert( (p->exclMask & mask)==0 ); + + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ for(ii=ofst; iiexclMask |= mask; + for(ii=ofst; ii=ofst; iMutex--){ + sqlite3_mutex_leave(pShmNode->aMutex[iMutex]); } +#else + sqlite3_mutex_leave(pShmNode->pShmMutex); +#endif } - assert( assertLockingArrayOk(pShmNode) ); - sqlite3_mutex_leave(pShmNode->pShmMutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; @@ -39617,11 +44192,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -40193,6 +44773,7 @@ static const char *unixTempFileDir(void){ static int unixGetTempname(int nBuf, char *zBuf){ const char *zDir; int iLimit = 0; + int rc = SQLITE_OK; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -40201,18 +44782,26 @@ static int unixGetTempname(int nBuf, char *zBuf){ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; - do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; - }while( osAccess(zBuf,0)==0 ); - return SQLITE_OK; + if( zDir==0 ){ + rc = SQLITE_IOERR_GETTEMPPATH; + }else{ + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){ + rc = SQLITE_ERROR; + break; + } + }while( osAccess(zBuf,0)==0 ); + } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) @@ -40355,20 +44944,23 @@ static int findCreateFileMode( ** ** where NN is a decimal number. The NN naming schemes are ** used by the test_multiplex.c module. + ** + ** In normal operation, the journal file name will always contain + ** a '-' character. However in 8+3 filename mode, or if a corrupt + ** rollback journal specifies a super-journal with a goofy name, then + ** the '-' might be missing or the '-' might be the first character in + ** the filename. In that case, just return SQLITE_OK with *pMode==0. */ nDb = sqlite3Strlen30(zPath) - 1; - while( zPath[nDb]!='-' ){ - /* In normal operation, the journal file name will always contain - ** a '-' character. However in 8+3 filename mode, or if a corrupt - ** rollback journal specifies a super-journal with a goofy name, then - ** the '-' might be missing. */ - if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; + while( nDb>0 && zPath[nDb]!='.' ){ + if( zPath[nDb]=='-' ){ + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + rc = getFileMode(zDb, pMode, pUid, pGid); + break; + } nDb--; } - memcpy(zDb, zPath, nDb); - zDb[nDb] = '\0'; - - rc = getFileMode(zDb, pMode, pUid, pGid); }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; }else if( flags & SQLITE_OPEN_URI ){ @@ -40553,12 +45145,20 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + sqlite3_log(SQLITE_WARNING, "Try open file readonly"); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ @@ -40617,6 +45217,9 @@ static int unixOpen( if( fstatfs(fd, &fsInfo) == -1 ){ storeLastErrno(p, errno); robust_close(p, fd, __LINE__); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_ACCESS, "unixOpen, fd[%d], flags[%d], errno[%d]", fd, errno, flags); +#endif /* LOG_DUMP */ return SQLITE_IOERR_ACCESS; } if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { @@ -40757,6 +45360,140 @@ static int unixAccess( return SQLITE_OK; } +#ifndef HARMONY_OS + +/* +** A pathname under construction +*/ +typedef struct DbPath DbPath; +struct DbPath { + int rc; /* Non-zero following any error */ + int nSymlink; /* Number of symlinks resolved */ + char *zOut; /* Write the pathname here */ + int nOut; /* Bytes of space available to zOut[] */ + int nUsed; /* Bytes of zOut[] currently being used */ +}; + +/* Forward reference */ +static void appendAllPathElements(DbPath*,const char*); + +/* +** Append a single path element to the DbPath under construction +*/ +static void appendOnePathElement( + DbPath *pPath, /* Path under construction, to which to append zName */ + const char *zName, /* Name to append to pPath. Not zero-terminated */ + int nName /* Number of significant bytes in zName */ +){ + assert( nName>0 ); + assert( zName!=0 ); + if( zName[0]=='.' ){ + if( nName==1 ) return; + if( zName[1]=='.' && nName==2 ){ + if( pPath->nUsed>1 ){ + assert( pPath->zOut[0]=='/' ); + while( pPath->zOut[--pPath->nUsed]!='/' ){} + } + return; + } + } + if( pPath->nUsed + nName + 2 >= pPath->nOut ){ + pPath->rc = SQLITE_ERROR; + return; + } + pPath->zOut[pPath->nUsed++] = '/'; + memcpy(&pPath->zOut[pPath->nUsed], zName, nName); + pPath->nUsed += nName; +#if defined(HAVE_READLINK) && defined(HAVE_LSTAT) + if( pPath->rc==SQLITE_OK ){ + const char *zIn; + struct stat buf; + pPath->zOut[pPath->nUsed] = 0; + zIn = pPath->zOut; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + pPath->rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else if( S_ISLNK(buf.st_mode) ){ + ssize_t got; + char zLnk[SQLITE_MAX_PATHLEN+2]; + if( pPath->nSymlink++ > SQLITE_MAX_SYMLINK ){ + pPath->rc = SQLITE_CANTOPEN_BKPT; + return; + } + got = osReadlink(zIn, zLnk, sizeof(zLnk)-2); + if( got<=0 || got>=(ssize_t)sizeof(zLnk)-2 ){ + pPath->rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + return; + } + zLnk[got] = 0; + if( zLnk[0]=='/' ){ + pPath->nUsed = 0; + }else{ + pPath->nUsed -= nName + 1; + } + appendAllPathElements(pPath, zLnk); + } + } +#endif +} + +/* +** Append all path elements in zPath to the DbPath under construction. +*/ +static void appendAllPathElements( + DbPath *pPath, /* Path under construction, to which to append zName */ + const char *zPath /* Path to append to pPath. Is zero-terminated */ +){ + int i = 0; + int j = 0; + do{ + while( zPath[i] && zPath[i]!='/' ){ i++; } + if( i>j ){ + appendOnePathElement(pPath, &zPath[j], i-j); + } + j = i+1; + }while( zPath[i++] ); +} + +/* +** Turn a relative pathname into a full pathname. The relative path +** is stored as a nul-terminated string in the buffer pointed to by +** zPath. +** +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** (in this case, MAX_PATHNAME bytes). The full-path is written to +** this buffer before returning. +*/ +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ + DbPath path; + UNUSED_PARAMETER(pVfs); + path.rc = 0; + path.nUsed = 0; + path.nSymlink = 0; + path.nOut = nOut; + path.zOut = zOut; + if( zPath[0]!='/' ){ + char zPwd[SQLITE_MAX_PATHLEN+2]; + if( osGetcwd(zPwd, sizeof(zPwd)-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + appendAllPathElements(&path, zPwd); + } + appendAllPathElements(&path, zPath); + zOut[path.nUsed] = 0; + if( path.rc || path.nUsed<2 ) return SQLITE_CANTOPEN_BKPT; + if( path.nSymlink ) return SQLITE_OK_SYMLINK; + return SQLITE_OK; +} + +#else + /* ** If the last component of the pathname in z[0]..z[j-1] is something ** other than ".." then back it out and return true. If the last @@ -40934,6 +45671,7 @@ static int unixFullPathname( #endif /* HAVE_READLINK && HAVE_LSTAT */ } +#endif /* !HARMONY_OS */ #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -41048,12 +45786,17 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS +#if !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP+0 struct timespec sp; - sp.tv_sec = microseconds / 1000000; sp.tv_nsec = (microseconds % 1000000) * 1000; + + /* Almost all modern unix systems support nanosleep(). But if you are + ** compiling for one of the rare exceptions, you can use + ** -DHAVE_NANOSLEEP=0 (perhaps in conjuction with -DHAVE_USLEEP if + ** usleep() is available) in order to bypass the use of nanosleep() */ nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); return microseconds; #elif defined(HAVE_USLEEP) && HAVE_USLEEP @@ -41340,6 +46083,9 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", lPath, errno, osGetpid(0))); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_LOCK, "proxyGetLockPath len[%d], dbLen[%d], i[%d]", len, dbLen, i); +#endif /* LOG_DUMP */ return SQLITE_IOERR_LOCK; } len = strlcat(lPath, "sqliteplocks", maxLen); @@ -41458,6 +46204,9 @@ static int proxyCreateUnixFile( case EACCES: return SQLITE_PERM; case EIO: +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_LOCK, "proxyCreateUnixFile-EIO, fd[%d]", fd); +#endif /* LOG_DUMP */ return SQLITE_IOERR_LOCK; /* even though it is the conch */ default: return SQLITE_CANTOPEN_BKPT; @@ -41626,6 +46375,9 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ struct stat buf; if( osFstat(conchFile->h, &buf) ){ storeLastErrno(pFile, errno); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_LOCK, "proxyConchLock pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, errno); +#endif /* LOG_DUMP */ return SQLITE_IOERR_LOCK; } @@ -41646,6 +46398,9 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ storeLastErrno(pFile, errno); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_LOCK, "proxyConchLock tries 2, pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, errno); +#endif /* LOG_DUMP */ return SQLITE_IOERR_LOCK; } if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ @@ -41720,6 +46475,9 @@ static int proxyTakeConch(unixFile *pFile){ if( readLen<0 ){ /* I/O error: lastErrno set by seekAndRead */ storeLastErrno(pFile, conchFile->lastErrno); +#ifdef LOG_DUMP + sqlite3_log(SQLITE_IOERR_READ, "proxyTakeConch pFile fd[%d], conchFile fd[%d], lastErrno[%d]", pFile->h, conchFile->h, conchFile->lastErrno); +#endif /* LOG_DUMP */ rc = SQLITE_IOERR_READ; goto end_takeconch; }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || @@ -42430,8 +47188,16 @@ SQLITE_API int sqlite3_os_init(void){ /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ +#ifdef SQLITE_DEFAULT_UNIX_VFS + sqlite3_vfs_register(&aVfs[i], + 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS)); +#else sqlite3_vfs_register(&aVfs[i], i==0); +#endif } +#ifdef SQLITE_OS_KV_OPTIONAL + sqlite3KvvfsInit(); +#endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); #ifndef SQLITE_OMIT_WAL @@ -43635,7 +48401,7 @@ static struct win_syscall { /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** "win32" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -44394,10 +49160,12 @@ SQLITE_API int sqlite3_win32_set_directory8( const char *zValue /* New value for directory being set or reset */ ){ char **ppDirectory = 0; + int rc; #ifndef SQLITE_OMIT_AUTOINIT - int rc = sqlite3_initialize(); + rc = sqlite3_initialize(); if( rc ) return rc; #endif + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ ppDirectory = &sqlite3_data_directory; }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ @@ -44412,14 +49180,19 @@ SQLITE_API int sqlite3_win32_set_directory8( if( zValue && zValue[0] ){ zCopy = sqlite3_mprintf("%s", zValue); if ( zCopy==0 ){ - return SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM_BKPT; + goto set_directory8_done; } } sqlite3_free(*ppDirectory); *ppDirectory = zCopy; - return SQLITE_OK; + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; } - return SQLITE_ERROR; +set_directory8_done: + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } /* @@ -45208,7 +49981,7 @@ static int winRead( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -45286,7 +50059,7 @@ static int winWrite( pFile->h, pBuf, amt, offset, pFile->locktype)); #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -45396,7 +50169,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ ** all references to memory-mapped content are closed. That is doable, ** but involves adding a few branches in the common write code path which ** could slow down normal operations slightly. Hence, we have decided for - ** now to simply make trancations a no-op if there are pending reads. We + ** now to simply make transactions a no-op if there are pending reads. We ** can maybe revisit this decision in the future. */ return SQLITE_OK; @@ -45455,7 +50228,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ #ifdef SQLITE_TEST /* ** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. +** that syncs and fullsyncs are occurring at the right times. */ SQLITE_API int sqlite3_sync_count = 0; SQLITE_API int sqlite3_fullsync_count = 0; @@ -45812,7 +50585,7 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); - res = winUnlockReadLock(pFile); + (void)winUnlockReadLock(pFile); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ @@ -46546,10 +51319,14 @@ static int winShmLock( winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ winShm *pX; /* For looping over all siblings */ - winShmNode *pShmNode = p->pShmNode; + winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ u16 mask; /* Mask of locks to take or release */ + if( p==0 ) return SQLITE_IOERR_SHMLOCK; + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); assert( n>=1 ); assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) @@ -46986,6 +51763,11 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ @@ -46994,7 +51776,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return rc; } } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; @@ -47189,6 +51971,19 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ return 0; } +/* +** If sqlite3_temp_directory is defined, take the mutex and return true. +** +** If sqlite3_temp_directory is NULL (undefined), omit the mutex and +** return false. +*/ +static int winTempDirDefined(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + if( sqlite3_temp_directory!=0 ) return 1; + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return 0; +} + /* ** Create a temporary file name and store the resulting pointer into pzBuf. ** The pointer returned in pzBuf must be freed via sqlite3_free(). @@ -47199,6 +51994,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"; size_t i, j; + DWORD pid; int nPre = sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX); int nMax, nBuf, nDir, nLen; char *zBuf; @@ -47225,20 +52021,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ */ nDir = nMax - (nPre + 15); assert( nDir>0 ); - if( sqlite3_temp_directory ){ + if( winTempDirDefined() ){ int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); if( nDirLen>0 ){ if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ nDirLen++; } if( nDirLen>nDir ){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); } sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); } + #if defined(__CYGWIN__) else{ static const char *azDirs[] = { @@ -47408,7 +52207,10 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); + pid = osGetCurrentProcessId(); for(i=0; i<15; i++, j++){ + zBuf[j] += pid & 0xff; + pid >>= 8; zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; @@ -47646,7 +52448,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -47663,7 +52465,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -47683,7 +52485,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -47906,6 +52708,13 @@ static int winAccess( OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); + if( zFilename==0 ){ + *pResOut = 0; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); @@ -48027,7 +52836,7 @@ static BOOL winIsVerbatimPathname( ** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname ** bytes in size. */ -static int winFullPathname( +static int winFullPathnameNoMutex( sqlite3_vfs *pVfs, /* Pointer to vfs object */ const char *zRelative, /* Possibly relative input path */ int nFull, /* Size of output buffer in bytes */ @@ -48206,6 +53015,20 @@ static int winFullPathname( } #endif } +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + int rc; + MUTEX_LOGIC( sqlite3_mutex *pMutex; ) + MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); ) + sqlite3_mutex_enter(pMutex); + rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); + sqlite3_mutex_leave(pMutex); + return rc; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -48742,6 +53565,7 @@ static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); static int memdbSync(sqlite3_file*, int flags); static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); static int memdbLock(sqlite3_file*, int); +static int memdbUnlock(sqlite3_file*, int); /* static int memdbCheckReservedLock(sqlite3_file*, int *pResOut);// not used */ static int memdbFileControl(sqlite3_file*, int op, void *pArg); /* static int memdbSectorSize(sqlite3_file*); // not used */ @@ -48800,7 +53624,7 @@ static const sqlite3_io_methods memdb_io_methods = { memdbSync, /* xSync */ memdbFileSize, /* xFileSize */ memdbLock, /* xLock */ - memdbLock, /* xUnlock - same as xLock in this case */ + memdbUnlock, /* xUnlock */ 0, /* memdbCheckReservedLock, */ /* xCheckReservedLock */ memdbFileControl, /* xFileControl */ 0, /* memdbSectorSize,*/ /* xSectorSize */ @@ -49001,39 +53825,81 @@ static int memdbLock(sqlite3_file *pFile, int eLock){ MemFile *pThis = (MemFile*)pFile; MemStore *p = pThis->pStore; int rc = SQLITE_OK; - if( eLock==pThis->eLock ) return SQLITE_OK; + if( eLock<=pThis->eLock ) return SQLITE_OK; memdbEnter(p); - if( eLock>SQLITE_LOCK_SHARED ){ - if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ - rc = SQLITE_READONLY; - }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ - if( p->nWrLock ){ - rc = SQLITE_BUSY; - }else{ - p->nWrLock = 1; + + assert( p->nWrLock==0 || p->nWrLock==1 ); + assert( pThis->eLock<=SQLITE_LOCK_SHARED || p->nWrLock==1 ); + assert( pThis->eLock==SQLITE_LOCK_NONE || p->nRdLock>=1 ); + + if( eLock>SQLITE_LOCK_SHARED && (p->mFlags & SQLITE_DESERIALIZE_READONLY) ){ + rc = SQLITE_READONLY; + }else{ + switch( eLock ){ + case SQLITE_LOCK_SHARED: { + assert( pThis->eLock==SQLITE_LOCK_NONE ); + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nRdLock++; + } + break; + }; + + case SQLITE_LOCK_RESERVED: + case SQLITE_LOCK_PENDING: { + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( ALWAYS(pThis->eLock==SQLITE_LOCK_SHARED) ){ + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; + } + } + break; + } + + default: { + assert( eLock==SQLITE_LOCK_EXCLUSIVE ); + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( p->nRdLock>1 ){ + rc = SQLITE_BUSY; + }else if( pThis->eLock==SQLITE_LOCK_SHARED ){ + p->nWrLock = 1; + } + break; } } - }else if( eLock==SQLITE_LOCK_SHARED ){ - if( pThis->eLock > SQLITE_LOCK_SHARED ){ - assert( p->nWrLock==1 ); - p->nWrLock = 0; - }else if( p->nWrLock ){ - rc = SQLITE_BUSY; - }else{ - p->nRdLock++; + } + if( rc==SQLITE_OK ) pThis->eLock = eLock; + memdbLeave(p); + return rc; +} + +/* +** Unlock an memdb-file. +*/ +static int memdbUnlock(sqlite3_file *pFile, int eLock){ + MemFile *pThis = (MemFile*)pFile; + MemStore *p = pThis->pStore; + if( eLock>=pThis->eLock ) return SQLITE_OK; + memdbEnter(p); + + assert( eLock==SQLITE_LOCK_SHARED || eLock==SQLITE_LOCK_NONE ); + if( eLock==SQLITE_LOCK_SHARED ){ + if( ALWAYS(pThis->eLock>SQLITE_LOCK_SHARED) ){ + p->nWrLock--; } }else{ - assert( eLock==SQLITE_LOCK_NONE ); if( pThis->eLock>SQLITE_LOCK_SHARED ){ - assert( p->nWrLock==1 ); - p->nWrLock = 0; + p->nWrLock--; } - assert( p->nRdLock>0 ); p->nRdLock--; } - if( rc==SQLITE_OK ) pThis->eLock = eLock; + + pThis->eLock = eLock; memdbLeave(p); - return rc; + return SQLITE_OK; } #if 0 @@ -49143,7 +54009,7 @@ static int memdbOpen( memset(pFile, 0, sizeof(*pFile)); szName = sqlite3Strlen30(zName); - if( szName>1 && zName[0]=='/' ){ + if( szName>1 && (zName[0]=='/' || zName[0]=='\\') ){ int i; #ifndef SQLITE_MUTEX_OMIT sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); @@ -49389,6 +54255,14 @@ SQLITE_API unsigned char *sqlite3_serialize( pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; @@ -49490,6 +54364,13 @@ end_deserialize: return rc; } +/* +** Return true if the VFS is the memvfs. +*/ +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs *pVfs){ + return pVfs==&memdb_vfs; +} + /* ** This routine is called when the extension is loaded. ** Register the new VFS. @@ -49702,7 +54583,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ h = BITVEC_HASH(i++); /* if there wasn't a hash collision, and this doesn't */ /* completely fill the hash, then just add it without */ - /* worring about sub-dividing and re-hashing. */ + /* worrying about sub-dividing and re-hashing. */ if( !p->u.aHash[h] ){ if (p->nSet<(BITVEC_NINT-1)) { goto bitvec_set_end; @@ -49969,7 +54850,7 @@ bitvec_end: struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRefSum; /* Sum of ref counts over all pages */ + i64 nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ @@ -49994,12 +54875,24 @@ struct PCache { int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ # define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} - void pcacheDump(PCache *pCache){ - int N; - int i, j; - sqlite3_pcache_page *pLower; + static void pcachePageTrace(int i, sqlite3_pcache_page *pLower){ PgHdr *pPg; unsigned char *a; + int j; + if( pLower==0 ){ + printf("%3d: NULL\n", i); + }else{ + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } + } + static void pcacheDump(PCache *pCache){ + int N; + int i; + sqlite3_pcache_page *pLower; if( sqlite3PcacheTrace<2 ) return; if( pCache->pCache==0 ) return; @@ -50007,22 +54900,42 @@ struct PCache { if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf("\n"); - if( pPg->pPage==0 ){ + pcachePageTrace(i, pLower); + if( pLower && ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } } - #else +#else # define pcacheTrace(X) +# define pcachePageTrace(PGNO, X) # define pcacheDump(X) #endif +/* +** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. +** This routine runs inside of assert() statements only. +*/ +#if defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) +static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 1; + } + return 0; +} +static int pageNotOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 0; + } + return 1; +} +#else +# define pageOnDirtyList(A,B) 1 +# define pageNotOnDirtyList(A,B) 1 +#endif + /* ** Check invariants on a PgHdr entry. Return true if everything is OK. ** Return false if any invariant is violated. @@ -50041,8 +54954,13 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ - assert( pCache->pDirtyTail!=pPg ); + assert( pageNotOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirtylist */ + }else{ + assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ + assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); + assert( pPg->pDirtyPrev==0 || pPg->pDirtyPrev->pDirtyNext==pPg ); + assert( pPg->pDirtyPrev!=0 || pCache->pDirty==pPg ); + assert( pageOnDirtyList(pCache, pPg) ); } /* WRITEABLE pages must also be DIRTY */ if( pPg->flags & PGHDR_WRITEABLE ){ @@ -50172,7 +55090,7 @@ static int numberOfCachePages(PCache *p){ return p->szCache; }else{ i64 n; - /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + /* IMPLEMENTATION-OF: R-59858-46238 If the argument N is negative, then the ** number of cache pages is adjusted to be a number of pages that would ** use approximately abs(N*1024) bytes of memory based on the current ** page size. */ @@ -50316,8 +55234,9 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + pcacheTrace(("%p.FETCH %d%s (result: %p) ",pCache,pgno, createFlag?" create":"",pRes)); + pcachePageTrace(pgno, pRes); return pRes; } @@ -50445,6 +55364,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ pcacheUnpin(p); }else{ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } } @@ -50488,6 +55408,7 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + assert( sqlite3PcachePageSanity(p) ); } assert( sqlite3PcachePageSanity(p) ); } @@ -50550,14 +55471,24 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; + sqlite3_pcache_page *pOther; assert( p->nRef>0 ); assert( newPgno>0 ); assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); + if( pOther ){ + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); + } sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } @@ -50647,7 +55578,7 @@ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ } /* -** Sort the list of pages in accending order by pgno. Pages are +** Sort the list of pages in ascending order by pgno. Pages are ** connected by pDirty pointers. The pDirtyPrev pointers are ** corrupted by this sort. ** @@ -50706,14 +55637,14 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ ** This is not the total number of pages referenced, but the sum of the ** reference count for all pages. */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache *pCache){ return pCache->nRefSum; } /* ** Return the number of references to the page supplied as an argument. */ -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr *p){ return p->nRef; } @@ -50855,12 +55786,13 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. +** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER +** was defined, then the page content would be held in a separate memory +** allocation from the PgHdr1. This was intended to avoid clownshoe memory +** allocations. However, the btree layer needs a small (16-byte) overrun +** area after the page content buffer. The header serves as that overrun +** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid +** any possibility of a memory error. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. @@ -50886,7 +55818,7 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. ** Or if N is negative, then -1024*N bytes of memory are allocated and used -** for as many pages as can be accomodated. +** for as many pages as can be accommodated. ** ** Only one of (2) or (3) can be used. Once the memory available to (2) or ** (3) is exhausted, subsequent allocations fail over to the general-purpose @@ -50905,30 +55837,40 @@ typedef struct PGroup PGroup; /* ** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure and is used to cache the page content. +** +** When reading a corrupt database file, it is possible that SQLite might +** read a few bytes (no more than 16 bytes) past the end of the page buffer. +** It will only read past the end of the page buffer, never write. This +** object is positioned immediately after the page buffer to serve as an +** overrun area, so that overreads are harmless. ** -** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** Variables isBulkLocal and isAnchor were once type "u8". That works, ** but causes a 2-byte gap in the structure for most architectures (since ** pointers must be either 4 or 8-byte aligned). As this structure is located ** in memory directly after the associated page data, if the database is ** corrupt, code at the b-tree layer may overread the page buffer and ** read part of this structure before the corruption is detected. This -** can cause a valgrind error if the unitialized gap is accessed. Using u16 -** ensures there is no such gap, and therefore no bytes of unitialized memory -** in the structure. +** can cause a valgrind error if the uninitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of uninitialized +** memory in the structure. +** +** The pLruNext and pLruPrev pointers form a double-linked circular list +** of all pages that are unpinned. The PGroup.lru element (which should be +** the only element on the list with PgHdr1.isAnchor set to 1) forms the +** beginning and the end of the list. */ struct PgHdr1 { - sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ - unsigned int iKey; /* Key value (page number) */ - u16 isBulkLocal; /* This page from bulk local storage */ - u16 isAnchor; /* This is the PGroup.lru element */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ - /* NB: pLruPrev is only valid if pLruNext!=0 */ + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in circular LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* @@ -51254,25 +56196,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ pcache1LeaveMutex(pCache->pGroup); #endif if( benignMalloc ){ sqlite3BeginBenignMalloc(); } -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else pPg = pcache1Alloc(pCache->szAlloc); -#endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; -#ifndef SQLITE_PCACHE_SEPARATE_HEADER p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; @@ -51296,9 +56226,6 @@ static void pcache1FreePage(PgHdr1 *p){ pCache->pFree = p; }else{ pcache1Free(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - sqlite3_free(p); -#endif } (*pCache->pnPurgeable)--; } @@ -51939,23 +56866,26 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int h; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); + assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ + hOld = iOld%pCache->nHash; + pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; } *pp = pPage->pNext; - h = iNew%pCache->nHash; + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ + hNew = iNew%pCache->nHash; pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; + pPage->pNext = pCache->apHash[hNew]; + pCache->apHash[hNew] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } @@ -52062,9 +56992,6 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ && p->isAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - nFree += sqlite3MemSize(p); -#endif assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); @@ -52145,7 +57072,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** The TEST primitive includes a "batch" number. The TEST primitive ** will only see elements that were inserted before the last change ** in the batch number. In other words, if an INSERT occurs between -** two TESTs where the TESTs have the same batch nubmer, then the +** two TESTs where the TESTs have the same batch number, then the ** value added by the INSERT will not be visible to the second TEST. ** The initial batch number is zero, so if the very first TEST contains ** a non-zero batch number, it will see all prior INSERTs. @@ -52677,6 +57604,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 +# undef SQLITE_USE_SEH #else #define WAL_SAVEPOINT_NDATA 4 @@ -52783,6 +57711,10 @@ SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); #endif +#ifdef SQLITE_USE_SEH +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal*); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ @@ -53068,7 +58000,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** outstanding transactions have been abandoned, the pager is able to ** transition back to OPEN state, discarding the contents of the ** page-cache and any other in-memory state at the same time. Everything -** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** is reloaded from disk (and, if necessary, hot-journal rollback performed) ** when a read-transaction is next opened on the pager (transitioning ** the pager into READER state). At that point the system has recovered ** from the error. @@ -53173,10 +58105,10 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) +#ifdef SQLITE_HAS_CODEC /* ** A macro used for invoking the codec if there is one */ -#ifdef SQLITE_HAS_CODEC # define CODEC1(P,D,N,X,E) \ if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } # define CODEC2(P,D,N,X,E,O) \ @@ -53185,7 +58117,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #else # define CODEC1(P,D,N,X,E) /* NO-OP */ # define CODEC2(P,D,N,X,E,O) O=(char*)D -#endif +#endif /* SQLITE_HAS_CODEC */ /* ** The maximum allowed sector size. 64KiB. If the xSectorsize() method @@ -53222,6 +58154,26 @@ struct PagerSavepoint { #endif }; +#if !defined(SQLITE_OS_UNIX) && defined(SQLITE_META_DWR) +#undef SQLITE_META_DWR +#endif + +#ifdef SQLITE_META_DWR +static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, const char *zRight); +typedef struct MetaDwrHdr MetaDwrHdr; +static int MetaDwrWriteHeader(Pager *pPager, MetaDwrHdr *hdr); +static int MetaDwrUpdateMetaPages(Btree *pBt); +static void MetaDwrPagerRelease(Pager *pPager); +static int MetaDwrOpenFile(Pager *pPager, u8 openCreate); +static void MetaDwrCheckVacuum(BtShared *pBt); +static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion); +static int MetaDwrOpenAndCheck(Btree *pBt); +static void MetaDwrDisable(Btree *pBt); +#define META_HEADER_CHANGED 1 +#define META_SCHEMA_CHANGED 2 +#define META_IN_RECOVERY 1 +#define META_RECOVER_SUCCESS 2 +#endif /* ** Bits of the Pager.doNotSpill flag. See further description below. */ @@ -53462,13 +58414,14 @@ struct Pager { u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ u32 sectorSize; /* Assumed sector size during rollback */ Pgno mxPgno; /* Maximum allowed size of the database */ + Pgno lckPgno; /* Page number for the locking page */ i64 pageSize; /* Number of bytes in a page */ i64 journalSizeLimit; /* Size limit for persistent journal files */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int aStat[4]; /* Total cache hits, misses, writes, spills */ + u32 aStat[4]; /* Total cache hits, misses, writes, spills */ #ifdef SQLITE_TEST int nRead; /* Database pages read */ #endif @@ -53479,13 +58432,20 @@ struct Pager { void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ void (*xCodecFree)(void*); /* Destructor for the codec */ void *pCodec; /* First argument to xCodec... methods */ -#endif +#endif /* SQLITE_HAS_CODEC */ char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ #ifndef SQLITE_OMIT_WAL Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ #endif +#ifdef SQLITE_META_DWR + u8 metaChanged; + sqlite3_file *metaFd; + MetaDwrHdr *metaHdr; + void *metaMapPage; + int (*xGetMethod)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ +#endif }; /* @@ -53603,13 +58563,12 @@ SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; #ifdef SQLITE_HAS_CODEC if( pPager->xCodec!=0 ) return 0; -#endif +#endif /* SQLITE_HAS_CODEC */ #ifndef SQLITE_OMIT_WAL if( pPager->pWal ){ u32 iRead = 0; - int rc; - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return (rc==SQLITE_OK && iRead==0); + (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return iRead==0; } #endif return 1; @@ -53840,12 +58799,16 @@ static void setGetterMethod(Pager *pPager){ }else if( USEFETCH(pPager) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 -#endif +#endif /* SQLITE_HAS_CODEC */ ){ pPager->xGet = getPageMMap; #endif /* SQLITE_MAX_MMAP_SIZE>0 */ }else{ +#ifdef SQLITE_META_DWR + pPager->xGet = pPager->xGetMethod ? pPager->xGetMethod : getPageNormal; +#else pPager->xGet = getPageNormal; +#endif } } @@ -54029,17 +58992,70 @@ static int jrnlBufferSize(Pager *pPager){ ** and debugging only. */ #ifdef SQLITE_CHECK_PAGES -/* -** Return a 32-bit hash of the page data for pPage. -*/ -static u32 pager_datahash(int nByte, unsigned char *pData){ +#if defined (__arm__) || defined (__aarch64__) +#include +u32 deep_fast_hash_arm(void *src, int srcLen){ + uint16_t chunkSize = srcLen/4; + uint8_t *u8p_src = (uint8_t *)src; + uint16x8_t m_prime = vdupq_n_u16(44497); + uint16x8_t m_res0 = vdupq_n_u16(0); + uint16x8_t m_res1 = vdupq_n_u16(0); + uint16x8_t m_res2 = vdupq_n_u16(0); + uint16x8_t m_res3 = vdupq_n_u16(0); + uint16x8_t m_res4 = vdupq_n_u16(0); + uint16x8_t m_res5 = vdupq_n_u16(0); + uint16x8_t m_res6 = vdupq_n_u16(0); + uint16x8_t m_res7 = vdupq_n_u16(0); + + for(int i=0; ipPager->pageSize, (unsigned char *)pPage->pData); } @@ -54055,8 +59071,15 @@ static void pager_set_pagehash(PgHdr *pPage){ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( pPager->eState!=PAGER_ERROR ); - assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); + if( pPager->eState==PAGER_ERROR ){ + return; + } + if( pPg->flags&PGHDR_DIRTY ) { + return; + } + if( pPg->pageHash!=pager_pagehash(pPg) ){ + sqlite3_log(SQLITE_CORRUPT, "cache corruption occurs through checking page(%u)", pPg->pgno); + } } #else @@ -54285,9 +59308,32 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } + + /* The random check-hash initializer */ - sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + } +#ifdef SQLITE_DEBUG + else{ + /* The Pager.cksumInit variable is usually randomized above to protect + ** against there being existing records in the journal file. This is + ** dangerous, as following a crash they may be mistaken for records + ** written by the current transaction and rolled back into the database + ** file, causing corruption. The following assert statements verify + ** that this is not required in "journal_mode=memory" mode, as in that + ** case the journal file is always 0 bytes in size at this point. + ** It is advantageous to avoid the sqlite3_randomness() call if possible + ** as it takes the global PRNG mutex. */ + i64 sz = 0; + sqlite3OsFileSize(pPager->jfd, &sz); + assert( sz==0 ); + assert( pPager->journalOff==journalHdrOffset(pPager) ); + assert( sqlite3JournalIsInMemory(pPager->jfd) ); + } +#endif put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); /* The assumed sector size for this process */ @@ -54461,13 +59507,13 @@ static int readJournalHdr( ** journal file descriptor is advanced to the next sector boundary before ** anything is written. The format is: ** -** + 4 bytes: PAGER_MJ_PGNO. +** + 4 bytes: PAGER_SJ_PGNO. ** + N bytes: super-journal filename in utf-8. ** + 4 bytes: N (length of super-journal name in bytes, no nul-terminator). ** + 4 bytes: super-journal name checksum. ** + 8 bytes: aJournalMagic[]. ** -** The super-journal page checksum is the sum of the bytes in thesuper-journal +** The super-journal page checksum is the sum of the bytes in the super-journal ** name, where each byte is interpreted as a signed 8-bit integer. ** ** If zSuper is a NULL pointer (occurs for a single database transaction), @@ -54509,7 +59555,7 @@ static int writeSuperJournal(Pager *pPager, const char *zSuper){ /* Write the super-journal data to the end of the journal file. If ** an error occurs, return the error code to the caller. */ - if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_SJ_PGNO(pPager)))) || (0 != (rc = sqlite3OsWrite(pPager->jfd, zSuper, nSuper, iHdrOff+4))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper, nSuper))) || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper+4, cksum))) @@ -54520,7 +59566,7 @@ static int writeSuperJournal(Pager *pPager, const char *zSuper){ } pPager->journalOff += (nSuper+20); - /* If the pager is in peristent-journal mode, then the physical + /* If the pager is in persistent-journal mode, then the physical ** journal-file may extend past the end of the super-journal name ** and 8 bytes of magic data just written to the file. This is ** dangerous because the code to rollback a hot-journal file @@ -54690,7 +59736,7 @@ static void pager_unlock(Pager *pPager){ /* ** This function is called whenever an IOERR or FULL error that requires -** the pager to transition into the ERROR state may ahve occurred. +** the pager to transition into the ERROR state may have occurred. ** The first argument is a pointer to the pager structure, the second ** the error-code about to be returned by a pager API function. The ** value returned is a copy of the second argument to this function. @@ -54896,7 +59942,14 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ } sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); } - +#ifdef SQLITE_META_DWR + if (bCommit && pPager->metaChanged != 0) { + sqlite3BeginBenignMalloc(); + (void)MetaDwrWriteHeader(pPager, pPager->metaHdr); + sqlite3EndBenignMalloc(); + pPager->metaChanged = 0; + } +#endif if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE @@ -54931,6 +59984,9 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ return (rc==SQLITE_OK?rc2:rc); } +/* Forward reference */ +static int pager_playback(Pager *pPager, int isHot); + /* ** Execute a rollback if a transaction is active and unlock the ** database file. @@ -54959,13 +60015,28 @@ static void pagerUnlockAndRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0, 0); } + }else if( pPager->eState==PAGER_ERROR + && pPager->journalMode==PAGER_JOURNALMODE_MEMORY + && isOpen(pPager->jfd) + ){ + /* Special case for a ROLLBACK due to I/O error with an in-memory + ** journal: We have to rollback immediately, before the journal is + ** closed, because once it is closed, all content is forgotten. */ + int errCode = pPager->errCode; + u8 eLock = pPager->eLock; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + pPager->eLock = EXCLUSIVE_LOCK; + pager_playback(pPager, 1); + pPager->errCode = errCode; + pPager->eLock = eLock; } pager_unlock(pPager); } /* ** Parameter aData must point to a buffer of pPager->pageSize bytes -** of data. Compute and return a checksum based ont the contents of the +** of data. Compute and return a checksum based on the contents of the ** page of data and the current value of pPager->cksumInit. ** ** This is not a real checksum. It is really just the sum of the @@ -54992,22 +60063,17 @@ static u32 pager_cksum(Pager *pPager, const u8 *aData){ return cksum; } +#ifdef SQLITE_HAS_CODEC /* ** Report the current page size and number of reserved bytes back ** to the codec. */ -#ifdef SQLITE_HAS_CODEC static void pagerReportSize(Pager *pPager){ if( pPager->xCodecSizeChng ){ pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, (int)pPager->nReserve); } } -#else -# define pagerReportSize(X) /* No-op if we do not support a codec */ -#endif - -#ifdef SQLITE_HAS_CODEC /* ** Make sure the number of reserved bits is the same in the destination ** pager as it is in the source. This comes up when a VACUUM changes the @@ -55019,6 +60085,8 @@ SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ pagerReportSize(pDest); } } +#else +# define pagerReportSize(X) /* No-op if we do not support a codec */ #endif /* @@ -55048,7 +60116,7 @@ SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ ** corrupted, SQLITE_DONE is returned. Data is considered corrupted in ** two circumstances: ** -** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** * If the record page-number is illegal (0 or PAGER_SJ_PGNO), or ** * If the record is being rolled back from the main journal file ** and the checksum field does not match the record content. ** @@ -55076,7 +60144,7 @@ static int pager_playback_one_page( /* The jrnlEnc flag is true if Journal pages should be passed through ** the codec. It is false for pure in-memory journals. */ const int jrnlEnc = (isMainJrnl || pPager->subjInMemory==0); -#endif +#endif /* SQLITE_HAS_CODEC */ assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ @@ -55113,7 +60181,7 @@ static int pager_playback_one_page( ** it could cause invalid data to be written into the journal. We need to ** detect this invalid data (with high probability) and ignore it. */ - if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + if( pgno==0 || pgno==PAGER_SJ_PGNO(pPager) ){ assert( !isSavepnt ); return SQLITE_DONE; } @@ -55212,7 +60280,7 @@ static int pager_playback_one_page( rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); }else -#endif +#endif /* SQLITE_HAS_CODEC */ rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ @@ -55225,7 +60293,7 @@ static int pager_playback_one_page( sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT,aData); }else -#endif +#endif /* SQLITE_HAS_CODEC */ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); } }else if( !isMainJrnl && pPg==0 ){ @@ -55276,11 +60344,10 @@ static int pager_playback_one_page( if( pgno==1 ){ memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); } - - /* Decode the page just read from disk */ #if SQLITE_HAS_CODEC + /* Decode the page just read from disk */ if( jrnlEnc ){ CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); } -#endif +#endif /* SQLITE_HAS_CODEC */ sqlite3PcacheRelease(pPg); } return rc; @@ -55451,6 +60518,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState!=PAGER_READER ); + PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); + if( isOpen(pPager->fd) && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) @@ -55469,6 +60538,7 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ memset(pTmp, 0, szPage); testcase( (newSize-szPage) == currentSize ); testcase( (newSize-szPage) > currentSize ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &newSize); rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); } if( rc==SQLITE_OK ){ @@ -55691,6 +60761,9 @@ static int pager_playback(Pager *pPager, int isHot){ goto end_playback; } pPager->dbSize = mxPg; + if( pPager->mxPgnomxPgno = mxPg; + } } /* Copy original pages out of the journal and back into the @@ -55777,7 +60850,7 @@ end_playback: ** see if it is possible to delete the super-journal. */ assert( zSuper==&pPager->pTmpSpace[4] ); - memset(&zSuper[-4], 0, 4); + memset(pPager->pTmpSpace, 0, 4); rc = pager_delsuper(pPager, zSuper); testcase( rc!=SQLITE_OK ); } @@ -55818,7 +60891,7 @@ static int readDbPage(PgHdr *pPg){ assert( isOpen(pPager->fd) ); if( pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); // find in wal-index if( rc ) return rc; } if( iFrame ){ @@ -55980,7 +61053,7 @@ static int pagerWalFrames( assert( pPager->pWal ); assert( pList ); #ifdef SQLITE_DEBUG - /* Verify that the page list is in accending order */ + /* Verify that the page list is in ascending order */ for(p=pList; p && p->pDirty; p=p->pDirty){ assert( p->pgno < p->pDirty->pgno ); } @@ -56111,7 +61184,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ #ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager -** exists if the database is not empy, or verify that the *-wal file does +** exists if the database is not empty, or verify that the *-wal file does ** not exist (by deleting it) if the database file is empty. ** ** If the database is not empty and the *-wal file exists, open the pager @@ -56400,7 +61473,6 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** Numeric values associated with these states are OFF==1, NORMAL=2, ** and FULL=3. */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS SQLITE_PRIVATE void sqlite3PagerSetFlags( Pager *pPager, /* The pager to set safety level for */ unsigned pgFlags /* Various flags */ @@ -56435,7 +61507,6 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( pPager->doNotSpill |= SPILLFLAG_OFF; } } -#endif /* ** The following global variable is incremented whenever the library @@ -56589,6 +61660,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR pPager->pTmpSpace = pNew; pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); pPager->pageSize = pageSize; + pPager->lckPgno = (Pgno)(PENDING_BYTE/pageSize) + 1; }else{ sqlite3PageFree(pNew); } @@ -56749,8 +61821,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** current database image, in pages, OR ** ** b) if the page content were written at this time, it would not -** be necessary to write the current content out to the sub-journal -** (as determined by function subjRequiresPage()). +** be necessary to write the current content out to the sub-journal. ** ** If the condition asserted by this function were not true, and the ** dirty page were to be discarded from the cache via the pagerStress() @@ -56765,8 +61836,16 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ */ #if defined(SQLITE_DEBUG) static void assertTruncateConstraintCb(PgHdr *pPg){ + Pager *pPager = pPg->pPager; assert( pPg->flags&PGHDR_DIRTY ); - assert( pPg->pgno<=pPg->pPager->dbSize || !subjRequiresPage(pPg) ); + if( pPg->pgno>pPager->dbSize ){ /* if (a) is false */ + Pgno pgno = pPg->pgno; + int i; + for(i=0; ipPager->nSavepoint; i++){ + PagerSavepoint *p = &pPager->aSavepoint[i]; + assert( p->nOrigpInSavepoint,pgno) ); + } + } } static void assertTruncateConstraint(Pager *pPager){ sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); @@ -56788,7 +61867,6 @@ static void assertTruncateConstraint(Pager *pPager){ */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSize>=nPage || CORRUPT_DB ); - testcase( pPager->dbSizeeState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; @@ -56966,6 +62044,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); pPager->pWal = 0; } +#endif +#ifdef SQLITE_META_DWR + MetaDwrPagerRelease(pPager); #endif pager_reset(pPager); if( MEMDB ){ @@ -56995,10 +62076,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3OsClose(pPager->fd); sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); - #ifdef SQLITE_HAS_CODEC if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); -#endif +#endif /* SQLITE_HAS_CODEC */ assert( !pPager->aSavepoint && !pPager->pInJournal ); assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); @@ -57068,6 +62148,7 @@ static int syncJournal(Pager *pPager, int newHdr){ assert( !pagerUseWal(pPager) ); rc = sqlite3PagerExclusiveLock(pPager); + MARK_LAST_BUSY_LINE(rc); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ @@ -57250,7 +62331,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); if( pList->pgno==1 ) pager_write_changecounter(pList); - /* Encode the database */ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); /* Write out the page data. */ @@ -57340,12 +62420,11 @@ static int subjournalPage(PgHdr *pPg){ void *pData = pPg->pData; i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - #if SQLITE_HAS_CODEC if( !pPager->subjInMemory ){ CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); }else -#endif +#endif /* SQLITE_HAS_CODEC */ pData2 = pData; PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); @@ -57528,11 +62607,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ -#ifndef SQLITE_OMIT_DESERIALIZE int memJM = 0; /* Memory journal mode */ -#else -# define memJM 0 -#endif int readOnly = 0; /* True if this is a read-only file */ int journalFileSize; /* Bytes to allocate for each journal fd */ char *zPathname = 0; /* Full path to database file */ @@ -57542,7 +62617,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ - int nUri = 0; /* Number of URI parameters */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -57590,7 +62664,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( while( *z ){ z += strlen(z)+1; z += strlen(z)+1; - nUri++; } nUriByte = (int)(&z[1] - zUri); assert( nUriByte>=1 ); @@ -57653,12 +62726,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** specific formatting and order of the various filenames, so if the format ** changes here, be sure to change it there as well. */ + assert( SQLITE_PTRSIZE==sizeof(Pager*) ); pPtr = (u8 *)sqlite3MallocZero( ROUND8(sizeof(*pPager)) + /* Pager structure */ ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ - sizeof(pPager) + /* Space to hold a pointer */ + SQLITE_PTRSIZE + /* Space to hold a pointer */ 4 + /* Database prefix */ nPathname + 1 + /* database filename */ nUriByte + /* query parameters */ @@ -57679,7 +62753,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); - memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + memcpy(pPtr, &pPager, SQLITE_PTRSIZE); pPtr += SQLITE_PTRSIZE; /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ pPtr += 4; /* Skip zero prefix */ @@ -57733,9 +62807,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); -#ifndef SQLITE_OMIT_DESERIALIZE pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; -#endif readOnly = (fout&SQLITE_OPEN_READONLY)!=0; /* If the file was successfully opened for read/write access, @@ -57814,7 +62886,11 @@ act_like_temp_file: rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } - +#ifdef SQLITE_META_DWR + if( rc==SQLITE_OK && !memDb && !readOnly){ + (void)MetaDwrOpenFile(pPager, 0); + } +#endif /* If an error occurred above, free the Pager structure and close the file. */ if( rc!=SQLITE_OK ){ @@ -57846,18 +62922,7 @@ act_like_temp_file: pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; assert( useJournal || pPager->tempFile ); - pPager->noSync = pPager->tempFile; - if( pPager->noSync ){ - assert( pPager->fullSync==0 ); - assert( pPager->extraSync==0 ); - assert( pPager->syncFlags==0 ); - assert( pPager->walSyncFlags==0 ); - }else{ - pPager->fullSync = 1; - pPager->extraSync = 0; - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); - } + sqlite3PagerSetFlags(pPager, (SQLITE_DEFAULT_SYNCHRONOUS+1)|PAGER_CACHESPILL); /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ @@ -57883,15 +62948,18 @@ act_like_temp_file: /* ** Return the sqlite3_file for the main database given the name -** of the corresonding WAL or Journal name as passed into +** of the corresponding WAL or Journal name as passed into ** xOpen. */ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ Pager *pPager; + const char *p; while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ zName--; } - pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + p = zName - 4 - sizeof(Pager*); + assert( EIGHT_BYTE_ALIGNMENT(p) ); + pPager = *(Pager**)p; return pPager->fd; } @@ -57977,6 +63045,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } + MARK_LAST_BUSY_LINE(rc); sqlite3EndBenignMalloc(); }else{ /* The journal file exists and no other connection has a reserved @@ -58066,6 +63135,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); + MARK_LAST_BUSY_LINE(rc); if( rc!=SQLITE_OK ){ assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); goto failed; @@ -58102,6 +63172,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** downgraded to SHARED_LOCK before this function returns. */ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + MARK_LAST_BUSY_LINE(rc); if( rc!=SQLITE_OK ){ goto failed; } @@ -58243,7 +63314,6 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } - failed: if( rc!=SQLITE_OK ){ assert( !MEMDB ); @@ -58344,7 +63414,12 @@ static int getPageNormal( assert( assert_pager_state(pPager) ); assert( pPager->hasHeldSharedLock==1 ); - if( pgno==0 ) return SQLITE_CORRUPT_BKPT; + if( pgno==0 ) { + const char *zMsg = "pgno should not be 0"; + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, 0, CORRUPT_TYPE_UNKOWN, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); + } pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); if( pBase==0 ){ pPg = 0; @@ -58364,7 +63439,7 @@ static int getPageNormal( if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ - assert( pgno!=PAGER_MJ_PGNO(pPager) ); + assert( pgno!=PAGER_SJ_PGNO(pPager) ); pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; @@ -58375,8 +63450,12 @@ static int getPageNormal( ** (*) obsolete. Was: maximum page number is 2^31 ** (2) Never try to fetch the locking page */ - if( pgno==PAGER_MJ_PGNO(pPager) ){ - rc = SQLITE_CORRUPT_BKPT; + if( pgno==PAGER_SJ_PGNO(pPager) ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "try fetching the locking page(%u)", pgno); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, pgno, CORRUPT_TYPE_UNKOWN, + -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT(&context); goto pager_acquire_err; } @@ -58386,6 +63465,10 @@ static int getPageNormal( if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; + if( pgno<=pPager->dbSize ){ + sqlite3PcacheRelease(pPg); + pPg = 0; + } goto pager_acquire_err; } if( noContent ){ @@ -58451,14 +63534,17 @@ static int getPageMMap( assert( USEFETCH(pPager) ); #ifdef SQLITE_HAS_CODEC assert( pPager->xCodec==0 ); -#endif +#endif /* SQLITE_HAS_CODEC */ /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here ** allows the compiler optimizer to reuse the results of the "pgno>1" ** test in the previous statement, and avoid testing pgno==0 in the ** common case where pgno is large. */ if( pgno<=1 && pgno==0 ){ - return SQLITE_CORRUPT_BKPT; + const char *zMsg = "pgno should not be 0"; + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, 0, CORRUPT_TYPE_UNKOWN, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); @@ -58524,7 +63610,20 @@ SQLITE_PRIVATE int sqlite3PagerGet( DbPage **ppPage, /* Write a pointer to the page here */ int flags /* PAGER_GET_XXX flags */ ){ +#if 0 /* Trace page fetch by setting to 1 */ + int rc; + printf("PAGE %u\n", pgno); + fflush(stdout); + rc = pPager->xGet(pPager, pgno, ppPage, flags); + if( rc ){ + printf("PAGE %u failed with 0x%02x\n", pgno, rc); + fflush(stdout); + } + return rc; +#else + /* Normal, high-speed version of sqlite3PagerGet() */ return pPager->xGet(pPager, pgno, ppPage, flags); +#endif } /* @@ -58552,10 +63651,12 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ /* ** Release a page reference. ** -** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be -** used if we know that the page being released is not the last page. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used +** if we know that the page being released is not the last reference to page1. ** The btree layer always holds page1 open until the end, so these first -** to routines can be used to release any page other than BtShared.pPage1. +** two routines can be used to release any page other than BtShared.pPage1. +** The assert() at tag-20230419-2 proves that this constraint is always +** honored. ** ** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine ** checks the total number of outstanding pages and if the number of @@ -58571,7 +63672,7 @@ SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ sqlite3PcacheRelease(pPg); } /* Do not use this routine to release the last reference to page1 */ - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */ } SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); @@ -58637,6 +63738,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + flags |= SQLITE_OPEN_EXCLUSIVE; nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; @@ -58672,6 +63774,7 @@ static int pager_open_journal(Pager *pPager){ if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + pPager->journalOff = 0; }else{ assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; @@ -58713,6 +63816,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + MARK_LAST_BUSY_LINE(rc); if( rc!=SQLITE_OK ){ return rc; } @@ -58725,6 +63829,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory ** holds the write-lock. If possible, the upper layer will call it. */ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + MARK_LAST_BUSY_LINE(rc); }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The @@ -58732,8 +63837,10 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory ** lock, but not when obtaining the RESERVED lock. */ rc = pagerLockDb(pPager, RESERVED_LOCK); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK && exFlag ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + MARK_LAST_BUSY_LINE(rc); } } @@ -58776,7 +63883,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + assert( pPg->pgno!=PAGER_SJ_PGNO(pPager) ); assert( pPager->journalHdr<=pPager->journalOff ); CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); @@ -58955,7 +64062,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ Pgno pg = pg1+ii; PgHdr *pPage; if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ + if( pg!=PAGER_SJ_PGNO(pPager) ){ rc = sqlite3PagerGet(pPager, pg, &pPage, 0); if( rc==SQLITE_OK ){ rc = pager_write(pPage); @@ -59118,7 +64225,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -59396,6 +64503,13 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); if( rc==SQLITE_OK ){ rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbFileSize ){ + char *pTmp = pPager->pTmpSpace; + int szPage = (int)pPager->pageSize; + memset(pTmp, 0, szPage); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, + ((i64)pPager->dbSize*pPager->pageSize)-szPage); + } if( rc==SQLITE_OK ){ rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); } @@ -59433,7 +64547,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** last page is never written out to disk, leaving the database file ** undersized. Fix this now if it is the case. */ if( pPager->dbSize>pPager->dbFileSize ){ - Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_SJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; @@ -59504,7 +64618,6 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ pPager->eState = PAGER_READER; return SQLITE_OK; } - PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); rc = pager_end_transaction(pPager, pPager->setSuper, 1); return pager_error(pPager, rc); @@ -59630,11 +64743,11 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->aStat[PAGER_STAT_HIT]; - a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[6] = (int)pPager->aStat[PAGER_STAT_HIT] & 0x7fffffff; + a[7] = (int)pPager->aStat[PAGER_STAT_MISS] & 0x7fffffff; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->aStat[PAGER_STAT_WRITE]; + a[10] = (int)pPager->aStat[PAGER_STAT_WRITE] & 0x7fffffff; return a; } #endif @@ -59650,7 +64763,7 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ ** reset parameter is non-zero, the cache hit or miss count is zeroed before ** returning. */ -SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, u64 *pnVal){ assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS @@ -59858,7 +64971,11 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ */ SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; + if( nullIfMemDb && (pPager->memDb || sqlite3IsMemdb(pPager->pVfs)) ){ + return &zFake[4]; + }else{ + return pPager->zFilename; + } } /* @@ -59882,7 +64999,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -60038,7 +65155,11 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i if( pPgOld ){ if( NEVER(pPgOld->nRef>1) ){ sqlite3PagerUnrefNotNull(pPgOld); - return SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "page(%u) should be no references, ref cnt:%d", pgno, (int)pPgOld->nRef); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPager->dbSize, pgno, CORRUPT_TYPE_UNKOWN, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( pPager->tempFile ){ @@ -60171,12 +65292,12 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ /* The eMode parameter is always valid */ - assert( eMode==PAGER_JOURNALMODE_DELETE - || eMode==PAGER_JOURNALMODE_TRUNCATE - || eMode==PAGER_JOURNALMODE_PERSIST - || eMode==PAGER_JOURNALMODE_OFF - || eMode==PAGER_JOURNALMODE_WAL - || eMode==PAGER_JOURNALMODE_MEMORY ); + assert( eMode==PAGER_JOURNALMODE_DELETE /* 0 */ + || eMode==PAGER_JOURNALMODE_PERSIST /* 1 */ + || eMode==PAGER_JOURNALMODE_OFF /* 2 */ + || eMode==PAGER_JOURNALMODE_TRUNCATE /* 3 */ + || eMode==PAGER_JOURNALMODE_MEMORY /* 4 */ + || eMode==PAGER_JOURNALMODE_WAL /* 5 */ ); /* This routine is only called from the OP_JournalMode opcode, and ** the logic there will never allow a temporary file to be changed @@ -60200,7 +65321,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; - /* When transistioning from TRUNCATE or PERSIST to any other journal + /* When transitioning from TRUNCATE or PERSIST to any other journal ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ @@ -60213,7 +65334,6 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( isOpen(pPager->fd) || pPager->exclusiveMode ); if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ - /* In this case we would like to delete the journal file. If it is ** not possible, then that is not a problem. Deleting the journal file ** here is an optimization only. @@ -60235,6 +65355,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ if( pPager->eState==PAGER_READER ){ assert( rc==SQLITE_OK ); rc = pagerLockDb(pPager, RESERVED_LOCK); + MARK_LAST_BUSY_LINE(rc); } if( rc==SQLITE_OK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); @@ -60246,7 +65367,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } - }else if( eMode==PAGER_JOURNALMODE_OFF ){ + }else if( eMode==PAGER_JOURNALMODE_OFF || eMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3OsClose(pPager->jfd); } } @@ -60325,6 +65446,18 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint( int *pnCkpt /* OUT: Final number of checkpointed frames */ ){ int rc = SQLITE_OK; + if( pPager->pWal==0 && pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + /* This only happens when a database file is zero bytes in size opened and + ** then "PRAGMA journal_mode=WAL" is run and then sqlite3_wal_checkpoint() + ** is invoked without any intervening transactions. We need to start + ** a transaction to initialize pWal. The PRAGMA table_list statement is + ** used for this since it starts transactions on every database file, + ** including all ATTACHed databases. This seems expensive for a single + ** sqlite3_wal_checkpoint() call, but it happens very rarely. + ** https://sqlite.org/forum/forumpost/fd0f19d229156939 + */ + sqlite3_exec(db, "PRAGMA table_list",0,0,0); + } if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), @@ -60356,13 +65489,15 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ */ static int pagerExclusiveLock(Pager *pPager){ int rc; /* Return code */ + u8 eOrigLock; /* Original lock */ - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->eLock>=SHARED_LOCK ); + eOrigLock = pPager->eLock; rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ /* If the attempt to grab the exclusive lock failed, release the ** pending lock that may have been obtained instead. */ - pagerUnlockDb(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, eOrigLock); } return rc; @@ -60387,6 +65522,7 @@ static int pagerOpenWal(Pager *pPager){ */ if( pPager->exclusiveMode ){ rc = pagerExclusiveLock(pPager); + MARK_LAST_BUSY_LINE(rc); } /* Open the connection to the log file. If this operation fails, @@ -60470,6 +65606,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ if( !pPager->pWal ){ int logexists = 0; rc = pagerLockDb(pPager, SHARED_LOCK); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists @@ -60485,6 +65622,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ */ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); @@ -60615,6 +65753,12 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){ + return sqlite3WalSystemErrno(pPager->pWal); +} +#endif + #endif /* SQLITE_OMIT_DISKIO */ /************** End of pager.c ***********************************************/ @@ -60905,7 +66049,7 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; ** ** Technically, the various VFSes are free to implement these locks however ** they see fit. However, compatibility is encouraged so that VFSes can -** interoperate. The standard implemention used on both unix and windows +** interoperate. The standard implementation used on both unix and windows ** is for the index number to indicate a byte offset into the ** WalCkptInfo.aLock[] array in the wal-index header. In other words, all ** locks are on the shm file. The WALINDEX_LOCK_OFFSET constant (which @@ -60981,7 +66125,7 @@ struct WalIndexHdr { ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) ** for any aReadMark[] means that entry is unused. aReadMark[0] is ** a special case; its value is never used and it exists as a place-holder -** to avoid having to offset aReadMark[] indexs by one. Readers holding +** to avoid having to offset aReadMark[] indexes by one. Readers holding ** WAL_READ_LOCK(0) always ignore the entire WAL and read all content ** directly from the database. ** @@ -61149,7 +66293,15 @@ struct Wal { u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_USE_SEH + u32 lockMask; /* Mask of locks held */ + void *pFree; /* Pointer to sqlite3_free() if exception thrown */ + u32 *pWiValue; /* Value to write into apWiData[iWiPg] */ + int iWiPg; /* Write pWiValue into apWiData[iWiPg] */ + int iSysErrno; /* System error code following exception */ +#endif #ifdef SQLITE_DEBUG + int nSehTry; /* Number of nested SEH_TRY{} blocks */ u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT @@ -61231,6 +66383,113 @@ struct WalIterator { sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ ) +/* +** Structured Exception Handling (SEH) is a Windows-specific technique +** for catching exceptions raised while accessing memory-mapped files. +** +** The -DSQLITE_USE_SEH compile-time option means to use SEH to catch and +** deal with system-level errors that arise during WAL -shm file processing. +** Without this compile-time option, any system-level faults that appear +** while accessing the memory-mapped -shm file will cause a process-wide +** signal to be deliver, which will more than likely cause the entire +** process to exit. +*/ +#ifdef SQLITE_USE_SEH +#include + +/* Beginning of a block of code in which an exception might occur */ +# define SEH_TRY __try { \ + assert( walAssertLockmask(pWal) && pWal->nSehTry==0 ); \ + VVA_ONLY(pWal->nSehTry++); + +/* The end of a block of code in which an exception might occur */ +# define SEH_EXCEPT(X) \ + VVA_ONLY(pWal->nSehTry--); \ + assert( pWal->nSehTry==0 ); \ + } __except( sehExceptionFilter(pWal, GetExceptionCode(), GetExceptionInformation() ) ){ X } + +/* Simulate a memory-mapping fault in the -shm file for testing purposes */ +# define SEH_INJECT_FAULT sehInjectFault(pWal) + +/* +** The second argument is the return value of GetExceptionCode() for the +** current exception. Return EXCEPTION_EXECUTE_HANDLER if the exception code +** indicates that the exception may have been caused by accessing the *-shm +** file mapping. Or EXCEPTION_CONTINUE_SEARCH otherwise. +*/ +static int sehExceptionFilter(Wal *pWal, int eCode, EXCEPTION_POINTERS *p){ + VVA_ONLY(pWal->nSehTry--); + if( eCode==EXCEPTION_IN_PAGE_ERROR ){ + if( p && p->ExceptionRecord && p->ExceptionRecord->NumberParameters>=3 ){ + /* From MSDN: For this type of exception, the first element of the + ** ExceptionInformation[] array is a read-write flag - 0 if the exception + ** was thrown while reading, 1 if while writing. The second element is + ** the virtual address being accessed. The "third array element specifies + ** the underlying NTSTATUS code that resulted in the exception". */ + pWal->iSysErrno = (int)p->ExceptionRecord->ExceptionInformation[2]; + } + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +/* +** If one is configured, invoke the xTestCallback callback with 650 as +** the argument. If it returns true, throw the same exception that is +** thrown by the system if the *-shm file mapping is accessed after it +** has been invalidated. +*/ +static void sehInjectFault(Wal *pWal){ + int res; + assert( pWal->nSehTry>0 ); + + res = sqlite3FaultSim(650); + if( res!=0 ){ + ULONG_PTR aArg[3]; + aArg[0] = 0; + aArg[1] = 0; + aArg[2] = (ULONG_PTR)res; + RaiseException(EXCEPTION_IN_PAGE_ERROR, 0, 3, (const ULONG_PTR*)aArg); + } +} + +/* +** There are two ways to use this macro. To set a pointer to be freed +** if an exception is thrown: +** +** SEH_FREE_ON_ERROR(0, pPtr); +** +** and to cancel the same: +** +** SEH_FREE_ON_ERROR(pPtr, 0); +** +** In the first case, there must not already be a pointer registered to +** be freed. In the second case, pPtr must be the registered pointer. +*/ +#define SEH_FREE_ON_ERROR(X,Y) \ + assert( (X==0 || Y==0) && pWal->pFree==X ); pWal->pFree = Y + +/* +** There are two ways to use this macro. To arrange for pWal->apWiData[iPg] +** to be set to pValue if an exception is thrown: +** +** SEH_SET_ON_ERROR(iPg, pValue); +** +** and to cancel the same: +** +** SEH_SET_ON_ERROR(0, 0); +*/ +#define SEH_SET_ON_ERROR(X,Y) pWal->iWiPg = X; pWal->pWiValue = Y + +#else +# define SEH_TRY VVA_ONLY(pWal->nSehTry++); +# define SEH_EXCEPT(X) VVA_ONLY(pWal->nSehTry--); assert( pWal->nSehTry==0 ); +# define SEH_INJECT_FAULT assert( pWal->nSehTry>0 ); +# define SEH_FREE_ON_ERROR(X,Y) +# define SEH_SET_ON_ERROR(X,Y) +#endif /* ifdef SQLITE_USE_SEH */ + + /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are @@ -61303,6 +66562,7 @@ static int walIndexPage( int iPage, /* The page we seek */ volatile u32 **ppPage /* Write the page pointer here */ ){ + SEH_INJECT_FAULT; if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ return walIndexPageRealloc(pWal, iPage, ppPage); } @@ -61314,6 +66574,7 @@ static int walIndexPage( */ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); } @@ -61322,6 +66583,7 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ */ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalIndexHdr*)pWal->apWiData[0]; } @@ -61367,19 +66629,40 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ + if( !nativeCksum ){ do { + s1 += BYTESWAP32(aData[0]) + s2; + s2 += BYTESWAP32(aData[1]) + s1; + aData += 2; + }while( aDatalockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ) pWal->lockMask |= (1 << lockIdx); +#endif return rc; } static void walUnlockShared(Wal *pWal, int lockIdx){ if( pWal->exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(1 << lockIdx); +#endif WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ @@ -61574,12 +66863,20 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, walLockName(lockIdx), n, rc ? "failed" : "ok")); VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ){ + pWal->lockMask |= (((1<exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(((1<apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; } @@ -61791,7 +67089,13 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ /* Write the aPgno[] array entry and the hash-table slot. */ nCollide = idx; for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ - if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; + if( (nCollide--)==0 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "no place for page(%u) to map into WAL, idx:%d", iPage, idx); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, iPage, CORRUPT_TYPE_FRAME_WAL, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); + } } sLoc.aPgno[idx-1] = iPage; AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); @@ -61859,6 +67163,7 @@ static int walIndexRecover(Wal *pWal){ iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); if( rc ){ + MARK_LAST_BUSY_LINE(rc); return rc; } @@ -61930,6 +67235,7 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + SEH_FREE_ON_ERROR(0, aFrame); if( !aFrame ){ rc = SQLITE_NOMEM_BKPT; goto recovery_error; @@ -61948,6 +67254,7 @@ static int walIndexRecover(Wal *pWal){ rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); assert( aShare!=0 || rc!=SQLITE_OK ); if( aShare==0 ) break; + SEH_SET_ON_ERROR(iPg, aShare); pWal->apWiData[iPg] = aPrivate; for(iFrame=iFirst; iFrame<=iLast; iFrame++){ @@ -61975,6 +67282,7 @@ static int walIndexRecover(Wal *pWal){ } } pWal->apWiData[iPg] = aShare; + SEH_SET_ON_ERROR(0,0); nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); nHdr32 = nHdr / sizeof(u32); #ifndef SQLITE_SAFER_WALINDEX_RECOVERY @@ -62005,9 +67313,11 @@ static int walIndexRecover(Wal *pWal){ } } #endif + SEH_INJECT_FAULT; if( iFrame<=iLast ) break; } + SEH_FREE_ON_ERROR(aFrame, 0); sqlite3_free(aFrame); } @@ -62035,6 +67345,7 @@ finished: }else{ pInfo->aReadMark[i] = READMARK_NOT_USED; } + SEH_INJECT_FAULT; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc!=SQLITE_BUSY ){ goto recovery_error; @@ -62192,7 +67503,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( } /* -** Change the size to which the WAL file is trucated on each reset. +** Change the size to which the WAL file is truncated on each reset. */ SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ if( pWal ) pWal->mxWalSize = iLimit; @@ -62418,23 +67729,16 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte + + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - - /* Allocate temporary space used by the merge-sort routine. This block - ** of memory will be freed before this function returns. - */ - aTmp = (ht_slot *)sqlite3_malloc64( - sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); - if( !aTmp ){ - rc = SQLITE_NOMEM_BKPT; - } - + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + SEH_FREE_ON_ERROR(0, p); for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && iaSegment[i].aPgno = (u32 *)sLoc.aPgno; } } - sqlite3_free(aTmp); - if( rc!=SQLITE_OK ){ + SEH_FREE_ON_ERROR(p, 0); walIteratorFree(p); p = 0; } @@ -62473,6 +67776,19 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + + +/* +** Attempt to enable blocking locks that block for nMs ms. Return 1 if +** blocking locks are successfully enabled, or 0 otherwise. +*/ +static int walEnableBlockingMs(Wal *pWal, int nMs){ + int rc = sqlite3OsFileControl( + pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs + ); + return (rc==SQLITE_OK); +} + /* ** Attempt to enable blocking locks. Blocking locks are enabled only if (a) ** they are supported by the VFS, and (b) the database handle is configured @@ -62484,11 +67800,7 @@ static int walEnableBlocking(Wal *pWal){ if( pWal->db ){ int tmout = pWal->db->busyTimeout; if( tmout ){ - int rc; - rc = sqlite3OsFileControl( - pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout - ); - res = (rc==SQLITE_OK); + res = walEnableBlockingMs(pWal, tmout); } } return res; @@ -62537,20 +67849,10 @@ SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db){ pWal->db = db; } -/* -** Take an exclusive WRITE lock. Blocking if so configured. -*/ -static int walLockWriter(Wal *pWal){ - int rc; - walEnableBlocking(pWal); - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - walDisableBlocking(pWal); - return rc; -} #else # define walEnableBlocking(x) 0 # define walDisableBlocking(x) -# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1) +# define walEnableBlockingMs(pWal, ms) 0 # define sqlite3WalDb(pWal, db) #endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ @@ -62622,6 +67924,11 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ assert( pInfo->aReadMark[0]==0 ); } +#ifdef SQLITE_HDR_CHECK +static int checkHeaderValid(Pager *pager, u8 *zBuf, const char *logStr); +static int checkDbHeaderValid(sqlite3 *db, int iDbpage, u8 *zBuf); +#endif /* SQLITE_HDR_CHECK */ + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -62690,13 +67997,14 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark+i); + u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK ){ u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - AtomicStore(pInfo->aReadMark+i, iMark); + AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -62717,8 +68025,7 @@ static int walCheckpoint( && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; - - pInfo->nBackfillAttempted = mxSafeFrame; + pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; /* Sync the WAL to disk */ rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); @@ -62737,7 +68044,13 @@ static int walCheckpoint( ** database plus the amount of data in the wal file, plus the ** maximum size of the pending-byte page (65536 bytes), then ** must be corruption somewhere. */ - rc = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, + "final db size unexpected,nSize=%lld,mxFrame=%u,pageSize=%d,nReq=%lld", + nSize, pWal->hdr.mxFrame, szPage, nReq); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, 0, CORRUPT_TYPE_FRAME_WAL, + -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT(&context); }else{ sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); } @@ -62749,6 +68062,7 @@ static int walCheckpoint( while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); + SEH_INJECT_FAULT; if( AtomicLoad(&db->u1.isInterrupted) ){ rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; break; @@ -62762,6 +68076,10 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); +#ifdef SQLITE_HDR_CHECK + rc = checkDbHeaderValid(db, iDbpage, zBuf); + if( rc!=SQLITE_OK ) break; +#endif /* SQLITE_HDR_CHECK */ rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -62778,12 +68096,14 @@ static int walCheckpoint( } } if( rc==SQLITE_OK ){ - AtomicStore(&pInfo->nBackfill, mxSafeFrame); + AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; } } /* Release the reader lock held while backfilling */ walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } else { + MARK_LAST_BUSY_LINE(rc); } if( rc==SQLITE_BUSY ){ @@ -62800,13 +68120,16 @@ static int walCheckpoint( */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); + SEH_INJECT_FAULT; if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; + MARK_LAST_BUSY_LINE(rc); }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; sqlite3_randomness(4, &salt1); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as @@ -62831,6 +68154,7 @@ static int walCheckpoint( } walcheckpoint_out: + SEH_FREE_ON_ERROR(pIter, 0); walIteratorFree(pIter); return rc; } @@ -62853,6 +68177,93 @@ static void walLimitSize(Wal *pWal, i64 nMax){ } } +#ifdef SQLITE_USE_SEH +/* +** This is the "standard" exception handler used in a few places to handle +** an exception thrown by reading from the *-shm mapping after it has become +** invalid in SQLITE_USE_SEH builds. It is used as follows: +** +** SEH_TRY { ... } +** SEH_EXCEPT( rc = walHandleException(pWal); ) +** +** This function does three things: +** +** 1) Determines the locks that should be held, based on the contents of +** the Wal.readLock, Wal.writeLock and Wal.ckptLock variables. All other +** held locks are assumed to be transient locks that would have been +** released had the exception not been thrown and are dropped. +** +** 2) Frees the pointer at Wal.pFree, if any, using sqlite3_free(). +** +** 3) Set pWal->apWiData[pWal->iWiPg] to pWal->pWiValue if not NULL +** +** 4) Returns SQLITE_IOERR. +*/ +static int walHandleException(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<lockMask & ~( + (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) + ); + for(ii=0; iipFree); + pWal->pFree = 0; + if( pWal->pWiValue ){ + pWal->apWiData[pWal->iWiPg] = pWal->pWiValue; + pWal->pWiValue = 0; + } + return SQLITE_IOERR_IN_PAGE; +} + +/* +** Assert that the Wal.lockMask mask, which indicates the locks held +** by the connenction, is consistent with the Wal.readLock, Wal.writeLock +** and Wal.ckptLock variables. To be used as: +** +** assert( walAssertLockmask(pWal) ); +*/ +static int walAssertLockmask(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) +#ifdef SQLITE_ENABLE_SNAPSHOT + | (pWal->pSnapshot ? (pWal->lockMask & (1 << WAL_CKPT_LOCK)) : 0) +#endif + ); + assert( mExpect==pWal->lockMask ); + } + return 1; +} + +/* +** Return and zero the "system error" field set when an +** EXCEPTION_IN_PAGE_ERROR exception is caught. +*/ +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal *pWal){ + int iRet = 0; + if( pWal ){ + iRet = pWal->iSysErrno; + pWal->iSysErrno = 0; + } + return iRet; +} + +#else +# define walAssertLockmask(x) 1 +#endif /* ifdef SQLITE_USE_SEH */ + /* ** Close a connection to a log file. */ @@ -62867,6 +68278,8 @@ SQLITE_PRIVATE int sqlite3WalClose( if( pWal ){ int isDelete = 0; /* True to unlink wal and wal-index files */ + assert( walAssertLockmask(pWal) ); + /* If an EXCLUSIVE lock can be obtained on the database file (using the ** ordinary, rollback-mode locking methods, this guarantees that the ** connection associated with this log file is the only connection to @@ -62891,7 +68304,7 @@ SQLITE_PRIVATE int sqlite3WalClose( ); if( bPersist!=1 ){ /* Try to delete the WAL file if the checkpoint completed and - ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** fsynced (rc==SQLITE_OK) and if we are not in persistent-wal ** mode (!bPersist) */ isDelete = 1; }else if( pWal->mxWalSize>=0 ){ @@ -62904,8 +68317,9 @@ SQLITE_PRIVATE int sqlite3WalClose( walLimitSize(pWal, 0); } } + } else { + MARK_LAST_BUSY_LINE(rc); } - walIndexClose(pWal, isDelete); sqlite3OsClose(pWal->pWalFd); if( isDelete ){ @@ -62958,7 +68372,7 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ ** give false-positive warnings about these accesses because the tools do not ** account for the double-read and the memory barrier. The use of mutexes ** here would be problematic as the memory being accessed is potentially - ** shared among multiple processes and not all mutex implementions work + ** shared among multiple processes and not all mutex implementations work ** reliably in that environment. */ aHdr = walIndexHdr(pWal); @@ -63058,9 +68472,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } + MARK_LAST_BUSY_LINE(rc); }else{ int bWriteLock = pWal->writeLock; - if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){ + if( bWriteLock + || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) + ){ pWal->writeLock = 1; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); @@ -63068,7 +68485,8 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ /* If the wal-index header is still malformed even while holding ** a WRITE lock, it can only mean that the header is corrupted and ** needs to be reconstructed. So run recovery to do exactly that. - */ + ** Disable blocking locks first. */ + walDisableBlocking(pWal); rc = walIndexRecover(pWal); *pChanged = 1; } @@ -63078,6 +68496,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } + MARK_LAST_BUSY_LINE(rc); } } @@ -63097,6 +68516,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** writer truncated the WAL out from under it. If that happens, it ** indicates that a writer has fixed the SHM file for us, so retry */ if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; + MARK_LAST_BUSY_LINE(rc); } pWal->exclusiveMode = WAL_NORMAL_MODE; } @@ -63224,7 +68644,9 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ } /* Allocate a buffer to read frames into */ - szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE; + assert( (pWal->szPage & (pWal->szPage-1))==0 ); + assert( pWal->szPage>=512 && pWal->szPage<=65536 ); + szFrame = pWal->szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame); if( aFrame==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -63238,7 +68660,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** the caller. */ aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; - for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->szPage); iOffset+szFrame<=szWal; iOffset+=szFrame ){ @@ -63276,6 +68698,37 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ return rc; } +/* +** The final argument passed to walTryBeginRead() is of type (int*). The +** caller should invoke walTryBeginRead as follows: +** +** int cnt = 0; +** do { +** rc = walTryBeginRead(..., &cnt); +** }while( rc==WAL_RETRY ); +** +** The final value of "cnt" is of no use to the caller. It is used by +** the implementation of walTryBeginRead() as follows: +** +** + Each time walTryBeginRead() is called, it is incremented. Once +** it reaches WAL_RETRY_PROTOCOL_LIMIT - indicating that walTryBeginRead() +** has many times been invoked and failed with WAL_RETRY - walTryBeginRead() +** returns SQLITE_PROTOCOL. +** +** + If SQLITE_ENABLE_SETLK_TIMEOUT is defined and walTryBeginRead() failed +** because a blocking lock timed out (SQLITE_BUSY_TIMEOUT from the OS +** layer), the WAL_RETRY_BLOCKED_MASK bit is set in "cnt". In this case +** the next invocation of walTryBeginRead() may omit an expected call to +** sqlite3OsSleep(). There has already been a delay when the previous call +** waited on a lock. +*/ +#define WAL_RETRY_PROTOCOL_LIMIT 100 +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +# define WAL_RETRY_BLOCKED_MASK 0x10000000 +#else +# define WAL_RETRY_BLOCKED_MASK 0 +#endif + /* ** Attempt to start a read transaction. This might fail due to a race or ** other transient condition. When that happens, it returns WAL_RETRY to @@ -63326,13 +68779,17 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** so it takes care to hold an exclusive lock on the corresponding ** WAL_READ_LOCK() while changing values. */ -static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ +static void DumpLocksByWal(Wal *pWal); +static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ u32 mxReadMark; /* Largest aReadMark[] value */ int mxI; /* Index of largest aReadMark[] value */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ u32 mxFrame; /* Wal frame to lock to */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int nBlockTmout = 0; +#endif assert( pWal->readLock<0 ); /* Not currently locked */ @@ -63356,14 +68813,37 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** so that on the 100th (and last) RETRY we delay for 323 milliseconds. ** The total delay time before giving up is less than 10 seconds. */ - if( cnt>5 ){ + (*pCnt)++; + if( *pCnt>5 ){ int nDelay = 1; /* Pause time in microseconds */ - if( cnt>100 ){ + int cnt = (*pCnt & ~WAL_RETRY_BLOCKED_MASK); + if( cnt>WAL_RETRY_PROTOCOL_LIMIT ){ VVA_ONLY( pWal->lockError = 1; ) return SQLITE_PROTOCOL; } - if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; + if( *pCnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; +#if SQLITE_OS_UNIX + if( cnt>=15 ) DumpLocksByWal(pWal); +#endif /* SQLITE_OS_UNIX */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor + ** to block for locks for approximately nDelay us. This affects three + ** locks: (a) the shared lock taken on the DMS slot in os_unix.c (if + ** using os_unix.c), (b) the WRITER lock taken in walIndexReadHdr() if the + ** first attempted read fails, and (c) the shared lock taken on the + ** read-mark. + ** + ** If the previous call failed due to an SQLITE_BUSY_TIMEOUT error, + ** then sleep for the minimum of 1us. The previous call already provided + ** an extra delay while it was blocking on the lock. + */ + nBlockTmout = (nDelay+998) / 1000; + if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){ + if( *pCnt & WAL_RETRY_BLOCKED_MASK ) nDelay = 1; + } +#endif sqlite3OsSleep(pWal->pVfs, nDelay); + *pCnt &= ~WAL_RETRY_BLOCKED_MASK; } if( !useWal ){ @@ -63371,6 +68851,13 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( pWal->bShmUnreliable==0 ){ rc = walIndexReadHdr(pWal, pChanged); } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + walDisableBlocking(pWal); + if( rc==SQLITE_BUSY_TIMEOUT ){ + rc = SQLITE_BUSY; + *pCnt |= WAL_RETRY_BLOCKED_MASK; + } +#endif if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to @@ -63389,11 +68876,14 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** must be zeroed before the requested page is returned. */ rc = WAL_RETRY; + MARK_LAST_BUSY_LINE(SQLITE_BUSY); }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ walUnlockShared(pWal, WAL_RECOVER_LOCK); rc = WAL_RETRY; + MARK_LAST_BUSY_LINE(SQLITE_BUSY); }else if( rc==SQLITE_BUSY ){ rc = SQLITE_BUSY_RECOVERY; + MARK_LAST_BUSY_LINE(SQLITE_BUSY); } } if( rc!=SQLITE_OK ){ @@ -63407,6 +68897,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->nWiData>0 ); assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); + SEH_INJECT_FAULT; if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) @@ -63416,6 +68907,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** and can be safely ignored. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); + MARK_LAST_BUSY_LINE(SQLITE_BUSY); walShmBarrier(pWal); if( rc==SQLITE_OK ){ if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ @@ -63433,6 +68925,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** it finished. Leaving a corrupt image in the database file. */ walUnlockShared(pWal, WAL_READ_LOCK(0)); + MARK_LAST_BUSY_LINE(SQLITE_BUSY); return WAL_RETRY; } pWal->readLock = 0; @@ -63456,7 +68949,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #endif for(i=1; iaReadMark+i); + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; @@ -63477,6 +68970,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ }else if( rc!=SQLITE_BUSY ){ return rc; } + MARK_LAST_BUSY_LINE(rc); } } if( mxI==0 ){ @@ -63484,9 +68978,20 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } + (void)walEnableBlockingMs(pWal, nBlockTmout); rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + walDisableBlocking(pWal); if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ){ + *pCnt |= WAL_RETRY_BLOCKED_MASK; + } +#else + assert( rc!=SQLITE_BUSY_TIMEOUT ); +#endif + assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); + MARK_LAST_BUSY_LINE(rc); + return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; } /* Now that the read-lock has been obtained, check that neither the ** value in the aReadMark[] array or the contents of the wal-index @@ -63522,12 +69027,13 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** we can guarantee that the checkpointer that set nBackfill could not ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ - pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT; walShmBarrier(pWal); if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + MARK_LAST_BUSY_LINE(rc); return WAL_RETRY; }else{ assert( mxReadMark<=pWal->hdr.mxFrame ); @@ -63537,6 +69043,54 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #ifdef SQLITE_ENABLE_SNAPSHOT +/* +** This function does the work of sqlite3WalSnapshotRecover(). +*/ +static int walSnapshotRecover( + Wal *pWal, /* WAL handle */ + void *pBuf1, /* Temp buffer pWal->szPage bytes in size */ + void *pBuf2 /* Temp buffer pWal->szPage bytes in size */ +){ + int szPage = (int)pWal->szPage; + int rc; + i64 szDb; /* Size of db file in bytes */ + + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ + + rc = walHashGet(pWal, walFramePage(i), &sLoc); + if( rc!=SQLITE_OK ) break; + assert( i - sLoc.iZero - 1 >=0 ); + pgno = sLoc.aPgno[i-sLoc.iZero-1]; + iDbOff = (i64)(pgno-1) * szPage; + + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); + + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } + + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + } + + pInfo->nBackfillAttempted = i-1; + } + } + + return rc; +} + /* ** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted ** variable so that older snapshots can be accessed. To do this, loop @@ -63562,50 +69116,21 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ assert( pWal->readLock>=0 ); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc==SQLITE_OK ){ - volatile WalCkptInfo *pInfo = walCkptInfo(pWal); - int szPage = (int)pWal->szPage; - i64 szDb; /* Size of db file in bytes */ - - rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); - if( rc==SQLITE_OK ){ - void *pBuf1 = sqlite3_malloc(szPage); - void *pBuf2 = sqlite3_malloc(szPage); - if( pBuf1==0 || pBuf2==0 ){ - rc = SQLITE_NOMEM; - }else{ - u32 i = pInfo->nBackfillAttempted; - for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ - WalHashLoc sLoc; /* Hash table location */ - u32 pgno; /* Page number in db file */ - i64 iDbOff; /* Offset of db file entry */ - i64 iWalOff; /* Offset of wal file entry */ - - rc = walHashGet(pWal, walFramePage(i), &sLoc); - if( rc!=SQLITE_OK ) break; - assert( i - sLoc.iZero - 1 >=0 ); - pgno = sLoc.aPgno[i-sLoc.iZero-1]; - iDbOff = (i64)(pgno-1) * szPage; - - if( iDbOff+szPage<=szDb ){ - iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; - rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); - } - - if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ - break; - } - } - - pInfo->nBackfillAttempted = i-1; - } + void *pBuf1 = sqlite3_malloc(pWal->szPage); + void *pBuf2 = sqlite3_malloc(pWal->szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + pWal->ckptLock = 1; + SEH_TRY { + rc = walSnapshotRecover(pWal, pBuf1, pBuf2); } - - sqlite3_free(pBuf1); - sqlite3_free(pBuf2); + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->ckptLock = 0; } + + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); } @@ -63614,28 +69139,20 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ #endif /* SQLITE_ENABLE_SNAPSHOT */ /* -** Begin a read transaction on the database. -** -** This routine used to be called sqlite3OpenSnapshot() and with good reason: -** it takes a snapshot of the state of the WAL and wal-index for the current -** instant in time. The current thread will continue to use this snapshot. -** Other threads might append new content to the WAL and wal-index but -** that extra content is ignored by the current thread. -** -** If the database contents have changes since the previous read -** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that its cache is stale and -** needs to be flushed. +** This function does the work of sqlite3WalBeginReadTransaction() (see +** below). That function simply calls this one inside an SEH_TRY{...} block. */ -SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ +static int walBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ #ifdef SQLITE_ENABLE_SNAPSHOT + int ckptLock = 0; int bChanged = 0; WalIndexHdr *pSnapshot = pWal->pSnapshot; #endif assert( pWal->ckptLock==0 ); + assert( pWal->nSehTry>0 ); #ifdef SQLITE_ENABLE_SNAPSHOT if( pSnapshot ){ @@ -63654,16 +69171,16 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ (void)walEnableBlocking(pWal); rc = walLockShared(pWal, WAL_CKPT_LOCK); walDisableBlocking(pWal); - + MARK_LAST_BUSY_LINE(rc); if( rc!=SQLITE_OK ){ return rc; } - pWal->ckptLock = 1; + ckptLock = 1; } #endif do{ - rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); + rc = walTryBeginRead(pWal, pChanged, 0, &cnt); }while( rc==WAL_RETRY ); testcase( (rc&0xff)==SQLITE_BUSY ); testcase( (rc&0xff)==SQLITE_IOERR ); @@ -63722,15 +69239,37 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ } /* Release the shared CKPT lock obtained above. */ - if( pWal->ckptLock ){ + if( ckptLock ){ assert( pSnapshot ); walUnlockShared(pWal, WAL_CKPT_LOCK); - pWal->ckptLock = 0; } #endif return rc; } +/* +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that its cache is stale and +** needs to be flushed. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; + SEH_TRY { + rc = walBeginReadTransaction(pWal, pChanged); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + /* ** Finish with a read transaction. All this does is release the ** read-lock. @@ -63751,7 +69290,7 @@ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ ** Return SQLITE_OK if successful, or an error code if an error occurs. If an ** error does occur, the final value of *piRead is undefined. */ -SQLITE_PRIVATE int sqlite3WalFindFrame( +static int walFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ u32 *piRead /* OUT: Frame number (or zero) */ @@ -63814,6 +69353,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( } nCollide = HASHTABLE_NSLOT; iKey = walHash(pgno); + SEH_INJECT_FAULT; while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ @@ -63821,7 +69361,12 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( iRead = iFrame; } if( (nCollide--)==0 ){ - return SQLITE_CORRUPT_BKPT; + *piRead = 0; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "mis-match page(%u) to map into WAL", pgno); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, pgno, CORRUPT_TYPE_FRAME_WAL, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } iKey = walNextHash(iKey); } @@ -63850,6 +69395,30 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( return SQLITE_OK; } +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +** +** The difference between this function and walFindFrame() is that this +** function wraps walFindFrame() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + int rc; + SEH_TRY { + rc = walFindFrame(pWal, pgno, piRead); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + return rc; +} + /* ** Read the contents of frame iRead from the wal file into buffer pOut ** (which is nOut bytes in size). Return SQLITE_OK if successful, or an @@ -63931,12 +69500,17 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** time the read transaction on this connection was started, then ** the write is disallowed. */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + SEH_TRY { + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + + if( rc!=SQLITE_OK ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; } - return rc; } @@ -63972,30 +69546,33 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; - /* Restore the clients cache of the wal-index header to the state it - ** was in before the client began writing to the database. - */ - memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); - - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ - /* This call cannot fail. Unless the page for which the page number - ** is passed as the second argument is (a) in the cache and - ** (b) has an outstanding reference, then xUndo is either a no-op - ** (if (a) is false) or simply expels the page from the cache (if (b) - ** is false). - ** - ** If the upper layer is doing a rollback, it is guaranteed that there - ** are no outstanding references to any page other than page 1. And - ** page 1 is never written to the log until the transaction is - ** committed. As a result, the call to xUndo may not fail. + SEH_TRY { + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. */ - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; } @@ -64039,7 +69616,10 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ pWal->hdr.mxFrame = aWalData[0]; pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; - walCleanupHash(pWal); + SEH_TRY { + walCleanupHash(pWal); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; @@ -64089,7 +69669,7 @@ static int walRestartLog(Wal *pWal){ cnt = 0; do{ int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); }while( rc==WAL_RETRY ); assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ testcase( (rc&0xff)==SQLITE_IOERR ); @@ -64154,11 +69734,18 @@ static int walWriteOneFrame( int rc; /* Result code from subfunctions */ void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ -#if defined(SQLITE_HAS_CODEC) +#ifdef SQLITE_HDR_CHECK + if( pPage->pgno==1 ){ + rc = checkHeaderValid(pPage->pPager, pPage->pData, "walWrite"); + if( rc!=SQLITE_OK ) return rc; + } +#endif /* SQLITE_HDR_CHECK */ + +#ifdef SQLITE_HAS_CODEC if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; #else pData = pPage->pData; -#endif +#endif /* SQLITE_HAS_CODEC */ walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); if( rc ) return rc; @@ -64224,7 +69811,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). */ -SQLITE_PRIVATE int sqlite3WalFrames( +static int walFrames( Wal *pWal, /* Wal handle to write to */ int szPage, /* Database page-size in bytes */ PgHdr *pList, /* List of dirty pages to write */ @@ -64312,7 +69899,9 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; } } - assert( (int)pWal->szPage==szPage ); + if( (int)pWal->szPage!=szPage ){ + return SQLITE_CORRUPT_BKPT; /* TH3 test case: cov1/corrupt155.test */ + } /* Setup information needed to write frames into the WAL */ w.pWal = pWal; @@ -64333,7 +69922,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + VVA_ONLY(rc =) walFindFrame(pWal, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; @@ -64341,11 +69930,11 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( pWal->iReCksum==0 || iWriteiReCksum ){ pWal->iReCksum = iWrite; } -#if defined(SQLITE_HAS_CODEC) +#ifdef SQLITE_HAS_CODEC if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; #else pData = p->pData; -#endif +#endif /* SQLITE_HAS_CODEC */ rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; @@ -64456,6 +70045,33 @@ SQLITE_PRIVATE int sqlite3WalFrames( return rc; } +#ifdef LOG_DUMP +static sqlite3_int64 g_lastCkptTime = 0; +#endif /* LOG_DUMP */ + +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +** +** The difference between this function and walFrames() is that this +** function wraps walFrames() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; + SEH_TRY { + rc = walFrames(pWal, szPage, pList, nTruncate, isCommit, sync_flags); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + /* ** This routine is called to implement sqlite3_wal_checkpoint() and ** related interfaces. @@ -64493,10 +70109,9 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); - /* Enable blocking locks, if possible. If blocking locks are successfully - ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */ + /* Enable blocking locks, if possible. */ sqlite3WalDb(pWal, db); - (void)walEnableBlocking(pWal); + if( xBusy2 ) (void)walEnableBlocking(pWal); /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive ** "checkpoint" lock on the database file. @@ -64507,6 +70122,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( ** it will not be invoked in this case. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + MARK_LAST_BUSY_LINE(rc); testcase( rc==SQLITE_BUSY ); testcase( rc!=SQLITE_OK && xBusy2!=0 ); if( rc==SQLITE_OK ){ @@ -64523,6 +70139,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + MARK_LAST_BUSY_LINE(rc); if( rc==SQLITE_OK ){ pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ @@ -64535,30 +70152,43 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Read the wal-index header. */ - if( rc==SQLITE_OK ){ - walDisableBlocking(pWal); - rc = walIndexReadHdr(pWal, &isChanged); - (void)walEnableBlocking(pWal); - if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ - sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + SEH_TRY { + if( rc==SQLITE_OK ){ + /* For a passive checkpoint, do not re-enable blocking locks after + ** reading the wal-index header. A passive checkpoint should not block + ** or invoke the busy handler. The only lock such a checkpoint may + ** attempt to obtain is a lock on a read-slot, and it should give up + ** immediately and do a partial checkpoint if it cannot obtain it. */ + walDisableBlocking(pWal); + rc = walIndexReadHdr(pWal, &isChanged); + if( eMode2!=SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } - } - - /* Copy data from the log to the database file. */ - if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); - } + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "mis-match between pageSize=%d and bufferSize=%d, mxFrame=%u", + walPagesize(pWal),nBuf,pWal->hdr.mxFrame); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pWal->hdr.nPage, 0, CORRUPT_TYPE_FRAME_WAL, + -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT(&context); + }else{ + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); + } - /* If no error occurred, set the output variables. */ - if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + SEH_INJECT_FAULT; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + } } } + SEH_EXCEPT( rc = walHandleException(pWal); ) if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was @@ -64579,6 +70209,12 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; } +#ifdef LOG_DUMP + if( rc ){ + sqlite3_log(SQLITE_NOTICE, "ckpt rc[%d]", rc); + } + sqlite3OsCurrentTimeInt64(db->pVfs, &g_lastCkptTime); +#endif /* LOG_DUMP */ WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); #ifdef SQLITE_ENABLE_SETLK_TIMEOUT if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; @@ -64635,7 +70271,9 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ ** locks are taken in this case). Nor should the pager attempt to ** upgrade to exclusive-mode following such an error. */ +#ifndef SQLITE_USE_SEH assert( pWal->readLock>=0 || pWal->lockError ); +#endif assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ @@ -64736,16 +70374,19 @@ SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ */ SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ int rc; - rc = walLockShared(pWal, WAL_CKPT_LOCK); - if( rc==SQLITE_OK ){ - WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; - if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) - || pNew->mxFramenBackfillAttempted - ){ - rc = SQLITE_ERROR_SNAPSHOT; - walUnlockShared(pWal, WAL_CKPT_LOCK); + SEH_TRY { + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } } } + SEH_EXCEPT( rc = walHandleException(pWal); ) return rc; } @@ -64868,7 +70509,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** 22 1 Min embedded payload fraction (must be 32) ** 23 1 Min leaf payload fraction (must be 32) ** 24 4 File change counter -** 28 4 Reserved for future use +** 28 4 The size of the database in pages ** 32 4 First freelist page ** 36 4 Number of freelist pages in the file ** 40 60 15 4-byte meta values passed to higher layers @@ -64976,7 +70617,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** byte are used. The integer consists of all bytes that have bit 8 set and ** the first byte with bit 8 clear. The most significant byte of the integer ** appears first. A variable-length integer may not be more than 9 bytes long. -** As a special case, all 8 bytes of the 9th byte are used as data. This +** As a special case, all 8 bits of the 9th byte are used as data. This ** allows a 64-bit integer to be encoded in 9 bytes. ** ** 0x00 becomes 0x00000000 @@ -64984,7 +70625,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** 0x81 0x00 becomes 0x00000080 ** 0x82 0x00 becomes 0x00000100 ** 0x80 0x7f becomes 0x0000007f -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x91 0xd1 0xac 0x78 becomes 0x12345678 ** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 ** ** Variable length integers are used for rowids and to hold the number of @@ -65067,7 +70708,7 @@ typedef struct CellInfo CellInfo; ** page that has been loaded into memory. The information in this object ** is derived from the raw on-disk page content. ** -** As each database page is loaded into memory, the pager allocats an +** As each database page is loaded into memory, the pager allocates an ** instance of this object and zeros the first 8 bytes. (This is the ** "extra" information associated with each page of the pager.) ** @@ -65097,7 +70738,9 @@ struct MemPage { u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ BtShared *pBt; /* Pointer to BtShared that this page is part of */ u8 *aData; /* Pointer to disk image of the page data */ - u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aDataEnd; /* One byte past the end of the entire page - not just + ** the usable space, the entire page. Used to prevent + ** corruption-induced buffer overflow. */ u8 *aCellIdx; /* The cell index area */ u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ @@ -65259,6 +70902,10 @@ struct BtShared { #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ int nPreformatSize; /* Size of last cell written by TransferRow() */ +#ifdef SQLITE_META_DWR + u32 maxMetaPage; + u32 metaRecoverStatus; +#endif }; /* @@ -65358,7 +71005,7 @@ struct BtCursor { #define BTCF_WriteFlag 0x01 /* True if a write cursor */ #define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ -#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_AtLast 0x08 /* Cursor is pointing to the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ #define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ @@ -65402,7 +71049,7 @@ struct BtCursor { /* ** The database page the PENDING_BYTE occupies. This page is never used. */ -# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) +#define PENDING_BYTE_PAGE(pBt) ((Pgno)((PENDING_BYTE/((pBt)->pageSize))+1)) /* ** These macros define the location of the pointer-map entry for a @@ -65476,15 +71123,15 @@ struct BtCursor { ** So, this macro is defined instead. */ #ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) +#define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else -#define ISAUTOVACUUM 0 +#define ISAUTOVACUUM(pBt) 0 #endif /* -** This structure is passed around through all the sanity checking routines -** in order to keep track of some global state information. +** This structure is passed around through all the PRAGMA integrity_check +** checking routines in order to keep track of some global state information. ** ** The aRef[] array is allocated so that there is 1 bit for each page in ** the database. As the integrity-check proceeds, for each page used in @@ -65497,16 +71144,19 @@ struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ u8 *aPgRef; /* 1 bit per page in the db (see above) */ - Pgno nPage; /* Number of pages in the database */ + Pgno nCkPage; /* Pages in the database. 0 for partial check */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ - int bOomFault; /* A memory allocation error has occurred */ + int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ + u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ - Pgno v1; /* Value for first %u substitution in zPfx */ - int v2; /* Value for second %d substitution in zPfx */ + Pgno v0; /* Value for first %u substitution in zPfx (root page) */ + Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ + int v2; /* Value for third %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -65519,7 +71169,7 @@ struct IntegrityCk { /* ** get2byteAligned(), unlike get2byte(), requires that its argument point to a -** two-byte aligned address. get2bytea() is only used for accessing the +** two-byte aligned address. get2byteAligned() is only used for accessing the ** cell addresses in a btree header. */ #if SQLITE_BYTEORDER==4321 @@ -65696,7 +71346,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ ** ** There is a corresponding leave-all procedures. ** -** Enter the mutexes in accending order by BtShared pointer address +** Enter the mutexes in ascending order by BtShared pointer address ** to avoid the possibility of deadlock when two threads with ** two or more btrees in common both try to lock all their btrees ** at the same instant. @@ -65770,6 +71420,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ Btree *p; assert( db!=0 ); + if( db->pVfs==0 && db->nDb==0 ) return 1; if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); assert( iDb>=0 && iDbnDb ); if( !sqlite3_mutex_held(db->mutex) ) return 0; @@ -65952,6 +71603,32 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ } #endif +#ifdef SQLITE_HDR_CHECK +static int checkHeaderValid(Pager *pager, u8 *zBuf, const char *logStr){ +#ifdef SQLITE_HAS_CODEC + if( pager==NULL || pager->pCodec ){ + return SQLITE_OK; + } +#endif /* SQLITE_HAS_CODEC */ + if( zBuf && strncmp((const char *)zBuf, zMagicHeader, 16)!=0 ){ + sqlite3_log(SQLITE_NOTADB, "[%s]wrong header format, memory might be overwritten!", logStr); + return SQLITE_NOTADB; + } + return SQLITE_OK; +} + +static int checkDbHeaderValid(sqlite3 *db, int iDbpage, u8 *zBuf){ + if( iDbpage==1 && db->aDb ){ + Btree *p = db->aDb[0].pBt; + if( p && p->pBt ){ + Pager *pager = sqlite3BtreePager(p); + return checkHeaderValid(pager, zBuf, "ckpt"); + } + } + return SQLITE_OK; +} +#endif /* SQLITE_HDR_CHECK */ + /* ** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single ** (MemPage*) as an argument. The (MemPage*) must not be NULL. @@ -65962,26 +71639,65 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ ** with the page number and filename associated with the (MemPage*). */ #ifdef SQLITE_DEBUG -int corruptPageError(int lineno, MemPage *p){ +int corruptPageError(int lineno, MemPage *p, sqlite3CorruptContext *context){ char *zMsg; sqlite3BeginBenignMalloc(); - zMsg = sqlite3_mprintf("database corruption page %d of %s", - (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + zMsg = sqlite3_mprintf("database corruption page %u of %s", + p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) ); sqlite3EndBenignMalloc(); if( zMsg ){ sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); } sqlite3_free(zMsg); - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_REPORT(context); } -# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) +# define SQLITE_CORRUPT_PAGE(context,pMemPage) corruptPageError(__LINE__, (pMemPage),(context)) #else -# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) +# define SQLITE_CORRUPT_PAGE(context,pMemPage) SQLITE_CORRUPT_PGNO((pMemPage)->pgno,(context)) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** @@ -66043,7 +71759,7 @@ static int hasSharedCacheTableLock( int bSeen = 0; for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ Index *pIdx = (Index *)sqliteHashData(p); - if( pIdx->tnum==(int)iRoot ){ + if( pIdx->tnum==iRoot ){ if( bSeen ){ /* Two or more indexes share the same root page. There must ** be imposter tables. So just return true. The assert is not @@ -66058,6 +71774,8 @@ static int hasSharedCacheTableLock( iTab = iRoot; } + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ @@ -66191,6 +71909,8 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); @@ -66258,6 +71978,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -66296,6 +72018,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -66636,7 +72361,7 @@ SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ /* ** In this version of BtreeMoveto, pKey is a packed index record ** such as is generated by the OP_MakeRecord opcode. Unpack the -** record and then call BtreeMovetoUnpacked() to do the work. +** record and then call sqlite3BtreeIndexMoveto() to do the work. */ static int btreeMoveto( BtCursor *pCur, /* Cursor open on the btree to be searched */ @@ -66655,7 +72380,12 @@ static int btreeMoveto( if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ - rc = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected fields in total:%u, should != 0 and < %u", + pIdxKey->nField,pKeyInfo->nAllField); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pCur->pBt->nPage, 0, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT(&context); }else{ rc = sqlite3BtreeIndexMoveto(pCur, pIdxKey, pRes); } @@ -66775,8 +72505,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) */ SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ /* Used only by system that substitute their own storage engine */ +#ifdef SQLITE_DEBUG + if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){ + va_list ap; + Expr *pExpr; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3CursorRangeHintExprCheck; + va_start(ap, eHintType); + pExpr = va_arg(ap, Expr*); + w.u.aMem = va_arg(ap, Mem*); + va_end(ap); + assert( pExpr!=0 ); + assert( w.u.aMem!=0 ); + sqlite3WalkExpr(&w, pExpr); + } +#endif /* SQLITE_DEBUG */ } -#endif +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* ** Provide flag hints to the cursor. @@ -66835,7 +72582,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); assert( pBt->autoVacuum ); - if( key==0 ){ + if( key==0 ){ // The pgno of each entry on ptrmap page starts from 3, an unexpected pgno indicates data corrupted *pRC = SQLITE_CORRUPT_BKPT; return; } @@ -66849,19 +72596,31 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ /* The first byte of the extra data is the MemPage.isInit byte. ** If that byte is set, it means this page is also being used ** as a btree page. */ - *pRC = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode((unsigned char *)sqlite3PagerGetExtra(pDbPage), 8, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "page(%u) been initialized before as a btree page, base16:%s", + iPtrmap, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, + -1, 0, zMsg, NULL); + *pRC = SQLITE_CORRUPT_REPORT(&context); goto ptrmap_exit; } offset = PTRMAP_PTROFFSET(iPtrmap, key); if( offset<0 ){ - *pRC = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect offset in ptrmap page(%u), target:%u, page usableSize=%u", + iPtrmap, key, pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, + -1, 0, zMsg, NULL); + *pRC = SQLITE_CORRUPT_REPORT(&context); goto ptrmap_exit; } assert( offset <= (int)pBt->usableSize-5 ); pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; @@ -66899,7 +72658,12 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ offset = PTRMAP_PTROFFSET(iPtrmap, key); if( offset<0 ){ sqlite3PagerUnref(pDbPage); - return SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect offset in ptrmap page(%d), target:%u, page usableSize=%u", + iPtrmap, key, pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } assert( offset <= (int)pBt->usableSize-5 ); assert( pEType!=0 ); @@ -66907,7 +72671,15 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); sqlite3PagerUnref(pDbPage); - if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap); + if( *pEType<1 || *pEType>5 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // 5 bytes for each entry on ptrmap page + (void)sqlite3base16Encode(pPtrmap, 5, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpect entry type:%d, base16:%s", (int)*pEType, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iPtrmap, CORRUPT_TYPE_PAGE_PTR_MAP, + offset, 5, zMsg, NULL); + return SQLITE_CORRUPT_REPORT_PGNO(&context); + } return SQLITE_OK; } @@ -67054,19 +72826,37 @@ static void btreeParseCellPtr( ** ** pIter += getVarint(pIter, (u64*)&pInfo->nKey); ** - ** The code is inlined to avoid a function call. + ** The code is inlined and the loop is unrolled for performance. + ** This routine is a high-runner. */ iKey = *pIter; if( iKey>=0x80 ){ - u8 *pEnd = &pIter[7]; - iKey &= 0x7f; - while(1){ - iKey = (iKey<<7) | (*++pIter & 0x7f); - if( (*pIter)<0x80 ) break; - if( pIter>=pEnd ){ - iKey = (iKey<<8) | *++pIter; - break; + u8 x; + iKey = (iKey<<7) ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter); + } + } + } + } + } + }else{ + iKey ^= 0x204000; } + }else{ + iKey ^= 0x4000; } } pIter++; @@ -67142,10 +72932,12 @@ static void btreeParseCell( ** the space used by the cell pointer. ** ** cellSizePtrNoPayload() => table internal nodes -** cellSizePtr() => all index nodes & table leaf nodes +** cellSizePtrTableLeaf() => table leaf nodes +** cellSizePtr() => index internal nodes +** cellSizeIdxLeaf() => index leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ u8 *pEnd; /* End mark for a varint */ u32 nSize; /* Size value to return */ @@ -67158,6 +72950,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif + assert( pPage->childPtrSize==4 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -67167,13 +72960,48 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ }while( *(pIter)>=0x80 && pIterintKey ){ - /* pIter now points at the 64-bit integer key value, a variable length - ** integer. The following block moves pIter to point at the first byte - ** past the end of the key value. */ - pEnd = &pIter[9]; - while( (*pIter++)&0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + assert( nSize>4 ); + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + assert( pPage->childPtrSize==0 ); + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); testcase( nSize==(u32)pPage->maxLocal+1 ); if( nSize<=pPage->maxLocal ){ @@ -67213,6 +73041,58 @@ static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ assert( debuginfo.nSize==(u16)(pIter - pCell) || CORRUPT_DB ); return (u16)(pIter - pCell); } +static u16 cellSizePtrTableLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} #ifdef SQLITE_DEBUG @@ -67226,7 +73106,7 @@ static u16 cellSize(MemPage *pPage, int iCell){ #ifndef SQLITE_OMIT_AUTOVACUUM /* ** The cell pCell is currently part of page pSrc but will ultimately be part -** of pPage. (pSrc and pPager are often the same.) If pCell contains a +** of pPage. (pSrc and pPage are often the same.) If pCell contains a ** pointer to an overflow page, insert an entry into the pointer-map for ** the overflow page that will be valid after pCell has been moved to pPage. */ @@ -67237,9 +73117,16 @@ static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ pPage->xParseCell(pPage, pCell, &info); if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + if( SQLITE_OVERFLOW(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ testcase( pSrc!=pPage ); - *pRC = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // Output cell header as much as possible, 4 bytes for overflow pgno + (void)sqlite3base16Encode(pCell, info.nSize - info.nLocal - 4, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "cell overflow, offset=%d, rest=%d, length=%u, base16:%s", + (int)(pCell - pPage->aData), (int)(pSrc->aDataEnd - pCell), info.nSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + pCell - pPage->aData, info.nSize, zMsg, NULL); + *pRC = SQLITE_CORRUPT_REPORT(&context); return; } ovfl = get4byte(&pCell[info.nSize-4]); @@ -67282,8 +73169,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = 0; - src = data = pPage->aData; + data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; @@ -67298,10 +73184,29 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** reconstruct the entire page. */ if( (int)data[hdr+7]<=nMaxFrag ){ int iFree = get2byte(&data[hdr+1]); - if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree>usableSize-4 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Output first 8 bytes as it's page header + sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset=%d overflow, usableSize=%d, base16:%s", + iFree, usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + 0, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } if( iFree ){ int iFree2 = get2byte(&data[iFree]); - if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree2>usableSize-4 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data, 4, xBuffer, sizeof(xBuffer)); // Output first freeblock's header 4 bytes + sqlite3_snprintf(sizeof(zMsg), zMsg, + "1st freeblock's next pointer overflow, point:%d, usableSize=%d, base16:%s", + iFree2, usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iFree, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ u8 *pEnd = &data[cellOffset + nCell*2]; u8 *pAddr; @@ -67309,16 +73214,51 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ int sz = get2byte(&data[iFree+2]); int top = get2byte(&data[hdr+5]); if( top>=iFree ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print first 8 bytes which is page header + sqlite3_snprintf(sizeof(zMsg), zMsg, + "1st freeblock's offset:%d should > CellContentArea's offset:%d, base16:%s", + iFree, top, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iFree, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } if( iFree2 ){ - if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree+sz>iFree2 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print first 8 bytes which is page header + sqlite3_snprintf(sizeof(zMsg), zMsg, + "the 1st 2 freeblocks mis-order, 1st block offset:%d, size:%d, 2nd block offset:%d, base16:%s", + iFree, sz, iFree2, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iFree, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } sz2 = get2byte(&data[iFree2+2]); - if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree2+sz2 > usableSize ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data + iFree2, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to 2nd block + sqlite3_snprintf(sizeof(zMsg), zMsg, + "the 2nd freeblock overflow, offset:%d, size:%d, usableSize:%d, base16:%s", + iFree2, sz2, usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iFree2, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; - }else if( NEVER(iFree+sz>usableSize) ){ - return SQLITE_CORRUPT_PAGE(pPage); + }else if( iFree+sz>usableSize ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data + iFree, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to 1st block + sqlite3_snprintf(sizeof(zMsg), zMsg, + "the 1st freeblock overflow, offset:%d, size:%d, usableSize:%d, base16:%s", iFree, sz, usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iFree, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } cbrk = top+sz; @@ -67337,42 +73277,58 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ cbrk = usableSize; iCellLast = usableSize - 4; iCellStart = get2byte(&data[hdr+5]); - for(i=0; iiCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( pc>=iCellStart && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); - cbrk -= size; - if( cbrkusableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( cbrk+size<=usableSize && cbrk>=iCellStart ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - put2byte(pAddr, cbrk); - if( temp==0 ){ - if( cbrk==pc ) continue; - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); - src = temp; - } - memcpy(&data[cbrk], &src[pc], size); + if( nCell>0 ){ + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(temp, data, usableSize); + src = temp; + for(i=0; iiCellLast ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; // Print 4 bytes belong to 1st block + (void)sqlite3base16Encode(data + cellOffset + i*2, 2, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "%d-th cell pointer:%d out of range[%d, %d], base16:%s", + i, pc, iCellStart, iCellLast, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + cellOffset + i*2, 2, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( pc>=0 && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "move %d-th cell from %d using unexpected size:%d", i, pc, size); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + memcpy(&data[cbrk], &src[pc], size); + } } data[hdr+7] = 0; - defragment_out: +defragment_out: assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, + "after defragment, free bytes should not change, fragment bytes:%d, free space:%d, total:%d", + (int)data[hdr+7], cbrk-iCellFirst, pPage->nFree); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -67397,11 +73353,12 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** will be ignored if adding the extra space to the fragmentation count ** causes the fragmentation count to exceed 60. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ // search on B-tree page const int hdr = pPg->hdrOffset; /* Offset to page header */ u8 * const aData = pPg->aData; /* Page data */ int iAddr = hdr + 1; /* Address of ptr to pc */ - int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + u8 *pTmp = &aData[iAddr]; /* Temporary ptr into aData[] */ + int pc = get2byte(pTmp); /* Address of a free slot */ int x; /* Excess size of the slot */ int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ int size; /* Size of the free slot */ @@ -67411,7 +73368,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ - size = get2byte(&aData[pc+2]); + pTmp = &aData[pc+2]; + size = get2byte(pTmp); if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); @@ -67424,9 +73382,18 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + return &aData[pc]; }else if( x+pc > maxPC ){ /* This slot extends off the end of the usable part of the page */ - *pRc = SQLITE_CORRUPT_PAGE(pPg); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(aData + pc, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to free block + sqlite3_snprintf(sizeof(zMsg), zMsg, + "freeblock rest bytes:%d begin at %d which cost %d, still exceed usableSize:%u, base16:%s", + x, pc, nByte, pPg->pBt->usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + pc, 4, zMsg, NULL); + *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); return 0; }else{ /* The slot remains on the free-list. Reduce its size to account @@ -67436,18 +73403,30 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ return &aData[pc + x]; } iAddr = pc; - pc = get2byte(&aData[pc]); - if( pc<=iAddr+size ){ + pTmp = &aData[pc]; + pc = get2byte(pTmp); + if( pc<=iAddr ){ if( pc ){ - /* The next slot in the chain is not past the end of the current slot */ - *pRc = SQLITE_CORRUPT_PAGE(pPg); + /* The next slot in the chain comes before the current slot */ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pTmp, 2, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to free block + sqlite3_snprintf(sizeof(zMsg), zMsg, + "the next slot:%d in chain comes before current slot:%d, base16:%s", pc, iAddr, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + iAddr, 2, zMsg, NULL); + *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); } return 0; } } if( pc>maxPC+nByte-4 ){ /* The free slot chain extends off the end of the page */ - *pRc = SQLITE_CORRUPT_PAGE(pPg); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "free slot:%d overflow, end:%d", pc, maxPC+nByte-4); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPg->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + *pRc = SQLITE_CORRUPT_PAGE(&context, pPg); } return 0; } @@ -67465,11 +73444,12 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** allocation is being made in order to insert a new cell, so we will ** also end up needing a new cell pointer. */ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ +static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ int rc = SQLITE_OK; /* Integer return code */ + u8 *pTmp; /* Temp ptr into data[] */ int gap; /* First byte of gap between cell pointers and cell content */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -67488,14 +73468,22 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** then the cell content offset of an empty page wants to be 65536. ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ - top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ + pTmp = &data[hdr+5]; + top = get2byte(pTmp); if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data, 8, xBuffer, sizeof(xBuffer)); // Print 8 bytes belong to page header + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected cellContentArea offset:%d, base16:%s", top, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } + }else if( top>(int)pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(NULL, pPage); } /* If there is enough space between gap and top for one more cell pointer, @@ -67512,7 +73500,11 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( pSpace+nByte<=data+pPage->pBt->usableSize ); *pIdx = g2 = (int)(pSpace-data); if( g2<=gap ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "cellpointers(end:%d) overlap with freeblock(%d)", gap, g2); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); }else{ return SQLITE_OK; } @@ -67557,7 +73549,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** ** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor -** does it detect cells or freeblocks that encrouch into the reserved bytes +** does it detect cells or freeblocks that encroach into the reserved bytes ** at the end of the page. So do additional corruption checks inside this ** routine and return SQLITE_CORRUPT if any problems are found. */ @@ -67570,6 +73562,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ u16 x; /* Offset to cell content area */ u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ + u8 *pTmp; /* Temporary ptr into data[] */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -67577,7 +73570,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -67588,16 +73581,27 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ while( (iFreeBlk = get2byte(&data[iPtr]))pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } iPtr = iFreeBlk; } if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data + iPtr, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to freeblock + sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d overflow, base16:%s", (int)iFreeBlk, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, iPtr, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } - assert( iFreeBlk>iPtr || iFreeBlk==0 ); + assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none @@ -67607,10 +73611,24 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ */ if( iFreeBlk && iEnd+3>=iFreeBlk ){ nFrag = iFreeBlk - iEnd; - if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); + if( iEnd>iFreeBlk ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d overlaps with pre block's end:%u", + (int)iFreeBlk, iEnd); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); if( iEnd > pPage->pBt->usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(data + iFreeBlk, 4, xBuffer, sizeof(xBuffer)); // Print 4 bytes belong to freeblock + sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock offset:%d, end:%u overflow, usableSize:%u, base16:%s", + (int)iFreeBlk, iEnd, pPage->pBt->usableSize, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, iFreeBlk, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); @@ -67623,35 +73641,63 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( iPtr>hdr+1 ){ int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); if( iPtrEnd+3>=iStart ){ - if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); + if( iPtrEnd>iStart ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "check pre freeblock end:%d overlaps with the pending free block:%d", + iPtrEnd, (int)iStart); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, iPtr, iPtrEnd - iPtr, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } nFrag += iStart - iPtrEnd; iSize = iEnd - iPtr; iStart = iPtr; } } - if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); + if( nFrag>data[hdr+7] ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "fragment free bytes:%d increase unexpectly, should be %d", + (int)nFrag, (int)data[hdr+7]); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } data[hdr+7] -= nFrag; } - x = get2byte(&data[hdr+5]); + pTmp = &data[hdr+5]; + x = get2byte(pTmp); + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ - if( iStart= the beginning of the CellContentArea:%d", (int)x, (int)iStart); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( iPtr!=hdr+1 ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "1st freeblock's pos incorrect, hdr:%d, iPtr:%d", (int)hdr, (int)iPtr); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, 0, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } put2byte(&data[hdr+1], iFreeBlk); put2byte(&data[hdr+5], iEnd); }else{ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[iStart], 0, iSize); - } - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); pPage->nFree += iOrigSize; return SQLITE_OK; } @@ -67663,57 +73709,93 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ ** Only the following combinations are supported. Anything different ** indicates a corrupt database files: ** -** PTF_ZERODATA -** PTF_ZERODATA | PTF_LEAF -** PTF_LEAFDATA | PTF_INTKEY -** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +** PTF_ZERODATA (0x02, 2) +** PTF_LEAFDATA | PTF_INTKEY (0x05, 5) +** PTF_ZERODATA | PTF_LEAF (0x0a, 10) +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF (0x0d, 13) */ static int decodeFlags(MemPage *pPage, int flagByte){ BtShared *pBt; /* A copy of pPage->pBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; - pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); - pPage->intKey = 1; - if( pPage->leaf ){ + pPage->max1bytePayload = pBt->max1bytePayload; + if( flagByte>=(PTF_ZERODATA | PTF_LEAF) ){ + pPage->childPtrSize = 0; + pPage->leaf = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF) ){ pPage->intKeyLeaf = 1; + pPage->xCellSize = cellSizePtrTableLeaf; pPage->xParseCell = btreeParseCellPtr; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; +#ifdef LOG_DUMP + sqlite3_log(SQLITE_CORRUPT, + "database corruption at page[%u], flagByte[%x], isInit[%u], intKey[%u], intKeyLeaf[%u], leaf[%u], " + "childPtrSize[%u], cellOffset[%u], nCell[%u], hdrOffset[%u], minLocal[%u], maxLocal[%u], last ckpt time[%lld]", + pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, + pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); +#endif /* LOG_DUMP */ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pPage->aData, 8, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "unrecognized flag:%d, base16:%s", flagByte, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } + }else{ + pPage->childPtrSize = 4; + pPage->leaf = 0; + if( flagByte==(PTF_ZERODATA) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; +#ifdef LOG_DUMP + sqlite3_log(SQLITE_CORRUPT, + "database corruption at page[%u], flagByte[%x], isInit[%u], intKey[%u], intKeyLeaf[%u], leaf[%u], " + "childPtrSize[%u], cellOffset[%u], nCell[%u], hdrOffset[%u], minLocal[%u], maxLocal[%u], last ckpt time[%lld]", + pPage->pgno, flagByte, pPage->isInit, pPage->intKey, pPage->intKeyLeaf, pPage->leaf, + pPage->childPtrSize, pPage->cellOffset, pPage->nCell, pPage->hdrOffset, pPage->minLocal, pPage->maxLocal, g_lastCkptTime); +#endif /* LOG_DUMP */ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pPage->aData, 8, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "unrecognized flag:%d, base16:%s", flagByte, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ - assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ - assert( (PTF_ZERODATA|PTF_LEAF)==10 ); - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xParseCell = btreeParseCellPtrIndex; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is - ** an error. */ - return SQLITE_CORRUPT_PAGE(pPage); } - pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; } @@ -67762,12 +73844,20 @@ static int btreeComputeFreeSpace(MemPage *pPage){ /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will ** always be at least one cell before the first freeblock. */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "the 1st freeblock:%d before all cells:%d", pc, top); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } while( 1 ){ if( pc>iCellLast ){ /* Freeblock off the end of the page */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "freeblock end:%d out of page range:%d", pc, iCellLast); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } next = get2byte(&data[pc]); size = get2byte(&data[pc+2]); @@ -67777,11 +73867,19 @@ static int btreeComputeFreeSpace(MemPage *pPage){ } if( next>0 ){ /* Freeblock not in ascending order */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "all freeblocks should order by asc, pre:%d, cur:%u", pc, next); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } if( pc+size>(unsigned int)usableSize ){ /* Last freeblock extends past page end */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "last freeblock overflow, offset:%d, size:%u", pc, size); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } } @@ -67793,7 +73891,13 @@ static int btreeComputeFreeSpace(MemPage *pPage){ ** area, according to the page header, lies within the page. */ if( nFree>usableSize || nFreepBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } pPage->nFree = (u16)(nFree - iCellFirst); return SQLITE_OK; @@ -67824,12 +73928,20 @@ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "cell pointer:%d indicate out of range:[%d, %d]", pc, iCellFirst, iCellLast); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "unexpected cell size:%d,offset:%d, out of range:%d", sz, pc, usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } } return SQLITE_OK; @@ -67861,21 +73973,26 @@ static int btreeInitPage(MemPage *pPage){ /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating ** the b-tree page type. */ if( decodeFlags(pPage, data[0]) ){ - return SQLITE_CORRUPT_PAGE(pPage); + return SQLITE_CORRUPT_PAGE(NULL, pPage); } assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; pPage->aCellIdx = data + pPage->childPtrSize + 8; - pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataEnd = pPage->aData + pBt->pageSize; pPage->aDataOfst = pPage->aData + pPage->childPtrSize; /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the ** number of cells on the page. */ pPage->nCell = get2byte(&data[3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "too many cells(%d) for the page:%u, offset:%d, out of range:%u", + (int)pPage->nCell, pPage->pgno, (int)pPage->hdrOffset + 3, MX_CELL(pBt)); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } testcase( pPage->nCell==MX_CELL(pBt) ); /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only @@ -67903,7 +74020,7 @@ static void zeroPage(MemPage *pPage, int flags){ u8 hdr = pPage->hdrOffset; u16 first; - assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -67919,7 +74036,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->cellOffset = first; - pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; @@ -68004,68 +74121,45 @@ SQLITE_PRIVATE Pgno sqlite3BtreeLastPage(Btree *p){ /* ** Get a page from the pager and initialize it. -** -** If pCur!=0 then the page is being fetched as part of a moveToChild() -** call. Do additional sanity checking on the page in this case. -** And if the fetch fails, this routine must decrement pCur->iPage. -** -** The page is fetched as read-write unless pCur is not NULL and is -** a read-only cursor. -** -** If an error occurs, then *ppPage is undefined. It -** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - BtCursor *pCur, /* Cursor to receive the page, or NULL */ int bReadOnly /* True for a read-only page */ ){ int rc; DbPage *pDbPage; + MemPage *pPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->pPage ); - assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); - assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ - rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error1; + *ppPage = 0; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "page number(%u) > db file size(%u)", pgno, btreePagecount(pBt)); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error1; + *ppPage = 0; + return rc; } - *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( (*ppPage)->isInit==0 ){ + pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pPage->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); - rc = btreeInitPage(*ppPage); + rc = btreeInitPage(pPage); if( rc!=SQLITE_OK ){ - goto getAndInitPage_error2; + releasePage(pPage); + *ppPage = 0; + return rc; } } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); - - /* If obtaining a child page for a cursor, we must verify that the page is - ** compatible with the root page. */ - if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ - rc = SQLITE_CORRUPT_PGNO(pgno); - goto getAndInitPage_error2; - } + assert( pPage->pgno==pgno || CORRUPT_DB ); + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + *ppPage = pPage; return SQLITE_OK; - -getAndInitPage_error2: - releasePage(*ppPage); -getAndInitPage_error1: - if( pCur ){ - pCur->iPage--; - pCur->pPage = pCur->apPage[pCur->iPage]; - } - testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT ); - return rc; } /* @@ -68148,12 +74242,13 @@ static void pageReinit(DbPage *pData){ ** call to btreeInitPage() will likely return SQLITE_CORRUPT. ** But no harm is done by this. And it is very important that ** btreeInitPage() be called on every btree page so we make - ** the call for every page that comes in for re-initing. */ + ** the call for every page that comes in for re-initializing. */ btreeInitPage(pPage); } } } +static void DumpLocksByPager(Pager *pPager); /* ** Invoke the busy handler for a btree. */ @@ -68161,7 +74256,13 @@ static int btreeInvokeBusyHandler(void *pArg){ BtShared *pBt = (BtShared*)pArg; assert( pBt->db ); assert( sqlite3_mutex_held(pBt->db->mutex) ); - return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); + int rc = sqlite3InvokeBusyHandler(&pBt->db->busyHandler); +#if SQLITE_OS_UNIX + if (rc == 0) { + DumpLocksByPager(pBt->pPager); + } +#endif /* SQLITE_OS_UNIX */ + return rc; } /* @@ -68327,6 +74428,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); + /* Suppress false-positive compiler warning from PVS-Studio */ + memset(&zDbHeader[16], 0, 8); + pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -68443,8 +74547,8 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( } } #endif - *ppBtree = p; + *ppBtree = p; btree_open_out: if( rc!=SQLITE_OK ){ if( pBt && pBt->pPager ){ @@ -68522,30 +74626,38 @@ static int removeFromSharingList(BtShared *pBt){ ** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child ** pointer. */ -static void allocateTempSpace(BtShared *pBt){ - if( !pBt->pTmpSpace ){ - pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); - - /* One of the uses of pBt->pTmpSpace is to format cells before - ** inserting them into a leaf page (function fillInCell()). If - ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes - ** by the various routines that manipulate binary cells. Which - ** can mean that fillInCell() only initializes the first 2 or 3 - ** bytes of pTmpSpace, but that the first 4 bytes are copied from - ** it into a database page. This is not actually a problem, but it - ** does cause a valgrind error when the 1 or 2 bytes of unitialized - ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. - ** - ** Also: Provide four bytes of initialized space before the - ** beginning of pTmpSpace as an area available to prepend the - ** left-child pointer to the beginning of a cell. - */ - if( pBt->pTmpSpace ){ - memset(pBt->pTmpSpace, 0, 8); - pBt->pTmpSpace += 4; - } +static SQLITE_NOINLINE int allocateTempSpace(BtShared *pBt){ + assert( pBt!=0 ); + assert( pBt->pTmpSpace==0 ); + /* This routine is called only by btreeCursor() when allocating the + ** first write cursor for the BtShared object */ + assert( pBt->pCursor!=0 && (pBt->pCursor->curFlags & BTCF_WriteFlag)!=0 ); + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + if( pBt->pTmpSpace==0 ){ + BtCursor *pCur = pBt->pCursor; + pBt->pCursor = pCur->pNext; /* Unlink the cursor */ + memset(pCur, 0, sizeof(*pCur)); + return SQLITE_NOMEM_BKPT; } + + /* One of the uses of pBt->pTmpSpace is to format cells before + ** inserting them into a leaf page (function fillInCell()). If + ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes + ** by the various routines that manipulate binary cells. Which + ** can mean that fillInCell() only initializes the first 2 or 3 + ** bytes of pTmpSpace, but that the first 4 bytes are copied from + ** it into a database page. This is not actually a problem, but it + ** does cause a valgrind error when the 1 or 2 bytes of uninitialized + ** data is passed to system call write(). So to avoid this error, + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + return SQLITE_OK; } /* @@ -68581,7 +74693,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ } } #endif - + ResetLockStatus(); /* Rollback any active transaction and free the handle structure. ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. @@ -68770,7 +74882,7 @@ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ /* ** Return the number of bytes of space at the end of every page that -** are intentually left unused. This is the "reserved" space that is +** are intentionally left unused. This is the "reserved" space that is ** sometimes used by extensions. ** ** The value returned is the larger of the current reserve size and @@ -69017,7 +75129,6 @@ static int lockBtree(BtShared *pBt){ ){ goto page1_init_failed; } - pBt->btsFlags |= BTS_PAGESIZE_FIXED; assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of @@ -69037,14 +75148,19 @@ static int lockBtree(BtShared *pBt){ releasePageOne(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); return rc; } - if( sqlite3WritableSchema(pBt->db)==0 && nPage>nPageFile ){ - rc = SQLITE_CORRUPT_BKPT; - goto page1_init_failed; + if( nPage>nPageFile ){ + if( sqlite3WritableSchema(pBt->db)==0 ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + }else{ + nPage = nPageFile; + } } /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to ** be less than 480. In other words, if the page size is 512, then the @@ -69052,6 +75168,7 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; pBt->pageSize = pageSize; pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -69230,14 +75347,18 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ +static SQLITE_NOINLINE int btreeBeginTrans( + Btree *p, /* The btree in which to start the transaction */ + int wrflag, /* True to start a write transaction */ + int *pSchemaVersion /* Put schema version number here, if not NULL */ +){ BtShared *pBt = p->pBt; Pager *pPager = pBt->pPager; int rc = SQLITE_OK; sqlite3BtreeEnter(p); btreeIntegrity(p); - + ResetLockStatus(); /* If the btree is already in a write-transaction, or it ** is already in a read-transaction and a read-transaction ** is requested, this is a no-op. @@ -69397,11 +75518,38 @@ trans_begun: rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); } } - +#ifdef SQLITE_META_DWR + if (rc == SQLITE_NOTADB || rc == SQLITE_CORRUPT) { + int rc1 = MetaDwrRecoverAndBeginTran(p, wrflag, pSchemaVersion); + rc = (rc1 == SQLITE_OK) ? SQLITE_OK : rc; + } +#endif btreeIntegrity(p); sqlite3BtreeLeave(p); return rc; } +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt; + if( p->sharable + || p->inTrans==TRANS_NONE + || (p->inTrans==TRANS_READ && wrflag!=0) + ){ + return btreeBeginTrans(p,wrflag,pSchemaVersion); + } + pBt = p->pBt; + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + return sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + }else{ + return SQLITE_OK; + } +} #ifndef SQLITE_OMIT_AUTOVACUUM @@ -69461,7 +75609,12 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW2 ){ /* The pointer is always the first 4 bytes of the page in this case. */ if( get4byte(pPage->aData)!=iFrom ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "1st 4 bytes of ovrflow page(%u) point to next(%u), should be %u", + pPage->pgno, get4byte(pPage->aData), iFrom); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_PTR_MAP, + 0, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } put4byte(pPage->aData, iTo); }else{ @@ -69480,7 +75633,13 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ pPage->xParseCell(pPage, pCell, &info); if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, + "btree cell contain ovrflow pointer overflow, offset:%d, size:%u, usableSize:%u", + (int)(pCell - pPage->aData), info.nSize, pPage->pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_PTR_MAP, pCell - pPage->aData, info.nSize, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } if( iFrom==get4byte(pCell+info.nSize-4) ){ put4byte(pCell+info.nSize-4, iTo); @@ -69488,6 +75647,15 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } } }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, + "btree cell contain child pointer overflow, offset:%d, size:4, usableSize:%u", + (int)(pCell - pPage->aData), pPage->pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_PTR_MAP, pCell - pPage->aData, 4, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } if( get4byte(pCell)==iFrom ){ put4byte(pCell, iTo); break; @@ -69498,7 +75666,11 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( i==nCell ){ if( eType!=PTRMAP_BTREE || get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "missing pointer point to overflow page on btree page(%u)", pPage->pgno); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); } @@ -69536,7 +75708,7 @@ static int relocatePage( if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -69631,7 +75803,11 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ return rc; } if( eType==PTRMAP_ROOTPAGE ){ - return SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "try vacuum root page(%u), should not happened", nFin); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, PTRMAP_PAGENO(pBt, iLastPg), + CORRUPT_TYPE_PAGE_PTR_MAP, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } if( eType==PTRMAP_FREEPAGE ){ @@ -69674,12 +75850,17 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ } do { MemPage *pFreePg; + Pgno dbSize = btreePagecount(pBt); rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); + if( iFreePg>dbSize ){ + releasePage(pLastPg); + return SQLITE_CORRUPT_BKPT; + } }while( bCommit && iFreePg>nFin ); assert( iFreePgpPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); +#ifdef SQLITE_META_DWR + MetaDwrCheckVacuum(pBt); +#endif } }else{ rc = SQLITE_DONE; @@ -69840,6 +76024,9 @@ static int autoVacuumCommit(Btree *p){ put4byte(&pBt->pPage1->aData[28], nFin); pBt->bDoTruncate = 1; pBt->nPage = nFin; +#ifdef SQLITE_META_DWR + MetaDwrCheckVacuum(pBt); +#endif } if( rc!=SQLITE_OK ){ sqlite3PagerRollback(pPager); @@ -69896,6 +76083,9 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ if( pBt->bDoTruncate ){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } +#endif +#ifdef SQLITE_META_DWR + (void)MetaDwrUpdateMetaPages(p); #endif rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); sqlite3BtreeLeave(p); @@ -70286,10 +76476,6 @@ static int btreeCursor( assert( pBt->pPage1 && pBt->pPage1->aData ); assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); - if( wrFlag ){ - allocateTempSpace(pBt); - if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; - } if( iTable<=1 ){ if( iTable<1 ){ return SQLITE_CORRUPT_BKPT; @@ -70306,19 +76492,25 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; - pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + pCur->curFlags = 0; /* If there are two or more cursors on the same btree, then all such ** cursors *must* have the BTCF_Multiple flag set. */ for(pX=pBt->pCursor; pX; pX=pX->pNext){ if( pX->pgnoRoot==iTable ){ pX->curFlags |= BTCF_Multiple; - pCur->curFlags |= BTCF_Multiple; + pCur->curFlags = BTCF_Multiple; } } + pCur->eState = CURSOR_INVALID; pCur->pNext = pBt->pCursor; pBt->pCursor = pCur; - pCur->eState = CURSOR_INVALID; + if( wrFlag ){ + pCur->curFlags |= BTCF_WriteFlag; + pCur->curPagerFlags = 0; + if( pBt->pTmpSpace==0 ) return allocateTempSpace(pBt); + }else{ + pCur->curPagerFlags = PAGER_GET_READONLY; + } return SQLITE_OK; } static int btreeCursorWithLock( @@ -70487,7 +76679,6 @@ SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ pCur->curFlags &= ~BTCF_Pinned; } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the ** payload to which the cursor is pointing. @@ -70499,7 +76690,6 @@ SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor *pCur){ return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + (i64)(pCur->info.pPayload - pCur->pPage->aData); } -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* ** Return the number of bytes of payload for the entry that pCur is @@ -70525,7 +76715,7 @@ SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ ** routine always returns 2147483647 (which is the largest record ** that SQLite can handle) or more. But returning a smaller value might ** prevent large memory allocations when trying to interpret a -** corrupt datrabase. +** corrupt database. ** ** The current implementation merely returns the size of the underlying ** database file. @@ -70693,7 +76883,12 @@ static int accessPayload( assert( eOp==0 || eOp==1 ); assert( pCur->eState==CURSOR_VALID ); if( pCur->ix>=pPage->nCell ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "cell index:%u exceed limit:%u on the page:%u", + pCur->ix, pPage->nCell, pPage->pgno); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } assert( cursorHoldsMutex(pCur) ); @@ -70708,7 +76903,12 @@ static int accessPayload( ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ** but is recast into its current form to avoid integer overflow problems */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "base on payload size:%u, the max offset(%d) should > %d", + pCur->info.nLocal, (int)(aPayload - pPage->aData), (int)(pBt->usableSize - pCur->info.nLocal)); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + 0, 8, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } /* Check if data must be read/written to/from the btree page itself. */ @@ -70744,9 +76944,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -70756,6 +76959,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -70770,7 +76979,16 @@ static int accessPayload( assert( rc==SQLITE_OK && amt>0 ); while( nextPage ){ /* If required, populate the overflow page-list cache. */ - if( nextPage > pBt->nPage ) return SQLITE_CORRUPT_BKPT; + if( nextPage > pBt->nPage ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(aPayload + pCur->info.nLocal, 4, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow page:%u should not exceed the size of database file, base16:%s", + nextPage, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + aPayload - pPage->aData + pCur->info.nLocal, 4, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); + } assert( pCur->aOverflow[iIdx]==0 || pCur->aOverflow[iIdx]==nextPage || CORRUPT_DB ); @@ -70825,7 +77043,6 @@ static int accessPayload( assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); - if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else @@ -70855,7 +77072,11 @@ static int accessPayload( if( rc==SQLITE_OK && amt>0 ){ /* Overflow chain ends prematurely */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow chain ends prematurely, rest:%d", amt); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } return rc; } @@ -70987,8 +77208,7 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - BtShared *pBt = pCur->pBt; - + int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); + rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); + assert( pCur->pPage!=0 || rc!=SQLITE_OK ); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(newPgno, NULL); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + } + return rc; } #ifdef SQLITE_DEBUG @@ -71093,7 +77324,7 @@ static int moveToRoot(BtCursor *pCur){ while( --pCur->iPage ){ releasePageNotNull(pCur->apPage[pCur->iPage]); } - pCur->pPage = pCur->apPage[0]; + pRoot = pCur->pPage = pCur->apPage[0]; goto skip_init; } }else if( pCur->pgnoRoot==0 ){ @@ -71108,8 +77339,8 @@ static int moveToRoot(BtCursor *pCur){ } sqlite3BtreeClearCursor(pCur); } - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, - 0, pCur->curPagerFlags); + rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, + pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -71118,7 +77349,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->curIntKey = pCur->pPage->intKey; } pRoot = pCur->pPage; - assert( pRoot->pgno==pCur->pgnoRoot ); + assert( pRoot->pgno==pCur->pgnoRoot || CORRUPT_DB ); /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is @@ -71132,7 +77363,12 @@ static int moveToRoot(BtCursor *pCur){ ** (or the freelist). */ assert( pRoot->intKey==1 || pRoot->intKey==0 ); if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ - return SQLITE_CORRUPT_PAGE(pCur->pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "the page(%u) state illegal, isInit:%u, pKeyInfo%s0, intKey:%u", + pRoot->pgno, pRoot->isInit, ((pCur->pKeyInfo==0)?"==":"!="), pRoot->intKey); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pCur->pBt->nPage, pRoot->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pCur->pPage); } skip_init: @@ -71140,7 +77376,6 @@ skip_init: pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); - pRoot = pCur->pPage; if( pRoot->nCell>0 ){ pCur->eState = CURSOR_VALID; }else if( !pRoot->leaf ){ @@ -71222,42 +77457,36 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ *pRes = 0; rc = moveToLeftmost(pCur); }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + assert( pCur->pgnoRoot==0 || (pCur->pPage!=0 && pCur->pPage->nCell==0) ); *pRes = 1; rc = SQLITE_OK; } return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ -SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - - assert( cursorOwnsBtShared(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - - /* If the cursor already points to the last entry, this is a no-op. */ - if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif - *pRes = 0; - return SQLITE_OK; - } - - rc = moveToRoot(pCur); +static SQLITE_NOINLINE int btreeLast(BtCursor *pCur, int *pRes){ + int rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); *pRes = 0; @@ -71274,6 +77503,18 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ } return rc; } +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor already points to the last entry, this is a no-op. */ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); + *pRes = 0; + return SQLITE_OK; + } + return btreeLast(pCur, pRes); +} /* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) ** table near the key intKey. Return a success code. @@ -71321,13 +77562,14 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } /* If the requested key is one more than the previous key, then ** try to get there using sqlite3BtreeNext() rather than a full ** binary search. This is an optimization only. The correct answer - ** is still obtained without this case, only a little more slowely */ + ** is still obtained without this case, only a little more slowly. */ if( pCur->info.nKey+1==intKey ){ *pRes = 0; rc = sqlite3BtreeNext(pCur, 0); @@ -71381,14 +77623,17 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( upr = pPage->nCell-1; assert( biasRight==0 || biasRight==1 ); idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ - pCur->ix = (u16)idx; for(;;){ i64 nCellKey; pCell = findCellPastPtr(pPage, idx); if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ){ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "cell idx(%d) point to a cell should not out of page", idx); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } } } @@ -71441,6 +77686,69 @@ moveto_table_finish: return rc; } +/* +** Compare the "idx"-th cell on the page the cursor pCur is currently +** pointing to to pIdxKey using xRecordCompare. Return negative or +** zero if the cell is less than or equal pIdxKey. Return positive +** if unknown. +** +** Return value negative: Cell at pCur[idx] less than pIdxKey +** +** Return value is zero: Cell at pCur[idx] equals pIdxKey +** +** Return value positive: Nothing is known about the relationship +** of the cell at pCur[idx] and pIdxKey. +** +** This routine is part of an optimization. It is always safe to return +** a positive value as that will cause the optimization to be skipped. +*/ +static int indexCellCompare( + BtCursor *pCur, + int idx, + UnpackedRecord *pIdxKey, + RecordCompare xRecordCompare +){ + MemPage *pPage = pCur->pPage; + int c; + int nCell; /* Size of the pCell cell in bytes */ + u8 *pCell = findCellPastPtr(pPage, idx); + + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* If the record extends into overflow pages, do not attempt + ** the optimization. */ + c = 99; + } + return c; +} + +/* +** Return true (non-zero) if pCur is current pointing to the last +** page of a table. +*/ +static int cursorOnLastPage(BtCursor *pCur){ + int i; + assert( pCur->eState==CURSOR_VALID ); + for(i=0; iiPage; i++){ + MemPage *pPage = pCur->apPage[i]; + if( pCur->aiIdx[i]nCell ) return 0; + } + return 1; +} + /* Move the cursor so that it points to an entry in an index table ** near the key pIdxKey. Return a success code. ** @@ -71491,6 +77799,43 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( || pIdxKey->default_rc==-1 ); + + /* Check to see if we can skip a lot of work. Two cases: + ** + ** (1) If the cursor is already pointing to the very last cell + ** in the table and the pIdxKey search key is greater than or + ** equal to that last cell, then no movement is required. + ** + ** (2) If the cursor is on the last page of the table and the first + ** cell on that last page is less than or equal to the pIdxKey + ** search key, then we can start the search on the current page + ** without needing to go back to root. + */ + if( pCur->eState==CURSOR_VALID + && pCur->pPage->leaf + && cursorOnLastPage(pCur) + ){ + int c; + if( pCur->ix==pCur->pPage->nCell-1 + && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && pIdxKey->errCode==SQLITE_OK + ){ + *pRes = c; + return SQLITE_OK; /* Cursor already pointing at the correct spot */ + } + if( pCur->iPage>0 + && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + && pIdxKey->errCode==SQLITE_OK + ){ + pCur->curFlags &= ~BTCF_ValidOvfl; + if( !pCur->pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } + goto bypass_moveto_root; /* Start search on the current page */ + } + pIdxKey->errCode = SQLITE_OK; + } + rc = moveToRoot(pCur); if( rc ){ if( rc==SQLITE_EMPTY ){ @@ -71500,12 +77845,14 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( } return rc; } + +bypass_moveto_root: assert( pCur->pPage ); assert( pCur->pPage->isInit ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->pPage->nCell > 0 ); - assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); - assert( pCur->curIntKey || pIdxKey ); + assert( pCur->curIntKey==0 ); + assert( pIdxKey!=0 ); for(;;){ int lwr, upr, idx, c; Pgno chldPg; @@ -71519,11 +77866,10 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( ** be the right kind (index or table) of b-tree page. Otherwise ** a moveToChild() or moveToRoot() call would have detected corruption. */ assert( pPage->nCell>0 ); - assert( pPage->intKey==(pIdxKey==0) ); + assert( pPage->intKey==0 ); lwr = 0; upr = pPage->nCell-1; idx = upr>>1; /* idx = (lwr+upr)/2; */ - pCur->ix = (u16)idx; for(;;){ int nCell; /* Size of the pCell cell in bytes */ pCell = findCellPastPtr(pPage, idx); @@ -71570,7 +77916,12 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ testcase( nCell==2 ); /* Minimum legal index key size */ if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ - rc = SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "nCell:%d illegal, usableSize:%u, nPage:%u", + nCell, pCur->pBt->usableSize, pCur->pBt->nPage); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_PAGE(&context, pPage); goto moveto_index_finish; } pCellKey = sqlite3Malloc( nCell+nOverrun ); @@ -71612,7 +77963,7 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); assert( pPage->isInit ); if( pPage->leaf ){ - assert( pCur->ixpPage->nCell ); + assert( pCur->ixpPage->nCell || CORRUPT_DB ); pCur->ix = (u16)idx; *pRes = c; rc = SQLITE_OK; @@ -71623,10 +77974,36 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( }else{ chldPg = get4byte(findCell(pPage, lwr)); } - pCur->ix = (u16)lwr; - rc = moveToChild(pCur, chldPg); - if( rc ) break; - } + + /* This block is similar to an in-lined version of: + ** + ** pCur->ix = (u16)lwr; + ** rc = moveToChild(pCur, chldPg); + ** if( rc ) break; + */ + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + rc = getAndInitPage(pCur->pBt, chldPg, &pCur->pPage, pCur->curPagerFlags); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(chldPg, NULL); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + break; + } + /* + ***** End of in-lined moveToChild() call */ + } moveto_index_finish: pCur->info.nSize = 0; assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); @@ -71661,10 +78038,10 @@ SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; @@ -71717,14 +78094,8 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( !pPage->isInit || sqlite3FaultSim(412) ){ - /* The only known way for this to happen is for there to be a - ** recursive SQL function that does a DELETE operation as part of a - ** SELECT which deletes content out from under an active cursor - ** in a corrupt database file where the table being DELETE-ed from - ** has pages in common with the table being queried. See TH3 - ** module cov1/btree78.test testcase 220 (2018-06-08) for an - ** example. */ + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } @@ -71816,7 +78187,10 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ } pPage = pCur->pPage; - assert( pPage->isInit ); + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( !pPage->leaf ){ int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); @@ -71900,12 +78274,18 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); - /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 - ** stores stores the total number of pages on the freelist. */ + /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36 + ** stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ - return SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE*3] = {0}; + (void)sqlite3base16Encode(pPage1->aData, 100, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, + "total pages(%u) in freelist should not over the total size of db file(%u), base16:%s", n, mxPage, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, 1, CORRUPT_TYPE_FILE_HEADER, 36, 4, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } if( n>0 ){ /* There are pages on the freelist. Reuse one of those pages. */ @@ -71961,7 +78341,12 @@ static int allocateBtreePage( } testcase( iTrunk==mxPage ); if( iTrunk>mxPage || nSearch++ > n ){ - rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "freelist trunk page(%u) should <= the size of db(%u)", + iTrunk, mxPage); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, (pPrevTrunk ? pPrevTrunk->pgno : 1), + CORRUPT_TYPE_PAGE_FREE_LIST, -1, 0, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT_PGNO(&context); }else{ rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); } @@ -71987,10 +78372,17 @@ static int allocateBtreePage( memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); *ppPage = pTrunk; pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ - rc = SQLITE_CORRUPT_PGNO(iTrunk); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pTrunk->aData, 8, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "total leaf pages(%u) on trunk page over limit(%u), base16:%s", + k, (u32)(pBt->usableSize/4 - 2), xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, + 0, 8, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT_PGNO(&context); goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM }else if( searchList @@ -72024,7 +78416,14 @@ static int allocateBtreePage( MemPage *pNewTrunk; Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); if( iNewTrunk>mxPage ){ - rc = SQLITE_CORRUPT_PGNO(iTrunk); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pTrunk->aData, 12, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, + "leaf page's pgno(%u) on trunk page exceed db file size(%u), base16:%s", iNewTrunk, mxPage, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, + 8, 4, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT_PGNO(&context); goto end_allocate_page; } testcase( iNewTrunk==mxPage ); @@ -72053,7 +78452,7 @@ static int allocateBtreePage( } } pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); #endif }else if( k>0 ){ /* Extract a leaf from the trunk */ @@ -72089,7 +78488,14 @@ static int allocateBtreePage( iPage = get4byte(&aData[8+closest*4]); testcase( iPage==mxPage ); if( iPage>mxPage || iPage<2 ){ - rc = SQLITE_CORRUPT_PGNO(iTrunk); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(aData+8+closest*4, 4, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "leaf page's pgno(%u) out of range:[3, %d], base16:%s", + iPage, mxPage, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(mxPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, + 8+closest*4, 4, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT_PGNO(&context); goto end_allocate_page; } testcase( iPage==mxPage ); @@ -72098,8 +78504,8 @@ static int allocateBtreePage( ){ int noContent; *pPgno = iPage; - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" - ": %d more free pages\n", + TRACE(("ALLOCATE: %u was leaf %u of %u on trunk %u" + ": %u more free pages\n", *pPgno, closest+1, k, pTrunk->pgno, n-1)); rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc ) goto end_allocate_page; @@ -72155,7 +78561,7 @@ static int allocateBtreePage( ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ @@ -72178,7 +78584,7 @@ static int allocateBtreePage( releasePage(*ppPage); *ppPage = 0; } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + TRACE(("ALLOCATE: %u from end of file\n", *pPgno)); } assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); @@ -72215,7 +78621,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( NEVER(iPage<2) || iPage>pBt->nPage ){ + if( iPage<2 || iPage>pBt->nPage ){ return SQLITE_CORRUPT_BKPT; } if( pMemPage ){ @@ -72246,7 +78652,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } @@ -72274,7 +78680,14 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ nLeaf = get4byte(&pTrunk->aData[4]); assert( pBt->usableSize>32 ); if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ - rc = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pTrunk->aData, 4, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "the number of leaf page(%u) on trunk page(%d) exceed limit, base16:%s", + nLeaf, iTrunk, xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pBt->nPage, iTrunk, CORRUPT_TYPE_PAGE_FREE_LIST, + 0, 4, zMsg, NULL); + rc = SQLITE_CORRUPT_REPORT(&context); goto freepage_out; } if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ @@ -72306,7 +78719,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno)); goto freepage_out; } } @@ -72327,7 +78740,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ put4byte(pPage->aData, iTrunk); put4byte(&pPage->aData[4], 0); put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk)); freepage_out: if( pPage ){ @@ -72363,7 +78776,12 @@ static SQLITE_NOINLINE int clearCellOverflow( testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd ); if( pCell + pInfo->nSize > pPage->aDataEnd ){ /* Cell extends past end of page */ - return SQLITE_CORRUPT_PAGE(pPage); + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow end of page, pgno:%u, offset:%d, size:%u, usableSize:%u", + pPage->pgno, (int)(pCell - pPage->aData), pInfo->nSize, pPage->pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(btreePagecount(pPage->pBt), pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); } ovflPgno = get4byte(pCell + pInfo->nSize - 4); pBt = pPage->pBt; @@ -72380,7 +78798,14 @@ static SQLITE_NOINLINE int clearCellOverflow( /* 0 is not a legal page number and page 1 cannot be an ** overflow page. Therefore if ovflPgno<2 or past the end of the ** file the database must be corrupt. */ - return SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + char xBuffer[SQLITE_PRINT_BUF_SIZE] = {0}; + (void)sqlite3base16Encode(pCell + pInfo->nSize - 4, 4, xBuffer, sizeof(xBuffer)); + sqlite3_snprintf(sizeof(zMsg), zMsg, "overflow page's pgno(%u) illegal, out of range:[2, %u], base16:%s", + ovflPgno, btreePagecount(pBt), xBuffer); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(btreePagecount(pBt), pPage->pgno, + CORRUPT_TYPE_PAGE_BTREE_LEAF, -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_REPORT(&context); } if( nOvfl ){ rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); @@ -72416,7 +78841,7 @@ static SQLITE_NOINLINE int clearCellOverflow( /* Call xParseCell to compute the size of a cell. If the cell contains ** overflow, then invoke cellClearOverflow to clear out that overflow. -** STore the result code (SQLITE_OK or some error code) in rc. +** Store the result code (SQLITE_OK or some error code) in rc. ** ** Implemented as macro to force inlining for performance. */ @@ -72489,7 +78914,10 @@ static int fillInCell( n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrcnFree>=0 ); data = pPage->aData; ptr = &pPage->aCellIdx[2*idx]; - assert( pPage->pBt->usableSize > (int)(ptr-data) ); + assert( pPage->pBt->usableSize > (u32)(ptr-data) ); pc = get2byte(ptr); hdr = pPage->hdrOffset; testcase( pc==(u32)get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); if( pc+sz > pPage->pBt->usableSize ){ - *pRC = SQLITE_CORRUPT_BKPT; + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, "cell offset:%u size:%d, idx:%d overflow, usableSize:%u", + pc, sz, idx, pPage->pBt->usableSize); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(pPage->pBt->nPage, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + *pRC = SQLITE_CORRUPT_REPORT(&context); return; } rc = freeSpace(pPage, pc, sz); @@ -72687,23 +79120,27 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. ** -** *pRC must be SQLITE_OK when this routine is called. +** The insertCellFast() routine below works exactly the same as +** insertCell() except that it lacks the pTemp and iChild parameters +** which are assumed zero. Other than that, the two routines are the +** same. +** +** Fixes or enhancements to this routine should be reflected in +** insertCellFast()! */ -static void insertCell( +static int insertCell( MemPage *pPage, /* Page into which we are copying */ int i, /* New cell becomes the i-th cell of the page */ u8 *pCell, /* Content of the new cell */ int sz, /* Bytes of content in pCell */ u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ + Pgno iChild /* If non-zero, replace first 4 bytes with this value */ ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -72712,14 +79149,103 @@ static void insertCell( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); assert( pPage->nFree>=0 ); + assert( iChild>0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); pCell = pTemp; } - if( iChild ){ - put4byte(pCell, iChild); + put4byte(pCell, iChild); + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( NEVER(rc!=SQLITE_OK) ){ + return rc; } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ return rc; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; + } +#endif + } + return SQLITE_OK; +} + +/* +** This variant of insertCell() assumes that the pTemp and iChild +** parameters are both zero. Use this variant in sqlite3BtreeInsert() +** for performance improvement, and also so that this variant is only +** called from that one place, and is thus inlined, and thus runs must +** faster. +** +** Fixes or enhancements to this routine should be reflected into +** the insertCell() routine. +*/ +static int insertCellFast( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz /* Bytes of content in pCell */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + assert( pPage->nOverflow==0 ); + if( sz+2>pPage->nFree ){ j = pPage->nOverflow++; /* Comparison against ArraySize-1 since we hold back one extra slot ** as a contingency. In other words, never need more than 3 overflow @@ -72738,31 +79264,20 @@ static void insertCell( }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; + return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } + if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - if( iChild ){ - /* In a corrupt database where an entry in the cell index section of - ** a btree page has a value of 3 or less, the pCell value might point - ** as many as 4 bytes in front of the start of the aData buffer for - ** the source page. Make sure this does not cause problems by not - ** reading the first 4 bytes */ - memcpy(&data[idx+4], pCell+4, sz-4); - put4byte(&data[idx], iChild); - }else{ - memcpy(&data[idx], pCell, sz); - } + memcpy(&data[idx], pCell, sz); pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); @@ -72772,13 +79287,16 @@ static void insertCell( assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } #endif } + return SQLITE_OK; } /* @@ -72879,14 +79397,16 @@ struct CellArray { ** computed. */ static void populateCellCache(CellArray *p, int idx, int N){ + MemPage *pRef = p->pRef; + u16 *szCell = p->szCell; assert( idx>=0 && idx+N<=p->nCell ); while( N>0 ){ assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + if( szCell[idx]==0 ){ + szCell[idx] = pRef->xCellSize(pRef, p->apCell[idx]); }else{ assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + szCell[idx]==pRef->xCellSize(pRef, p->apCell[idx]) ); } idx++; N--; @@ -72940,12 +79460,13 @@ static int rebuildPage( int k; /* Current slot in pCArray->apEnd[] */ u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + assert( nCell>0 ); assert( i(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[k]<=i; k++){} pSrcEnd = pCArray->apEnd[k]; pData = pEnd; @@ -73008,7 +79529,7 @@ static int rebuildPage( ** Finally, argument pBegin points to the byte immediately following the ** end of the space required by this page for the cell-pointer area (for ** all cells - not just those inserted by the current call). If the content -** area must be extended to before this point in order to accomodate all +** area must be extended to before this point in order to accommodate all ** cells in apCell[], then the cells do not fit and non-zero is returned. */ static int pageInsertArray( @@ -73028,7 +79549,7 @@ static int pageInsertArray( u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ if( iEnd<=iFirst ) return 0; - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[k]<=i ; k++){} pEnd = pCArray->apEnd[k]; while( 1 /*Exit by break*/ ){ int sz, rc; @@ -73086,39 +79607,50 @@ static int pageFreeArray( u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; - int i; + int i, j; int iEnd = iFirst + nCell; - u8 *pFree = 0; - int szFree = 0; + int nFree = 0; + int aOfst[10]; + int aAfter[10]; for(i=iFirst; iapCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; + int iAfter; + int iOfst; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which ** cells need freeing */ sz = pCArray->szCell[i]; assert( sz>0 ); - if( pFree!=(pCell + sz) ){ - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); - } - pFree = pCell; - szFree = sz; - if( pFree+sz>pEnd ){ - return 0; + iOfst = (u16)(pCell - aData); + iAfter = iOfst+sz; + for(j=0; j=nFree ){ + if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){ + for(j=0; jpEnd ) return 0; + nFree++; } nRet++; } } - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + for(j=0; jpPg->aDataEnd ) goto editpage_fail; + if( NEVER(pData>pPg->aDataEnd) ) goto editpage_fail; /* Add cells to the start of the page */ if( iNewpgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); @@ -73345,8 +79878,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ /* Insert the new divider cell into pParent. */ if( rc==SQLITE_OK ){ - insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + rc = insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno); } /* Set the right-child pointer of pParent to point to the new page. */ @@ -73455,7 +79988,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ *pRC = setChildPtrmaps(pTo); } } @@ -73533,8 +80066,6 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ - Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ - u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); @@ -73590,7 +80121,7 @@ static int balance_nonroot( pgno = get4byte(pRight); while( 1 ){ if( rc==SQLITE_OK ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); } if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); @@ -73697,7 +80228,7 @@ static int balance_nonroot( ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(NULL, pOld); goto balance_cleanup; } @@ -73721,7 +80252,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limitaiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(NULL, pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -73881,15 +80412,17 @@ static int balance_nonroot( d = r + 1 - leafData; (void)cachedCellSize(&b, d); do{ + int szR, szD; assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + && (bBulk || szRight+szD+2 > szLeft-(szR+(i==k-1?0:2)))){ break; } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; + szRight += szD + 2; + szLeft -= szR + 2; cntNew[i-1] = r; r--; d--; @@ -73902,7 +80435,7 @@ static int balance_nonroot( } } - /* Sanity check: For a non-corrupt database file one of the follwing + /* Sanity check: For a non-corrupt database file one of the following ** must be true: ** (1) We found one or more cells (cntNew[0])>0), or ** (2) pPage is a virtual root page. A virtual root page is when @@ -73910,7 +80443,7 @@ static int balance_nonroot( ** that page. */ assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); - TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 @@ -73943,7 +80476,7 @@ static int balance_nonroot( cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; @@ -73958,47 +80491,44 @@ static int balance_nonroot( ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** - ** An O(n^2) insertion sort algorithm is used, but since n is never more - ** than (NB+2) (a small constant), that should not be a problem. + ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 + ** (5), that is not a performance concern. ** ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; ipgno; - aPgFlags[i] = apNew[i]->pDbPage->flags; - for(j=0; jpgno; + assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE ); + assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY ); } - for(i=0; ii ){ - sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); - } - sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); - apNew[i]->pgno = pgno; + for(i=0; ipgno < apNew[iB]->pgno ) iB = j; } - } - TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " - "%d(%d nc=%d) %d(%d nc=%d)\n", + /* If apNew[i] has a page number that is bigger than any of the + ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent + ** entry that has the smallest page number (which we know to be + ** entry apNew[iB]). + */ + if( iB!=i ){ + Pgno pgnoA = apNew[i]->pgno; + Pgno pgnoB = apNew[iB]->pgno; + Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1; + u16 fgA = apNew[i]->pDbPage->flags; + u16 fgB = apNew[iB]->pDbPage->flags; + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB); + sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA); + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB); + apNew[i]->pgno = pgnoB; + apNew[iB]->pgno = pgnoA; + } + } + + TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) " + "%u(%u nc=%u) %u(%u nc=%u)\n", apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, @@ -74039,7 +80569,7 @@ static int balance_nonroot( ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; @@ -74130,13 +80660,13 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; b.ixNx[k]<=i && ALWAYS(kpgno, &rc); + rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); } @@ -74166,6 +80696,8 @@ static int balance_nonroot( for(i=1-nNew; i=0 && iPg=1 || i>=0 ); + assert( iPg=0 /* On the upwards pass, or... */ || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ @@ -74232,7 +80764,7 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ @@ -74243,7 +80775,7 @@ static int balance_nonroot( } assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n", nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. @@ -74253,7 +80785,7 @@ static int balance_nonroot( } #if 0 - if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ + if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may @@ -74315,7 +80847,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } @@ -74328,7 +80860,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ memcpy(pChild->aiOvfl, pRoot->aiOvfl, @@ -74363,7 +80895,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pCur->pPage); } } return SQLITE_OK; @@ -74381,7 +80913,6 @@ static int anotherValidCursor(BtCursor *pCur){ */ static int balance(BtCursor *pCur){ int rc = SQLITE_OK; - const int nMin = pCur->pBt->usableSize * 2 / 3; u8 aBalanceQuickSpace[13]; u8 *pFree = 0; @@ -74393,7 +80924,11 @@ static int balance(BtCursor *pCur){ MemPage *pPage = pCur->pPage; if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; - if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + if( pPage->nOverflow==0 && pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){ + /* No rebalance required as long as: + ** (1) There are no overflow cells + ** (2) The amount of free space on the page is less than 2/3rds of + ** the total usable space on the page. */ break; }else if( (iPage = pCur->iPage)==0 ){ if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ @@ -74416,6 +80951,11 @@ static int balance(BtCursor *pCur){ }else{ break; } + }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ + rc = SQLITE_CORRUPT_PAGE(NULL, pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -74514,7 +81054,7 @@ static int btreeOverwriteContent( ){ int nData = pX->nData - iOffset; if( nData<=0 ){ - /* Overwritting with zeros */ + /* Overwriting with zeros */ int i; for(i=0; ipData to write */ int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ int rc; /* Return code */ @@ -74557,16 +81101,12 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ Pgno ovflPgno; /* Next overflow page to write */ u32 ovflPageSize; /* Size to write on overflow page */ - if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd - || pCur->info.pPayload < pPage->aData + pPage->cellOffset - ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pCur->info.nLocalinfo.pPayload, pX, 0, pCur->info.nLocal); if( rc ) return rc; - if( pCur->info.nLocal==nTotal ) return SQLITE_OK; /* Now overwrite the overflow pages */ iOffset = pCur->info.nLocal; @@ -74579,7 +81119,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(NULL, pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -74596,6 +81136,36 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ return SQLITE_OK; } +/* +** Overwrite the cell that cursor pCur is pointing to with fresh content +** contained in pX. +*/ +static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ + int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ + MemPage *pPage = pCur->pPage; /* Page being written */ + + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ + char zMsg[SQLITE_PRINT_CORRUPT_SIZE] = {0}; + sqlite3_snprintf(sizeof(zMsg), zMsg, + "cell payload cursor point to(%d), size:%u overlaps with non-cell content area:[%u, %d]", + (int)(pCur->info.pPayload - pPage->aData), pCur->info.nLocal, pPage->cellOffset, + (int)(pPage->aDataEnd - pPage->aData)); + sqlite3CorruptContext context = SQLITE_CORRUPT_CONTEXT(0, pPage->pgno, CORRUPT_TYPE_PAGE_BTREE_LEAF, + -1, 0, zMsg, NULL); + return SQLITE_CORRUPT_PAGE(&context, pPage); + } + if( pCur->info.nLocal==nTotal ){ + /* The entire cell is local */ + return btreeOverwriteContent(pPage, pCur->info.pPayload, pX, + 0, pCur->info.nLocal); + }else{ + /* The cell contains overflow content */ + return btreeOverwriteOverflowCell(pCur, pX); + } +} + /* ** Insert a new record into the BTree. The content of the new record @@ -74613,7 +81183,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ ** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to -** MovetoUnpacked() to seek cursor pCur to (pKey,nKey) has already +** sqlite3BtreeIndexMoveto() to seek cursor pCur to (pKey,nKey) has already ** been performed. In other words, if seekResult!=0 then the cursor ** is currently pointing to a cell that will be adjacent to the cell ** to be inserted. If seekResult<0 then pCur points to a cell that is @@ -74631,7 +81201,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ const BtreePayload *pX, /* Content of the row to be inserted */ int flags, /* True if this is likely an append */ - int seekResult /* Result of prior MovetoUnpacked() call */ + int seekResult /* Result of prior IndexMoveto() call */ ){ int rc; int loc = seekResult; /* -1: before desired location +1: after */ @@ -74639,31 +81209,12 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( int idx; MemPage *pPage; Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags ); assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 ); - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } - - assert( cursorOwnsBtShared(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); - assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); - - /* Assert that the caller has been consistent. If this cursor was opened - ** expecting an index b-tree, then the caller should be inserting blob - ** keys with no associated data. If the cursor was opened expecting an - ** intkey table, the caller should be inserting integer keys with a - ** blob of associated data. */ - assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); - /* Save the positions of any other cursors open on this table. ** ** In some cases, the call to btreeMoveto() below is a no-op. For @@ -74676,7 +81227,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** not to clear the cursor here. */ if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + rc = saveAllCursors(p->pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; if( loc && pCur->iPage<0 ){ /* This can only happen if the schema is corrupt such that there is more @@ -74684,10 +81235,33 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot, NULL); } } + /* Ensure that the cursor is not in the CURSOR_FAULT state and that it + ** points to a valid cell. + */ + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + testcase( pCur->eState==CURSOR_REQUIRESEEK ); + testcase( pCur->eState==CURSOR_FAULT ); + rc = moveToRoot(pCur); + if( rc && rc!=SQLITE_EMPTY ) return rc; + } + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && p->pBt->inTransaction==TRANS_WRITE + && (p->pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); + if( pCur->pKeyInfo==0 ){ assert( pX->pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob @@ -74776,51 +81350,57 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } assert( pCur->eState==CURSOR_VALID - || (pCur->eState==CURSOR_INVALID && loc) - || CORRUPT_DB ); + || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); assert( pPage->leaf || !pPage->intKey ); if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ - rc = SQLITE_CORRUPT_BKPT; + /* ^^^^^--- due to the moveToRoot() call above */ + rc = SQLITE_CORRUPT_PAGE(NULL, pPage); }else{ rc = btreeComputeFreeSpace(pPage); } if( rc ) return rc; } - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); - assert( pPage->isInit ); - newCell = pBt->pTmpSpace; + assert( pPage->isInit || CORRUPT_DB ); + newCell = p->pBt->pTmpSpace; assert( newCell!=0 ); + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; - szNew = pBt->nPreformatSize; - if( szNew<4 ) szNew = 4; - if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + szNew = p->pBt->nPreformatSize; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } + if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); if( info.nPayload!=info.nLocal ){ Pgno ovfl = get4byte(&newCell[szNew-4]); - ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + ptrmapPut(p->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + if( NEVER(rc) ) goto end_insert; } } }else{ rc = fillInCell(pPage, newCell, pX, &szNew); + if( rc ) goto end_insert; } - if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); - assert( szNew <= MX_CELL_SIZE(pBt) ); + assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; + pCur->info.nSize = 0; if( loc==0 ){ CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -74834,7 +81414,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM || szNewminLocal) + && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add @@ -74847,10 +81427,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -74860,11 +81440,11 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); @@ -74888,10 +81468,9 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ - pCur->info.nSize = 0; if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -74937,7 +81516,6 @@ end_insert: ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ - int rc = SQLITE_OK; BtShared *pBt = pDest->pBt; u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ const u8 *aIn; /* Pointer to next input buffer */ @@ -74945,18 +81523,24 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 u32 nRem; /* Bytes of data still to copy */ getCellInfo(pSrc); - aOut += putVarint32(aOut, pSrc->info.nPayload); + if( pSrc->info.nPayload<0x80 ){ + *(aOut++) = pSrc->info.nPayload; + }else{ + aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); + } if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey); nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + return SQLITE_OK; }else{ + int rc = SQLITE_OK; Pager *pSrcPager = pSrc->pBt->pPager; u8 *pPgnoOut = 0; Pgno ovflIn = 0; @@ -74973,7 +81557,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } @@ -75008,7 +81592,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); - if( ISAUTOVACUUM && pPageOut ){ + if( ISAUTOVACUUM(pBt) && pPageOut ){ ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); } releasePage(pPageOut); @@ -75024,9 +81608,8 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 releasePage(pPageOut); sqlite3PagerUnref(pPageIn); + return rc; } - - return rc; } /* @@ -75049,14 +81632,13 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; - int rc; /* Return code */ - MemPage *pPage; /* Page to delete cell from */ - unsigned char *pCell; /* Pointer to cell to delete */ - int iCellIdx; /* Index of cell to delete */ - int iCellDepth; /* Depth of node containing pCell */ - CellInfo info; /* Size of the cell being deleted */ - int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ - u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + CellInfo info; /* Size of the cell being deleted */ + u8 bPreserve; /* Keep cursor valid. 2 for CURSOR_SKIPNEXT */ assert( cursorOwnsBtShared(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -75065,36 +81647,52 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); - if( pCur->eState==CURSOR_REQUIRESEEK ){ - rc = btreeRestoreCursorPosition(pCur); - assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); - if( rc || pCur->eState!=CURSOR_VALID ) return rc; + if( pCur->eState!=CURSOR_VALID ){ + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); + if( rc || pCur->eState!=CURSOR_VALID ) return rc; + }else{ + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot, NULL); + } } - assert( CORRUPT_DB || pCur->eState==CURSOR_VALID ); + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; pPage = pCur->pPage; + if( pPage->nCell<=iCellIdx ){ + return SQLITE_CORRUPT_PAGE(NULL, pPage); + } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(NULL, pPage); } - if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ + return SQLITE_CORRUPT_PAGE(NULL, pPage); } - /* If the bPreserve flag is set to true, then the cursor position must + /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete ** will cause a b-tree rebalance, then this is done by saving the cursor ** key and leaving the cursor in CURSOR_REQUIRESEEK state before ** returning. ** - ** Or, if the current delete will not cause a rebalance, then the cursor + ** If the current delete will not cause a rebalance, then the cursor ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately - ** before or after the deleted entry. In this case set bSkipnext to true. */ + ** before or after the deleted entry. + ** + ** The bPreserve value records which path is required: + ** + ** bPreserve==0 Not necessary to save the cursor position + ** bPreserve==1 Use CURSOR_REQUIRESEEK to save the cursor position + ** bPreserve==2 Cursor won't move. Set CURSOR_SKIPNEXT. + */ + bPreserve = (flags & BTREE_SAVEPOSITION)!=0; if( bPreserve ){ if( !pPage->leaf - || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + || (pPage->nFree+pPage->xCellSize(pPage,pCell)+2) > + (int)(pBt->usableSize*2/3) || pPage->nCell==1 /* See dbfuzz001.test for a test case */ ){ /* A b-tree rebalance will be required after deleting this entry. @@ -75102,7 +81700,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ rc = saveCursorKey(pCur); if( rc ) return rc; }else{ - bSkipnext = 1; + bPreserve = 2; } } @@ -75162,14 +81760,14 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(NULL, pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); if( rc==SQLITE_OK ){ - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; @@ -75190,7 +81788,15 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** been corrected, so be it. Otherwise, after balancing the leaf node, ** walk the cursor up the tree to the internal node and balance it as ** well. */ - rc = balance(pCur); + assert( pCur->pPage->nOverflow==0 ); + assert( pCur->pPage->nFree>=0 ); + if( pCur->pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){ + /* Optimization: If the free space is less than 2/3rds of the page, + ** then balance() will always be a no-op. No need to invoke it. */ + rc = SQLITE_OK; + }else{ + rc = balance(pCur); + } if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ releasePageNotNull(pCur->pPage); pCur->iPage--; @@ -75202,8 +81808,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ } if( rc==SQLITE_OK ){ - if( bSkipnext ){ - assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); + if( bPreserve>1 ){ + assert( (pCur->iPage==iCellDepth || CORRUPT_DB) ); assert( pPage==pCur->pPage || CORRUPT_DB ); assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); pCur->eState = CURSOR_SKIPNEXT; @@ -75241,7 +81847,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ MemPage *pRoot; Pgno pgnoRoot; int rc; - int ptfFlags; /* Page-type flage for the root page of new table */ + int ptfFlags; /* Page-type flags for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -75270,7 +81876,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot, NULL); } pgnoRoot++; @@ -75318,7 +81924,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot, NULL); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -75408,14 +82014,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno, NULL); } - rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 - && sqlite3PagerPageRefcount(pPage->pDbPage)!=1 + && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(NULL, pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -75519,7 +82125,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable, NULL); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -75672,6 +82278,11 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ assert( iMeta==0 || iMeta==1 ); pBt->incrVacuum = (u8)iMeta; } +#endif +#ifdef SQLITE_META_DWR + if (idx == 1 && pBt->pPager->metaFd) { + pBt->pPager->metaChanged = META_SCHEMA_CHANGED; + } #endif } sqlite3BtreeLeave(p); @@ -75760,6 +82371,41 @@ SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ } #ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Record an OOM error during integrity_check +*/ +static void checkOom(IntegrityCk *pCheck){ + pCheck->rc = SQLITE_NOMEM; + pCheck->mxErr = 0; /* Causes integrity_check processing to stop */ + if( pCheck->nErr==0 ) pCheck->nErr++; +} + +/* +** Invoke the progress handler, if appropriate. Also check for an +** interrupt. +*/ +static void checkProgress(IntegrityCk *pCheck){ + sqlite3 *db = pCheck->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + assert( db->nProgressOps>0 ); + pCheck->nStep++; + if( (pCheck->nStep % db->nProgressOps)==0 + && db->xProgress(db->pProgressArg) + ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } + } +#endif +} + /* ** Append a message to the error message string. */ @@ -75769,6 +82415,7 @@ static void checkAppendMsg( ... ){ va_list ap; + checkProgress(pCheck); if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -75777,12 +82424,13 @@ static void checkAppendMsg( sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, + pCheck->v0, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==SQLITE_NOMEM ){ - pCheck->bOomFault = 1; + checkOom(pCheck); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -75794,7 +82442,8 @@ static void checkAppendMsg( ** corresponds to page iPg is already set. */ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); } @@ -75802,7 +82451,8 @@ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. */ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); } @@ -75816,15 +82466,14 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage>pCheck->nPage || iPage==0 ){ - checkAppendMsg(pCheck, "invalid page number %d", iPage); + if( iPage>pCheck->nCkPage || iPage==0 ){ + checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %u", iPage); return 1; } - if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; setPageReferenced(pCheck, iPage); return 0; } @@ -75847,14 +82496,14 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->bOomFault = 1; - checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -75879,7 +82528,7 @@ static void checkList( if( checkRef(pCheck, iPage) ) break; N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ - checkAppendMsg(pCheck, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %u", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -75892,7 +82541,7 @@ static void checkList( #endif if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, - "freelist leaf count too big on page %d", iPage); + "freelist leaf count too big on page %u", iPage); N--; }else{ for(i=0; i<(int)n; i++){ @@ -75924,7 +82573,7 @@ static void checkList( } if( N && nErrAtStart==pCheck->nErr ){ checkAppendMsg(pCheck, - "%s is %d but should be %d", + "%s is %u but should be %u", isFreeList ? "size" : "overflow list length", expected-N, expected); } @@ -75954,7 +82603,9 @@ static void checkList( ** lower 16 bits are the index of the last byte of that range. */ static void btreeHeapInsert(u32 *aHeap, u32 x){ - u32 j, i = ++aHeap[0]; + u32 j, i; + assert( aHeap!=0 ); + i = ++aHeap[0]; aHeap[i] = x; while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ x = aHeap[j]; @@ -76031,18 +82682,20 @@ static int checkTreePage( /* Check that the page exists */ + checkProgress(pCheck); + if( pCheck->mxErr==0 ) goto end_of_check; pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; - pCheck->zPfx = "Page %u: "; + pCheck->zPfx = "Tree %u page %u: "; pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); + if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; goto end_of_check; } - /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ savedIsInit = pPage->isInit; @@ -76062,7 +82715,7 @@ static int checkTreePage( hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %u cell %d: "; + pCheck->zPfx = "Tree %u page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -76070,6 +82723,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -76082,7 +82738,7 @@ static int checkTreePage( pgno = get4byte(&data[hdr+8]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %u at right child: "; + pCheck->zPfx = "Tree %u page %u right child: "; checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif @@ -76106,7 +82762,7 @@ static int checkTreePage( pc = get2byteAligned(pCellIdx); pCellIdx -= 2; if( pcusableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + checkAppendMsg(pCheck, "Offset %u out of range %u..%u", pc, contentOffset, usableSize-4); doCoverageCheck = 0; continue; @@ -76181,6 +82837,7 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header @@ -76238,7 +82895,7 @@ static int checkTreePage( */ if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, - "Fragmentation of %d bytes reported as %d on page %u", + "Fragmentation of %u bytes reported as %u on page %u", nFrag, data[hdr+7], iPage); } } @@ -76276,13 +82933,15 @@ end_of_check: ** the unverified btrees. Except, if aRoot[1] is 1, then the freelist ** checks are still performed. */ -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ - int *pnErr /* Write number of errors seen to this variable */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ ){ Pgno i; IntegrityCk sCheck; @@ -76292,7 +82951,9 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -76305,42 +82966,36 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); + memset(&sCheck, 0, sizeof(sCheck)); sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; - sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.nCkPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; - sCheck.nErr = 0; - sCheck.bOomFault = 0; - sCheck.zPfx = 0; - sCheck.v1 = 0; - sCheck.v2 = 0; - sCheck.aPgRef = 0; - sCheck.heap = 0; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; - if( sCheck.nPage==0 ){ + if( sCheck.nCkPage==0 ){ goto integrity_ck_cleanup; } - sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + sCheck.aPgRef = sqlite3MallocZero((sCheck.nCkPage / 8)+ 1); if( !sCheck.aPgRef ){ - sCheck.bOomFault = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); if( sCheck.heap==0 ){ - sCheck.bOomFault = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + if( i<=sCheck.nCkPage ) setPageReferenced(&sCheck, i); /* Check the integrity of the freelist */ if( bCkFreelist ){ - sCheck.zPfx = "Main freelist: "; + sCheck.zPfx = "Freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36])); sCheck.zPfx = 0; @@ -76357,7 +83012,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( mxInHdr = get4byte(&pBt->pPage1->aData[52]); if( mx!=mxInHdr ){ checkAppendMsg(&sCheck, - "max rootpage (%d) disagrees with header (%d)", + "max rootpage (%u) disagrees with header (%u)", mx, mxInHdr ); } @@ -76371,24 +83026,28 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ if( !bPartial ){ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ + for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -76396,11 +83055,11 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); } #endif } @@ -76411,16 +83070,17 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( integrity_ck_cleanup: sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); - if( sCheck.bOomFault ){ + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ){ sqlite3_str_reset(&sCheck.errMsg); - sCheck.nErr++; + *pzOut = 0; + }else{ + *pzOut = sqlite3StrAccumFinish(&sCheck.errMsg); } - *pnErr = sCheck.nErr; - if( sCheck.nErr==0 ) sqlite3_str_reset(&sCheck.errMsg); /* Make sure this analysis did not leave any unref() pages. */ assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); sqlite3BtreeLeave(p); - return sqlite3StrAccumFinish(&sCheck.errMsg); + return sCheck.rc; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -76472,8 +83132,10 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int * if( p ){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); + ResetLockStatus(); if( pBt->inTransaction!=TRANS_NONE ){ rc = SQLITE_LOCKED; + MARK_LAST_BUSY_LINE(rc); }else{ rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); } @@ -76685,6 +83347,17 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ */ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** If no transaction is active and the database is not a temp-db, clear +** the in-memory pager cache. +*/ +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->inTransaction==TRANS_NONE ){ + sqlite3PagerClearCache(pBt->pPager); + } +} + #if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** Return true if the Btree passed as the only argument is sharable. @@ -76793,14 +83466,13 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ if( i==1 ){ Parse sParse; int rc = 0; - memset(&sParse, 0, sizeof(sParse)); - sParse.db = pDb; + sqlite3ParseObjectInit(&sParse,pDb); if( sqlite3OpenTempDatabase(&sParse) ){ sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); rc = SQLITE_ERROR; } sqlite3DbFree(pErrorDb, sParse.zErrMsg); - sqlite3ParserReset(&sParse); + sqlite3ParseObjectReset(&sParse); if( rc ){ return 0; } @@ -76949,7 +83621,7 @@ static int backupOnePage( ** p->pSrc may not actually be the owner. */ int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); int nDestReserve = sqlite3BtreeGetRequestedReserve(p->pDest); -#endif +#endif /* SQLITE_HAS_CODEC */ int rc = SQLITE_OK; i64 iOff; @@ -76958,13 +83630,8 @@ static int backupOnePage( assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); assert( zSrcData ); + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } #ifdef SQLITE_HAS_CODEC /* Backup is not possible if the page size of the destination is changing ** and a codec is in use. @@ -76983,7 +83650,8 @@ static int backupOnePage( rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); if( rc==SQLITE_OK && newPgsz!=(u32)nSrcPgsz ) rc = SQLITE_READONLY; } -#endif +#endif /* SQLITE_HAS_CODEC */ + /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. @@ -77115,7 +83783,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); pgszDest = sqlite3BtreeGetPageSize(p->pDest); destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + if( SQLITE_OK==rc + && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager)) + && pgszSrc!=pgszDest + ){ rc = SQLITE_READONLY; } @@ -77235,6 +83906,12 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } } } + #ifdef SQLITE_META_DWR + if (rc == SQLITE_OK && p->pDest->pBt->pPager->metaFd) { + p->pDest->pBt->pPager->metaChanged = META_SCHEMA_CHANGED; + (void)MetaDwrUpdateMetaPages(p->pDest); + } + #endif if( rc==SQLITE_OK ){ rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); } @@ -77265,6 +83942,12 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } }else{ sqlite3PagerTruncateImage(pDestPager, nDestTruncate); + #ifdef SQLITE_META_DWR + if (rc == SQLITE_OK && p->pDest->pBt->pPager->metaFd) { + p->pDest->pBt->pPager->metaChanged = META_SCHEMA_CHANGED; + (void)MetaDwrUpdateMetaPages(p->pDest); + } + #endif rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } @@ -77481,7 +84164,7 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ #ifdef SQLITE_HAS_CODEC sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); -#endif +#endif /* SQLITE_HAS_CODEC */ /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to @@ -77625,9 +84308,9 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ i64 x; assert( (p->flags&MEM_Int)*2==sizeof(x) ); memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); - sqlite3Int64ToText(x, zBuf); + p->n = sqlite3Int64ToText(x, zBuf); #else - sqlite3Int64ToText(p->u.i, zBuf); + p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); @@ -77635,6 +84318,7 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ + p->n = acc.nChar; } } @@ -77662,10 +84346,12 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ ** This routine is for use inside of assert() statements only. */ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ + Mem tmp; char zBuf[100]; char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; if( p->flags & MEM_Term ){ /* Insure that the string is properly zero-terminated. Pay particular ** attention to the case where p->n is odd */ @@ -77678,7 +84364,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; - vdbeMemRenderNum(sizeof(zBuf), zBuf, p); + memcpy(&tmp, p, sizeof(tmp)); + vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; i = j = 0; incr = 1; @@ -77715,7 +84402,11 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); - if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + if( !(pMem->flags&MEM_Str) ){ + pMem->enc = desiredEnc; + return SQLITE_OK; + } + if( pMem->enc==desiredEnc ){ return SQLITE_OK; } assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -77817,6 +84508,40 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ return SQLITE_OK; } +/* +** If pMem is already a string, detect if it is a zero-terminated +** string, or make it into one if possible, and mark it as such. +** +** This is an optimization. Correct operation continues even if +** this routine is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ + if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ + /* pMem must be a string, and it cannot be an ephemeral or static string */ + return; + } + if( pMem->enc!=SQLITE_UTF8 ) return; + if( NEVER(pMem->z==0) ) return; + if( pMem->flags & MEM_Dyn ){ + if( pMem->xDel==sqlite3_free + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) + ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } + if( pMem->xDel==sqlite3RCStrUnref ){ + /* Blindly assume that all RCStr objects are zero-terminated */ + pMem->flags |= MEM_Term; + return; + } + }else if( pMem->szMalloc >= pMem->n+1 ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } +} + /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. @@ -77943,7 +84668,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30NN(pMem->z); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); @@ -77964,9 +84689,10 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ Mem t; assert( pFunc!=0 ); assert( pMem!=0 ); + assert( pMem->db!=0 ); assert( pFunc->xFinalize!=0 ); assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( sqlite3_mutex_held(pMem->db->mutex) ); memset(&ctx, 0, sizeof(ctx)); memset(&t, 0, sizeof(t)); t.flags = MEM_Null; @@ -77974,6 +84700,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ctx.pOut = &t; ctx.pMem = pMem; ctx.pFunc = pFunc; + ctx.enc = ENC(t.db); pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( (pMem->flags & MEM_Dyn)==0 ); if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); @@ -77995,12 +84722,14 @@ SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc assert( pFunc!=0 ); assert( pFunc->xValue!=0 ); assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); - assert( pAccum->db==0 || sqlite3_mutex_held(pAccum->db->mutex) ); + assert( pAccum->db!=0 ); + assert( sqlite3_mutex_held(pAccum->db->mutex) ); memset(&ctx, 0, sizeof(ctx)); sqlite3VdbeMemSetNull(pOut); ctx.pOut = pOut; ctx.pMem = pAccum; ctx.pFunc = pFunc; + ctx.enc = ENC(pAccum->db); pFunc->xValue(&ctx); return ctx.isError; } @@ -78066,34 +84795,12 @@ SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ } } -/* -** Convert a 64-bit IEEE double into a 64-bit signed integer. -** If the double is out of range of a 64-bit signed integer then -** return the closest available 64-bit signed integer. +/* Like sqlite3VdbeMemRelease() but faster for cases where we +** know in advance that the Mem is not MEM_Dyn or MEM_Agg. */ -static SQLITE_NOINLINE i64 doubleToInt64(double r){ -#ifdef SQLITE_OMIT_FLOATING_POINT - /* When floating-point is omitted, double and int64 are the same thing */ - return r; -#else - /* - ** Many compilers we encounter do not define constants for the - ** minimum and maximum 64-bit integers, or they define them - ** inconsistently. And many do not understand the "LL" notation. - ** So we define our own static constants here using nothing - ** larger than a 32-bit integer constant. - */ - static const i64 maxInt = LARGEST_INT64; - static const i64 minInt = SMALLEST_INT64; - - if( r<=(double)minInt ){ - return minInt; - }else if( r>=(double)maxInt ){ - return maxInt; - }else{ - return (i64)r; - } -#endif +SQLITE_PRIVATE void sqlite3VdbeMemReleaseMalloc(Mem *p){ + assert( !VdbeMemDynamic(p) ); + if( p->szMalloc ) vdbeMemClear(p); } /* @@ -78107,12 +84814,12 @@ static SQLITE_NOINLINE i64 doubleToInt64(double r){ ** ** If pMem represents a string value, its encoding might be changed. */ -static SQLITE_NOINLINE i64 memIntValue(Mem *pMem){ +static SQLITE_NOINLINE i64 memIntValue(const Mem *pMem){ i64 value = 0; sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; } -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem *pMem){ int flags; assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -78122,7 +84829,7 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->u.r); + return sqlite3RealToI64(pMem->u.r); }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ return memIntValue(pMem); }else{ @@ -78171,32 +84878,35 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ } /* -** The MEM structure is already a MEM_Real. Try to also make it a -** MEM_Int if we can. +** The MEM structure is already a MEM_Real or MEM_IntReal. Try to +** make it a MEM_Int if we can. */ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ - i64 ix; assert( pMem!=0 ); - assert( pMem->flags & MEM_Real ); + assert( pMem->flags & (MEM_Real|MEM_IntReal) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - ix = doubleToInt64(pMem->u.r); - - /* Only mark the value as an integer if - ** - ** (1) the round-trip conversion real->int->real is a no-op, and - ** (2) The integer is neither the largest nor the smallest - ** possible integer (ticket #3922) - ** - ** The second and third terms in the following conditional enforces - ** the second condition under the assumption that addition overflow causes - ** values to wrap around. - */ - if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + if( pMem->flags & MEM_IntReal ){ MemSetTypeFlag(pMem, MEM_Int); + }else{ + i64 ix = sqlite3RealToI64(pMem->u.r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + } } } @@ -78244,6 +84954,16 @@ SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ && i >= -2251799813685248LL && i < 2251799813685248LL); } +/* Convert a floating point value to its closest integer. Do so in +** a way that avoids 'outside the range of representable values' warnings +** from UBSAN. +*/ +SQLITE_PRIVATE i64 sqlite3RealToI64(double r){ + if( r<-9223372036854774784.0 ) return SMALLEST_INT64; + if( r>+9223372036854774784.0 ) return LARGEST_INT64; + return (i64)r; +} + /* ** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. @@ -78265,7 +84985,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) - || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); @@ -78311,13 +85031,17 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ break; } default: { + int rc; assert( aff==SQLITE_AFF_TEXT ); assert( MEM_Str==(MEM_Blob>>3) ); pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); - return sqlite3VdbeChangeEncoding(pMem, encoding); + if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; + rc = sqlite3VdbeChangeEncoding(pMem, encoding); + if( rc ) return rc; + sqlite3VdbeMemZeroTerminateIfAble(pMem); } } return SQLITE_OK; @@ -78413,6 +85137,13 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + /* A no-op destructor */ SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } @@ -78427,6 +85158,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetPointer( void (*xDestructor)(void*) ){ assert( pMem->flags==MEM_Null ); + vdbeMemClear(pMem); pMem->u.zPType = zPType ? zPType : ""; pMem->z = pPtr; pMem->flags = MEM_Null|MEM_Dyn|MEM_Subtype|MEM_Term; @@ -78609,6 +85341,13 @@ SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ ** stored without allocating memory, then it is. If a memory allocation ** is required to store the string, then value of pMem is unchanged. In ** either case, SQLITE_TOOBIG is returned. +** +** The "enc" parameter is the text encoding for the string, or zero +** to store a blob. +** +** If n is negative, then the string consists of all bytes up to but +** excluding the first zero character. The n parameter must be +** non-negative for blobs. */ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( Mem *pMem, /* Memory cell to set to string value */ @@ -78619,11 +85358,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ){ i64 nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ - u16 flags = 0; /* New value for pMem->flags */ + u16 flags; /* New value for pMem->flags */ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( enc!=0 || n>=0 ); /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ if( !z ){ @@ -78636,7 +85376,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( }else{ iLimit = SQLITE_MAX_LENGTH; } - flags = (enc==0?MEM_Blob:MEM_Str); if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ @@ -78644,7 +85383,23 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } - flags |= MEM_Term; + flags= MEM_Str|MEM_Term; + }else if( enc==0 ){ + flags = MEM_Blob; + enc = SQLITE_UTF8; + }else{ + flags = MEM_Str; + } + if( nByte>iLimit ){ + if( xDel && xDel!=SQLITE_TRANSIENT ){ + if( xDel==SQLITE_DYNAMIC ){ + sqlite3DbFree(pMem->db, (void*)z); + }else{ + xDel((void*)z); + } + } + sqlite3VdbeMemSetNull(pMem); + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } /* The following block sets the new values of Mem.z and Mem.xDel. It @@ -78656,9 +85411,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } - if( nByte>iLimit ){ - return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); - } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); @@ -78680,16 +85432,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( pMem->n = (int)(nByte & 0x7fffffff); pMem->flags = flags; - if( enc ){ - pMem->enc = enc; -#ifdef SQLITE_ENABLE_SESSION - }else if( pMem->db==0 ){ - pMem->enc = SQLITE_UTF8; -#endif - }else{ - assert( pMem->db!=0 ); - pMem->enc = ENC(pMem->db); - } + pMem->enc = enc; #ifndef SQLITE_OMIT_UTF16 if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ @@ -78697,9 +85440,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( } #endif - if( nByte>iLimit ){ - return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); - } return SQLITE_OK; } @@ -78832,6 +85572,24 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ @@ -78899,6 +85657,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ } pRec->nField = p->iVal+1; + sqlite3VdbeMemSetNull(&pRec->aMem[p->iVal]); return &pRec->aMem[p->iVal]; } #else @@ -78952,9 +85711,12 @@ static int valueFromFunction( if( pList ) nVal = pList->nExpr; assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 - || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + || (pFunc->funcFlags & (SQLITE_FUNC_NEEDCOLL|SQLITE_FUNC_RUNONLY))!=0 ){ return SQLITE_OK; } @@ -78977,10 +85739,10 @@ static int valueFromFunction( goto value_from_function_out; } - assert( pCtx->pParse->rc==SQLITE_OK ); memset(&ctx, 0, sizeof(ctx)); ctx.pOut = pVal; ctx.pFunc = pFunc; + ctx.enc = ENC(db); pFunc->xSFunc(&ctx, nVal, apVal); if( ctx.isError ){ rc = ctx.isError; @@ -78989,16 +85751,16 @@ static int valueFromFunction( sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } } - pCtx->pParse->rc = rc; value_from_function_out: if( rc!=SQLITE_OK ){ pVal = 0; + pCtx->pParse->rc = rc; } if( apVal ){ for(i=0; iop)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; -#if defined(SQLITE_ENABLE_STAT4) if( op==TK_REGISTER ) op = pExpr->op2; -#else - if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; -#endif /* Compressed expressions only appear when parsing the DEFAULT clause ** on a table column definition, and hence only when pCtx==0. This @@ -79060,21 +85818,34 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ - sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); - sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); +#ifdef SQLITE_ENABLE_STAT4 + rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif + sqlite3VdbeMemCast(*ppVal, aff, enc); + sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } return rc; } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -79083,12 +85854,26 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } @@ -79152,6 +85937,7 @@ static int valueFromExpr( if( pVal ){ pVal->flags = MEM_Int; pVal->u.i = pExpr->u.zToken[4]==0; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } } @@ -79160,7 +85946,7 @@ static int valueFromExpr( no_mem: #ifdef SQLITE_ENABLE_STAT4 - if( pCtx==0 || pCtx->pParse->nErr==0 ) + if( pCtx==0 || NEVER(pCtx->pParse->nErr==0) ) #endif sqlite3OomFault(db); sqlite3DbFree(db, zVal); @@ -79357,17 +86143,17 @@ SQLITE_PRIVATE int sqlite3Stat4Column( sqlite3_value **ppVal /* OUT: Extracted value */ ){ u32 t = 0; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField = 0; /* Size of the current data field */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); @@ -79445,6 +86231,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ return p->n; } + if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){ + return p->n; + } if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; @@ -79490,12 +86279,12 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ - db->pVdbe->pPrev = p; + db->pVdbe->ppVPrev = &p->pVNext; } - p->pNext = db->pVdbe; - p->pPrev = 0; + p->pVNext = db->pVdbe; + p->ppVPrev = &db->pVdbe; db->pVdbe = p; - p->iVdbeMagic = VDBE_MAGIC_INIT; + assert( p->eVdbeState==VDBE_INIT_STATE ); p->pParse = pParse; pParse->pVdbe = p; assert( pParse->aLabel==0 ); @@ -79575,21 +86364,28 @@ SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( #endif /* -** Swap all content between two VDBE structures. +** Swap byte-code between two VDBE structures. +** +** This happens after pB was previously run and returned +** SQLITE_SCHEMA. The statement was then reprepared in pA. +** This routine transfers the new bytecode in pA over to pB +** so that pB can be run again. The old pB byte code is +** moved back to pA so that it will be cleaned up when pA is +** finalized. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ - Vdbe tmp, *pTmp; + Vdbe tmp, *pTmp, **ppTmp; char *zTmp; assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; - pTmp = pA->pNext; - pA->pNext = pB->pNext; - pB->pNext = pTmp; - pTmp = pA->pPrev; - pA->pPrev = pB->pPrev; - pB->pPrev = pTmp; + pTmp = pA->pVNext; + pA->pVNext = pB->pVNext; + pB->pVNext = pTmp; + ppTmp = pA->ppVPrev; + pA->ppVPrev = pB->ppVPrev; + pB->ppVPrev = ppTmp; zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; @@ -79640,7 +86436,7 @@ static int growOpArray(Vdbe *v, int nOp){ return SQLITE_NOMEM; } - assert( nOp<=(1024/sizeof(Op)) ); + assert( nOp<=(int)(1024/sizeof(Op)) ); assert( nNew>=(v->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ @@ -79659,16 +86455,48 @@ static int growOpArray(Vdbe *v, int nOp){ ** ** Other useful labels for breakpoints include: ** test_trace_breakpoint(pc,pOp) -** sqlite3CorruptError(lineno) +** sqlite3CorruptError(lineno,context) ** sqlite3MisuseError(lineno) ** sqlite3CantopenError(lineno) */ static void test_addop_breakpoint(int pc, Op *pOp){ - static int n = 0; + static u64 n = 0; + (void)pc; + (void)pOp; n++; + if( n==LARGEST_UINT64 ) abort(); /* so that n is used, preventing a warning */ } #endif +/* +** Slow paths for sqlite3VdbeAddOp3() and sqlite3VdbeAddOp4Int() for the +** unusual case when we need to increase the size of the Vdbe.aOp[] array +** before adding the new opcode. +*/ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} +static SQLITE_NOINLINE int addOp4IntSlow( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + if( p->db->mallocFailed==0 ){ + VdbeOp *pOp = &p->aOp[addr]; + pOp->p4type = P4_INT32; + pOp->p4.i = p4; + } + return addr; +} + + /* ** Add a new instruction to the list of instructions current in the ** VDBE. Return the address of the new instruction. @@ -79679,24 +86507,23 @@ static void test_addop_breakpoint(int pc, Op *pOp){ ** ** op The opcode for this instruction ** -** p1, p2, p3 Operands -** -** Use the sqlite3VdbeResolveLabel() function to fix an address and -** the sqlite3VdbeChangeP4() function to change the value of the P4 -** operand. +** p1, p2, p3, p4 Operands */ -static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->nOpAlloc<=p->nOp ); - if( growOpArray(p, 1) ) return 1; - assert( p->nOpAlloc>p->nOp ); - return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ + return sqlite3VdbeAddOp3(p, op, 0, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ + return sqlite3VdbeAddOp3(p, op, p1, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ + return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ int i; VdbeOp *pOp; i = p->nOp; - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( op>=0 && op<0xff ); if( p->nOpAlloc<=i ){ return growOp3(p, op, p1, p2, p3); @@ -79712,32 +86539,78 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; + + /* Replicate this logic in sqlite3VdbeAddOp4Int() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; #endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i, &p->aOp[i]); test_addop_breakpoint(i, &p->aOp[i]); } #endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif #ifdef SQLITE_VDBE_COVERAGE pOp->iSrcLine = 0; #endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp4Int() */ + return i; } -SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ - return sqlite3VdbeAddOp3(p, op, 0, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ - return sqlite3VdbeAddOp3(p, op, p1, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ - return sqlite3VdbeAddOp3(p, op, p1, p2, 0); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int i; + VdbeOp *pOp; + + i = p->nOp; + if( p->nOpAlloc<=i ){ + return addOp4IntSlow(p, op, p1, p2, p3, p4); + } + p->nOp++; + pOp = &p->aOp[i]; + assert( pOp!=0 ); + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.i = p4; + pOp->p4type = P4_INT32; + + /* Replicate this logic in sqlite3VdbeAddOp3() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(i, &p->aOp[i]); + } +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp3() */ + + return i; } /* Generate code for an unconditional jump to instruction iDest @@ -79841,6 +86714,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, p1, p2, p3, (char*)pCtx, P4_FUNCCTX); sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + sqlite3MayAbort(pParse); return addr; } @@ -79891,11 +86765,12 @@ SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ ** If the bPush flag is true, then make this opcode the parent for ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ -SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ -#ifndef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ + int addr = 0; +#if !defined(SQLITE_DEBUG) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ - if( pParse->explain==2 ) + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { char *zMsg; @@ -79907,13 +86782,15 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt va_end(ap); v = pParse->pVdbe; iThis = v->nOp; - sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + addr = sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); if( bPush){ pParse->addrExplain = iThis; } + sqlite3VdbeScanStatus(v, iThis, -1, -1, 0, 0); } + return addr; } /* @@ -79941,26 +86818,6 @@ SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, sqlite3MayAbort(p->pParse); } -/* -** Add an opcode that includes the p4 value as an integer. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - int p4 /* The P4 operand as an integer */ -){ - int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); - if( p->db->mallocFailed==0 ){ - VdbeOp *pOp = &p->aOp[addr]; - pOp->p4type = P4_INT32; - pOp->p4.i = p4; - } - return addr; -} - /* Insert the end of a co-routine */ SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ @@ -80021,6 +86878,9 @@ static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ int i; for(i=p->nLabelAlloc; iaLabel[i] = -1; #endif + if( nNewSize>=100 && (nNewSize/100)>(p->nLabelAlloc/100) ){ + sqlite3ProgressCheck(p); + } p->nLabelAlloc = nNewSize; p->aLabel[j] = v->nOp; } @@ -80028,7 +86888,7 @@ static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; int j = ADDR(x); - assert( v->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( v->eVdbeState==VDBE_INIT_STATE ); assert( j<-p->nLabel ); assert( j>=0 ); #ifdef SQLITE_DEBUG @@ -80048,14 +86908,20 @@ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ ** Mark the VDBE as one that can only be run one time. */ SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ - p->runOnlyOnce = 1; + sqlite3VdbeAddOp2(p, OP_Expire, 1, 1); } /* -** Mark the VDBE as one that can only be run multiple times. +** Mark the VDBE as one that can be run multiple times. */ SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ - p->runOnlyOnce = 0; + int i; + for(i=1; ALWAYS(inOp); i++){ + if( ALWAYS(p->aOp[i].opcode==OP_Expire) ){ + p->aOp[1].opcode = OP_Noop; + break; + } + } } #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ @@ -80159,6 +87025,8 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; + + if( v==0 ) return 0; memset(&sIter, 0, sizeof(sIter)); sIter.v = v; @@ -80168,6 +87036,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || opcode==OP_VDestroy || opcode==OP_VCreate || opcode==OP_ParseSchema + || opcode==OP_Function || opcode==OP_PureFunc || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ @@ -80242,7 +87111,7 @@ SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe *p){ ** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately ** indicate what the prepared statement actually does. ** -** (4) Initialize the p4.xAdvance pointer on opcodes that use it. +** (4) (discontinued) ** ** (5) Reclaim the memory allocated for storing labels. ** @@ -80255,11 +87124,13 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; - while(1){ - + assert( p->aOp[0].opcode==OP_Init ); + while( 1 /* Loop terminates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -80288,24 +87159,9 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 1; break; } - case OP_Next: - case OP_SorterNext: { - pOp->p4.xAdvance = sqlite3BtreeNext; - pOp->p4type = P4_ADVANCE; - /* The code generator never codes any of these opcodes as a jump - ** to a label. They are always coded as a jump backwards to a - ** known address */ + case OP_Init: { assert( pOp->p2>=0 ); - break; - } - case OP_Prev: { - pOp->p4.xAdvance = sqlite3BtreePrevious; - pOp->p4type = P4_ADVANCE; - /* The code generator never codes any of these opcodes as a jump - ** to a label. They are always coded as a jump backwards to a - ** known address */ - assert( pOp->p2>=0 ); - break; + goto resolve_p2_values_loop_exit; } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { @@ -80329,8 +87185,18 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } @@ -80339,21 +87205,112 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } - if( pOp==p->aOp ) break; + assert( pOp>p->aOp ); pOp--; } - sqlite3DbFree(p->db, pParse->aLabel); - pParse->aLabel = 0; +resolve_p2_values_loop_exit: + if( aLabel ){ + sqlite3DbNNFreeNN(p->db, pParse->aLabel); + pParse->aLabel = 0; + } pParse->nLabel = 0; *pMaxFuncArgs = nMaxArgs; assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } +#ifdef SQLITE_DEBUG +/* +** Check to see if a subroutine contains a jump to a location outside of +** the subroutine. If a jump outside the subroutine is detected, add code +** that will cause the program to halt with an error message. +** +** The subroutine consists of opcodes between iFirst and iLast. Jumps to +** locations within the subroutine are acceptable. iRetReg is a register +** that contains the return address. Jumps to outside the range of iFirst +** through iLast are also acceptable as long as the jump destination is +** an OP_Return to iReturnAddr. +** +** A jump to an unresolved label means that the jump destination will be +** beyond the current address. That is normally a jump to an early +** termination and is consider acceptable. +** +** This routine only runs during debug builds. The purpose is (of course) +** to detect invalid escapes out of a subroutine. The OP_Halt opcode +** is generated rather than an assert() or other error, so that ".eqp full" +** will still work to show the original bytecode, to aid in debugging. +*/ +SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn( + Vdbe *v, /* The byte-code program under construction */ + int iFirst, /* First opcode of the subroutine */ + int iLast, /* Last opcode of the subroutine */ + int iRetReg /* Subroutine return address register */ +){ + VdbeOp *pOp; + Parse *pParse; + int i; + sqlite3_str *pErr = 0; + assert( v!=0 ); + pParse = v->pParse; + assert( pParse!=0 ); + if( pParse->nErr ) return; + assert( iLast>=iFirst ); + assert( iLastnOp ); + pOp = &v->aOp[iFirst]; + for(i=iFirst; i<=iLast; i++, pOp++){ + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ){ + int iDest = pOp->p2; /* Jump destination */ + if( iDest==0 ) continue; + if( pOp->opcode==OP_Gosub ) continue; + if( pOp->p3==20230325 && pOp->opcode==OP_NotNull ){ + /* This is a deliberately taken illegal branch. tag-20230325-2 */ + continue; + } + if( iDest<0 ){ + int j = ADDR(iDest); + assert( j>=0 ); + if( j>=-pParse->nLabel || pParse->aLabel[j]<0 ){ + continue; + } + iDest = pParse->aLabel[j]; + } + if( iDestiLast ){ + int j = iDest; + for(; jnOp; j++){ + VdbeOp *pX = &v->aOp[j]; + if( pX->opcode==OP_Return ){ + if( pX->p1==iRetReg ) break; + continue; + } + if( pX->opcode==OP_Noop ) continue; + if( pX->opcode==OP_Explain ) continue; + if( pErr==0 ){ + pErr = sqlite3_str_new(0); + }else{ + sqlite3_str_appendchar(pErr, 1, '\n'); + } + sqlite3_str_appendf(pErr, + "Opcode at %d jumps to %d which is outside the " + "subroutine at %d..%d", + i, iDest, iFirst, iLast); + break; + } + } + } + } + if( pErr ){ + char *zErr = sqlite3_str_finish(pErr); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_INTERNAL, OE_Abort, 0, zErr, 0); + sqlite3_free(zErr); + sqlite3MayAbort(pParse); + } +} +#endif /* SQLITE_DEBUG */ + /* ** Return the address of the next instruction to be inserted. */ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); return p->nOp; } @@ -80438,7 +87395,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( int i; VdbeOp *pOut, *pFirst; assert( nOp>0 ); - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); if( p->nOp + nOp > p->nOpAlloc && growOpArray(p, nOp) ){ return 0; } @@ -80485,20 +87442,83 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } } } -#endif + +/* +** Add the range of instructions from addrStart to addrEnd (inclusive) to +** the set of those corresponding to the sqlite3_stmt_scanstatus() counters +** associated with the OP_Explain instruction at addrExplain. The +** sum of the sqlite3Hwtime() values for each of these instructions +** will be returned for SQLITE_SCANSTAT_NCYCLE requests. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange( + Vdbe *p, + int addrExplain, + int addrStart, + int addrEnd +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } + } + } + } +} + +/* +** Set the addresses for the SQLITE_SCANSTAT_NLOOP and SQLITE_SCANSTAT_NROW +** counters for the query element associated with the OP_Explain at +** addrExplain. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters( + Vdbe *p, + int addrExplain, + int addrLoop, + int addrVisit +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrLoop>0 ) pScan->addrLoop = addrLoop; + if( addrVisit>0 ) pScan->addrVisit = addrVisit; + } + } +} +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* @@ -80506,15 +87526,19 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( ** for a specific instruction. */ SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p1 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( addr>=0 || p->db->mallocFailed ); sqlite3VdbeGetOp(p,addr)->p2 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p3 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -80522,6 +87546,18 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; } +/* +** If the previous opcode is an OP_Column that delivers results +** into register iDest, then add the OPFLAG_TYPEOFARG flag to that +** opcode. +*/ +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ + VdbeOp *pOp = sqlite3VdbeGetLastOp(p); + if( pOp->p3==iDest && pOp->opcode==OP_Column ){ + pOp->p5 |= OPFLAG_TYPEOFARG; + } +} + /* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. @@ -80550,7 +87586,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ || p->aOp[addr].opcode==OP_FkIfZero ); assert( p->aOp[addr].p4type==0 ); #ifdef SQLITE_VDBE_COVERAGE - sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ + sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ #endif p->nOp--; }else{ @@ -80561,11 +87597,12 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ /* ** If the input FuncDef structure is ephemeral, then free it. If -** the FuncDef is not ephermal, then do nothing. +** the FuncDef is not ephemeral, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + assert( db!=0 ); if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFreeNN(db, pDef); + sqlite3DbNNFreeNN(db, pDef); } } @@ -80574,11 +87611,12 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ */ static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + assert( db!=0 ); freeEphemeralFunction(db, p->pFunc); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); @@ -80590,9 +87628,8 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ case P4_REAL: case P4_INT64: case P4_DYNAMIC: - case P4_DYNBLOB: case P4_INTARRAY: { - sqlite3DbFree(db, p4); + if( p4 ) sqlite3DbNNFreeNN(db, p4); break; } case P4_KEYINFO: { @@ -80621,6 +87658,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } + case P4_TABLEREF: { + if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); + break; + } } } @@ -80630,15 +87671,19 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ ** nOp entries. */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ + assert( nOp>=0 ); + assert( db!=0 ); if( aOp ){ - Op *pOp; - for(pOp=&aOp[nOp-1]; pOp>=aOp; pOp--){ + Op *pOp = &aOp[nOp-1]; + while(1){ /* Exit via break */ if( pOp->p4type <= P4_FREE_IF_LE ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif + if( pOp==aOp ) break; + pOp--; } - sqlite3DbFreeNN(db, aOp); + sqlite3DbNNFreeNN(db, aOp); } } @@ -80698,7 +87743,7 @@ SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( u32 mask, /* Mask of registers to NOT release */ int bUndefine /* If true, mark registers as undefined */ ){ - if( N==0 ) return; + if( N==0 || OptimizationDisabled(pParse->db, SQLITE_ReleaseReg) ) return; assert( pParse->pVdbe ); assert( iFirst>=1 ); assert( iFirst+N-1<=pParse->nMem ); @@ -80720,7 +87765,6 @@ SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( } #endif /* SQLITE_DEBUG */ - /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -80745,7 +87789,7 @@ static void SQLITE_NOINLINE vdbeChangeP4Full( int n ){ if( pOp->p4type ){ - freeP4(p->db, pOp->p4type, pOp->p4.p); + assert( pOp->p4type > P4_FREE_IF_LE ); pOp->p4type = 0; pOp->p4.p = 0; } @@ -80762,7 +87806,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int sqlite3 *db; assert( p!=0 ); db = p->db; - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( p->aOp!=0 || db->mallocFailed ); if( db->mallocFailed ){ if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); @@ -80807,7 +87851,7 @@ SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){ if( p->db->mallocFailed ){ freeP4(p->db, n, pP4); }else{ - assert( pP4!=0 ); + assert( pP4!=0 || n==P4_DYNAMIC ); assert( p->nOp>0 ); pOp = &p->aOp[p->nOp-1]; assert( pOp->p4type==P4_NOTUSED ); @@ -80838,8 +87882,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ */ static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed - || p->pParse->nErr>0 ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->pParse->nErr>0 ); if( p->nOp ){ assert( p->aOp ); sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); @@ -80870,13 +87913,13 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** Set the value if the iSrcLine field for the previously coded instruction. */ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ - sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; + sqlite3VdbeGetLastOp(v)->iSrcLine = iLine; } #endif /* SQLITE_VDBE_COVERAGE */ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** Return the opcode for a given address. The address must be non-negative. +** See sqlite3VdbeGetLastOp() to get the most recently added opcode. ** ** If a memory allocation error has occurred prior to the calling of this ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode @@ -80891,10 +87934,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* C89 specifies that the constant "dummy" will be initialized to all ** zeros, which is correct. MSVC generates a warning, nevertheless. */ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); - if( addr<0 ){ - addr = p->nOp - 1; - } + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); if( p->db->mallocFailed ){ return (VdbeOp*)&dummy; @@ -80903,6 +87943,12 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ } } +/* Return the most recently added opcode +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ + return sqlite3VdbeGetOp(p, p->nOp - 1); +} + #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) /* ** Return an integer value for one of the parameters to the opcode pOp @@ -80958,8 +88004,11 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( if( c=='4' ){ sqlite3_str_appendall(&x, zP4); }else if( c=='X' ){ - sqlite3_str_appendall(&x, pOp->zComment); - seenCom = 1; + if( pOp->zComment && pOp->zComment[0] ){ + sqlite3_str_appendall(&x, pOp->zComment); + seenCom = 1; + break; + } }else{ int v1 = translateP(c, pOp); int v2; @@ -81188,10 +88237,6 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = "program"; break; } - case P4_DYNBLOB: - case P4_ADVANCE: { - break; - } case P4_TABLE: { zP4 = pOp->p4.pTab->zName; break; @@ -81323,21 +88368,40 @@ SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ /* ** Initialize an array of N Mem element. +** +** This is a high-runner, so only those fields that really do need to +** be initialized are set. The Mem structure is organized so that +** the fields that get initialized are nearby and hopefully on the same +** cache line. +** +** Mem.flags = flags +** Mem.db = db +** Mem.szMalloc = 0 +** +** All other fields of Mem can safely remain uninitialized for now. They +** will be initialized before use. */ static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ - while( (N--)>0 ){ - p->db = db; - p->flags = flags; - p->szMalloc = 0; + if( N>0 ){ + do{ + p->flags = flags; + p->db = db; + p->szMalloc = 0; #ifdef SQLITE_DEBUG - p->pScopyFrom = 0; + p->pScopyFrom = 0; #endif - p++; + p++; + }while( (--N)>0 ); } } /* -** Release an array of N Mem elements +** Release auxiliary memory held in an array of N Mem elements. +** +** After this routine returns, all Mem elements in the array will still +** be valid. Those Mem elements that were not holding auxiliary resources +** will be unchanged. Mem elements which had something freed will be +** set to MEM_Undefined. */ static void releaseMemArray(Mem *p, int N){ if( p && N ){ @@ -81370,12 +88434,17 @@ static void releaseMemArray(Mem *p, int N){ if( p->flags&(MEM_Agg|MEM_Dyn) ){ testcase( (p->flags & MEM_Dyn)!=0 && p->xDel==sqlite3VdbeFrameMemDel ); sqlite3VdbeMemRelease(p); + p->flags = MEM_Undefined; }else if( p->szMalloc ){ - sqlite3DbFreeNN(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p->zMalloc); p->szMalloc = 0; + p->flags = MEM_Undefined; } - - p->flags = MEM_Undefined; +#ifdef SQLITE_DEBUG + else{ + p->flags = MEM_Undefined; + } +#endif }while( (++p)nChildMem]; assert( sqlite3VdbeFrameIsValid(p) ); for(i=0; inChildCsr; i++){ - sqlite3VdbeFreeCursor(p->v, apCsr[i]); + if( apCsr[i] ) sqlite3VdbeFreeCursorNN(p->v, apCsr[i]); } releaseMemArray(aMem, p->nChildMem); sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); @@ -81573,7 +88642,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( Op *pOp; /* Current opcode */ assert( p->explain ); - assert( p->iVdbeMagic==VDBE_MAGIC_RUN ); + assert( p->eVdbeState==VDBE_RUN_STATE ); assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); /* Even though this opcode does not use dynamic strings for @@ -81581,7 +88650,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); - p->pResultSet = 0; if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or @@ -81617,7 +88685,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 4; + assert( p->nResColumn==4 ); }else{ sqlite3VdbeMemSetInt64(pMem+0, i); sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), @@ -81636,9 +88704,9 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3VdbeMemSetNull(pMem+7); #endif sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 8; + assert( p->nResColumn==8 ); } - p->pResultSet = pMem; + p->pResultRow = pMem; if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; rc = SQLITE_ERROR; @@ -81728,11 +88796,11 @@ struct ReusableSpace { static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - sqlite3_int64 nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed. */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ - nByte = ROUND8(nByte); + nByte = ROUND8P(nByte); if( nByte <= p->nFree ){ p->nFree -= nByte; pBuf = &p->pSpace[p->nFree]; @@ -81749,18 +88817,19 @@ static void *allocSpace( ** running it. */ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#if defined(SQLITE_DEBUG) int i; #endif assert( p!=0 ); - assert( p->iVdbeMagic==VDBE_MAGIC_INIT || p->iVdbeMagic==VDBE_MAGIC_RESET ); + assert( p->eVdbeState==VDBE_INIT_STATE + || p->eVdbeState==VDBE_READY_STATE + || p->eVdbeState==VDBE_HALT_STATE ); /* There should be at least one opcode. */ assert( p->nOp>0 ); - /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ - p->iVdbeMagic = VDBE_MAGIC_RUN; + p->eVdbeState = VDBE_READY_STATE; #ifdef SQLITE_DEBUG for(i=0; inMem; i++){ @@ -81777,8 +88846,8 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->nFkConstraint = 0; #ifdef VDBE_PROFILE for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; } #endif } @@ -81816,7 +88885,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( assert( p!=0 ); assert( p->nOp>0 ); assert( pParse!=0 ); - assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( pParse==p->pParse ); p->pVList = pParse->pVList; pParse->pVList = 0; @@ -81839,7 +88908,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( ** opcode array. This extra memory will be reallocated for other elements ** of the prepared statement. */ - n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + n = ROUND8P(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ @@ -81849,26 +88918,9 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "id", "parent", "notused", "detail" - }; - int iFirst, mx, i; if( nMem<10 ) nMem = 10; p->explain = pParse->explain; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(p, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(p, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; inResColumn = 12 - 4*p->explain; } p->expired = 0; @@ -81887,9 +88939,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); -#endif if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; @@ -81898,9 +88947,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); -#endif } } @@ -81915,9 +88961,6 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->nMem = nMem; initMemArray(p->aMem, nMem, db, MEM_Undefined); memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - memset(p->anExec, 0, p->nOp*sizeof(i64)); -#endif } sqlite3VdbeRewind(p); } @@ -81927,11 +88970,25 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( ** happens to hold. */ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ - if( pCx==0 ){ + if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); +} +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ + VdbeTxtBlbCache *pCache = pCx->pCache; + assert( pCx->colCache ); + pCx->colCache = 0; + pCx->pCache = 0; + if( pCache->pCValue ){ + sqlite3RCStrUnref(pCache->pCValue); + pCache->pCValue = 0; + } + sqlite3DbFree(p->db, pCache); + sqlite3VdbeFreeCursorNN(p, pCx); +} +SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ + if( pCx->colCache ){ + freeCursorWithCache(p, pCx); return; } - assert( pCx->pBtx==0 || pCx->eCurType==CURTYPE_BTREE ); - assert( pCx->pBtx==0 || pCx->isEphemeral ); switch( pCx->eCurType ){ case CURTYPE_SORTER: { sqlite3VdbeSorterClose(p->db, pCx); @@ -81959,14 +89016,12 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ ** Close all cursors in the current frame. */ static void closeCursorsInFrame(Vdbe *p){ - if( p->apCsr ){ - int i; - for(i=0; inCursor; i++){ - VdbeCursor *pC = p->apCsr[i]; - if( pC ){ - sqlite3VdbeFreeCursor(p, pC); - p->apCsr[i] = 0; - } + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursorNN(p, pC); + p->apCsr[i] = 0; } } } @@ -81979,9 +89034,6 @@ static void closeCursorsInFrame(Vdbe *p){ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - v->anExec = pFrame->anExec; -#endif v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -82015,9 +89067,7 @@ static void closeAllCursors(Vdbe *p){ } assert( p->nFrame==0 ); closeCursorsInFrame(p); - if( p->aMem ){ - releaseMemArray(p->aMem, p->nMem); - } + releaseMemArray(p->aMem, p->nMem); while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; p->pDelFrame = pDel->pParent; @@ -82039,12 +89089,12 @@ SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ int n; sqlite3 *db = p->db; - if( p->nResColumn ){ - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + if( p->nResAlloc ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); sqlite3DbFree(db, p->aColName); } n = nResColumn*COLNAME_N; - p->nResColumn = (u16)nResColumn; + p->nResColumn = p->nResAlloc = (u16)nResColumn; p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); if( p->aColName==0 ) return; initMemArray(p->aColName, n, db, MEM_Null); @@ -82069,14 +89119,14 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( ){ int rc; Mem *pColName; - assert( idxnResColumn ); + assert( idxnResAlloc ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); - pColName = &(p->aColName[idx+var*p->nResColumn]); + pColName = &(p->aColName[idx+var*p->nResAlloc]); rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; @@ -82143,6 +89193,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ nTrans++; } rc = sqlite3PagerExclusiveLock(pPager); + MARK_LAST_BUSY_LINE(rc); sqlite3BtreeLeave(pBt); } } @@ -82364,7 +89415,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; } - p = p->pNext; + p = p->pVNext; } assert( cnt==db->nVdbeActive ); assert( nWrite==db->nVdbeWrite ); @@ -82457,7 +89508,8 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - return SQLITE_ERROR; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; } return SQLITE_OK; } @@ -82496,9 +89548,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ - if( p->iVdbeMagic!=VDBE_MAGIC_RUN ){ - return SQLITE_OK; - } + assert( p->eVdbeState==VDBE_RUN_STATE ); if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; } @@ -82507,7 +89557,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* No commit or rollback needed if the program never started or if the ** SQL statement does not read or write a database file. */ - if( p->pc>=0 && p->bIsReader ){ + if( p->bIsReader ){ int mrc; /* Primary error code from p->rc */ int eStatementOp = 0; int isSpecialError; /* Set to true if a 'special' error */ @@ -82555,7 +89605,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFk(p, 0); } /* If the auto-commit flag is set and this is the only active writer @@ -82590,6 +89640,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeLeave(p); return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ + sqlite3SystemError(db, rc); p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -82599,6 +89650,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -82655,15 +89708,13 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } /* We have successfully halted and closed the VM. Record this fact. */ - if( p->pc>=0 ){ - db->nVdbeActive--; - if( !p->readOnly ) db->nVdbeWrite--; - if( p->bIsReader ) db->nVdbeRead--; - assert( db->nVdbeActive>=db->nVdbeRead ); - assert( db->nVdbeRead>=db->nVdbeWrite ); - assert( db->nVdbeWrite>=0 ); - } - p->iVdbeMagic = VDBE_MAGIC_HALT; + db->nVdbeActive--; + if( !p->readOnly ) db->nVdbeWrite--; + if( p->bIsReader ) db->nVdbeRead--; + assert( db->nVdbeActive>=db->nVdbeRead ); + assert( db->nVdbeRead>=db->nVdbeWrite ); + assert( db->nVdbeWrite>=0 ); + p->eVdbeState = VDBE_HALT_STATE; checkActiveVdbeCnt(db); if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; @@ -82712,6 +89763,7 @@ SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ sqlite3ValueSetNull(db->pErr); } db->errCode = rc; + db->errByteOffset = -1; return rc; } @@ -82744,8 +89796,8 @@ static void vdbeInvokeSqllog(Vdbe *v){ ** again. ** ** To look at it another way, this routine resets the state of the -** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to -** VDBE_MAGIC_INIT. +** virtual machine from VDBE_RUN_STATE or VDBE_HALT_STATE back to +** VDBE_READY_STATE. */ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) @@ -82759,7 +89811,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ ** error, then it might not have been halted properly. So halt ** it now. */ - sqlite3VdbeHalt(p); + if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); /* If the VDBE has been run even partially, then transfer the error code ** and error message from the VDBE into the main database structure. But @@ -82773,13 +89825,6 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ }else{ db->errCode = p->rc; } - if( p->runOnlyOnce ) p->expired = 1; - }else if( p->rc && p->expired ){ - /* The expired flag was set on the VDBE before the first call - ** to sqlite3_step(). For consistency (since sqlite3_step() was - ** called), set the database error in this case as well. - */ - sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); } /* Reset register contents and reclaim error message memory. @@ -82796,7 +89841,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - p->pResultSet = 0; + p->pResultRow = 0; #ifdef SQLITE_DEBUG p->nWrite = 0; #endif @@ -82824,10 +89869,12 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } for(i=0; inOp; i++){ char zHdr[100]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + cnt, + cycles, + cnt>0 ? cycles/cnt : 0 ); fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); @@ -82836,7 +89883,6 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } } #endif - p->iVdbeMagic = VDBE_MAGIC_RESET; return p->rc & db->errMask; } @@ -82846,7 +89892,10 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ */ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ int rc = SQLITE_OK; - if( p->iVdbeMagic==VDBE_MAGIC_RUN || p->iVdbeMagic==VDBE_MAGIC_HALT ){ + assert( VDBE_RUN_STATE>VDBE_READY_STATE ); + assert( VDBE_HALT_STATE>VDBE_READY_STATE ); + assert( VDBE_INIT_STATEeVdbeState>=VDBE_READY_STATE ){ rc = sqlite3VdbeReset(p); assert( (rc & p->db->errMask)==rc ); } @@ -82898,29 +89947,32 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, ** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with ** the database connection and frees the object itself. */ -SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ +static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + assert( db!=0 ); assert( p->db==0 || p->db==db ); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + if( p->aColName ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); + sqlite3DbNNFreeNN(db, p->aColName); + } for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } - if( p->iVdbeMagic!=VDBE_MAGIC_INIT ){ + if( p->eVdbeState!=VDBE_INIT_STATE ){ releaseMemArray(p->aVar, p->nVar); - sqlite3DbFree(db, p->pVList); - sqlite3DbFree(db, p->pFree); + if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList); + if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree); } vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); + if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { - DblquoteStr *pThis, *pNext; - for(pThis=p->pDblStr; pThis; pThis=pNext){ - pNext = pThis->pNextStr; + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; sqlite3DbFree(db, pThis); } } @@ -82944,20 +89996,17 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ assert( p!=0 ); db = p->db; + assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); - if( p->pPrev ){ - p->pPrev->pNext = p->pNext; - }else{ - assert( db->pVdbe==p ); - db->pVdbe = p->pNext; - } - if( p->pNext ){ - p->pNext->pPrev = p->pPrev; + if( db->pnBytesFreed==0 ){ + assert( p->ppVPrev!=0 ); + *p->ppVPrev = p->pVNext; + if( p->pVNext ){ + p->pVNext->ppVPrev = p->ppVPrev; + } } - p->iVdbeMagic = VDBE_MAGIC_DEAD; - p->db = 0; - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -82991,7 +90040,7 @@ SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor *p){ ** is supposed to be pointing. If the row was deleted out from under the ** cursor, set the cursor to point to a NULL row. */ -static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeHandleMovedCursor(VdbeCursor *p){ int isDifferentRow, rc; assert( p->eCurType==CURTYPE_BTREE ); assert( p->uc.pCursor!=0 ); @@ -83007,41 +90056,9 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ ** if need be. Return any I/O error from the restore operation. */ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ - assert( p->eCurType==CURTYPE_BTREE ); - if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); - } - return SQLITE_OK; -} - -/* -** Make sure the cursor p is ready to read or write the row to which it -** was last positioned. Return an error code if an OOM fault or I/O error -** prevents us from positioning the cursor to its correct position. -** -** If a MoveTo operation is pending on the given cursor, then do that -** MoveTo now. If no move is pending, check to see if the row has been -** deleted out from under the cursor and if it has, mark the row as -** a NULL row. -** -** If the cursor is already pointing to the correct row and that row has -** not been deleted out from under the cursor, then this routine is a no-op. -*/ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, u32 *piCol){ - VdbeCursor *p = *pp; - assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO ); - if( p->deferredMoveto ){ - u32 iMap; - assert( !p->isEphemeral ); - if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 && !p->nullRow ){ - *pp = p->pAltCursor; - *piCol = iMap - 1; - return SQLITE_OK; - } - return sqlite3VdbeFinishMoveto(p); - } + assert( p->eCurType==CURTYPE_BTREE || IsNullCursor(p) ); if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); + return sqlite3VdbeHandleMovedCursor(p); } return SQLITE_OK; } @@ -83052,7 +90069,7 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, u32 *piCol){ ** sqlite3VdbeSerialType() ** sqlite3VdbeSerialTypeLen() ** sqlite3VdbeSerialLen() -** sqlite3VdbeSerialPut() +** sqlite3VdbeSerialPut() <--- in-lined into OP_MakeRecord as of 2022-04-02 ** sqlite3VdbeSerialGet() ** ** encapsulate the code that serializes values for storage in SQLite @@ -83164,7 +90181,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ /* ** The sizes for serial types less than 128 */ -static const u8 sqlite3SmallTypeSizes[] = { +SQLITE_PRIVATE const u8 sqlite3SmallTypeSizes[128] = { /* 0 1 2 3 4 5 6 7 8 9 */ /* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, /* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, @@ -83233,7 +90250,7 @@ SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ ** so we trust him. */ #ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT -static u64 floatSwap(u64 in){ +SQLITE_PRIVATE u64 sqlite3FloatSwap(u64 in){ union { u64 r; u32 i[2]; @@ -83246,59 +90263,8 @@ static u64 floatSwap(u64 in){ u.i[1] = t; return u.r; } -# define swapMixedEndianFloat(X) X = floatSwap(X) -#else -# define swapMixedEndianFloat(X) -#endif - -/* -** Write the serialized data blob for the value stored in pMem into -** buf. It is assumed that the caller has allocated sufficient space. -** Return the number of bytes written. -** -** nBuf is the amount of space left in buf[]. The caller is responsible -** for allocating enough space to buf[] to hold the entire field, exclusive -** of the pMem->u.nZero bytes for a MEM_Zero value. -** -** Return the number of bytes actually written into buf[]. The number -** of bytes in the zero-filled tail is included in the return value only -** if those bytes were zeroed in buf[]. -*/ -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ - u32 len; - - /* Integer and Real */ - if( serial_type<=7 && serial_type>0 ){ - u64 v; - u32 i; - if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->u.r) ); - memcpy(&v, &pMem->u.r, sizeof(v)); - swapMixedEndianFloat(v); - }else{ - v = pMem->u.i; - } - len = i = sqlite3SmallTypeSizes[serial_type]; - assert( i>0 ); - do{ - buf[--i] = (u8)(v&0xFF); - v >>= 8; - }while( i ); - return len; - } +#endif /* SQLITE_MIXED_ENDIAN_64BIT_FLOAT */ - /* String or blob */ - if( serial_type>=12 ){ - assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) - == (int)sqlite3VdbeSerialTypeLen(serial_type) ); - len = pMem->n; - if( len>0 ) memcpy(buf, pMem->z, len); - return len; - } - - /* NULL or constants 0 or 1 */ - return 0; -} /* Input "x" is a sequence of unsigned characters that represent a ** big-endian integer. Return the equivalent native integer @@ -83311,14 +90277,14 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ /* ** Deserialize the data blob pointed to by buf as serial type serial_type -** and store the result in pMem. Return the number of bytes read. +** and store the result in pMem. ** ** This function is implemented as two separate routines for performance. ** The few cases that require local variables are broken out into a separate ** routine so that in most cases the overhead of moving the stack pointer ** is avoided. */ -static u32 serialGet( +static void serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -83352,9 +90318,25 @@ static u32 serialGet( memcpy(&pMem->u.r, &x, sizeof(x)); pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } - return 8; } -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; +} +SQLITE_PRIVATE void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -83365,13 +90347,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->flags = MEM_Null|MEM_Zero; pMem->n = 0; pMem->u.nZero = 0; - break; + return; } case 11: /* Reserved for future use */ case 0: { /* Null */ /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; - break; + return; } case 1: { /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement @@ -83379,7 +90361,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 1; + return; } case 2: { /* 2-byte signed integer */ /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit @@ -83387,7 +90369,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 2; + return; } case 3: { /* 3-byte signed integer */ /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit @@ -83395,7 +90377,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 3; + return; } case 4: { /* 4-byte signed integer */ /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit @@ -83407,7 +90389,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( #endif pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 4; + return; } case 5: { /* 6-byte signed integer */ /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit @@ -83415,13 +90397,14 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 6; + return; } case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ /* These use local variables, so do them in a separate routine ** to avoid having to move the frame pointer in the common case */ - return serialGet(buf,serial_type,pMem); + serialGet(buf,serial_type,pMem); + return; } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ @@ -83429,7 +90412,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; - return 0; + return; } default: { /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in @@ -83440,10 +90423,10 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; pMem->flags = aFlag[serial_type&1]; - return pMem->n; + return; } } - return 0; + return; } /* ** This routine is used to allocate sufficient space for an UnpackedRecord @@ -83464,10 +90447,10 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ){ UnpackedRecord *p; /* Unpacked record to return */ int nByte; /* Number of bytes required for *p */ - nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); + nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; - p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; @@ -83506,7 +90489,8 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->szMalloc = 0; pMem->z = 0; - d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + d += sqlite3VdbeSerialTypeLen(serial_type); pMem++; if( (++u)>=p->nField ) break; } @@ -83585,12 +90569,22 @@ static int vdbeRecordCompareDebug( if( d1+(u64)serial_type1+2>(u64)nKey1 && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 ){ + if( serial_type1>=1 + && serial_type1<=7 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)<=(u64)nKey1+8 + && CORRUPT_DB + ){ + return 1; /* corrupt record not detected by + ** sqlite3VdbeRecordCompareWithSkip(). Return true + ** to avoid firing the assert() */ + } break; } /* Extract the values to be compared. */ - d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + d1 += sqlite3VdbeSerialTypeLen(serial_type1); /* Do the comparison */ @@ -83701,8 +90695,8 @@ static int vdbeCompareMemString( }else{ rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); } - sqlite3VdbeMemRelease(&c1); - sqlite3VdbeMemRelease(&c2); + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); return rc; } } @@ -83752,32 +90746,44 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem return n1 - n2; } +/* The following two functions are used only within testcase() to prove +** test coverage. These functions do no exist for production builds. +** We must use separate SQLITE_NOINLINE functions here, since otherwise +** optimizer code movement causes gcov to become very confused. +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) +static int SQLITE_NOINLINE doubleLt(double a, double b){ return a8 ){ + if( sqlite3IsNaN(r) ){ + /* SQLite considers NaN to be a NULL. And all integer values are greater + ** than NULL */ + return 1; + } + if( sqlite3Config.bUseLongDouble ){ LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; testcase( xr ); testcase( x==r ); - if( xr ) return +1; /*NO_TEST*/ /* work around bugs in gcov */ - return 0; /*NO_TEST*/ /* work around bugs in gcov */ + return (xr); }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( iy ) return +1; - s = (double)i; - if( sr ) return +1; - return 0; + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)r); } } @@ -83963,14 +90969,22 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** two elements in the keys are equal. Fix the various stack variables so ** that this routine begins comparing at the second field. */ if( bSkip ){ - u32 s1; - idx1 = 1 + getVarint32(&aKey1[1], s1); + u32 s1 = aKey1[1]; + if( s1<0x80 ){ + idx1 = 2; + }else{ + idx1 = 1 + sqlite3GetVarint32(&aKey1[1], &s1); + } szHdr1 = aKey1[0]; d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); i = 1; pRhs++; }else{ - idx1 = getVarint32(aKey1, szHdr1); + if( (szHdr1 = aKey1[0])<0x80 ){ + idx1 = 1; + }else{ + idx1 = sqlite3GetVarint32(aKey1, &szHdr1); + } d1 = szHdr1; i = 0; } @@ -83985,7 +90999,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ + while( 1 /*exit-by-break*/ ){ u32 serial_type; /* RHS is an integer */ @@ -83995,11 +91009,11 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); @@ -84019,19 +91033,23 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* Serial types 12 or greater are strings and blobs (greater than ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing - ** them to numberic values are. */ - rc = +1; + ** them to numeric values are. */ + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else{ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.ru.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.ru.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } @@ -84101,7 +91119,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ @@ -84123,8 +91148,13 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); + if( d1>(unsigned)nKey1 ) break; idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); + if( idx1>=(unsigned)szHdr1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corrupt index */ + } + } /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -84226,7 +91256,8 @@ static int vdbeRecordCompareInt( return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } - v = pPKey2->aMem[0].u.i; + assert( pPKey2->u.i == pPKey2->aMem[0].u.i ); + v = pPKey2->u.i; if( v>lhs ){ res = pPKey2->r1; }else if( vaMem[0].flags & MEM_Str ); + assert( pPKey2->aMem[0].n == pPKey2->n ); + assert( pPKey2->aMem[0].z == pPKey2->u.z ); vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); - serial_type = (u8)(aKey1[1]); - if( serial_type >= 0x80 ){ - sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); - } + serial_type = (signed char)(aKey1[1]); + +vrcs_restart: if( serial_type<12 ){ + if( serial_type<0 ){ + sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); + if( serial_type>=12 ) goto vrcs_restart; + assert( CORRUPT_DB ); + } res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ }else if( !(serial_type & 0x01) ){ res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ @@ -84280,15 +91317,15 @@ static int vdbeRecordCompareString( pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } - nCmp = MIN( pPKey2->aMem[0].n, nStr ); - res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + nCmp = MIN( pPKey2->n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->u.z, nCmp); if( res>0 ){ res = pPKey2->r2; }else if( res<0 ){ res = pPKey2->r1; }else{ - res = nStr - pPKey2->aMem[0].n; + res = nStr - pPKey2->n; if( res==0 ){ if( pPKey2->nField>1 ){ res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); @@ -84343,6 +91380,7 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ p->r2 = 1; } if( (flags & MEM_Int) ){ + p->u.i = p->aMem[0].u.i; return vdbeRecordCompareInt; } testcase( flags & MEM_Real ); @@ -84352,6 +91390,8 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ && p->pKeyInfo->aColl[0]==0 ){ assert( flags & MEM_Str ); + p->u.z = p->aMem[0].z; + p->n = p->aMem[0].n; return vdbeRecordCompareString; } } @@ -84424,14 +91464,14 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Fetch the integer off the end of the index record */ sqlite3VdbeSerialGet((u8*)&m.z[m.n-lenRowid], typeRowid, &v); *rowid = v.u.i; - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_OK; /* Jump here if database corruption is detected after m has been ** allocated. Free the m object and return SQLITE_CORRUPT. */ idx_rowid_corruption: testcase( m.szMalloc!=0 ); - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_CORRUPT_BKPT; } @@ -84473,7 +91513,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( return rc; } *res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, pUnpacked, 0); - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_OK; } @@ -84515,7 +91555,7 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ */ SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; - for(p = db->pVdbe; p; p=p->pNext){ + for(p = db->pVdbe; p; p=p->pVNext){ p->expired = iCode+1; } } @@ -84546,7 +91586,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -84566,7 +91607,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff */ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ @@ -84608,6 +91650,20 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ return 1; } +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored @@ -84636,13 +91692,14 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ ** the vdbeUnpackRecord() function found in vdbeapi.c. */ static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + assert( db!=0 ); if( p ){ int i; for(i=0; iaMem[i]; - if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); + if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem); } - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -84669,6 +91726,16 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); @@ -84685,8 +91752,8 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( assert( pCsr!=0 ); assert( pCsr->eCurType==CURTYPE_BTREE ); - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; @@ -84713,7 +91780,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } - sqlite3DbFreeNN(db, preupdate.aNew); + sqlite3DbNNFreeNN(db, preupdate.aNew); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -84737,6 +91804,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( */ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* #include "opcodes.h" */ #ifndef SQLITE_OMIT_DEPRECATED /* @@ -84830,7 +91898,9 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); - rc = sqlite3VdbeFinalize(v); + assert( v->eVdbeState>=VDBE_READY_STATE ); + rc = sqlite3VdbeReset(v); + sqlite3VdbeDelete(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); } @@ -84871,7 +91941,15 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; + sqlite3_mutex *mutex; +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif +#if SQLITE_THREADSAFE + mutex = p->db->mutex; #endif sqlite3_mutex_enter(mutex); for(i=0; inVar; i++){ @@ -84990,7 +92068,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_NULL, /* 0x1f (not possible) */ SQLITE_FLOAT, /* 0x20 INTREAL */ SQLITE_NULL, /* 0x21 (not possible) */ - SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ SQLITE_NULL, /* 0x23 (not possible) */ SQLITE_FLOAT, /* 0x24 (not possible) */ SQLITE_NULL, /* 0x25 (not possible) */ @@ -85038,6 +92116,9 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ #endif return aType[pVal->flags&MEM_AffMask]; } +SQLITE_API int sqlite3_value_encoding(sqlite3_value *pVal){ + return pVal->enc; +} /* Return true if a parameter to xUpdate represents an unchanged column */ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ @@ -85067,6 +92148,9 @@ SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ sqlite3ValueFree(pNew); pNew = 0; } + }else if( pNew->flags & MEM_Null ){ + /* Do not duplicate pointer values */ + pNew->flags &= ~(MEM_Term|MEM_Subtype); } return pNew; } @@ -85088,7 +92172,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ ** is too big or if an OOM occurs. ** ** The invokeValueDestructor(P,X) routine invokes destructor function X() -** on value P is not going to be used and need to be destroyed. +** on value P if P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -85097,7 +92181,8 @@ static void setResultStrOrError( u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - int rc = sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel); + Mem *pOut = pCtx->pOut; + int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); if( rc ){ if( rc==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); @@ -85107,12 +92192,17 @@ static void setResultStrOrError( assert( rc==SQLITE_NOMEM ); sqlite3_result_error_nomem(pCtx); } + return; + } + sqlite3VdbeChangeEncoding(pOut, pCtx->enc); + if( sqlite3VdbeMemTooBig(pOut) ){ + sqlite3_result_error_toobig(pCtx); } } static int invokeValueDestructor( const void *p, /* Value to destroy */ void (*xDel)(void*), /* The destructor */ - sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if not NULL */ ){ assert( xDel!=SQLITE_DYNAMIC ); if( xDel==0 ){ @@ -85122,7 +92212,14 @@ static int invokeValueDestructor( }else{ xDel((void*)p); } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx!=0 ){ + sqlite3_result_error_toobig(pCtx); + } +#else + assert( pCtx!=0 ); sqlite3_result_error_toobig(pCtx); +#endif return SQLITE_TOOBIG; } SQLITE_API void sqlite3_result_blob( @@ -85131,6 +92228,12 @@ SQLITE_API void sqlite3_result_blob( int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 || n<0 ){ + invokeValueDestructor(z, xDel, pCtx); + return; + } +#endif assert( n>=0 ); assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); @@ -85141,8 +92244,14 @@ SQLITE_API void sqlite3_result_blob64( sqlite3_uint64 n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -85150,30 +92259,48 @@ SQLITE_API void sqlite3_result_blob64( } } SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } @@ -85183,14 +92310,37 @@ SQLITE_API void sqlite3_result_pointer( const char *zPType, void (*xDestructor)(void*) ){ - Mem *pOut = pCtx->pOut; + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(pPtr, xDestructor, 0); + return; + } +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); sqlite3VdbeMemRelease(pOut); pOut->flags = MEM_Null; sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor); } SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ - Mem *pOut = pCtx->pOut; + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif +#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0 + if( pCtx->pFunc!=0 + && (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0 + ){ + char zErr[200]; + sqlite3_snprintf(sizeof(zErr), zErr, + "misuse of sqlite3_result_subtype() by %s()", + pCtx->pFunc->zName); + sqlite3_result_error(pCtx, zErr, -1); + return; + } +#endif /* SQLITE_STRICT_SUBTYPE */ + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); pOut->eSubtype = eSubtype & 0xff; pOut->flags |= MEM_Subtype; @@ -85201,6 +92351,12 @@ SQLITE_API void sqlite3_result_text( int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } @@ -85211,13 +92367,23 @@ SQLITE_API void sqlite3_result_text64( void (*xDel)(void *), unsigned char enc ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + n &= ~(u64)1; + } if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); } } #ifndef SQLITE_OMIT_UTF16 @@ -85228,7 +92394,7 @@ SQLITE_API void sqlite3_result_text16( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel); } SQLITE_API void sqlite3_result_text16be( sqlite3_context *pCtx, @@ -85237,7 +92403,7 @@ SQLITE_API void sqlite3_result_text16be( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel); } SQLITE_API void sqlite3_result_text16le( sqlite3_context *pCtx, @@ -85246,21 +92412,40 @@ SQLITE_API void sqlite3_result_text16le( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; + if( pValue==0 ){ + sqlite3_result_null(pCtx); + return; + } +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - sqlite3VdbeMemCopy(pCtx->pOut, pValue); + sqlite3VdbeMemCopy(pOut, pValue); + sqlite3VdbeChangeEncoding(pOut, pCtx->enc); + if( sqlite3VdbeMemTooBig(pOut) ){ + sqlite3_result_error_toobig(pCtx); + } } SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); + sqlite3_result_zeroblob64(pCtx, n>0 ? n : 0); } SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ - Mem *pOut = pCtx->pOut; + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return SQLITE_MISUSE_BKPT; +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(pCtx); return SQLITE_TOOBIG; } #ifndef SQLITE_OMIT_INCRBLOB @@ -85271,18 +92456,24 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ #endif } SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif pCtx->isError = errCode ? errCode : -1; #ifdef SQLITE_DEBUG if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; #endif if( pCtx->pOut->flags & MEM_Null ){ - sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, - SQLITE_UTF8, SQLITE_STATIC); + setResultStrOrError(pCtx, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, + SQLITE_STATIC); } } /* Force an SQLITE_TOOBIG error. */ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, @@ -85291,6 +92482,9 @@ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ /* An SQLITE_NOMEM error. */ SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM_BKPT; @@ -85351,80 +92545,83 @@ static int sqlite3Step(Vdbe *p){ int rc; assert(p); - if( p->iVdbeMagic!=VDBE_MAGIC_RUN ){ - /* We used to require that sqlite3_reset() be called before retrying - ** sqlite3_step() after any error or after SQLITE_DONE. But beginning - ** with version 3.7.0, we changed this so that sqlite3_reset() would - ** be called automatically instead of throwing the SQLITE_MISUSE error. - ** This "automatic-reset" change is not technically an incompatibility, - ** since any application that receives an SQLITE_MISUSE is broken by - ** definition. - ** - ** Nevertheless, some published applications that were originally written - ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE - ** returns, and those were broken by the automatic-reset change. As a - ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the - ** legacy behavior of returning SQLITE_MISUSE for cases where the - ** previous sqlite3_step() returned something other than a SQLITE_LOCKED - ** or SQLITE_BUSY error. - */ -#ifdef SQLITE_OMIT_AUTORESET - if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ - sqlite3_reset((sqlite3_stmt*)p); - }else{ - return SQLITE_MISUSE_BKPT; - } -#else - sqlite3_reset((sqlite3_stmt*)p); -#endif - } - - /* Check that malloc() has not failed. If it has, return early. */ db = p->db; - if( db->mallocFailed ){ - p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM_BKPT; - } + if( p->eVdbeState!=VDBE_RUN_STATE ){ + restart_step: + if( p->eVdbeState==VDBE_READY_STATE ){ + if( p->expired ){ + p->rc = SQLITE_SCHEMA; + rc = SQLITE_ERROR; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ + /* If this statement was prepared using saved SQL and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same + ** value. + */ + rc = sqlite3VdbeTransferError(p); + } + goto end_of_step; + } - if( p->pc<0 && p->expired ){ - p->rc = SQLITE_SCHEMA; - rc = SQLITE_ERROR; - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ - /* If this statement was prepared using saved SQL and an - ** error has occurred, then return the error code in p->rc to the - ** caller. Set the error code in the database handle to the same value. + /* If there are no other statements currently running, then + ** reset the interrupt flag. This prevents a call to sqlite3_interrupt + ** from interrupting a statement that has not yet started. */ - rc = sqlite3VdbeTransferError(p); - } - goto end_of_step; - } - if( p->pc<0 ){ - /* If there are no other statements currently running, then - ** reset the interrupt flag. This prevents a call to sqlite3_interrupt - ** from interrupting a statement that has not yet started. - */ - if( db->nVdbeActive==0 ){ - AtomicStore(&db->u1.isInterrupted, 0); - } + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } - assert( db->nVdbeWrite>0 || db->autoCommit==0 - || (db->nDeferredCons==0 && db->nDeferredImmCons==0) - ); + assert( db->nVdbeWrite>0 || db->autoCommit==0 + || (db->nDeferredCons==0 && db->nDeferredImmCons==0) + ); #ifndef SQLITE_OMIT_TRACE - if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 - && !db->init.busy && p->zSql ){ - sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); - }else{ - assert( p->startTime==0 ); - } + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 + && !db->init.busy && p->zSql ){ + sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); + } #endif - db->nVdbeActive++; - if( p->readOnly==0 ) db->nVdbeWrite++; - if( p->bIsReader ) db->nVdbeRead++; - p->pc = 0; + db->nVdbeActive++; + if( p->readOnly==0 ) db->nVdbeWrite++; + if( p->bIsReader ) db->nVdbeRead++; + p->pc = 0; + p->eVdbeState = VDBE_RUN_STATE; + }else + + if( ALWAYS(p->eVdbeState==VDBE_HALT_STATE) ){ + /* We used to require that sqlite3_reset() be called before retrying + ** sqlite3_step() after any error or after SQLITE_DONE. But beginning + ** with version 3.7.0, we changed this so that sqlite3_reset() would + ** be called automatically instead of throwing the SQLITE_MISUSE error. + ** This "automatic-reset" change is not technically an incompatibility, + ** since any application that receives an SQLITE_MISUSE is broken by + ** definition. + ** + ** Nevertheless, some published applications that were originally written + ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE + ** returns, and those were broken by the automatic-reset change. As a + ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the + ** legacy behavior of returning SQLITE_MISUSE for cases where the + ** previous sqlite3_step() returned something other than a SQLITE_LOCKED + ** or SQLITE_BUSY error. + */ +#ifdef SQLITE_OMIT_AUTORESET + if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ + sqlite3_reset((sqlite3_stmt*)p); + }else{ + return SQLITE_MISUSE_BKPT; + } +#else + sqlite3_reset((sqlite3_stmt*)p); +#endif + assert( p->eVdbeState==VDBE_READY_STATE ); + goto restart_step; + } } + #ifdef SQLITE_DEBUG p->rcApp = SQLITE_OK; #endif @@ -85439,12 +92636,17 @@ static int sqlite3Step(Vdbe *p){ db->nVdbeExec--; } - if( rc!=SQLITE_ROW ){ + if( rc==SQLITE_ROW ){ + assert( p->rc==SQLITE_OK ); + assert( db->mallocFailed==0 ); + db->errCode = SQLITE_ROW; + return SQLITE_ROW; + }else{ #ifndef SQLITE_OMIT_TRACE /* If the statement completed successfully, invoke the profile callback */ checkProfileCallback(db, p); #endif - + p->pResultRow = 0; if( rc==SQLITE_DONE && db->autoCommit ){ assert( p->rc==SQLITE_OK ); p->rc = doWalCallbacks(db); @@ -85482,7 +92684,7 @@ SQLITE_API int sqlite3_set_droptable_handle(sqlite3 *db, void (*xFunc)(sqlite3*, sqlite3_mutex_leave(db->mutex); return 0; } -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ /* ** This is the top-level implementation of sqlite3_step(). Call @@ -85500,7 +92702,6 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ } db = v->db; sqlite3_mutex_enter(db->mutex); - v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ int savedPc = v->pc; @@ -85510,7 +92711,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ int blockFull = v->blockFull; int startPos = v->startPos; int addedRows = v->addedRows; -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ rc = sqlite3Reprepare(v); if( rc!=SQLITE_OK ){ /* This case occurs after failing to recompile an sql statement. @@ -85539,10 +92740,16 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ v->blockFull = blockFull; v->startPos = startPos; v->addedRows = addedRows; -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ sqlite3_reset(pStmt); - if( savedPc>=0 ) v->doingRerun = 1; + if( savedPc>=0 ){ + /* Setting minWriteFileFormat to 254 is a signal to the OP_Init and + ** OP_Trace opcodes to *not* perform SQLITE_TRACE_STMT because it has + ** already been done once on a prior invocation that failed due to + ** SQLITE_SCHEMA. tag-20220401a */ + v->minWriteFileFormat = 254; + } assert( v->expired==0 ); } #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK @@ -85558,7 +92765,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ sqlite3_free(db->mDropSchemaName); db->mDropSchemaName = NULL; } -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ sqlite3_mutex_leave(db->mutex); return rc; } @@ -85569,6 +92776,9 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** pointer to it. */ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#endif assert( p && p->pFunc ); return p->pFunc->pUserData; } @@ -85584,7 +92794,11 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p && p->pOut ); +#endif return p->pOut->db; } @@ -85603,10 +92817,96 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ ** value, as a signal to the xUpdate routine that the column is unchanged. */ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p ); +#endif return sqlite3_value_nochange(p->pOut); } +/* +** The destructor function for a ValueList object. This needs to be +** a separate function, unknowable to the application, to ensure that +** calls to sqlite3_vtab_in_first()/sqlite3_vtab_in_next() that are not +** preceded by activation of IN processing via sqlite3_vtab_int() do not +** try to access a fake ValueList object inserted by a hostile extension. +*/ +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void *pToDelete){ + sqlite3_free(pToDelete); +} + +/* +** Implementation of sqlite3_vtab_in_first() (if bNext==0) and +** sqlite3_vtab_in_next() (if bNext!=0). +*/ +static int valueFromValueList( + sqlite3_value *pVal, /* Pointer to the ValueList object */ + sqlite3_value **ppOut, /* Store the next value from the list here */ + int bNext /* 1 for _next(). 0 for _first() */ +){ + int rc; + ValueList *pRhs; + + *ppOut = 0; + if( pVal==0 ) return SQLITE_MISUSE_BKPT; + if( (pVal->flags & MEM_Dyn)==0 || pVal->xDel!=sqlite3VdbeValueListFree ){ + return SQLITE_ERROR; + }else{ + assert( (pVal->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) ); + assert( pVal->eSubtype=='p' ); + assert( pVal->u.zPType!=0 && strcmp(pVal->u.zPType,"ValueList")==0 ); + pRhs = (ValueList*)pVal->z; + } + if( bNext ){ + rc = sqlite3BtreeNext(pRhs->pCsr, 0); + }else{ + int dummy = 0; + rc = sqlite3BtreeFirst(pRhs->pCsr, &dummy); + assert( rc==SQLITE_OK || sqlite3BtreeEof(pRhs->pCsr) ); + if( sqlite3BtreeEof(pRhs->pCsr) ) rc = SQLITE_DONE; + } + if( rc==SQLITE_OK ){ + u32 sz; /* Size of current row in bytes */ + Mem sMem; /* Raw content of current row */ + memset(&sMem, 0, sizeof(sMem)); + sz = sqlite3BtreePayloadSize(pRhs->pCsr); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem); + if( rc==SQLITE_OK ){ + u8 *zBuf = (u8*)sMem.z; + u32 iSerial; + sqlite3_value *pOut = pRhs->pOut; + int iOff = 1 + getVarint32(&zBuf[1], iSerial); + sqlite3VdbeSerialGet(&zBuf[iOff], iSerial, pOut); + pOut->enc = ENC(pOut->db); + if( (pOut->flags & MEM_Ephem)!=0 && sqlite3VdbeMemMakeWriteable(pOut) ){ + rc = SQLITE_NOMEM; + }else{ + *ppOut = pOut; + } + } + sqlite3VdbeMemRelease(&sMem); + } + return rc; +} + +/* +** Set the iterator value pVal to point to the first value in the set. +** Set (*ppOut) to point to this value before returning. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut){ + return valueFromValueList(pVal, ppOut, 0); +} + +/* +** Set the iterator value pVal to point to the next value in the set. +** Set (*ppOut) to point to this value before returning. +*/ +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut){ + return valueFromValueList(pVal, ppOut, 1); +} + /* ** Return the current time for a statement. If the current time ** is requested more than once within the same run of a single prepared @@ -85680,6 +92980,9 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return 0; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); #if SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 0; @@ -85712,8 +93015,12 @@ SQLITE_API void sqlite3_set_auxdata( void (*xDelete)(void*) ){ AuxData *pAuxData; - Vdbe *pVdbe = pCtx->pVdbe; + Vdbe *pVdbe; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif + pVdbe= pCtx->pVdbe; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); #ifdef SQLITE_ENABLE_STAT4 if( pVdbe==0 ) goto failed; @@ -85769,7 +93076,8 @@ SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - return pVm ? pVm->nResColumn : 0; + if( pVm==0 ) return 0; + return pVm->nResColumn; } /* @@ -85778,7 +93086,7 @@ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - if( pVm==0 || pVm->pResultSet==0 ) return 0; + if( pVm==0 || pVm->pResultRow==0 ) return 0; return pVm->nResColumn; } @@ -85801,15 +93109,15 @@ static const Mem *columnNullValue(void){ #endif = { /* .u = */ {0}, + /* .z = */ (char*)0, + /* .n = */ (int)0, /* .flags = */ (u16)MEM_Null, /* .enc = */ (u8)0, /* .eSubtype = */ (u8)0, - /* .n = */ (int)0, - /* .z = */ (char*)0, - /* .zMalloc = */ (char*)0, + /* .db = */ (sqlite3*)0, /* .szMalloc = */ (int)0, /* .uTemp = */ (u32)0, - /* .db = */ (sqlite3*)0, + /* .zMalloc = */ (char*)0, /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG /* .pScopyFrom = */ (Mem*)0, @@ -85833,8 +93141,8 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ if( pVm==0 ) return (Mem*)columnNullValue(); assert( pVm->db ); sqlite3_mutex_enter(pVm->db->mutex); - if( pVm->pResultSet!=0 && inResColumn && i>=0 ){ - pOut = &pVm->pResultSet[i]; + if( pVm->pResultRow!=0 && inResColumn && i>=0 ){ + pOut = &pVm->pResultRow[i]; }else{ sqlite3Error(pVm->db, SQLITE_RANGE); pOut = (Mem*)columnNullValue(); @@ -85858,7 +93166,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** sqiite3_column_blob() +** sqlite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { @@ -85942,6 +93250,32 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ return iType; } +/* +** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN. +*/ +static const char * const azExplainColNames8[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", /* EXPLAIN */ + "id", "parent", "notused", "detail" /* EQP */ +}; +static const u16 azExplainColNames16data[] = { + /* 0 */ 'a', 'd', 'd', 'r', 0, + /* 5 */ 'o', 'p', 'c', 'o', 'd', 'e', 0, + /* 12 */ 'p', '1', 0, + /* 15 */ 'p', '2', 0, + /* 18 */ 'p', '3', 0, + /* 21 */ 'p', '4', 0, + /* 24 */ 'p', '5', 0, + /* 27 */ 'c', 'o', 'm', 'm', 'e', 'n', 't', 0, + /* 35 */ 'i', 'd', 0, + /* 38 */ 'p', 'a', 'r', 'e', 'n', 't', 0, + /* 45 */ 'n', 'o', 't', 'u', 's', 'e', 'd', 0, + /* 53 */ 'd', 'e', 't', 'a', 'i', 'l', 0 +}; +static const u8 iExplainColNames16[] = { + 0, 5, 12, 15, 18, 21, 24, 27, + 35, 38, 45, 53 +}; + /* ** Convert the N-th element of pStmt->pColName[] into a string using ** xFunc() then return that string. If N is out of range, return 0. @@ -85974,15 +93308,29 @@ static const void *columnName( return 0; } #endif + if( N<0 ) return 0; ret = 0; p = (Vdbe *)pStmt; db = p->db; assert( db!=0 ); - n = sqlite3_column_count(pStmt); - if( N=0 ){ + sqlite3_mutex_enter(db->mutex); + + if( p->explain ){ + if( useType>0 ) goto columnName_end; + n = p->explain==1 ? 8 : 4; + if( N>=n ) goto columnName_end; + if( useUtf16 ){ + int i = iExplainColNames16[N + 8*p->explain - 8]; + ret = (void*)&azExplainColNames16data[i]; + }else{ + ret = (void*)azExplainColNames8[N + 8*p->explain - 8]; + } + goto columnName_end; + } + n = p->nResColumn; + if( NmallocFailed; N += useType*n; - sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -85994,12 +93342,14 @@ static const void *columnName( /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } - sqlite3_mutex_leave(db->mutex); } +columnName_end: + sqlite3_mutex_leave(db->mutex); return ret; } @@ -86092,7 +93442,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ /* ** Unbind the value bound to variable i in virtual machine p. This is the ** the same as binding a NULL value to the column. If the "i" parameter is -** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** out of range, then SQLITE_RANGE is returned. Otherwise SQLITE_OK. ** ** A successful evaluation of this routine acquires the mutex on p. ** the mutex is released if any kind of error occurs. @@ -86100,25 +93450,24 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ ** The error code stored in database p->db is overwritten with the return ** value in any case. */ -static int vdbeUnbind(Vdbe *p, int i){ +static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; if( vdbeSafetyNotNull(p) ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(p->db->mutex); - if( p->iVdbeMagic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE); + if( p->eVdbeState!=VDBE_READY_STATE ){ + sqlite3Error(p->db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(p->db->mutex); sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i<1 || i>p->nVar ){ + if( i>=(unsigned int)p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } - i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; @@ -86155,7 +93504,7 @@ static int bindText( Mem *pVar; int rc; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; @@ -86204,7 +93553,7 @@ SQLITE_API int sqlite3_bind_blob64( SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); @@ -86217,7 +93566,7 @@ SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); @@ -86227,7 +93576,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } @@ -86242,7 +93591,7 @@ SQLITE_API int sqlite3_bind_pointer( ){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); @@ -86269,7 +93618,10 @@ SQLITE_API int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + nData &= ~(u16)1; + } return bindText(pStmt, i, zData, nData, xDel, enc); } #ifndef SQLITE_OMIT_UTF16 @@ -86277,10 +93629,10 @@ SQLITE_API int sqlite3_bind_text16( sqlite3_stmt *pStmt, int i, const void *zData, - int nData, + int n, void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); + return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ @@ -86291,7 +93643,10 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu break; } case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->u.r); + assert( pValue->flags & (MEM_Real|MEM_IntReal) ); + rc = sqlite3_bind_double(pStmt, i, + (pValue->flags & MEM_Real) ? pValue->u.r : (double)pValue->u.i + ); break; } case SQLITE_BLOB: { @@ -86317,7 +93672,7 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); @@ -86331,6 +93686,9 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->db->mutex); if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ rc = SQLITE_TOOBIG; @@ -86451,12 +93809,48 @@ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->explain : 0; } +/* +** Set the explain mode for a statement. +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ + Vdbe *v = (Vdbe*)pStmt; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(v->db->mutex); + if( ((int)v->explain)==eMode ){ + rc = SQLITE_OK; + }else if( eMode<0 || eMode>2 ){ + rc = SQLITE_ERROR; + }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ + rc = SQLITE_ERROR; + }else if( v->eVdbeState!=VDBE_READY_STATE ){ + rc = SQLITE_BUSY; + }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){ + /* No reprepare necessary */ + v->explain = eMode; + rc = SQLITE_OK; + }else{ + v->explain = eMode; + rc = sqlite3Reprepare(v); + v->haveEqpOps = eMode==2; + } + if( v->explain ){ + v->nResColumn = 12 - 4*v->explain; + }else{ + v->nResColumn = v->nResAlloc; + } + sqlite3_mutex_leave(v->db->mutex); + return rc; +} + /* ** Return true if the prepared statement is in need of being reset. */ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->iVdbeMagic==VDBE_MAGIC_RUN && v->pc>=0; + return v!=0 && v->eVdbeState==VDBE_RUN_STATE; } /* @@ -86477,7 +93871,7 @@ SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; }else{ - pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pVNext; } sqlite3_mutex_leave(pDb->mutex); return pNext; @@ -86502,9 +93896,11 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ sqlite3_mutex_enter(db->mutex); v = 0; db->pnBytesFreed = (int*)&v; - sqlite3VdbeClearObject(db, pVdbe); - sqlite3DbFree(db, pVdbe); + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + sqlite3VdbeDelete(pVdbe); db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3_mutex_leave(db->mutex); }else{ v = pVdbe->aCounter[op]; @@ -86588,10 +93984,16 @@ static UnpackedRecord *vdbeUnpackRecord( ** a field of the row currently being updated or deleted. */ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; Mem *pMem; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; /* Test that this call is being made from within an SQLITE_DELETE or ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ if( !p || p->op==SQLITE_INSERT ){ @@ -86652,7 +94054,12 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa ** the number of columns in the row being updated, deleted or inserted. */ SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->keyinfo.nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -86670,7 +94077,12 @@ SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ ** or SET DEFAULT action is considered a trigger. */ SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->v->nFrame : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -86681,7 +94093,12 @@ SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ ** only. */ SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->iBlobWrite : -1); } #endif @@ -86692,10 +94109,16 @@ SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ ** a field of the row currently being updated or inserted. */ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; int rc = SQLITE_OK; Mem *pMem; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; if( !p || p->op==SQLITE_DELETE ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; @@ -86766,23 +94189,79 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa /* ** Return status data for a single loop within query pStmt. */ -SQLITE_API int sqlite3_stmt_scanstatus( +SQLITE_API int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement being queried */ - int idx, /* Index of loop to report on */ + int iScan, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ + int flags, void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanStatus *pScan; - if( idx<0 || idx>=p->nScan ) return 1; + VdbeOp *aOp; + int nOp; + ScanStatus *pScan = 0; + int idx; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 || pOut==0 + || iScanStatusOpSQLITE_SCANSTAT_NCYCLE ){ + return 1; + } +#endif + aOp = p->aOp; + nOp = p->nOp; + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + + if( iScan<0 ){ + int ii; + if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ + i64 res = 0; + for(ii=0; iinScan; idx++){ + pScan = &p->aScan[idx]; + if( pScan->zName ){ + iScan--; + if( iScan<0 ) break; + } + } + } + if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + if( pScan->addrLoop>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_NVISIT: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + if( pScan->addrVisit>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_EST: { @@ -86801,7 +94280,7 @@ SQLITE_API int sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -86809,12 +94288,51 @@ SQLITE_API int sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + case SQLITE_SCANSTAT_PARENTID: { + if( pScan->addrExplain ){ + *(int*)pOut = aOp[ pScan->addrExplain ].p2; }else{ *(int*)pOut = -1; } break; } + case SQLITE_SCANSTAT_NCYCLE: { + i64 res = 0; + if( pScan->aAddrRange[0]==0 ){ + res = -1; + }else{ + int ii; + for(ii=0; iiaAddrRange); ii+=2){ + int iIns = pScan->aAddrRange[ii]; + int iEnd = pScan->aAddrRange[ii+1]; + if( iIns==0 ) break; + if( iIns>0 ){ + while( iIns<=iEnd ){ + res += aOp[iIns].nCycle; + iIns++; + } + }else{ + int iOp; + for(iOp=0; iOpp1!=iEnd ) continue; + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ + continue; + } + res += aOp[iOp].nCycle; + } + } + } + } + *(i64*)pOut = res; + break; + } default: { return 1; } @@ -86822,12 +94340,29 @@ SQLITE_API int sqlite3_stmt_scanstatus( return 0; } +/* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int iScan, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); +} + /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); + int ii; + for(ii=0; p!=0 && iinOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ @@ -87157,13 +94692,17 @@ SQLITE_API int sqlite3_found_count = 0; ** ** Other useful labels for breakpoints include: ** test_addop_breakpoint(pc,pOp) -** sqlite3CorruptError(lineno) +** sqlite3CorruptError(lineno,context) ** sqlite3MisuseError(lineno) ** sqlite3CantopenError(lineno) */ static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ - static int n = 0; + static u64 n = 0; + (void)pc; + (void)pOp; + (void)v; n++; + if( n==LARGEST_UINT64 ) abort(); /* So that n is used, preventing a warning */ } #endif @@ -87271,7 +94810,6 @@ static VdbeCursor *allocateCursor( Vdbe *p, /* The virtual machine */ int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ - int iDb, /* Database the cursor belongs to, or -1 */ u8 eCurType /* Type of the new cursor */ ){ /* Find the memory cell that will be used to store the blob of memory @@ -87297,12 +94835,12 @@ static VdbeCursor *allocateCursor( int nByte; VdbeCursor *pCx = 0; nByte = - ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + + ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ - sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + sqlite3VdbeFreeCursorNN(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } @@ -87328,12 +94866,11 @@ static VdbeCursor *allocateCursor( p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc; memset(pCx, 0, offsetof(VdbeCursor,pAltCursor)); pCx->eCurType = eCurType; - pCx->iDb = iDb; pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; if( eCurType==CURTYPE_BTREE ){ pCx->uc.pCursor = (BtCursor*) - &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; sqlite3BtreeCursorZero(pCx->uc.pCursor); } return pCx; @@ -87346,7 +94883,8 @@ static VdbeCursor *allocateCursor( ** return false. */ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ - i64 iValue = (double)rValue; + i64 iValue; + iValue = sqlite3RealToI64(rValue); if( sqlite3RealSameAsInt(rValue,iValue) ){ *piValue = iValue; return 1; @@ -87402,6 +94940,10 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** always preferred, even if the affinity is REAL, because ** an integer representation is more space efficient on disk. ** +** SQLITE_AFF_FLEXNUM: +** If the value is text, then try to convert it into a number of +** some kind (integer or real) but do not make any other changes. +** ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** @@ -87416,11 +94958,11 @@ static void applyAffinity( ){ if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); + || affinity==SQLITE_AFF_NUMERIC || affinity==SQLITE_AFF_FLEXNUM ); if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags & MEM_Real)==0 ){ + if( (pRec->flags & (MEM_Real|MEM_IntReal))==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); - }else{ + }else if( affinity<=SQLITE_AFF_REAL ){ sqlite3VdbeIntegerAffinity(pRec); } } @@ -87508,17 +95050,18 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + assert( (pMem->flags & MEM_Null)==0 + || pMem->db==0 || pMem->db->mallocFailed ); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){ testcase( pMem->flags & MEM_Int ); testcase( pMem->flags & MEM_Real ); testcase( pMem->flags & MEM_IntReal ); - return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); - } - if( pMem->flags & (MEM_Str|MEM_Blob) ){ - testcase( pMem->flags & MEM_Str ); - testcase( pMem->flags & MEM_Blob ); - return computeNumericType(pMem); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null); } + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); return 0; } @@ -87579,6 +95122,9 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); } sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + if( f & MEM_Term ){ + sqlite3_str_appendf(pStr, "(0-term)"); + } } } #endif @@ -87647,17 +95193,6 @@ SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ # define REGISTER_TRACE(R,M) #endif - -#ifdef VDBE_PROFILE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/* #include "hwtime.h" */ - -#endif - #ifndef NDEBUG /* ** This function is only called from within an assert() expression. It @@ -87731,7 +95266,7 @@ no_mem: rc = SQLITE_NOMEM; return rc; } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ /* ** Return the register of pOp->p2 after first preparing it to be @@ -87756,6 +95291,118 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ } } +/* +** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning +** with pOp->p3. Return the hash. +*/ +static u64 filterHash(const Mem *aMem, const Op *pOp){ + int i, mx; + u64 h = 0; + + assert( pOp->p4type==P4_INT32 ); + for(i=pOp->p3, mx=i+pOp->p4.i; iflags & (MEM_Int|MEM_IntReal) ){ + h += p->u.i; + }else if( p->flags & MEM_Real ){ + h += sqlite3VdbeIntValue(p); + }else if( p->flags & (MEM_Str|MEM_Blob) ){ + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); + } + } + return h; +} + + +/* +** For OP_Column, factor out the case where content is loaded from +** overflow pages, so that the code to implement this case is separate +** the common case where all content fits on the page. Factoring out +** the code reduces register pressure and helps the common case +** to run faster. +*/ +static SQLITE_NOINLINE int vdbeColumnFromOverflow( + VdbeCursor *pC, /* The BTree cursor from which we are reading */ + int iCol, /* The column to read */ + int t, /* The serial-type code for the column value */ + i64 iOffset, /* Offset to the start of the content value */ + u32 cacheStatus, /* Current Vdbe.cacheCtr value */ + u32 colCacheCtr, /* Current value of the column cache counter */ + Mem *pDest /* Store the value into this register. */ +){ + int rc; + sqlite3 *db = pDest->db; + int encoding = pDest->enc; + int len = sqlite3VdbeSerialTypeLen(t); + assert( pC->eCurType==CURTYPE_BTREE ); + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; + if( len > 4000 && pC->pKeyInfo==0 ){ + /* Cache large column values that are on overflow pages using + ** an RCStr (reference counted string) so that if they are reloaded, + ** that do not have to be copied a second time. The overhead of + ** creating and managing the cache is such that this is only + ** profitable for larger TEXT and BLOB values. + ** + ** Only do this on table-btrees so that writes to index-btrees do not + ** need to clear the cache. This buys performance in the common case + ** in exchange for generality. + */ + VdbeTxtBlbCache *pCache; + char *pBuf; + if( pC->colCache==0 ){ + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); + if( pC->pCache==0 ) return SQLITE_NOMEM; + pC->colCache = 1; + } + pCache = pC->pCache; + if( pCache->pCValue==0 + || pCache->iCol!=iCol + || pCache->cacheStatus!=cacheStatus + || pCache->colCacheCtr!=colCacheCtr + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) + ){ + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); + if( pBuf==0 ) return SQLITE_NOMEM; + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); + if( rc ) return rc; + pBuf[len] = 0; + pBuf[len+1] = 0; + pBuf[len+2] = 0; + pCache->iCol = iCol; + pCache->cacheStatus = cacheStatus; + pCache->colCacheCtr = colCacheCtr; + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); + }else{ + pBuf = pCache->pCValue; + } + assert( t>=12 ); + sqlite3RCStrRef(pBuf); + if( t&1 ){ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + sqlite3RCStrUnref); + pDest->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, + sqlite3RCStrUnref); + } + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); + if( rc ) return rc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->z[len] = 0; + pDest->flags |= MEM_Term; + } + } + pDest->flags &= ~MEM_Ephem; + return rc; +} + + /* ** Return the symbolic name for the data type of a pMem */ @@ -87779,11 +95426,10 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ){ Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp = aOp; /* Current operation */ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) - Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif #ifdef SQLITE_DEBUG + Op *pOrigOp; /* Value of pOp at the top of the loop */ int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ + u8 iCompareIsInit = 0; /* iCompare is initialized */ #endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ @@ -87799,13 +95445,17 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ + u32 colCacheCtr = 0; /* Column cache counter */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ - assert( p->iVdbeMagic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ - sqlite3VdbeEnter(p); + assert( p->eVdbeState==VDBE_RUN_STATE ); /* sqlite3_step() verifies this */ + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeEnter(p); + } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; @@ -87826,7 +95476,6 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); - p->pResultSet = 0; db->busyHandler.nBusy = 0; if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); @@ -87863,12 +95512,18 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( rc==SQLITE_OK ); assert( pOp>=aOp && pOp<&aOp[p->nOp]); -#ifdef VDBE_PROFILE - start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); -#endif nVmStep++; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; + +#if defined(VDBE_PROFILE) + pOp->nExec++; + pnCycle = &pOp->nCycle; + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; + *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -87930,7 +95585,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } } #endif -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG pOrigOp = pOp; #endif @@ -87986,8 +95641,8 @@ SQLITE_PRIVATE int sqlite3VdbeExec( case OP_Goto: { /* jump */ #ifdef SQLITE_DEBUG - /* In debuggging mode, when the p5 flags is set on an OP_Goto, that - ** means we should really jump back to the preceeding OP_ReleaseReg + /* In debugging mode, when the p5 flags is set on an OP_Goto, that + ** means we should really jump back to the preceding OP_ReleaseReg ** instruction. */ if( pOp->p5 ){ assert( pOp->p2 < (int)(pOp - aOp) ); @@ -88047,24 +95702,39 @@ case OP_Gosub: { /* jump */ pIn1->flags = MEM_Int; pIn1->u.i = (int)(pOp-aOp); REGISTER_TRACE(pOp->p1, pIn1); - - /* Most jump operations do a goto to this spot in order to update - ** the pOp pointer. */ -jump_to_p2: - pOp = &aOp[pOp->p2 - 1]; - break; + goto jump_to_p2_and_check_for_interrupt; } -/* Opcode: Return P1 * * * * +/* Opcode: Return P1 P2 P3 * * +** +** Jump to the address stored in register P1. If P1 is a return address +** register, then this accomplishes a return from a subroutine. ** -** Jump to the next instruction after the address in register P1. After -** the jump, register P1 becomes undefined. +** If P3 is 1, then the jump is only taken if register P1 holds an integer +** values, otherwise execution falls through to the next opcode, and the +** OP_Return becomes a no-op. If P3 is 0, then register P1 must hold an +** integer or else an assert() is raised. P3 should be set to 1 when +** this opcode is used in combination with OP_BeginSubrtn, and set to 0 +** otherwise. +** +** The value in register P1 is unchanged by this opcode. +** +** P2 is not used by the byte-code engine. However, if P2 is positive +** and also less than the current address, then the "EXPLAIN" output +** formatter in the CLI will indent all opcodes from the P2 opcode up +** to be not including the current Return. P2 should be the first opcode +** in the subroutine from which this opcode is returning. Thus the P2 +** value is a byte-code indentation hint. See tag-20220407a in +** wherecode.c and shell.c. */ case OP_Return: { /* in1 */ pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags==MEM_Int ); - pOp = &aOp[pIn1->u.i]; - pIn1->flags = MEM_Undefined; + if( pIn1->flags & MEM_Int ){ + if( pOp->p3 ){ VdbeBranchTaken(1, 2); } + pOp = &aOp[pIn1->u.i]; + }else if( ALWAYS(pOp->p3) ){ + VdbeBranchTaken(0, 2); + } break; } @@ -88079,7 +95749,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); @@ -88087,7 +95757,14 @@ case OP_InitCoroutine: { /* jump */ assert( !VdbeMemDynamic(pOut) ); pOut->u.i = pOp->p3 - 1; pOut->flags = MEM_Int; - if( pOp->p2 ) goto jump_to_p2; + if( pOp->p2==0 ) break; + + /* Most jump operations do a goto to this spot in order to update + ** the pOp pointer. */ +jump_to_p2: + assert( pOp->p2>0 ); /* There are never any jumps to instruction 0 */ + assert( pOp->p2nOp ); /* Jumps must be in range */ + pOp = &aOp[pOp->p2 - 1]; break; } @@ -88095,7 +95772,9 @@ case OP_InitCoroutine: { /* jump */ ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -88107,8 +95786,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -88125,7 +95804,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -88173,7 +95852,7 @@ case OP_HaltIfNull: { /* in3 */ ** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. ** ** 0: (no change) -** 1: NOT NULL contraint failed: P4 +** 1: NOT NULL constraint failed: P4 ** 2: UNIQUE constraint failed: P4 ** 3: CHECK constraint failed: P4 ** 4: FOREIGN KEY constraint failed: P4 @@ -88193,13 +95872,18 @@ case OP_Halt: { if( p->pSharedBlock!=NULL ){ p->pSharedBlock->xFinish(p->pSharedBlock->pContext, p->addedRows, p->totalRows); } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ - pcx = (int)(pOp - aOp); #ifdef SQLITE_DEBUG if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } #endif - if( pOp->p1==SQLITE_OK && p->pFrame ){ + + /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates + ** something is wrong with the code generator. Raise an assertion in order + ** to bring this to the attention of fuzzers and other testing tools. */ + assert( pOp->p1!=SQLITE_INTERNAL ); + + if( p->pFrame && pOp->p1==SQLITE_OK ){ /* Halt the sub-program. Return control to the parent frame. */ pFrame = p->pFrame; p->pFrame = pFrame->pParent; @@ -88221,7 +95905,6 @@ case OP_Halt: { } p->rc = pOp->p1; p->errorAction = (u8)pOp->p2; - p->pc = pcx; assert( pOp->p5<=4 ); if( p->rc ){ if( pOp->p5 ){ @@ -88238,6 +95921,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } + pcx = (int)(pOp - aOp); sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); } rc = sqlite3VdbeHalt(p); @@ -88363,6 +96047,28 @@ case OP_String: { /* out2 */ break; } +/* Opcode: BeginSubrtn * P2 * * * +** Synopsis: r[P2]=NULL +** +** Mark the beginning of a subroutine that can be entered in-line +** or that can be called using OP_Gosub. The subroutine should +** be terminated by an OP_Return instruction that has a P1 operand that +** is the same as the P2 operand to this opcode and that has P3 set to 1. +** If the subroutine is entered in-line, then the OP_Return will simply +** fall through. But if the subroutine is entered using OP_Gosub, then +** the OP_Return will jump back to the first instruction after the OP_Gosub. +** +** This routine works by loading a NULL into the P2 register. When the +** return address register contains a NULL, the OP_Return instruction is +** a no-op that simply falls through to the next instruction (assuming that +** the OP_Return opcode has a P3 value of 1). Thus if the subroutine is +** entered in-line, then the OP_Return will cause in-line execution to +** continue. But if the subroutine is entered via OP_Gosub, then the +** OP_Return will cause a return to the address following the OP_Gosub. +** +** This opcode is identical to OP_Null. It has a different name +** only to make the byte code easier to read and verify. +*/ /* Opcode: Null P1 P2 P3 * * ** Synopsis: r[P2..P3]=NULL ** @@ -88375,6 +96081,7 @@ case OP_String: { /* out2 */ ** NULL values will not compare equal even if SQLITE_NULLEQ is set on ** OP_Ne or OP_Eq. */ +case OP_BeginSubrtn: case OP_Null: { /* out2 */ int cnt; u16 nullFlag; @@ -88416,30 +96123,32 @@ case OP_SoftNull: { ** Synopsis: r[P2]=P4 (len=P1) ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. +** blob in register P2. If P4 is a NULL pointer, then construct +** a zero-filled blob that is P1 bytes long in P2. */ case OP_Blob: { /* out2 */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); pOut = out2Prerelease(p, pOp); - sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + if( pOp->p4.z==0 ){ + sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1); + if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem; + }else{ + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + } pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -88499,11 +96208,16 @@ case OP_Move: { break; } -/* Opcode: Copy P1 P2 P3 * * +/* Opcode: Copy P1 P2 P3 * P5 ** Synopsis: r[P2@P3+1]=r[P1@P3+1] ** ** Make a copy of registers P1..P1+P3 into registers P2..P2+P3. ** +** If the 0x0002 bit of P5 is set then also clear the MEM_Subtype flag in the +** destination. The 0x0001 bit of P5 indicates that this Copy opcode cannot +** be merged. The 0x0001 bit is used by the query planner and does not +** come into play during query execution. +** ** This instruction makes a deep copy of the value. A duplicate ** is made of any string or blob constant. See also OP_SCopy. */ @@ -88518,6 +96232,9 @@ case OP_Copy: { memAboutToChange(p, pOut); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); Deephemeralize(pOut); + if( (pOut->flags & MEM_Subtype)!=0 && (pOp->p5 & 0x0002)!=0 ){ + pOut->flags &= ~MEM_Subtype; + } #ifdef SQLITE_DEBUG pOut->pScopyFrom = 0; #endif @@ -88570,24 +96287,22 @@ case OP_IntCopy: { /* out2 */ break; } -/* Opcode: ChngCntRow P1 P2 * * * -** Synopsis: output=r[P1] +/* Opcode: FkCheck * * * * * ** -** Output value in register P1 as the chance count for a DML statement, -** due to the "PRAGMA count_changes=ON" setting. Or, if there was a -** foreign key error in the statement, trigger the error now. +** Halt with an SQLITE_CONSTRAINT error if there are any unresolved +** foreign key constraint violations. If there are no foreign key +** constraint violations, this is a no-op. ** -** This opcode is a variant of OP_ResultRow that checks the foreign key -** immediate constraint count and throws an error if the count is -** non-zero. The P2 opcode must be 1. +** FK constraint violations are also checked when the prepared statement +** exits. This opcode is used to raise foreign key constraint errors prior +** to returning results such as a row change count or the result of a +** RETURNING clause. */ -case OP_ChngCntRow: { - assert( pOp->p2==1 ); +case OP_FkCheck: { if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ goto abort_due_to_error; } - /* Fall through to the next case, OP_ResultRow */ - /* no break */ deliberate_fall_through + break; } /* Opcode: ResultRow P1 P2 * * * @@ -88600,8 +96315,6 @@ case OP_ChngCntRow: { ** the result row. */ case OP_ResultRow: { - Mem *pMem; - int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 || CORRUPT_DB ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); @@ -88613,7 +96326,7 @@ case OP_ResultRow: { if( p->totalRows<=p->startPos || p->blockFull ){ break; } - pMem = &aMem[pOp->p1]; + Mem *pMem = &aMem[pOp->p1]; rc = copySharedBlockRow(p, pOp, pMem, pCtx); if( rc==SQLITE_FULL && p->addedRows && (p->startPos + p->addedRows) <= p->pSharedBlock->requiredPos ){ p->startPos += p->addedRows; @@ -88641,41 +96354,30 @@ case OP_ResultRow: { } break; } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ - /* Invalidate all ephemeral cursor row caches */ p->cacheCtr = (p->cacheCtr + 2)|1; - - /* Make sure the results of the current row are \000 terminated - ** and have an assigned type. The results are de-ephemeralized as - ** a side effect. - */ - pMem = p->pResultSet = &aMem[pOp->p1]; - for(i=0; ip2; i++){ - assert( memIsValid(&pMem[i]) ); - Deephemeralize(&pMem[i]); - assert( (pMem[i].flags & MEM_Ephem)==0 - || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); - sqlite3VdbeMemNulTerminate(&pMem[i]); - REGISTER_TRACE(pOp->p1+i, &pMem[i]); + p->pResultRow = &aMem[pOp->p1]; #ifdef SQLITE_DEBUG - /* The registers in the result will not be used again when the - ** prepared statement restarts. This is because sqlite3_column() - ** APIs might have caused type conversions of made other changes to - ** the register values. Therefore, we can go ahead and break any - ** OP_SCopy dependencies. */ - pMem[i].pScopyFrom = 0; -#endif + { + Mem *pMem = p->pResultRow; + int i; + for(i=0; ip2; i++){ + assert( memIsValid(&pMem[i]) ); + REGISTER_TRACE(pOp->p1+i, &pMem[i]); + /* The registers in the result will not be used again when the + ** prepared statement restarts. This is because sqlite3_column() + ** APIs might have caused type conversions of made other changes to + ** the register values. Therefore, we can go ahead and break any + ** OP_SCopy dependencies. */ + pMem[i].pScopyFrom = 0; + } } +#endif if( db->mallocFailed ) goto no_mem; - if( db->mTrace & SQLITE_TRACE_ROW ){ db->trace.xV2(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); } - - - /* Return SQLITE_ROW - */ p->pc = (int)(pOp - aOp) + 1; rc = SQLITE_ROW; goto vdbe_return; @@ -88730,7 +96432,7 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - if( sqlite3VdbeMemGrow(pOut, (int)nByte+3, pOut==pIn2) ){ + if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } MemSetTypeFlag(pOut, MEM_Str); @@ -88742,9 +96444,9 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); pIn1->flags = flags1; + if( encoding>SQLITE_UTF8 ) nByte &= ~1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; - pOut->z[nByte+2] = 0; pOut->flags |= MEM_Term; pOut->n = (int)nByte; pOut->enc = encoding; @@ -88795,7 +96497,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ @@ -88804,12 +96505,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - type1 = numericType(pIn1); + type1 = pIn1->flags; pIn2 = &aMem[pOp->p2]; - type2 = numericType(pIn2); + type2 = pIn2->flags; pOut = &aMem[pOp->p3]; - flags = pIn1->flags | pIn2->flags; if( (type1 & type2 & MEM_Int)!=0 ){ +int_math: iA = pIn1->u.i; iB = pIn2->u.i; switch( pOp->opcode ){ @@ -88831,9 +96532,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); - }else if( (flags & MEM_Null)!=0 ){ + }else if( ((type1 | type2) & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ + type1 = numericType(pIn1); + type2 = numericType(pIn2); + if( (type1 & type2 & MEM_Int)!=0 ) goto int_math; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -88991,7 +96695,7 @@ case OP_AddImm: { /* in1 */ pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); sqlite3VdbeMemIntegerify(pIn1); - pIn1->u.i += pOp->p2; + *(u64*)&pIn1->u.i += (u64)pOp->p2; break; } @@ -89002,7 +96706,7 @@ case OP_AddImm: { /* in1 */ ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); @@ -89043,7 +96747,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** @@ -89186,26 +96890,28 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = pIn1->flags; flags3 = pIn3->flags; if( (flags1 & flags3 & MEM_Int)!=0 ){ - assert( (pOp->p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_TEXT || CORRUPT_DB ); /* Common case of comparison of two integers */ if( pIn3->u.i > pIn1->u.i ){ - iCompare = +1; if( sqlite3aGTb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } + iCompare = +1; + VVA_ONLY( iCompareIsInit = 1; ) }else if( pIn3->u.i < pIn1->u.i ){ - iCompare = -1; if( sqlite3aLTb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } + iCompare = -1; + VVA_ONLY( iCompareIsInit = 1; ) }else{ - iCompare = 0; if( sqlite3aEQb[pOp->opcode] ){ VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); goto jump_to_p2; } + iCompare = 0; + VVA_ONLY( iCompareIsInit = 1; ) } VdbeBranchTaken(0, (pOp->p5 & SQLITE_NULLEQ)?2:3); break; @@ -89232,11 +96938,12 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** then the result is always NULL. ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. */ - iCompare = 1; /* Operands are not equal */ VdbeBranchTaken(2,3); if( pOp->p5 & SQLITE_JUMPIFNULL ){ goto jump_to_p2; } + iCompare = 1; /* Operands are not equal */ + VVA_ONLY( iCompareIsInit = 1; ) break; } }else{ @@ -89247,15 +96954,17 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( (flags1 | flags3)&MEM_Str ){ if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); - testcase( flags3==pIn3->flags ); + assert( flags3==pIn3->flags || CORRUPT_DB ); flags3 = pIn3->flags; } if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } - }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); @@ -89264,7 +96973,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); @@ -89293,6 +97004,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ res2 = sqlite3aGTb[pOp->opcode]; } iCompare = res; + VVA_ONLY( iCompareIsInit = 1; ) /* Undo any changes made by applyAffinity() to the input registers. */ assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); @@ -89314,10 +97026,10 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** opcodes are allowed to occur between this instruction and the previous ** OP_Lt or OP_Gt. ** -** If result of an OP_Eq comparison on the same two operands as the -** prior OP_Lt or OP_Gt would have been true, then jump to P2. -** If the result of an OP_Eq comparison on the two previous -** operands would have been false or NULL, then fall through. +** If the result of an OP_Eq comparison on the same two operands as +** the prior OP_Lt or OP_Gt would have been true, then jump to P2. If +** the result of an OP_Eq comparison on the two previous operands +** would have been false or NULL, then fall through. */ case OP_ElseEq: { /* same as TK_ESCAPE, jump */ @@ -89331,6 +97043,7 @@ case OP_ElseEq: { /* same as TK_ESCAPE, jump */ break; } #endif /* SQLITE_DEBUG */ + assert( iCompareIsInit ); VdbeBranchTaken(iCompare==0, 2); if( iCompare==0 ) goto jump_to_p2; break; @@ -89342,9 +97055,8 @@ case OP_ElseEq: { /* same as TK_ESCAPE, jump */ ** Set the permutation used by the OP_Compare operator in the next ** instruction. The permutation is stored in the P4 operand. ** -** The permutation is only valid until the next OP_Compare that has -** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should -** occur immediately prior to the OP_Compare. +** The permutation is only valid for the next opcode which must be +** an OP_Compare that has the OPFLAG_PERMUTE bit set in P5. ** ** The first integer in the P4 integer array is the length of the array ** and does not become part of the permutation. @@ -89376,6 +97088,8 @@ case OP_Permutation: { ** The comparison is a sort comparison, so NULLs compare equal, ** NULLs are less than numbers, numbers are less than strings, ** and strings are less than blobs. +** +** This opcode must be immediately followed by an OP_Jump opcode. */ case OP_Compare: { int n; @@ -89424,6 +97138,7 @@ case OP_Compare: { pColl = pKeyInfo->aColl[i]; bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + VVA_ONLY( iCompareIsInit = 1; ) if( iCompare ){ if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) @@ -89434,14 +97149,17 @@ case OP_Compare: { break; } } + assert( pOp[1].opcode==OP_Jump ); break; } /* Opcode: Jump P1 P2 P3 * * ** ** Jump to the instruction at address P1, P2, or P3 depending on whether -** in the most recent OP_Compare instruction the P1 vector was less than +** in the most recent OP_Compare instruction the P1 vector was less than, ** equal to, or greater than the P2 vector, respectively. +** +** This opcode must immediately follow an OP_Compare opcode. */ case OP_Jump: { /* jump */ #ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION @@ -89454,7 +97172,9 @@ case OP_Jump: { /* jump */ } break; } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ + assert( pOp>aOp && pOp[-1].opcode==OP_Compare ); + assert( iCompareIsInit ); if( iCompare<0 ){ VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ @@ -89654,26 +97374,103 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ break; } -/* Opcode: IsNullOrType P1 P2 P3 * * -** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2 +/* Opcode: IsType P1 P2 P3 P4 P5 +** Synopsis: if typeof(P1.P3) in P5 goto P2 +** +** Jump to P2 if the type of a column in a btree is one of the types specified +** by the P5 bitmask. +** +** P1 is normally a cursor on a btree for which the row decode cache is +** valid through at least column P3. In other words, there should have been +** a prior OP_Column for column P3 or greater. If the cursor is not valid, +** then this opcode might give spurious results. +** The the btree row has fewer than P3 columns, then use P4 as the +** datatype. +** +** If P1 is -1, then P3 is a register number and the datatype is taken +** from the value in that register. +** +** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant +** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. +** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. +** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** +** Take the jump to address P2 if and only if the datatype of the +** value determined by P1 and P3 corresponds to one of the bits in the +** P5 bitmask. ** -** Jump to P2 if the value in register P1 is NULL or has a datatype P3. -** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT, -** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT. */ -case OP_IsNullOrType: { /* jump, in1 */ - int doTheJump; - pIn1 = &aMem[pOp->p1]; - doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3; - VdbeBranchTaken( doTheJump, 2); - if( doTheJump ) goto jump_to_p2; +case OP_IsType: { /* jump */ + VdbeCursor *pC; + u16 typeMask; + u32 serialType; + + assert( pOp->p1>=(-1) && pOp->p1nCursor ); + assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) ); + if( pOp->p1>=0 ){ + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=0 ); + if( pOp->p3nHdrParsed ){ + serialType = pC->aType[pOp->p3]; + if( serialType>=12 ){ + if( serialType&1 ){ + typeMask = 0x04; /* SQLITE_TEXT */ + }else{ + typeMask = 0x08; /* SQLITE_BLOB */ + } + }else{ + static const unsigned char aMask[] = { + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2, + 0x01, 0x01, 0x10, 0x10 + }; + testcase( serialType==0 ); + testcase( serialType==1 ); + testcase( serialType==2 ); + testcase( serialType==3 ); + testcase( serialType==4 ); + testcase( serialType==5 ); + testcase( serialType==6 ); + testcase( serialType==7 ); + testcase( serialType==8 ); + testcase( serialType==9 ); + testcase( serialType==10 ); + testcase( serialType==11 ); + typeMask = aMask[serialType]; + } + }else{ + typeMask = 1 << (pOp->p4.i - 1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + }else{ + assert( memIsValid(&aMem[pOp->p3]) ); + typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2); + if( typeMask & pOp->p5 ){ + goto jump_to_p2; + } break; } /* Opcode: ZeroOrNull P1 P2 P3 * * ** Synopsis: r[P2] = 0 OR NULL ** -** If all both registers P1 and P3 are NOT NULL, then store a zero in +** If both registers P1 and P3 are NOT NULL, then store a zero in ** register P2. If either registers P1 or P3 are NULL then put ** a NULL in register P2. */ @@ -89709,11 +97506,14 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ ** If it is, then set register P3 to NULL and jump immediately to P2. ** If P1 is not on a NULL row, then fall through without making any ** changes. +** +** If P1 is not an open cursor, then this opcode is a no-op. */ case OP_IfNullRow: { /* jump */ + VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( p->apCsr[pOp->p1]!=0 ); - if( p->apCsr[pOp->p1]->nullRow ){ + pC = p->apCsr[pOp->p1]; + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } @@ -89741,22 +97541,30 @@ case OP_Offset: { /* out3 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; pOut = &p->aMem[pOp->p3]; - if( NEVER(pC==0) || pC->eCurType!=CURTYPE_BTREE ){ + if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){ sqlite3VdbeMemSetNull(pOut); }else{ - sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + if( sqlite3BtreeEof(pC->uc.pCursor) ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + } } break; } #endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* Opcode: Column P1 P2 P3 P4 P5 -** Synopsis: r[P3]=PX +** Synopsis: r[P3]=PX cursor P1 column P2 ** ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional ** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) +** from this record. If there are less than (P2+1) ** values in the record, extract a NULL. ** ** The value extracted is stored in register P3. @@ -89765,15 +97573,17 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then -** the result is guaranteed to only be used as the argument of a length() -** or typeof() function, respectively. The loading of large blobs can be -** skipped for length() and all content loading can be skipped for typeof(). +** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed +** to only be used by the length() function or the equivalent. The content +** of large blobs is not loaded, thus saving CPU cycles. If the +** OPFLAG_TYPEOFARG bit is set then the result will only be used by the +** typeof() function or the IS NULL or IS NOT NULL operators or the +** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { +case OP_Column: { /* ncycle */ u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ - BtCursor *pCrsr; /* The BTree cursor */ + BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ @@ -89787,43 +97597,53 @@ case OP_Column: { Mem *pReg; /* PseudoTable input register */ assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); p2 = (u32)pOp->p2; - /* If the cursor cache is stale (meaning it is not currently point at - ** the correct row) then bring it up-to-date by doing the necessary - ** B-Tree seek. */ - rc = sqlite3VdbeCursorMoveto(&pC, &p2); - if( rc ) goto abort_due_to_error; - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pDest = &aMem[pOp->p3]; - memAboutToChange(p, pDest); +op_column_restart: assert( pC!=0 ); - assert( p2<(u32)pC->nField ); + assert( p2<(u32)pC->nField + || (pC->eCurType==CURTYPE_PSEUDO && pC->seekResult==0) ); aOffset = pC->aOffset; + assert( aOffset==pC->aType+pC->nField ); assert( pC->eCurType!=CURTYPE_VTAB ); assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); assert( pC->eCurType!=CURTYPE_SORTER ); if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ if( pC->nullRow ){ - if( pC->eCurType==CURTYPE_PSEUDO ){ + if( pC->eCurType==CURTYPE_PSEUDO && pC->seekResult>0 ){ /* For the special case of as pseudo-cursor, the seekResult field ** identifies the register that holds the record */ - assert( pC->seekResult>0 ); pReg = &aMem[pC->seekResult]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = pReg->n; pC->aRow = (u8*)pReg->z; }else{ + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ pCrsr = pC->uc.pCursor; + if( pC->deferredMoveto ){ + u32 iMap; + assert( !pC->isEphemeral ); + if( pC->ub.aAltMap && (iMap = pC->ub.aAltMap[1+p2])>0 ){ + pC = pC->pAltCursor; + p2 = iMap - 1; + goto op_column_restart; + } + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + }else if( sqlite3BtreeCursorHasMoved(pCrsr) ){ + rc = sqlite3VdbeHandleMovedCursor(pC); + if( rc ) goto abort_due_to_error; + goto op_column_restart; + } assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -89831,15 +97651,15 @@ case OP_Column: { pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &pC->szRow); assert( pC->szRow<=pC->payloadSize ); assert( pC->szRow<=65536 ); /* Maximum page size is 64KiB */ - if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } } pC->cacheStatus = p->cacheCtr; - pC->iHdrOffset = getVarint32(pC->aRow, aOffset[0]); + if( (aOffset[0] = pC->aRow[0])<0x80 ){ + pC->iHdrOffset = 1; + }else{ + pC->iHdrOffset = sqlite3GetVarint32(pC->aRow, aOffset); + } pC->nHdrParsed = 0; - if( pC->szRowaRow does not have to hold the entire row, but it does at least ** need to cover the header of the record. If pC->aRow does not contain @@ -89879,6 +97699,10 @@ case OP_Column: { testcase( aOffset[0]==0 ); goto op_column_read_header; } + }else if( sqlite3BtreeCursorHasMoved(pC->uc.pCursor) ){ + rc = sqlite3VdbeHandleMovedCursor(pC); + if( rc ) goto abort_due_to_error; + goto op_column_restart; } /* Make sure at least the first p2+1 entries of the header have been @@ -89947,6 +97771,8 @@ case OP_Column: { ** columns. So the result will be either the default value or a NULL. */ if( pC->nHdrParsed<=p2 ){ + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ @@ -89964,6 +97790,8 @@ case OP_Column: { */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ){ sqlite3VdbeMemSetNull(pDest); @@ -89984,6 +97812,7 @@ case OP_Column: { pDest->n = len = (t-12)/2; pDest->enc = encoding; if( pDest->szMalloc < len+2 ){ + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; pDest->flags = MEM_Null; if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; }else{ @@ -89995,11 +97824,16 @@ case OP_Column: { pDest->flags = aFlag[t&1]; } }else{ + u8 p5; pDest->enc = encoding; + assert( pDest->db==db ); /* This branch happens only when content is on overflow pages */ - if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 - && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) - || (len = sqlite3VdbeSerialTypeLen(t))==0 + if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 + && (p5==OPFLAG_TYPEOFARG + || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG)) + ) + ) + || sqlite3VdbeSerialTypeLen(t)==0 ){ /* Content is irrelevant for ** 1. the typeof() function, @@ -90016,10 +97850,13 @@ case OP_Column: { */ sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); }else{ - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - pDest->flags &= ~MEM_Ephem; + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], + p->cacheCtr, colCacheCtr, pDest); + if( rc ){ + if( rc==SQLITE_NOMEM ) goto no_mem; + if( rc==SQLITE_TOOBIG ) goto too_big; + goto abort_due_to_error; + } } } @@ -90098,6 +97935,8 @@ case OP_TypeCheck: { break; } case COLTYPE_REAL: { + testcase( (pIn1->flags & (MEM_Real|MEM_IntReal))==MEM_Real ); + assert( (pIn1->flags & MEM_IntReal)==0 ); if( pIn1->flags & MEM_Int ){ /* When applying REAL affinity, if the result is still an MEM_Int ** that will fit in 6 bytes, then change the type to MEM_IntReal @@ -90115,7 +97954,7 @@ case OP_TypeCheck: { pIn1->flags |= MEM_Real; pIn1->flags &= ~MEM_Int; } - }else if( (pIn1->flags & MEM_Real)==0 ){ + }else if( (pIn1->flags & (MEM_Real|MEM_IntReal))==0 ){ goto vdbe_type_error; } break; @@ -90176,7 +98015,7 @@ case OP_Affinity: { }else{ pIn1->u.r = (double)pIn1->u.i; pIn1->flags |= MEM_Real; - pIn1->flags &= ~MEM_Int; + pIn1->flags &= ~(MEM_Int|MEM_Str); } } REGISTER_TRACE((int)(pIn1-aMem), pIn1); @@ -90226,7 +98065,6 @@ case OP_MakeRecord: { Mem *pLast; /* Last field of the record */ int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ - int file_format; /* File format to use for encoding */ u32 len; /* Length of a field */ u8 *zHdr; /* Where to write next byte of the header */ u8 *zPayload; /* Where to write next byte of the payload */ @@ -90255,7 +98093,6 @@ case OP_MakeRecord: { pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; - file_format = p->minWriteFileFormat; /* Identify the output register */ assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); @@ -90354,10 +98191,10 @@ case OP_MakeRecord: { testcase( uu==127 ); testcase( uu==128 ); testcase( uu==32767 ); testcase( uu==32768 ); testcase( uu==8388607 ); testcase( uu==8388608 ); - testcase( uu==2147483647 ); testcase( uu==2147483648 ); + testcase( uu==2147483647 ); testcase( uu==2147483648LL ); testcase( uu==140737488355327LL ); testcase( uu==140737488355328LL ); if( uu<=127 ){ - if( (i&1)==i && file_format>=4 ){ + if( (i&1)==i && p->minWriteFileFormat>=4 ){ pRec->uTemp = 8+(u32)uu; }else{ nData++; @@ -90462,18 +98299,69 @@ case OP_MakeRecord: { zPayload = zHdr + nHdr; /* Write the record */ - zHdr += putVarint32(zHdr, nHdr); + if( nHdr<0x80 ){ + *(zHdr++) = nHdr; + }else{ + zHdr += sqlite3PutVarint(zHdr,nHdr); + } assert( pData0<=pLast ); pRec = pData0; - do{ + while( 1 /*exit-by-break*/ ){ serial_type = pRec->uTemp; /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more - ** additional varints, one per column. */ - zHdr += putVarint32(zHdr, serial_type); /* serial type */ - /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** additional varints, one per column. + ** EVIDENCE-OF: R-64536-51728 The values for each column in the record ** immediately follow the header. */ - zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ - }while( (++pRec)<=pLast ); + if( serial_type<=7 ){ + *(zHdr++) = serial_type; + if( serial_type==0 ){ + /* NULL value. No change in zPayload */ + }else{ + u64 v; + if( serial_type==7 ){ + assert( sizeof(v)==sizeof(pRec->u.r) ); + memcpy(&v, &pRec->u.r, sizeof(v)); + swapMixedEndianFloat(v); + }else{ + v = pRec->u.i; + } + len = sqlite3SmallTypeSizes[serial_type]; + assert( len>=1 && len<=8 && len!=5 && len!=7 ); + switch( len ){ + default: zPayload[7] = (u8)(v&0xff); v >>= 8; + zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; + zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 1: zPayload[0] = (u8)(v&0xff); + } + zPayload += len; + } + }else if( serial_type<0x80 ){ + *(zHdr++) = serial_type; + if( serial_type>=14 && pRec->n>0 ){ + assert( pRec->z!=0 ); + memcpy(zPayload, pRec->z, pRec->n); + zPayload += pRec->n; + } + }else{ + zHdr += sqlite3PutVarint(zHdr, serial_type); + if( pRec->n ){ + assert( pRec->z!=0 ); + memcpy(zPayload, pRec->z, pRec->n); + zPayload += pRec->n; + } + } + if( pRec==pLast ) break; + pRec++; + } assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); assert( nByte==(int)(zPayload - (u8*)pOut->z) ); @@ -90482,7 +98370,7 @@ case OP_MakeRecord: { break; } -/* Opcode: Count P1 P2 p3 * * +/* Opcode: Count P1 P2 P3 * * ** Synopsis: r[P2]=count() ** ** Store the number of entries (an integer value) in the table or index @@ -90692,7 +98580,10 @@ case OP_Savepoint: { } } if( rc ) goto abort_due_to_error; - + if( p->eVdbeState==VDBE_HALT_STATE ){ + rc = SQLITE_DONE; + goto vdbe_return; + } break; } @@ -90796,6 +98687,7 @@ case OP_AutoCommit: { */ case OP_Transaction: { Btree *pBt; + Db *pDb; int iMeta = 0; assert( p->bIsReader ); @@ -90815,7 +98707,8 @@ case OP_Transaction: { } goto abort_due_to_error; } - pBt = db->aDb[pOp->p1].pBt; + pDb = &db->aDb[pOp->p1]; + pBt = pDb->pBt; if( pBt ){ rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta); @@ -90856,8 +98749,7 @@ case OP_Transaction: { assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); if( rc==SQLITE_OK && pOp->p5 - && (iMeta!=pOp->p3 - || db->aDb[pOp->p1].pSchema->iGeneration!=pOp->p4.i) + && (iMeta!=pOp->p3 || pDb->pSchema->iGeneration!=pOp->p4.i) ){ /* ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema @@ -90884,6 +98776,11 @@ case OP_Transaction: { } p->expired = 1; rc = SQLITE_SCHEMA; + + /* Set changeCntOn to 0 to prevent the value returned by sqlite3_changes() + ** from being modified in sqlite3VdbeHalt(). If this statement is + ** reprepared, changeCntOn will be set again. */ + p->changeCntOn = 0; } if( rc ) goto abort_due_to_error; break; @@ -90950,7 +98847,7 @@ case OP_SetCookie: { rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = pOp->p3 - pOp->p5; + *(u32*)&pDb->pSchema->schema_cookie = *(u32*)&pOp->p3 - pOp->p5; db->mDbFlags |= DBFLAG_SchemaChange; sqlite3FkClearTriggerCache(db, pOp->p1); }else if( pOp->p2==BTREE_FILE_FORMAT ){ @@ -91051,7 +98948,7 @@ case OP_SetCookie: { ** ** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; u32 p2; @@ -91072,7 +98969,7 @@ case OP_ReopenIdx: { } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); @@ -91130,8 +99027,9 @@ case OP_OpenWrite: assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ - pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); + pCur = allocateCursor(p, pOp->p1, nField, CURTYPE_BTREE); if( pCur==0 ) goto no_mem; + pCur->iDb = iDb; pCur->nullRow = 1; pCur->isOrdered = 1; pCur->pgnoRoot = p2; @@ -91165,7 +99063,7 @@ open_cursor_set_hints: ** ** Duplicate ephemeral cursors are used for self-joins of materialized views. */ -case OP_OpenDup: { +case OP_OpenDup: { /* ncycle */ VdbeCursor *pOrig; /* The original cursor to be duplicated */ VdbeCursor *pCx; /* The new cursor */ @@ -91173,7 +99071,7 @@ case OP_OpenDup: { assert( pOrig ); assert( pOrig->isEphemeral ); /* Only ephemeral cursors can be duplicated */ - pCx = allocateCursor(p, pOp->p1, pOrig->nField, -1, CURTYPE_BTREE); + pCx = allocateCursor(p, pOp->p1, pOrig->nField, CURTYPE_BTREE); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->isEphemeral = 1; @@ -91181,10 +99079,10 @@ case OP_OpenDup: { pCx->isTable = pOrig->isTable; pCx->pgnoRoot = pOrig->pgnoRoot; pCx->isOrdered = pOrig->isOrdered; - pCx->pBtx = pOrig->pBtx; - pCx->hasBeenDuped = 1; - pOrig->hasBeenDuped = 1; - rc = sqlite3BtreeCursor(pCx->pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pCx->ub.pBtx = pOrig->ub.pBtx; + pCx->noReuse = 1; + pOrig->noReuse = 1; + rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR, pCx->pKeyInfo, pCx->uc.pCursor); /* The sqlite3BtreeCursor() routine can only fail for the first cursor ** opened for a database. Since there is already an open cursor when this @@ -91227,8 +99125,8 @@ case OP_OpenDup: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; @@ -91250,23 +99148,23 @@ case OP_OpenEphemeral: { aMem[pOp->p3].z = ""; } pCx = p->apCsr[pOp->p1]; - if( pCx && !pCx->hasBeenDuped && ALWAYS(pOp->p2<=pCx->nField) ){ - /* If the ephermeral table is already open and has no duplicates from + if( pCx && !pCx->noReuse && ALWAYS(pOp->p2<=pCx->nField) ){ + /* If the ephemeral table is already open and has no duplicates from ** OP_OpenDup, then erase all existing content so that the table is ** empty again, rather than creating a new table. */ assert( pCx->isEphemeral ); pCx->seqCount = 0; pCx->cacheStatus = CACHE_STALE; - rc = sqlite3BtreeClearTable(pCx->pBtx, pCx->pgnoRoot, 0); + rc = sqlite3BtreeClearTable(pCx->ub.pBtx, pCx->pgnoRoot, 0); }else{ - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); + pCx = allocateCursor(p, pOp->p1, pOp->p2, CURTYPE_BTREE); if( pCx==0 ) goto no_mem; pCx->isEphemeral = 1; - rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->ub.pBtx, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1, 0); + rc = sqlite3BtreeBeginTrans(pCx->ub.pBtx, 1, 0); if( rc==SQLITE_OK ){ /* If a transient index is required, create it by calling ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before @@ -91275,26 +99173,26 @@ case OP_OpenEphemeral: { */ if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBtx, &pCx->pgnoRoot, + rc = sqlite3BtreeCreateTable(pCx->ub.pBtx, &pCx->pgnoRoot, BTREE_BLOBKEY | pOp->p5); if( rc==SQLITE_OK ){ assert( pCx->pgnoRoot==SCHEMA_ROOT+1 ); assert( pKeyInfo->db==db ); assert( pKeyInfo->enc==ENC(db) ); - rc = sqlite3BtreeCursor(pCx->pBtx, pCx->pgnoRoot, BTREE_WRCSR, + rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR, pKeyInfo, pCx->uc.pCursor); } pCx->isTable = 0; }else{ pCx->pgnoRoot = SCHEMA_ROOT; - rc = sqlite3BtreeCursor(pCx->pBtx, SCHEMA_ROOT, BTREE_WRCSR, + rc = sqlite3BtreeCursor(pCx->ub.pBtx, SCHEMA_ROOT, BTREE_WRCSR, 0, pCx->uc.pCursor); pCx->isTable = 1; } } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); if( rc ){ - sqlite3BtreeClose(pCx->pBtx); + sqlite3BtreeClose(pCx->ub.pBtx); } } } @@ -91318,7 +99216,7 @@ case OP_SorterOpen: { assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); + pCx = allocateCursor(p, pOp->p1, pOp->p2, CURTYPE_SORTER); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); @@ -91360,14 +99258,15 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); + pCx = allocateCursor(p, pOp->p1, pOp->p3, CURTYPE_PSEUDO); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; pCx->seekResult = pOp->p2; @@ -91386,7 +99285,7 @@ case OP_OpenPseudo: { ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; @@ -91503,10 +99402,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group */ -case OP_SeekLE: /* jump, in3, group */ -case OP_SeekGE: /* jump, in3, group */ -case OP_SeekGT: { /* jump, in3, group */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -91635,7 +99534,13 @@ case OP_SeekGT: { /* jump, in3, group */ r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; i0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]); + } + } #endif r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); @@ -91698,7 +99603,7 @@ seek_not_found: } -/* Opcode: SeekScan P1 P2 * * * +/* Opcode: SeekScan P1 P2 * * P5 ** Synopsis: Scan-ahead up to P1 rows ** ** This opcode is a prefix opcode to OP_SeekGE. In other words, this @@ -91708,8 +99613,8 @@ seek_not_found: ** This opcode uses the P1 through P4 operands of the subsequent ** OP_SeekGE. In the text that follows, the operands of the subsequent ** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only -** the P1 and P2 operands of this opcode are also used, and are called -** This.P1 and This.P2. +** the P1, P2 and P5 operands of this opcode are also used, and are called +** This.P1, This.P2 and This.P5. ** ** This opcode helps to optimize IN operators on a multi-column index ** where the IN operator is on the later terms of the index by avoiding @@ -91719,32 +99624,54 @@ seek_not_found: ** ** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which ** is the desired entry that we want the cursor SeekGE.P1 to be pointing -** to. Call this SeekGE.P4/P5 row the "target". +** to. Call this SeekGE.P3/P4 row the "target". ** ** If the SeekGE.P1 cursor is not currently pointing to a valid row, ** then this opcode is a no-op and control passes through into the OP_SeekGE. ** ** If the SeekGE.P1 cursor is pointing to a valid row, then that row ** might be the target row, or it might be near and slightly before the -** target row. This opcode attempts to position the cursor on the target -** row by, perhaps by invoking sqlite3BtreeStep() on the cursor -** between 0 and This.P1 times. -** -** There are three possible outcomes from this opcode:
      -** -**
    1. If after This.P1 steps, the cursor is still pointing to a place that -** is earlier in the btree than the target row, then fall through -** into the subsquence OP_SeekGE opcode. -** -**
    2. If the cursor is successfully moved to the target row by 0 or more -** sqlite3BtreeNext() calls, then jump to This.P2, which will land just -** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. -** -**
    3. If the cursor ends up past the target row (indicating the the target -** row does not exist in the btree) then jump to SeekOP.P2. +** target row, or it might be after the target row. If the cursor is +** currently before the target row, then this opcode attempts to position +** the cursor on or after the target row by invoking sqlite3BtreeStep() +** on the cursor between 1 and This.P1 times. +** +** The This.P5 parameter is a flag that indicates what to do if the +** cursor ends up pointing at a valid row that is past the target +** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If +** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 +** case occurs when there are no inequality constraints to the right of +** the IN constraint. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** occurs when there are inequality constraints to the right of the IN +** operator. In that case, the This.P2 will point either directly to or +** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for +** loop terminate. +** +** Possible outcomes from this opcode:
        +** +**
      1. If the cursor is initially not pointed to any valid row, then +** fall through into the subsequent OP_SeekGE opcode. +** +**
      2. If the cursor is left pointing to a row that is before the target +** row, even after making as many as This.P1 calls to +** sqlite3BtreeNext(), then also fall through into OP_SeekGE. +** +**
      3. If the cursor is left pointing at the target row, either because it +** was at the target row to begin with or because one or more +** sqlite3BtreeNext() calls moved the cursor to the target row, +** then jump to This.P2.., +** +**
      4. If the cursor started out before the target row and a call to +** to sqlite3BtreeNext() moved the cursor off the end of the index +** (indicating that the target row definitely does not exist in the +** btree) then jump to SeekGE.P2, ending the loop. +** +**
      5. If the cursor ends up on a valid row that is past the target row +** (indicating that the target row does not exist in the btree) then +** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. **
      */ -case OP_SeekScan: { +case OP_SeekScan: { /* ncycle */ VdbeCursor *pC; int res; int nStep; @@ -91752,14 +99679,25 @@ case OP_SeekScan: { assert( pOp[1].opcode==OP_SeekGE ); - /* pOp->p2 points to the first instruction past the OP_IdxGT that - ** follows the OP_SeekGE. */ + /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the + ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first + ** opcode past the OP_SeekGE itself. */ assert( pOp->p2>=(int)(pOp-aOp)+2 ); - assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE ); - testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); - assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); - assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); - assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); +#ifdef SQLITE_DEBUG + if( pOp->p5==0 ){ + /* There are no inequality constraints following the IN constraint. */ + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT + || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + }else{ + /* There are inequality constraints. */ + assert( pOp->p2==(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_SeekGE ); + } +#endif assert( pOp->p1>0 ); pC = p->apCsr[pOp[1].p1]; @@ -91793,8 +99731,9 @@ case OP_SeekScan: { while(1){ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); if( rc ) goto abort_due_to_error; - if( res>0 ){ + if( res>0 && pOp->p5==0 ){ seekscan_search_fail: + /* Jump to SeekGE.P2, ending the loop */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then skip\n", pOp->p1 - nStep); @@ -91804,7 +99743,8 @@ case OP_SeekScan: { pOp++; goto jump_to_p2; } - if( res==0 ){ + if( res>=0 ){ + /* Jump to This.P2, bypassing the OP_SeekGE opcode */ #ifdef SQLITE_DEBUG if( db->flags&SQLITE_VdbeTrace ){ printf("... %d steps and then success\n", pOp->p1 - nStep); @@ -91824,6 +99764,7 @@ case OP_SeekScan: { break; } nStep--; + pC->cacheStatus = CACHE_STALE; rc = sqlite3BtreeNext(pC->uc.pCursor, 0); if( rc ){ if( rc==SQLITE_DONE ){ @@ -91853,7 +99794,7 @@ case OP_SeekScan: { ** ** P1 must be a valid b-tree cursor. */ -case OP_SeekHit: { +case OP_SeekHit: { /* ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -91880,12 +99821,16 @@ case OP_SeekHit: { /* Opcode: IfNotOpen P1 P2 * * * ** Synopsis: if( !csr[P1] ) goto P2 ** -** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +** If cursor P1 is not open or if P1 is set to a NULL row using the +** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through. */ case OP_IfNotOpen: { /* jump */ + VdbeCursor *pCur; + assert( pOp->p1>=0 && pOp->p1nCursor ); - VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); - if( !p->apCsr[pOp->p1] ){ + pCur = p->apCsr[pOp->p1]; + VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); + if( pCur==0 || pCur->nullRow ){ goto jump_to_p2_and_check_for_interrupt; } break; @@ -91936,13 +99881,13 @@ case OP_IfNotOpen: { /* jump */ ** operands to OP_NotFound and OP_IdxGT. ** ** This opcode is an optimization attempt only. If this opcode always -** falls through, the correct answer is still obtained, but extra works +** falls through, the correct answer is still obtained, but extra work ** is performed. ** ** A value of N in the seekHit flag of cursor P1 means that there exists ** a key P3:N that will match some record in the index. We want to know ** if it is possible for a record P3:P4 to match some record in the -** index. If it is not possible, we can skips some work. So if seekHit +** index. If it is not possible, we can skip some work. So if seekHit ** is less than P4, attempt to find out if a match is possible by running ** OP_NotFound. ** @@ -91981,7 +99926,7 @@ case OP_IfNotOpen: { /* jump */ ** ** See also: NotFound, Found, NotExists */ -case OP_IfNoHope: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -91995,15 +99940,12 @@ case OP_IfNoHope: { /* jump, in3 */ /* Fall through into OP_NotFound */ /* no break */ deliberate_fall_through } -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; - int takeJump; int ii; VdbeCursor *pC; - int res; - UnpackedRecord *pFree; UnpackedRecord *pIdxKey; UnpackedRecord r; @@ -92018,14 +99960,15 @@ case OP_Found: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif - pIn3 = &aMem[pOp->p3]; + r.aMem = &aMem[pOp->p3]; assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); - if( pOp->p4.i>0 ){ + r.nField = (u16)pOp->p4.i; + if( r.nField>0 ){ + /* Key values in an array of registers */ r.pKeyInfo = pC->pKeyInfo; - r.nField = (u16)pOp->p4.i; - r.aMem = pIn3; + r.default_rc = 0; #ifdef SQLITE_DEBUG for(ii=0; iip3+ii, &r.aMem[ii]); } #endif - pIdxKey = &r; - pFree = 0; + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &pC->seekResult); }else{ - assert( pIn3->flags & MEM_Blob ); - rc = ExpandBlob(pIn3); + /* Composite key generated by OP_MakeRecord */ + assert( r.aMem->flags & MEM_Blob ); + assert( pOp->opcode!=OP_NoConflict ); + rc = ExpandBlob(r.aMem); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc ) goto no_mem; - pFree = pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); if( pIdxKey==0 ) goto no_mem; - sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); + sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + pIdxKey->default_rc = 0; + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); + sqlite3DbFreeNN(db, pIdxKey); } - pIdxKey->default_rc = 0; - takeJump = 0; - if( pOp->opcode==OP_NoConflict ){ - /* For the OP_NoConflict opcode, take the jump if any of the - ** input fields are NULL, since any key with a NULL will not - ** conflict */ - for(ii=0; iinField; ii++){ - if( pIdxKey->aMem[ii].flags & MEM_Null ){ - takeJump = 1; - break; - } - } - } - rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &res); - if( pFree ) sqlite3DbFreeNN(db, pFree); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pC->seekResult = res; - alreadyExists = (res==0); + alreadyExists = (pC->seekResult==0); pC->nullRow = 1-alreadyExists; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -92071,9 +100002,25 @@ case OP_Found: { /* jump, in3 */ VdbeBranchTaken(alreadyExists!=0,2); if( alreadyExists ) goto jump_to_p2; }else{ - VdbeBranchTaken(takeJump||alreadyExists==0,2); - if( takeJump || !alreadyExists ) goto jump_to_p2; - if( pOp->opcode==OP_IfNoHope ) pC->seekHit = pOp->p4.i; + if( !alreadyExists ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + } + if( pOp->opcode==OP_NoConflict ){ + /* For the OP_NoConflict opcode, take the jump if any of the + ** input fields are NULL, since any key with a NULL will not + ** conflict */ + for(ii=0; iiopcode==OP_IfNoHope ){ + pC->seekHit = pOp->p4.i; + } } break; } @@ -92125,7 +100072,7 @@ case OP_Found: { /* jump, in3 */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3 */ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -92150,7 +100097,7 @@ case OP_SeekRowid: { /* jump, in3 */ } /* Fall through into OP_NotExists */ /* no break */ deliberate_fall_through -case OP_NotExists: /* jump, in3 */ +case OP_NotExists: /* jump, in3, ncycle */ pIn3 = &aMem[pOp->p3]; assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -92430,8 +100377,11 @@ case OP_Insert: { if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pOp->p5 & OPFLAG_LASTROWID)==0 || (pOp->p5 & OPFLAG_NCHANGE)!=0 ); + if( pOp->p5 & OPFLAG_NCHANGE ){ + p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + } assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); x.pData = pData->z; x.nData = pData->n; @@ -92442,12 +100392,14 @@ case OP_Insert: { x.nZero = 0; } x.pKey = 0; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), seekResult ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; + colCacheCtr++; /* Invoke the update-hook if required. */ if( rc ) goto abort_due_to_error; @@ -92501,13 +100453,18 @@ case OP_RowCell: { ** left in an undefined state. ** ** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this -** delete one of several associated with deleting a table row and all its -** associated index entries. Exactly one of those deletes is the "primary" -** delete. The others are all on OPFLAG_FORDELETE cursors or else are -** marked with the AUXDELETE flag. +** delete is one of several associated with deleting a table row and +** all its associated index entries. Exactly one of those deletes is +** the "primary" delete. The others are all on OPFLAG_FORDELETE +** cursors or else are marked with the AUXDELETE flag. +** +** If the OPFLAG_NCHANGE (0x01) flag of P2 (NB: P2 not P5) is set, then +** the row change count is incremented (otherwise not). ** -** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row -** change count is incremented (otherwise not). +** If the OPFLAG_ISNOOP (0x40) flag of P2 (not P5!) is set, then the +** pre-update-hook for deletes is run, but the btree is otherwise unchanged. +** This happens when the OP_Delete is to be shortly followed by an OP_Insert +** with the same key, causing the btree entry to be overwritten. ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. @@ -92608,6 +100565,7 @@ case OP_Delete: { rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; + colCacheCtr++; pC->seekResult = 0; if( rc ) goto abort_due_to_error; @@ -92675,13 +100633,13 @@ case OP_SorterCompare: { ** Write into register P2 the current sorter data for sorter cursor P1. ** Then clear the column header cache on cursor P3. ** -** This opcode is normally use to move a record out of the sorter and into +** This opcode is normally used to move a record out of the sorter and into ** a register that is the source for a pseudo-table cursor created using ** OpenPseudo. That pseudo-table cursor is the one that is identified by ** parameter P3. Clearing the P3 column cache as part of this opcode saves ** us from having to issue a separate NullRow instruction to clear that cache. */ -case OP_SorterData: { +case OP_SorterData: { /* ncycle */ VdbeCursor *pC; pOut = &aMem[pOp->p2]; @@ -92764,7 +100722,7 @@ case OP_RowData: { } /* Opcode: Rowid P1 P2 * * * -** Synopsis: r[P2]=rowid +** Synopsis: r[P2]=PX rowid of P1 ** ** Store in register P2 an integer which is the key of the table entry that ** P1 is currently point to. @@ -92773,7 +100731,7 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; @@ -92819,13 +100777,25 @@ case OP_Rowid: { /* out2 */ ** Move the cursor P1 to a null row. Any OP_Column operations ** that occur while the cursor is on the null row will always ** write a NULL. +** +** If cursor P1 is not previously opened, open it now to a special +** pseudo-cursor that always returns NULL for every column. */ case OP_NullRow: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); + if( pC==0 ){ + /* If the cursor is not already open, create a special kind of + ** pseudo-cursor that always gives null rows. */ + pC = allocateCursor(p, pOp->p1, 1, CURTYPE_PSEUDO); + if( pC==0 ) goto no_mem; + pC->seekResult = 0; + pC->isTable = 1; + pC->noReuse = 1; + pC->uc.pCursor = sqlite3BtreeFakeValidCursor(); + } pC->nullRow = 1; pC->cacheStatus = CACHE_STALE; if( pC->eCurType==CURTYPE_BTREE ){ @@ -92860,8 +100830,8 @@ case OP_NullRow: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_SeekEnd: -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -92895,28 +100865,38 @@ case OP_Last: { /* jump */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; @@ -92944,8 +100924,8 @@ case OP_IfSmaller: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ -case OP_SorterSort: /* jump */ -case OP_Sort: { /* jump */ +case OP_SorterSort: /* jump ncycle */ +case OP_Sort: { /* jump ncycle */ #ifdef SQLITE_TEST sqlite3_sort_count++; sqlite3_search_count--; @@ -92962,17 +100942,22 @@ case OP_Sort: { /* jump */ ** If the table or index is not empty, fall through to the following ** instruction. ** +** If P2 is zero, that is an assertion that the P1 table is never +** empty and hence the jump will never be taken. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5==0 ); + assert( pOp->p2>=0 && pOp->p2nOp ); + pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); @@ -92992,13 +100977,14 @@ case OP_Rewind: { /* jump */ } if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; - assert( pOp->p2>0 && pOp->p2nOp ); - VdbeBranchTaken(res!=0,2); - if( res ) goto jump_to_p2; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } break; } -/* Opcode: Next P1 P2 P3 P4 P5 +/* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through @@ -93017,15 +101003,12 @@ case OP_Rewind: { /* jump */ ** omitted if that index had been unique. P3 is usually 0. P3 is ** always either 0 or 1. ** -** P4 is always of type P4_ADVANCE. The function pointer points to -** sqlite3BtreeNext(). -** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** ** See also: Prev */ -/* Opcode: Prev P1 P2 P3 P4 P5 +/* Opcode: Prev P1 P2 P3 * P5 ** ** Back up cursor P1 so that it points to the previous key/data pair in its ** table or index. If there is no previous key/value pairs then fall through @@ -93045,9 +101028,6 @@ case OP_Rewind: { /* jump */ ** omitted if that index had been unique. P3 is usually 0. P3 is ** always either 0 or 1. ** -** P4 is always of type P4_ADVANCE. The function pointer points to -** sqlite3BtreePrevious(). -** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ @@ -93065,30 +101045,37 @@ case OP_SorterNext: { /* jump */ assert( isSorter(pC) ); rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_Prev: /* jump */ -case OP_Next: /* jump */ + +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p5aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); assert( pC->eCurType==CURTYPE_BTREE ); - assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); - assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); + assert( pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope + || pC->seekOp==OP_NullRow); + rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); + goto next_tail; - /* The Next opcode is only used after SeekGT, SeekGE, Rewind, and Found. - ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ - assert( pOp->opcode!=OP_Next - || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE +case OP_Next: /* jump, ncycle */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->deferredMoveto==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid || pC->seekOp==OP_IfNoHope); - assert( pOp->opcode!=OP_Prev - || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE - || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope - || pC->seekOp==OP_NullRow); + rc = sqlite3BtreeNext(pC->uc.pCursor, pOp->p3); - rc = pOp->p4.xAdvance(pC->uc.pCursor, pOp->p3); next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(rc==SQLITE_OK,2); @@ -93266,8 +101253,8 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_DeferredSeek: -case OP_IdxRowid: { /* out2 */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ VdbeCursor *pC; /* The P1 index cursor */ VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ i64 rowid; /* Rowid that P1 current points to */ @@ -93275,9 +101262,9 @@ case OP_IdxRowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->eCurType==CURTYPE_BTREE || IsNullCursor(pC) ); assert( pC->uc.pCursor!=0 ); - assert( pC->isTable==0 ); + assert( pC->isTable==0 || IsNullCursor(pC) ); assert( pC->deferredMoveto==0 ); assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); @@ -93285,10 +101272,10 @@ case OP_IdxRowid: { /* out2 */ ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ rc = sqlite3VdbeCursorRestore(pC); - /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happens for an IdxRowid - ** or Seek opcode */ - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + /* sqlite3VdbeCursorRestore() may fail if the cursor has been disturbed + ** since it was last positioned and an error (e.g. OOM or an IO error) + ** occurs while trying to reposition it. */ + if( rc!=SQLITE_OK ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ @@ -93306,10 +101293,11 @@ case OP_IdxRowid: { /* out2 */ pTabCur->nullRow = 0; pTabCur->movetoTarget = rowid; pTabCur->deferredMoveto = 1; + pTabCur->cacheStatus = CACHE_STALE; assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); - pTabCur->aAltMap = pOp->p4.ai; - assert( !pC->isEphemeral ); assert( !pTabCur->isEphemeral ); + pTabCur->ub.aAltMap = pOp->p4.ai; + assert( !pC->isEphemeral ); pTabCur->pAltCursor = pC; }else{ pOut = out2Prerelease(p, pOp); @@ -93328,8 +101316,8 @@ case OP_IdxRowid: { /* out2 */ ** seek operation now, without further delay. If the cursor seek has ** already occurred, this instruction is a no-op. */ -case OP_FinishSeek: { - VdbeCursor *pC; /* The P1 index cursor */ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; @@ -93384,10 +101372,10 @@ case OP_FinishSeek: { ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -93440,7 +101428,7 @@ case OP_IdxGE: { /* jump */ rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); if( rc ) goto abort_due_to_error; res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, &r, 0); - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); } /* End of inlined sqlite3VdbeIdxKeyCompare() */ @@ -93464,7 +101452,7 @@ case OP_IdxGE: { /* jump */ ** file is given by P1. ** ** The table being destroyed is in the main database file if P3==0. If -** P3==1 then the table to be clear is in the auxiliary database file +** P3==1 then the table to be destroyed is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** ** If AUTOVACUUM is enabled then it is possible that another root page @@ -93524,8 +101512,8 @@ case OP_Destroy: { /* out2 */ ** in the database file is given by P1. But, unlike Destroy, do not ** remove the table or index from the database file. ** -** The table being clear is in the main database file if P2==0. If -** P2==1 then the table to be clear is in the auxiliary database file +** The table being cleared is in the main database file if P2==0. If +** P2==1 then the table to be cleared is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** ** If the P3 value is non-zero, then the row change count is incremented @@ -93608,16 +101596,57 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { + char *zErr; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; +#endif + u8 mTrace; + int savedAnalysisLimit; + sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; - rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + zErr = 0; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; +#endif + mTrace = db->mTrace; + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + db->mTrace = 0; + } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; - if( rc ) goto abort_due_to_error; +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; + if( zErr || rc ){ + sqlite3VdbeError(p, "%s", zErr); + sqlite3_free(zErr); + if( rc==SQLITE_NOMEM ) goto no_mem; + goto abort_due_to_error; + } break; } @@ -93763,11 +101792,11 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains one less than the maximum number of allowed errors. -** At most reg(P3) errors will be reported. +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** @@ -93787,24 +101816,27 @@ case OP_IntegrityCk: { Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); + assert( aRoot!=0 ); assert( aRoot[0]==(Pgno)nRoot ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - z = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); - }else if( z==0 ){ - goto no_mem; + }else if( rc ){ + sqlite3_free(z); + goto abort_due_to_error; }else{ pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); @@ -93925,7 +101957,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** P1 contains the address of the memory cell that contains the first memory ** cell in an array of values used as arguments to the sub-program. P2 ** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address ** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** @@ -93933,7 +101967,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ int nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ @@ -94008,9 +102042,6 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pFrame->anExec = p->anExec; -#endif #ifdef SQLITE_DEBUG pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif @@ -94047,9 +102078,6 @@ case OP_Program: { /* jump */ memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = 0; -#endif #ifdef SQLITE_DEBUG /* Verify that second and subsequent executions of the same trigger do not ** try to reuse register values from the first use. */ @@ -94189,7 +102217,7 @@ case OP_IfPos: { /* jump, in1 */ ** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** ** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3] ** holds the offset counter. The opcode computes the combined value ** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] ** value computed is the total number of rows that will need to be @@ -94265,7 +102293,7 @@ case OP_DecrJumpZero: { /* jump, in1 */ } break; } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--; @@ -94335,6 +102363,7 @@ case OP_AggStep: { pCtx->pVdbe = p; pCtx->skipFlag = 0; pCtx->isError = 0; + pCtx->enc = encoding; pCtx->argc = n; pOp->p4type = P4_FUNCCTX; pOp->p4.pCtx = pCtx; @@ -94369,7 +102398,7 @@ case OP_AggStep1: { /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ + ** reinitializes the relevant parts of the sqlite3_context object */ if( pCtx->pMem != pMem ){ pCtx->pMem = pMem; for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; @@ -94464,9 +102493,7 @@ case OP_AggFinal: { } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); - if( sqlite3VdbeMemTooBig(pMem) ){ - goto too_big; - } + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } @@ -94822,7 +102849,7 @@ case OP_VDestroy: { ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -94845,7 +102872,7 @@ case OP_VOpen: { pVCur->pVtab = pVtab; /* Initialize vdbe cursor object */ - pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); + pCur = allocateCursor(p, pOp->p1, 0, CURTYPE_VTAB); if( pCur ){ pCur->uc.pVCur = pVCur; pVtab->nRef++; @@ -94858,6 +102885,80 @@ case OP_VOpen: { } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCheck P1 P2 P3 P4 * +** +** P4 is a pointer to a Table object that is a virtual table in schema P1 +** that supports the xIntegrity() method. This opcode runs the xIntegrity() +** method for that virtual table, using P3 as the integer argument. If +** an error is reported back, the table name is prepended to the error +** message and that message is stored in P2. If no errors are seen, +** register P2 is set to NULL. +*/ +case OP_VCheck: { /* out2 */ + Table *pTab; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + char *zErr = 0; + + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetNull(pOut); /* Innocent until proven guilty */ + assert( pOp->p4type==P4_TABLEREF ); + pTab = pOp->p4.pTab; + assert( pTab!=0 ); + assert( pTab->nTabRef>0 ); + assert( IsVirtual(pTab) ); + if( pTab->u.vtab.p==0 ) break; + pVtab = pTab->u.vtab.p->pVtab; + assert( pVtab!=0 ); + pModule = pVtab->pModule; + assert( pModule!=0 ); + assert( pModule->iVersion>=4 ); + assert( pModule->xIntegrity!=0 ); + sqlite3VtabLock(pTab->u.vtab.p); + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName, + pOp->p3, &zErr); + sqlite3VtabUnlock(pTab->u.vtab.p); + if( rc ){ + sqlite3_free(zErr); + goto abort_due_to_error; + } + if( zErr ){ + sqlite3VdbeMemSetStr(pOut, zErr, -1, SQLITE_UTF8, sqlite3_free); + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VInitIn P1 P2 P3 * * +** Synopsis: r[P2]=ValueList(P1,P3) +** +** Set register P2 to be a pointer to a ValueList object for cursor P1 +** with cache register P3 and output register P3+1. This ValueList object +** can be used as the first argument to sqlite3_vtab_in_first() and +** sqlite3_vtab_in_next() to extract all of the values stored in the P1 +** cursor. Register P3 is used to hold the values returned by +** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). +*/ +case OP_VInitIn: { /* out2, ncycle */ + VdbeCursor *pC; /* The cursor containing the RHS values */ + ValueList *pRhs; /* New ValueList object to put in reg[P2] */ + + pC = p->apCsr[pOp->p1]; + pRhs = sqlite3_malloc64( sizeof(*pRhs) ); + if( pRhs==0 ) goto no_mem; + pRhs->pCsr = pC->uc.pCursor; + pRhs->pOut = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + pOut->flags = MEM_Null; + sqlite3VdbeMemSetPointer(pOut, pRhs, "ValueList", sqlite3VdbeValueListFree); + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VFilter P1 P2 P3 P4 * ** Synopsis: iplan=r[P3] zplan='P4' @@ -94878,7 +102979,7 @@ case OP_VOpen: { ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; @@ -94938,15 +103039,15 @@ case OP_VFilter: { /* jump */ ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are ** unused by OP_VColumn. */ -case OP_VColumn: { +case OP_VColumn: { /* ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; Mem *pDest; sqlite3_context sContext; + FuncDef nullFunc; VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur!=0 ); - assert( pCur->eCurType==CURTYPE_VTAB ); assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); @@ -94954,11 +103055,16 @@ case OP_VColumn: { sqlite3VdbeMemSetNull(pDest); break; } + assert( pCur->eCurType==CURTYPE_VTAB ); pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; + sContext.enc = encoding; + nullFunc.pUserData = 0; + nullFunc.funcFlags = SQLITE_RESULT_SUBTYPE; + sContext.pFunc = &nullFunc; assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 ); if( pOp->p5 & OPFLAG_NOCHNG ){ sqlite3VdbeMemSetNull(pDest); @@ -94977,9 +103083,6 @@ case OP_VColumn: { REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); - if( sqlite3VdbeMemTooBig(pDest) ){ - goto too_big; - } if( rc ) goto abort_due_to_error; break; } @@ -94992,7 +103095,7 @@ case OP_VColumn: { ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; @@ -95223,7 +103326,7 @@ case OP_MaxPgcnt: { /* out2 */ ** This opcode works exactly like OP_Function. The only difference is in ** its name. This opcode is used in places where the function must be ** purely non-deterministic. Some built-in date/time functions can be -** either determinitic of non-deterministic, depending on their arguments. +** either deterministic of non-deterministic, depending on their arguments. ** When those function are used in a non-deterministic way, they will check ** to see if they were called using OP_PureFunc instead of OP_Function, and ** if they were, they throw an error. @@ -95241,11 +103344,12 @@ case OP_Function: { /* group */ /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ + ** reinitializes the relevant parts of the sqlite3_context object */ pOut = &aMem[pOp->p3]; if( pCtx->pOut != pOut ){ pCtx->pVdbe = p; pCtx->pOut = pOut; + pCtx->enc = encoding; for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } assert( pCtx->pVdbe==p ); @@ -95272,17 +103376,134 @@ case OP_Function: { /* group */ if( rc ) goto abort_due_to_error; } - /* Copy the result of the function into register P3 */ - if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pOut, encoding); - if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; - } + assert( (pOut->flags&MEM_Str)==0 + || pOut->enc==encoding + || db->mallocFailed ); + assert( !sqlite3VdbeMemTooBig(pOut) ); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); break; } +/* Opcode: ClrSubtype P1 * * * * +** Synopsis: r[P1].subtype = 0 +** +** Clear the subtype from register P1. +*/ +case OP_ClrSubtype: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + pIn1->flags &= ~MEM_Subtype; + break; +} + +/* Opcode: GetSubtype P1 P2 * * * +** Synopsis: r[P2] = r[P1].subtype +** +** Extract the subtype value from register P1 and write that subtype +** into register P2. If P1 has no subtype, then P1 gets a NULL. +*/ +case OP_GetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Subtype ){ + sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype); + }else{ + sqlite3VdbeMemSetNull(pOut); + } + break; +} + +/* Opcode: SetSubtype P1 P2 * * * +** Synopsis: r[P2].subtype = r[P1] +** +** Set the subtype value of register P2 to the integer from register P1. +** If P1 is NULL, clear the subtype from p2. +*/ +case OP_SetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + pOut->flags &= ~MEM_Subtype; + }else{ + assert( pIn1->flags & MEM_Int ); + pOut->flags |= MEM_Subtype; + pOut->eSubtype = (u8)(pIn1->u.i & 0xff); + } + break; +} + +/* Opcode: FilterAdd P1 * P3 P4 * +** Synopsis: filter(P1) += key(P3@P4) +** +** Compute a hash on the P4 registers starting with r[P3] and +** add that hash to the bloom filter contained in r[P1]. +*/ +case OP_FilterAdd: { + u64 h; + + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags & MEM_Blob ); + assert( pIn1->n>0 ); + h = filterHash(aMem, pOp); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + int ii; + for(ii=pOp->p3; iip3+pOp->p4.i; ii++){ + registerTrace(ii, &aMem[ii]); + } + printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); + } +#endif + h %= (pIn1->n*8); + pIn1->z[h/8] |= 1<<(h&7); + break; +} + +/* Opcode: Filter P1 P2 P3 P4 * +** Synopsis: if key(P3@P4) not in filter(P1) goto P2 +** +** Compute a hash on the key contained in the P4 registers starting +** with r[P3]. Check to see if that hash is found in the +** bloom filter hosted by register P1. If it is not present then +** maybe jump to P2. Otherwise fall through. +** +** False negatives are harmless. It is always safe to fall through, +** even if the value is in the bloom filter. A false negative causes +** more CPU cycles to be used, but it should still yield the correct +** answer. However, an incorrect answer may well arise from a +** false positive - if the jump is taken when it should fall through. +*/ +case OP_Filter: { /* jump */ + u64 h; + + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Blob)!=0 ); + assert( pIn1->n >= 1 ); + h = filterHash(aMem, pOp); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + int ii; + for(ii=pOp->p3; iip3+pOp->p4.i; ii++){ + registerTrace(ii, &aMem[ii]); + } + printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); + } +#endif + h %= (pIn1->n*8); + if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){ + VdbeBranchTaken(1, 2); + p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++; + goto jump_to_p2; + }else{ + p->aCounter[SQLITE_STMTSTATUS_FILTER_MISS]++; + VdbeBranchTaken(0, 2); + } + break; +} + /* Opcode: Trace P1 P2 * P4 * ** ** Write P4 on the statement trace output if statement tracing is @@ -95309,7 +103530,7 @@ case OP_Function: { /* group */ ** error is encountered. */ case OP_Trace: -case OP_Init: { /* jump */ +case OP_Init: { /* jump0 */ int i; #ifndef SQLITE_OMIT_TRACE char *zTrace; @@ -95331,7 +103552,7 @@ case OP_Init: { /* jump */ #ifndef SQLITE_OMIT_TRACE if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0 - && !p->doingRerun + && p->minWriteFileFormat!=254 /* tag-20220401a */ && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ #ifndef SQLITE_OMIT_DEPRECATED @@ -95493,11 +103714,13 @@ default: { /* This is really OP_Noop, OP_Explain */ *****************************************************************************/ } -#ifdef VDBE_PROFILE - { - u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); - if( endTime>start ) pOrigOp->cycles += endTime - start; - pOrigOp->cnt++; +#if defined(VDBE_PROFILE) + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; } #endif @@ -95521,7 +103744,7 @@ default: { /* This is really OP_Noop, OP_Explain */ } if( opProperty==0xff ){ /* Never happens. This code exists to avoid a harmless linkage - ** warning aboud sqlite3VdbeRegisterDump() being defined but not + ** warning about sqlite3VdbeRegisterDump() being defined but not ** used. */ sqlite3VdbeRegisterDump(p); } @@ -95560,7 +103783,7 @@ abort_due_to_error: testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); - sqlite3VdbeHalt(p); + if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ db->flags |= SQLITE_CorruptRdOnly; @@ -95574,6 +103797,18 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: +#if defined(VDBE_PROFILE) + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } +#endif + #ifndef SQLITE_OMIT_PROGRESS_CALLBACK while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ nProgressLimit += db->nProgressOps; @@ -95585,7 +103820,9 @@ vdbe_return: } #endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; - sqlite3VdbeLeave(p); + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeLeave(p); + } assert( rc!=SQLITE_OK || nExtraDelete==0 || sqlite3_strlike("DELETE%",p->zSql,0)!=0 ); @@ -95680,8 +103917,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ /* Set the value of register r[1] in the SQL statement to integer iRow. ** This is done directly as a performance optimization */ - v->aMem[1].flags = MEM_Int; - v->aMem[1].u.i = iRow; + sqlite3VdbeMemSetInt64(&v->aMem[1], iRow); /* If the statement has been run before (and is paused at the OP_ResultRow) ** then back it up to the point where it does the OP_NotExists. This could @@ -95764,7 +104000,7 @@ SQLITE_API int sqlite3_blob_open( #endif *ppBlob = 0; #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + if( !sqlite3SafetyCheckOk(db) || zTable==0 || zColumn==0 ){ return SQLITE_MISUSE_BKPT; } #endif @@ -95773,10 +104009,9 @@ SQLITE_API int sqlite3_blob_open( sqlite3_mutex_enter(db->mutex); pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - do { - memset(&sParse, 0, sizeof(Parse)); + while(1){ + sqlite3ParseObjectInit(&sParse,db); if( !pBlob ) goto blob_open_out; - sParse.db = db; sqlite3DbFree(db, zErr); zErr = 0; @@ -95953,7 +104188,9 @@ SQLITE_API int sqlite3_blob_open( goto blob_open_out; } rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)=SQLITE_MAX_SCHEMA_RETRY || rc!=SQLITE_SCHEMA ) break; + sqlite3ParseObjectReset(&sParse); + } blob_open_out: if( rc==SQLITE_OK && db->mallocFailed==0 ){ @@ -95962,9 +104199,9 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); - sqlite3ParserReset(&sParse); + sqlite3ParseObjectReset(&sParse); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -96121,7 +104358,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ((Vdbe*)p->pStmt)->rc = SQLITE_OK; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); @@ -96224,7 +104461,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** The threshold for the amount of main memory to use before flushing ** records to a PMA is roughly the same as the limit configured for the ** page-cache of the main database. Specifically, the threshold is set to -** the value returned by "PRAGMA main.page_size" multipled by +** the value returned by "PRAGMA main.page_size" multiplied by ** that returned by "PRAGMA main.cache_size", in bytes. ** ** If the sorter is running in single-threaded mode, then all PMAs generated @@ -96247,7 +104484,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged -** incrementally as keys are retreived from the sorter by the VDBE. The +** incrementally as keys are retrieved from the sorter by the VDBE. The ** MergeEngine object, described in further detail below, performs this ** merge. ** @@ -96325,7 +104562,7 @@ struct SorterFile { struct SorterList { SorterRecord *pList; /* Linked list of records */ u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ - int szPMA; /* Size of pList as PMA in bytes */ + i64 szPMA; /* Size of pList as PMA in bytes */ }; /* @@ -96410,7 +104647,7 @@ struct MergeEngine { ** ** Essentially, this structure contains all those fields of the VdbeSorter ** structure for which each thread requires a separate instance. For example, -** each thread requries its own UnpackedRecord object to unpack records in +** each thread requeries its own UnpackedRecord object to unpack records in ** as part of comparison operations. ** ** Before a background thread is launched, variable bDone is set to 0. Then, @@ -96434,10 +104671,10 @@ typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ + int nPMA; /* Number of PMAs currently in file */ VdbeSorter *pSorter; /* Sorter that owns this sub-task */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ - int nPMA; /* Number of PMAs currently in file */ SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ @@ -96482,7 +104719,7 @@ struct VdbeSorter { ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. ** aKey might point into aMap or into aBuffer. If neither of those locations ** contain a contiguous representation of the key, then aAlloc is allocated -** and the key is copied into aAlloc and aKey is made to poitn to aAlloc. +** and the key is copied into aAlloc and aKey is made to point to aAlloc. ** ** pFd==0 at EOF. */ @@ -97099,7 +105336,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } #endif - assert( pCsr->pKeyInfo && pCsr->pBtx==0 ); + assert( pCsr->pKeyInfo ); + assert( !pCsr->isEphemeral ); assert( pCsr->eCurType==CURTYPE_SORTER ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); @@ -97852,7 +106090,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ ** the background thread from a sub-tasks previous turn is still running, ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, ** fall back to using the final sub-task. The first (pSorter->nTask-1) - ** sub-tasks are prefered as they use background threads - the final + ** sub-tasks are preferred as they use background threads - the final ** sub-task uses the main thread. */ for(i=0; iiPrev + i + 1) % nWorker; @@ -97910,8 +106148,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ int bFlush; /* True to flush contents of memory to PMA */ - int nReq; /* Bytes of memory required */ - int nPMA; /* Bytes of PMA space required */ + i64 nReq; /* Bytes of memory required */ + i64 nPMA; /* Bytes of PMA space required */ int t; /* serial type of first record field */ assert( pCsr->eCurType==CURTYPE_SORTER ); @@ -98336,7 +106574,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr. A multi-theaded IncrMerge object + /* Set up the required files for pIncr. A multi-threaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ @@ -98976,6 +107214,8 @@ static int bytecodevtabConnect( "p5 INT," "comment TEXT," "subprog TEXT," + "nexec INT," + "ncycle INT," "stmt HIDDEN" ");", @@ -98990,6 +107230,9 @@ static int bytecodevtabConnect( ");" }; + (void)argc; + (void)argv; + (void)pzErr; rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); if( rc==SQLITE_OK ){ pNew = sqlite3_malloc( sizeof(*pNew) ); @@ -99135,7 +107378,7 @@ static int bytecodevtabColumn( } } } - i += 10; + i += 20; } } switch( i ){ @@ -99185,16 +107428,31 @@ static int bytecodevtabColumn( } break; } - case 10: /* tables_used.type */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + case 9: /* nexec */ + sqlite3_result_int64(ctx, pOp->nExec); + break; + case 10: /* ncycle */ + sqlite3_result_int64(ctx, pOp->nCycle); + break; +#else + case 9: /* nexec */ + case 10: /* ncycle */ + sqlite3_result_int(ctx, 0); + break; +#endif + + case 20: /* tables_used.type */ sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); break; - case 11: /* tables_used.schema */ + case 21: /* tables_used.schema */ sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); break; - case 12: /* tables_used.name */ + case 22: /* tables_used.name */ sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); break; - case 13: /* tables_used.wr */ + case 23: /* tables_used.wr */ sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); break; } @@ -99225,6 +107483,7 @@ static int bytecodevtabFilter( bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; int rc = SQLITE_OK; + (void)idxStr; bytecodevtabCursorClear(pCur); pCur->iRowid = 0; @@ -99267,7 +107526,7 @@ static int bytecodevtabBestIndex( int rc = SQLITE_CONSTRAINT; struct sqlite3_index_constraint *p; bytecodevtab *pVTab = (bytecodevtab*)tab; - int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; pIdxInfo->estimatedCost = (double)100; pIdxInfo->estimatedRows = 100; pIdxInfo->idxNum = 0; @@ -99314,7 +107573,8 @@ static sqlite3_module bytecodevtabModule = { /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; @@ -99512,6 +107772,9 @@ static int memjrnlCreateFile(MemJournal *p){ } +/* Forward reference */ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size); + /* ** Write data to the file. */ @@ -99542,22 +107805,20 @@ static int memjrnlWrite( ** the in-memory journal is being used by a connection using the ** atomic-write optimization. In this case the first 28 bytes of the ** journal file may be written as part of committing the transaction. */ - assert( iOfst==p->endpoint.iOffset || iOfst==0 ); -#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ - || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + assert( iOfst<=p->endpoint.iOffset ); + if( iOfst>0 && iOfst!=p->endpoint.iOffset ){ + memjrnlTruncate(pJfd, iOfst); + } if( iOfst==0 && p->pFirst ){ assert( p->nChunkSize>iAmt ); memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); - }else -#else - assert( iOfst>0 || p->pFirst==0 ); -#endif - { + }else{ while( nWrite>0 ){ FileChunk *pChunk = p->endpoint.pChunk; int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + assert( pChunk!=0 || iChunkOffset==0 ); if( iChunkOffset==0 ){ /* New chunk is required to extend the file. */ FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); @@ -99572,10 +107833,11 @@ static int memjrnlWrite( assert( !p->pFirst ); p->pFirst = pNew; } - p->endpoint.pChunk = pNew; + pChunk = p->endpoint.pChunk = pNew; } - memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + assert( pChunk!=0 ); + memcpy((u8*)pChunk->zChunk + iChunkOffset, zWrite, iSpace); zWrite += iSpace; nWrite -= iSpace; p->endpoint.iOffset += iSpace; @@ -99691,6 +107953,8 @@ SQLITE_PRIVATE int sqlite3JournalOpen( ){ MemJournal *p = (MemJournal*)pJfd; + assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) ); + /* Zero the file-handle object. If nSpill was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls @@ -99834,7 +108098,7 @@ static int walkWindowList(Walker *pWalker, Window *pList, int bOneOnly){ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3WalkExprNN(Walker *pWalker, Expr *pExpr){ int rc; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); @@ -99843,7 +108107,9 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ if( rc ) return rc & WRC_Abort; if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ assert( pExpr->x.pList==0 || pExpr->pRight==0 ); - if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( pExpr->pLeft && sqlite3WalkExprNN(pWalker, pExpr->pLeft) ){ + return WRC_Abort; + } if( pExpr->pRight ){ assert( !ExprHasProperty(pExpr, EP_WinFunc) ); pExpr = pExpr->pRight; @@ -99867,7 +108133,7 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ - return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; + return pExpr ? sqlite3WalkExprNN(pWalker,pExpr) : WRC_Continue; } /* @@ -99993,7 +108259,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ } /* Increase the walkerDepth when entering a subquery, and -** descrease when leaving the subquery. +** decrease when leaving the subquery. */ SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){ UNUSED_PARAMETER(pSelect); @@ -100112,75 +108378,63 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); pDup = 0; }else{ + Expr temp; incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - - /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This - ** prevents ExprDelete() from deleting the Expr structure itself, - ** allowing it to be repopulated by the memcpy() on the following line. - ** The pExpr->u.zToken might point into memory that will be freed by the - ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to - ** make a copy of the token before doing the sqlite3DbFree(). - */ - ExprSetProperty(pExpr, EP_Static); - sqlite3ExprDelete(db, pExpr); - memcpy(pExpr, pDup, sizeof(*pExpr)); - if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ - assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); - pExpr->flags |= EP_MemToken; - } + memcpy(&temp, pDup, sizeof(Expr)); + memcpy(pDup, pExpr, sizeof(Expr)); + memcpy(pExpr, &temp, sizeof(Expr)); if( ExprHasProperty(pExpr, EP_WinFunc) ){ if( ALWAYS(pExpr->y.pWin!=0) ){ pExpr->y.pWin->pOwner = pExpr; } } - sqlite3DbFree(db, pDup); + sqlite3ExprDeferredDelete(pParse, pDup); } } - /* -** Return TRUE if the name zCol occurs anywhere in the USING clause. +** Subqueries store the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN", +** and mark the expression-list item by setting ExprList.a[].fg.eEName +** to ENAME_TAB. ** -** Return FALSE if the USING clause is NULL or if it does not contain -** zCol. -*/ -static int nameInUsingClause(IdList *pUsing, const char *zCol){ - if( pUsing ){ - int k; - for(k=0; knId; k++){ - if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; - } - } - return 0; -} - -/* -** Subqueries stores the original database, table and column names for their -** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". -** Check to see if the zSpan given to this routine matches the zDb, zTab, -** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will -** match anything. +** Check to see if the zSpan/eEName of the expression-list item passed to this +** routine matches the zDb, zTab, and zCol. If any of zDb, zTab, and zCol are +** NULL then those fields will match anything. Return true if there is a match, +** or false otherwise. +** +** SF_NestedFrom subqueries also store an entry for the implicit rowid (or +** _rowid_, or oid) column by setting ExprList.a[].fg.eEName to ENAME_ROWID, +** and setting zSpan to "DATABASE.TABLE.". This type of pItem +** argument matches if zCol is a rowid alias. If it is not NULL, (*pbRowid) +** is set to 1 if there is this kind of match. */ SQLITE_PRIVATE int sqlite3MatchEName( const struct ExprList_item *pItem, const char *zCol, const char *zTab, - const char *zDb + const char *zDb, + int *pbRowid ){ int n; const char *zSpan; - if( pItem->eEName!=ENAME_TAB ) return 0; + int eEName = pItem->fg.eEName; + if( eEName!=ENAME_TAB && (eEName!=ENAME_ROWID || NEVER(pbRowid==0)) ){ + return 0; + } + assert( pbRowid==0 || *pbRowid==0 ); zSpan = pItem->zEName; for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ @@ -100192,9 +108446,11 @@ SQLITE_PRIVATE int sqlite3MatchEName( return 0; } zSpan += n+1; - if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ - return 0; + if( zCol ){ + if( eEName==ENAME_TAB && sqlite3StrICmp(zSpan, zCol)!=0 ) return 0; + if( eEName==ENAME_ROWID && sqlite3IsRowid(zCol)==0 ) return 0; } + if( eEName==ENAME_ROWID ) *pbRowid = 1; return 1; } @@ -100227,6 +108483,7 @@ SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr *pExpr){ assert( ExprUseYTab(pExpr) ); pExTab = pExpr->y.pTab; assert( pExTab!=0 ); + assert( n < pExTab->nCol ); if( (pExTab->tabFlags & TF_HasGenerated)!=0 && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 ){ @@ -100241,6 +108498,55 @@ SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr *pExpr){ } } +/* +** Create a new expression term for the column specified by pMatch and +** iColumn. Append this new expression term to the FULL JOIN Match set +** in *ppList. Create a new *ppList if this is the first term in the +** set. +*/ +static void extendFJMatch( + Parse *pParse, /* Parsing context */ + ExprList **ppList, /* ExprList to extend */ + SrcItem *pMatch, /* Source table containing the column */ + i16 iColumn /* The column number */ +){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); + if( pNew ){ + pNew->iTable = pMatch->iCursor; + pNew->iColumn = iColumn; + pNew->y.pTab = pMatch->pTab; + assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); + ExprSetProperty(pNew, EP_CanBeNull); + *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); + } +} + +/* +** Return TRUE (non-zero) if zTab is a valid name for the schema table pTab. +*/ +static SQLITE_NOINLINE int isValidSchemaTableName( + const char *zTab, /* Name as it appears in the SQL */ + Table *pTab, /* The schema table we are trying to match */ + const char *zDb /* non-NULL if a database qualifier is present */ +){ + const char *zLegacy; + assert( pTab!=0 ); + assert( pTab->tnum==1 ); + if( sqlite3StrNICmp(zTab, "sqlite_", 7)!=0 ) return 0; + zLegacy = pTab->zName; + if( strcmp(zLegacy+7, &LEGACY_TEMP_SCHEMA_TABLE[7])==0 ){ + if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ + return 1; + } + if( zDb==0 ) return 0; + if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + }else{ + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + } + return 0; +} + /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up ** that name in the set of source tables in pSrcList and make the pExpr @@ -100272,13 +108578,13 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ int i, j; /* Loop counters */ int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ + int cntTab = 0; /* Number of potential "rowid" matches */ int nSubquery = 0; /* How many levels of subquery */ sqlite3 *db = pParse->db; /* The database connection */ SrcItem *pItem; /* Use for looping over pSrcList items */ @@ -100286,11 +108592,14 @@ static int lookupName( NameContext *pTopNC = pNC; /* First namecontext in the list */ Schema *pSchema = 0; /* Schema of the expression */ int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ - Table *pTab = 0; /* Table hold the row */ + Table *pTab = 0; /* Table holding the row */ Column *pCol; /* A column of pTab */ + ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( zDb==0 || zTab!=0 ); assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); /* Initialize the node to no-match */ @@ -100339,28 +108648,80 @@ static int lookupName( pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + if( pItem->fg.isNestedFrom ){ + /* In this case, pItem is a subquery that has been formed from a + ** parenthesized subset of the FROM clause terms. Example: + ** .... FROM t1 LEFT JOIN (t2 RIGHT JOIN t3 USING(x)) USING(y) ... + ** \_________________________/ + ** This pItem -------------^ + */ int hit = 0; + assert( pItem->pSelect!=0 ); pEList = pItem->pSelect->pEList; + assert( pEList!=0 ); + assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ - if( sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ + int bRowid = 0; /* True if possible rowid match */ + if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb, &bRowid) ){ + continue; + } + if( bRowid==0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } + } cnt++; - cntTab = 2; - pMatch = pItem; - pExpr->iColumn = j; hit = 1; + }else if( cnt>0 ){ + /* This is a potential rowid match, but there has already been + ** a real match found. So this can be ignored. */ + continue; } + cntTab++; + pMatch = pItem; + pExpr->iColumn = j; + pEList->a[j].fg.bUsed = 1; + + /* rowid cannot be part of a USING clause - assert() this. */ + assert( bRowid==0 || pEList->a[j].fg.bUsingTerm==0 ); + if( pEList->a[j].fg.bUsingTerm ) break; } if( hit || zTab==0 ) continue; } - if( zDb && pTab->pSchema!=pSchema ){ - continue; - } + assert( zDb==0 || zTab!=0 ); if( zTab ){ - const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; - assert( zTabName!=0 ); - if( sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; + if( zDb ){ + if( pTab->pSchema!=pSchema ) continue; + if( pSchema==0 && strcmp(zDb,"*")!=0 ) continue; + } + if( pItem->zAlias!=0 ){ + if( sqlite3StrICmp(zTab, pItem->zAlias)!=0 ){ + continue; + } + }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ + if( pTab->tnum!=1 ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ @@ -100372,33 +108733,79 @@ static int lookupName( if( pCol->hName==hCol && sqlite3StrICmp(pCol->zCnName, zCol)==0 ){ - /* If there has been exactly one prior match and this match - ** is for the right-hand table of a NATURAL JOIN or is in a - ** USING clause, then skip this match. - */ - if( cnt==1 ){ - if( pItem->fg.jointype & JT_NATURAL ) continue; - if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } } cnt++; pMatch = pItem; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( pItem->fg.isNestedFrom ){ + sqlite3SrcItemColumnUsed(pItem, j); + } break; } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && ALWAYS(pMatch!=0) + && ALWAYS(pMatch->pTab!=0) + && (pMatch->pTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); pExpr->y.pTab = pMatch->pTab; - /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); - if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } pSchema = pExpr->y.pTab->pSchema; @@ -100419,7 +108826,9 @@ static int lookupName( assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 - && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + && ALWAYS(zTab==0 + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -100485,6 +108894,8 @@ static int lookupName( pExpr->y.pTab = pTab; if( pParse->bReturning ){ eNewExprOp = TK_REGISTER; + pExpr->op2 = TK_COLUMN; + pExpr->iColumn = iCol; pExpr->iTable = pNC->uNC.iBaseReg + (pTab->nCol+1)*pExpr->iTable + sqlite3TableColumnToStorage(pTab, iCol) + 1; }else{ @@ -100514,14 +108925,19 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab)) + && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; - pExpr->iColumn = -1; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pTab!=0 && IsView(pMatch->pTab) ){ + eNewExprOp = TK_NULL; + } +#endif + if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -100551,7 +108967,7 @@ static int lookupName( assert( pEList!=0 ); for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zEName; - if( pEList->a[j].eEName==ENAME_NAME + if( pEList->a[j].fg.eEName==ENAME_NAME && sqlite3_stricmp(zAs, zCol)==0 ){ Expr *pOrig; @@ -100638,21 +109054,63 @@ static int lookupName( } /* - ** cnt==0 means there was not match. cnt>1 means there were two or - ** more matches. Either way, we have an error. + ** cnt==0 means there was not match. + ** cnt>1 means there were two or more matches. + ** + ** cnt==0 is always an error. cnt>1 is often an error, but might + ** be multiple matches for a NATURAL LEFT JOIN or a LEFT JOIN USING. */ + assert( pFJMatch==0 || cnt>0 ); + assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); if( cnt!=1 ){ const char *zErr; + if( pFJMatch ){ + if( pFJMatch->nExpr==cnt-1 ){ + if( ExprHasProperty(pExpr,EP_Leaf) ){ + ExprClearProperty(pExpr,EP_Leaf); + }else{ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + } + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + pExpr->op = TK_FUNCTION; + pExpr->u.zToken = "coalesce"; + pExpr->x.pList = pFJMatch; + cnt = 1; + goto lookupname_end; + }else{ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + } + } zErr = cnt==0 ? "no such column" : "ambiguous column name"; if( zDb ){ sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); pParse->checkSchema = 1; pTopNC->nNcErr++; + eNewExprOp = TK_NULL; + } + assert( pFJMatch==0 ); + + /* Remove all substructure from pExpr */ + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + ExprSetProperty(pExpr, EP_Leaf); } /* If a column from a table in pSrcList is referenced, then record @@ -100669,20 +109127,15 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } - /* Clean up and return - */ - if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ - sqlite3ExprDelete(db, pExpr->pLeft); - pExpr->pLeft = 0; - sqlite3ExprDelete(db, pExpr->pRight); - pExpr->pRight = 0; - } pExpr->op = eNewExprOp; - ExprSetProperty(pExpr, EP_Leaf); lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); @@ -100759,7 +109212,8 @@ static void notValidImpl( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ const char *zMsg, /* Type of error */ - Expr *pExpr /* Invalidate this expression on error */ + Expr *pExpr, /* Invalidate this expression on error */ + Expr *pError /* Associate error with this expression */ ){ const char *zIn = "partial index WHERE clauses"; if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; @@ -100771,10 +109225,11 @@ static void notValidImpl( #endif sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); if( pExpr ) pExpr->op = TK_NULL; + sqlite3RecordErrorOffsetOfExpr(pParse->db, pError); } -#define sqlite3ResolveNotValid(P,N,M,X,E) \ +#define sqlite3ResolveNotValid(P,N,M,X,E,R) \ assert( ((X)&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); \ - if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E); + if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E,R); /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -100854,6 +109309,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { @@ -100864,23 +109332,36 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_FromJoin) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->op==TK_NOTNULL ){ - pExpr->u.zToken = "true"; - ExprSetProperty(pExpr, EP_IsTrue); - }else{ - pExpr->u.zToken = "false"; - ExprSetProperty(pExpr, EP_IsFalse); - } - pExpr->op = TK_TRUEFALSE; - for(i=0, p=pNC; p && ipNext, i++){ - p->nRef = anRef[i]; + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; + } + + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } @@ -100894,7 +109375,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -100903,13 +109383,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); sqlite3ResolveNotValid(pParse, pNC, "the \".\" operator", - NC_IdxExpr|NC_GenCol, 0); + NC_IdxExpr|NC_GenCol, 0, pExpr); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -100922,14 +109402,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names @@ -100940,7 +109419,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ - int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ @@ -100949,8 +109427,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); #endif assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); + assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); @@ -100967,8 +109445,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ sqlite3ErrorMsg(pParse, - "second argument to likelihood() must be a " - "constant between 0.0 and 1.0"); + "second argument to %#T() must be a " + "constant between 0.0 and 1.0", pExpr); pNC->nNcErr++; } }else{ @@ -100989,8 +109467,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); if( auth!=SQLITE_OK ){ if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); + sqlite3ErrorMsg(pParse, "not authorized to use function: %#T", + pExpr); pNC->nNcErr++; } pExpr->op = TK_NULL; @@ -101013,7 +109491,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all ** all this. */ sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", - NC_IdxExpr|NC_PartIdx|NC_GenCol, 0); + NC_IdxExpr|NC_PartIdx|NC_GenCol, 0, pExpr); }else{ assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ pExpr->op2 = pNC->ncFlags & NC_SelfRef; @@ -101026,7 +109504,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* Internal-use-only functions are disallowed unless the ** SQL is being compiled using sqlite3NestedParse() or ** the SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test-control has be - ** used to activate internal functionsn for testing purposes */ + ** used to activate internal functions for testing purposes */ no_such_func = 1; pDef = 0; }else @@ -101045,7 +109523,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ); if( pDef && pDef->xValue==0 && pWin ){ sqlite3ErrorMsg(pParse, - "%.*s() may not be used as a window function", nId, zId + "%#T() may not be used as a window function", pExpr ); pNC->nNcErr++; }else if( @@ -101059,13 +109537,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ }else{ zType = "aggregate"; } - sqlite3ErrorMsg(pParse, "misuse of %s function %.*s()",zType,nId,zId); + sqlite3ErrorMsg(pParse, "misuse of %s function %#T()",zType,pExpr); pNC->nNcErr++; is_agg = 0; } #else if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ - sqlite3ErrorMsg(pParse,"misuse of aggregate function %.*s()",nId,zId); + sqlite3ErrorMsg(pParse,"misuse of aggregate function %#T()",pExpr); pNC->nNcErr++; is_agg = 0; } @@ -101075,22 +109553,26 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ && pParse->explain==0 #endif ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + sqlite3ErrorMsg(pParse, "no such function: %#T", pExpr); pNC->nNcErr++; }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %#T()", + pExpr); pNC->nNcErr++; } #ifndef SQLITE_OMIT_WINDOWFUNC else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ sqlite3ErrorMsg(pParse, - "FILTER may not be used with non-aggregate %.*s()", - nId, zId + "FILTER may not be used with non-aggregate %#T()", + pExpr ); pNC->nNcErr++; } #endif + else if( is_agg==0 && pExpr->pLeft ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + pNC->nNcErr++; + } if( is_agg ){ /* Window functions may not be arguments of aggregate functions. ** Or arguments of other window functions. But aggregate functions @@ -101102,13 +109584,16 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pWin ){ Select *pSel = pNC->pWinSelect; @@ -101137,11 +109622,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ while( pNC2 && sqlite3ReferencesSrcList(pParse, pExpr, pNC2->pSrcList)==0 ){ - pExpr->op2++; + pExpr->op2 += (1 + pNC2->nNestedSelect); pNC2 = pNC2->pNext; } assert( pDef!=0 || IN_RENAME_OBJECT ); if( pNC2 && pDef ){ + pExpr->op2 += pNC2->nNestedSelect; assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); assert( SQLITE_FUNC_ANYORDER==NC_OrderAgg ); testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); @@ -101170,16 +109656,18 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); if( pNC->ncFlags & NC_SelfRef ){ - notValidImpl(pParse, pNC, "subqueries", pExpr); + notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ sqlite3WalkSelect(pWalker, pExpr->x.pSelect); } assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; + pExpr->x.pSelect->selFlags |= SF_Correlated; } + pNC->ncFlags |= NC_Subquery; } break; } @@ -101189,7 +109677,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); sqlite3ResolveNotValid(pParse, pNC, "parameters", - NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr, pExpr); break; } case TK_IS: @@ -101241,11 +109729,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_ISNOT ); testcase( pExpr->op==TK_BETWEEN ); sqlite3ErrorMsg(pParse, "row value misused"); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); } break; } } - return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + return pParse->nErr ? WRC_Abort : WRC_Continue; } /* @@ -101274,7 +109764,7 @@ static int resolveAsName( assert( !ExprHasProperty(pE, EP_IntValue) ); zCol = pE->u.zToken; for(i=0; inExpr; i++){ - if( pEList->a[i].eEName==ENAME_NAME + if( pEList->a[i].fg.eEName==ENAME_NAME && sqlite3_stricmp(pEList->a[i].zEName, zCol)==0 ){ return i+1; @@ -101353,11 +109843,13 @@ static void resolveOutOfRangeError( Parse *pParse, /* The error context into which to write the error */ const char *zType, /* "ORDER" or "GROUP" */ int i, /* The index (1-based) of the term out of range */ - int mx /* Largest permissible value of i */ + int mx, /* Largest permissible value of i */ + Expr *pError /* Associate the error with the expression */ ){ sqlite3ErrorMsg(pParse, "%r %s BY term out of range - should be " "between 1 and %d", i, zType, mx); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pError); } /* @@ -101393,7 +109885,7 @@ static int resolveCompoundOrderBy( return 1; } for(i=0; inExpr; i++){ - pOrderBy->a[i].done = 0; + pOrderBy->a[i].fg.done = 0; } pSelect->pNext = 0; while( pSelect->pPrior ){ @@ -101408,12 +109900,12 @@ static int resolveCompoundOrderBy( for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ int iCol = -1; Expr *pE, *pDup; - if( pItem->done ) continue; + if( pItem->fg.done ) continue; pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( NEVER(pE==0) ) continue; if( sqlite3ExprIsInteger(pE, &iCol) ){ if( iCol<=0 || iCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; } }else{ @@ -101461,7 +109953,7 @@ static int resolveCompoundOrderBy( sqlite3ExprDelete(db, pE); pItem->u.x.iOrderByCol = (u16)iCol; } - pItem->done = 1; + pItem->fg.done = 1; }else{ moreToDo = 1; } @@ -101469,7 +109961,7 @@ static int resolveCompoundOrderBy( pSelect = pSelect->pNext; } for(i=0; inExpr; i++){ - if( pOrderBy->a[i].done==0 ){ + if( pOrderBy->a[i].fg.done==0 ){ sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " "column in the result set", i+1); return 1; @@ -101509,7 +110001,7 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ if( pItem->u.x.iOrderByCol ){ if( pItem->u.x.iOrderByCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr, 0); return 1; } resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,0); @@ -101601,7 +110093,7 @@ static int resolveOrderGroupBy( ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ if( iCol<1 || iCol>0xffff ){ - resolveOutOfRangeError(pParse, zType, i+1, nResult); + resolveOutOfRangeError(pParse, zType, i+1, nResult, pE2); return 1; } pItem->u.x.iOrderByCol = (u16)iCol; @@ -101615,7 +110107,7 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ - /* Since this expresion is being changed into a reference + /* Since this expression is being changed into a reference ** to an identical expression in the result set, remove all Window ** objects belonging to the expression from the Select.pWin list. */ windowRemoveExprFromSelect(pSelect, pE); @@ -101659,7 +110151,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ if( (p->selFlags & SF_Expanded)==0 ){ sqlite3SelectPrep(pParse, p, pOuterNC); - return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + return pParse->nErr ? WRC_Abort : WRC_Prune; } isCompound = p->pPrior!=0; @@ -101668,10 +110160,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ while( p ){ assert( (p->selFlags & SF_Expanded)!=0 ); assert( (p->selFlags & SF_Resolved)==0 ); - assert( db->suppressErr==0 ); /* SF_Resolved not set if errors suppressed */ p->selFlags |= SF_Resolved; - /* Resolve the expressions in the LIMIT and OFFSET clauses. These ** are not allowed to refer to any names, so pass an empty NameContext. */ @@ -101698,8 +110188,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ /* Recursively resolve names in all subqueries in the FROM clause */ + if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; + assert( pItem->zName!=0 || pItem->pSelect!=0 );/* Test of tag-20240424-1*/ if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; @@ -101707,7 +110199,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pItem->zName ) pParse->zAuthContext = pItem->zName; sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; - if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + if( pParse->nErr ) return WRC_Abort; + assert( db->mallocFailed==0 ); /* If the number of references to the outer context changed when ** expressions in the sub-select were resolved, the sub-select @@ -101721,6 +110214,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } } + if( pOuterNC && ALWAYS(pOuterNC->nNestedSelect>0) ){ + pOuterNC->nNestedSelect--; + } /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. @@ -101758,13 +110254,15 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.uNC.pEList = p->pEList; sNC.ncFlags |= NC_UEList; if( p->pHaving ){ - if( !pGroupBy ){ - sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + if( (p->selFlags & SF_Aggregate)==0 ){ + sqlite3ErrorMsg(pParse, "HAVING clause on a non-aggregate query"); return WRC_Abort; } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ @@ -101937,7 +110435,8 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return SQLITE_ERROR; } #endif - sqlite3WalkExpr(&w, pExpr); + assert( pExpr!=0 ); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -101954,6 +110453,9 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ SQLITE_PRIVATE int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ @@ -101962,7 +110464,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -101976,10 +110478,10 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif - sqlite3WalkExpr(&w, pExpr); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -101993,15 +110495,15 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* ** Resolve all names in all expressions of a SELECT and in all -** decendents of the SELECT, including compounds off of p->pPrior, +** descendants of the SELECT, including compounds off of p->pPrior, ** subqueries in expressions, and subqueries used as FROM clause ** terms. ** @@ -102128,49 +110630,122 @@ SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table *pTab, int iCol){ */ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ int op; - while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ - assert( pExpr->op==TK_COLLATE - || pExpr->op==TK_IF_NULL_ROW - || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); - pExpr = pExpr->pLeft; - assert( pExpr!=0 ); - } op = pExpr->op; - if( op==TK_REGISTER ) op = pExpr->op2; - if( op==TK_COLUMN || op==TK_AGG_COLUMN ){ - assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ + while( 1 /* exit-by-break */ ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); } - } - if( op==TK_SELECT ){ - assert( ExprUseXSelect(pExpr) ); - assert( pExpr->x.pSelect!=0 ); - assert( pExpr->x.pSelect->pEList!=0 ); - assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); - } + if( op==TK_SELECT ){ + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } #ifndef SQLITE_OMIT_CAST - if( op==TK_CAST ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken, 0); - } + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } #endif - if( op==TK_SELECT_COLUMN ){ - assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); - assert( pExpr->iColumn < pExpr->iTable ); - assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); - return sqlite3ExprAffinity( - pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr - ); - } - if( op==TK_VECTOR ){ - assert( ExprUseXList(pExpr) ); - return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); + assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iColumn >= 0 ); + assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + if( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + op = pExpr->op; + continue; + } + if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; } return pExpr->affExpr; } +/* +** Make a guess at all the possible datatypes of the result that could +** be returned by an expression. Return a bitmask indicating the answer: +** +** 0x01 Numeric +** 0x02 Text +** 0x04 Blob +** +** If the expression must return NULL, then 0x00 is returned. +*/ +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr){ + while( pExpr ){ + switch( pExpr->op ){ + case TK_COLLATE: + case TK_IF_NULL_ROW: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + break; + } + case TK_NULL: { + pExpr = 0; + break; + } + case TK_STRING: { + return 0x02; + } + case TK_BLOB: { + return 0x04; + } + case TK_CONCAT: { + return 0x06; + } + case TK_VARIABLE: + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + return 0x07; + } + case TK_COLUMN: + case TK_AGG_COLUMN: + case TK_SELECT: + case TK_CAST: + case TK_SELECT_COLUMN: + case TK_VECTOR: { + int aff = sqlite3ExprAffinity(pExpr); + if( aff>=SQLITE_AFF_NUMERIC ) return 0x05; + if( aff==SQLITE_AFF_TEXT ) return 0x06; + return 0x07; + } + case TK_CASE: { + int res = 0; + int ii; + ExprList *pList = pExpr->x.pList; + assert( ExprUseXList(pExpr) && pList!=0 ); + assert( pList->nExpr > 0); + for(ii=1; iinExpr; ii+=2){ + res |= sqlite3ExprDataType(pList->a[ii].pExpr); + } + if( pList->nExpr % 2 ){ + res |= sqlite3ExprDataType(pList->a[pList->nExpr-1].pExpr); + } + return res; + } + default: { + return 0x01; + } + } /* End of switch(op) */ + } /* End of while(pExpr) */ + return 0x00; +} + /* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to a new Expr node that @@ -102229,9 +110804,10 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; @@ -102258,18 +110834,17 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){ + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ + int j; assert( ExprUseYTab(p) ); - if( p->y.pTab!=0 ){ - /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = p->iColumn; - if( j>=0 ){ - const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - } - break; + assert( p->y.pTab!=0 ); + if( (j = p->iColumn)>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); } + break; } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; @@ -102291,11 +110866,10 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ }else{ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( ExprUseXList(p) ); - assert( p->x.pList==0 || p->pRight==0 ); - if( p->x.pList!=0 && !db->mallocFailed ){ + assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); + if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; - for(i=0; ALWAYS(ix.pList->nExpr); i++){ + for(i=0; ix.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; @@ -102317,7 +110891,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ /* ** Return the collation sequence for the expression pExpr. If ** there is no defined collating sequence, return a pointer to the -** defautl collation sequence. +** default collation sequence. ** ** See also: sqlite3ExprCollSeq() ** @@ -102447,7 +111021,7 @@ SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq( return pColl; } -/* Expresssion p is a comparison operator. Return a collation sequence +/* Expression p is a comparison operator. Return a collation sequence ** appropriate for the comparison operator. ** ** This is normally just a wrapper around sqlite3BinaryCompareCollSeq(). @@ -102604,6 +111178,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( */ pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); if( pRet ){ + ExprSetProperty(pRet, EP_FullSize); pRet->iTable = nField; pRet->iColumn = iField; pRet->pLeft = pVector; @@ -102853,9 +111428,10 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){ ** if appropriate. */ static void exprSetHeight(Expr *p){ - int nHeight = 0; - heightOfExpr(p->pLeft, &nHeight); - heightOfExpr(p->pRight, &nHeight); + int nHeight = p->pLeft ? p->pLeft->nHeight : 0; + if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){ + nHeight = p->pRight->nHeight; + } if( ExprUseXSelect(p) ){ heightOfSelect(p->x.pSelect, &nHeight); }else if( p->x.pList ){ @@ -102902,6 +111478,15 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ #define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ +/* +** Set the error offset for an Expr node, if possible. +*/ +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ + if( pExpr==0 ) return; + if( NEVER(ExprUseWJoin(pExpr)) ) return; + pExpr->w.iOfst = iOfst; +} + /* ** This routine is the core allocator for Expr nodes. ** @@ -102916,11 +111501,12 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -102936,7 +111522,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -102998,15 +111584,26 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); }else{ + assert( ExprUseXList(pRoot) ); + assert( pRoot->x.pSelect==0 ); if( pRight ){ pRoot->pRight = pRight; pRoot->flags |= EP_Propagate & pRight->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + pRoot->nHeight = pRight->nHeight+1; + }else{ + pRoot->nHeight = 1; +#endif } if( pLeft ){ pRoot->pLeft = pLeft; pRoot->flags |= EP_Propagate & pLeft->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pLeft->nHeight>=pRoot->nHeight ){ + pRoot->nHeight = pLeft->nHeight+1; + } +#endif } - exprSetHeight(pRoot); } } @@ -103115,9 +111712,9 @@ SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprLis ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. ** -** If one side or the other of the AND is known to be false, then instead -** of returning an AND expression, just return a constant expression with -** a value of false. +** If one side or the other of the AND is known to be false, and neither side +** is part of an ON clause, then instead of returning an AND expression, +** just return a constant expression with a value of false. */ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ sqlite3 *db = pParse->db; @@ -103125,14 +111722,17 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) - && !IN_RENAME_OBJECT - ){ - sqlite3ExprDeferredDelete(pParse, pLeft); - sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + u32 f = pLeft->flags | pRight->flags; + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } } } @@ -103154,6 +111754,8 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction( sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ return 0; } + assert( !ExprHasProperty(pNew, EP_InnerON|EP_OuterON) ); + pNew->w.iOfst = (int)(pToken->z - pParse->zTail); if( pList && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] && !pParse->nested @@ -103168,6 +111770,67 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction( return pNew; } +/* +** Report an error when attempting to use an ORDER BY clause within +** the arguments of a non-aggregate function. +*/ +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse *pParse, Expr *p){ + sqlite3ErrorMsg(pParse, + "ORDER BY may not be used with non-aggregate %#T()", p + ); +} + +/* +** Attach an ORDER BY clause to a function call. +** +** functionname( arguments ORDER BY sortlist ) +** \_____________________/ \______/ +** pExpr pOrderBy +** +** The ORDER BY clause is inserted into a new Expr node of type TK_ORDER +** and added to the Expr.pLeft field of the parent TK_FUNCTION node. +*/ +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The function call to which ORDER BY is to be added */ + ExprList *pOrderBy /* The ORDER BY clause to add */ +){ + Expr *pOB; + sqlite3 *db = pParse->db; + if( NEVER(pOrderBy==0) ){ + assert( db->mallocFailed ); + return; + } + if( pExpr==0 ){ + assert( db->mallocFailed ); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + assert( pExpr->op==TK_FUNCTION ); + assert( pExpr->pLeft==0 ); + assert( ExprUseXList(pExpr) ); + if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){ + /* Ignore ORDER BY on zero-argument aggregates */ + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy); + return; + } + if( IsWindowFunc(pExpr) ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + + pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); + if( pOB==0 ){ + sqlite3ExprListDelete(db, pOrderBy); + return; + } + pOB->x.pList = pOrderBy; + assert( ExprUseXList(pOB) ); + pExpr->pLeft = pOB; + ExprSetProperty(pOB, EP_FullSize); +} + /* ** Check to see if a function is usable according to current access ** rules: @@ -103197,7 +111860,7 @@ SQLITE_PRIVATE void sqlite3ExprFunctionUsable( ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning ** that the schema is possibly tainted). */ - sqlite3ErrorMsg(pParse, "unsafe use of %s()", pDef->zName); + sqlite3ErrorMsg(pParse, "unsafe use of %#T()", pExpr); } } } @@ -103253,6 +111916,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); return; } x = (ynVar)i; @@ -103280,6 +111944,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n pExpr->iColumn = x; if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); } } @@ -103288,6 +111953,8 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n */ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); + assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -103303,7 +111970,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -103318,33 +111984,56 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } - } - if( ExprHasProperty(p, EP_MemToken) ){ - assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3DbFree(db, p->u.zToken); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } } SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p); +} +/* +** Clear both elements of an OnOrUsing object +*/ +SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ + if( p==0 ){ + /* Nothing to clear */ + }else if( p->pOn ){ + sqlite3ExprDeleteNN(db, p->pOn); + }else if( p->pUsing ){ + sqlite3IdListDelete(db, p->pUsing); + } +} /* ** Arrange to cause pExpr to be deleted when the pParse is deleted. ** This is similar to sqlite3ExprDelete() except that the delete is -** deferred untilthe pParse is deleted. +** deferred until the pParse is deleted. ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - pParse->pConstExpr = - sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -103409,16 +112098,11 @@ static int dupedExprStructSize(const Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN -#ifndef SQLITE_OMIT_WINDOWFUNC - || ExprHasProperty(p, EP_WinFunc) -#endif - ){ + if( 0==flags || ExprHasProperty(p, EP_FullSize) ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasProperty(p, EP_FromJoin) ); - assert( !ExprHasProperty(p, EP_MemToken) ); + assert( !ExprHasProperty(p, EP_OuterON) ); assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; @@ -103445,56 +112129,93 @@ static int dupedExprNodeSize(const Expr *p, int flags){ /* ** Return the number of bytes required to create a duplicate of the -** expression passed as the first argument. The second argument is a -** mask containing EXPRDUP_XXX flags. +** expression passed as the first argument. ** ** The value returned includes space to create a copy of the Expr struct ** itself and the buffer referred to by Expr.u.zToken, if any. ** -** If the EXPRDUP_REDUCE flag is set, then the return value includes -** space to duplicate all Expr nodes in the tree formed by Expr.pLeft -** and Expr.pRight variables (but not for any structures pointed to or -** descended from the Expr.x.pList or Expr.x.pSelect variables). +** The return value includes space to duplicate all Expr nodes in the +** tree formed by Expr.pLeft and Expr.pRight, but not any other +** substructure such as Expr.x.pList, Expr.x.pSelect, and Expr.y.pWin. */ -static int dupedExprSize(const Expr *p, int flags){ - int nByte = 0; - if( p ){ - nByte = dupedExprNodeSize(p, flags); - if( flags&EXPRDUP_REDUCE ){ - nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); - } - } +static int dupedExprSize(const Expr *p){ + int nByte; + assert( p!=0 ); + nByte = dupedExprNodeSize(p, EXPRDUP_REDUCE); + if( p->pLeft ) nByte += dupedExprSize(p->pLeft); + if( p->pRight ) nByte += dupedExprSize(p->pRight); + assert( nByte==ROUND8(nByte) ); return nByte; } /* -** This function is similar to sqlite3ExprDup(), except that if pzBuffer -** is not NULL then *pzBuffer is assumed to point to a buffer large enough -** to store the copy of expression p, the copies of p->u.zToken -** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte past the -** portion of the buffer copied into by this function. +** An EdupBuf is a memory allocation used to stored multiple Expr objects +** together with their Expr.zToken content. This is used to help implement +** compression while doing sqlite3ExprDup(). The top-level Expr does the +** allocation for itself and many of its decendents, then passes an instance +** of the structure down into exprDup() so that they decendents can have +** access to that memory. +*/ +typedef struct EdupBuf EdupBuf; +struct EdupBuf { + u8 *zAlloc; /* Memory space available for storage */ +#ifdef SQLITE_DEBUG + u8 *zEnd; /* First byte past the end of memory */ +#endif +}; + +/* +** This function is similar to sqlite3ExprDup(), except that if pEdupBuf +** is not NULL then it points to memory that can be used to store a copy +** of the input Expr p together with its p->u.zToken (if any). pEdupBuf +** is updated with the new buffer tail prior to returning. */ -static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ +static Expr *exprDup( + sqlite3 *db, /* Database connection (for memory allocation) */ + const Expr *p, /* Expr tree to be duplicated */ + int dupFlags, /* EXPRDUP_REDUCE for compression. 0 if not */ + EdupBuf *pEdupBuf /* Preallocated storage space, or NULL */ +){ Expr *pNew; /* Value to return */ - u8 *zAlloc; /* Memory space from which to build Expr object */ + EdupBuf sEdupBuf; /* Memory space from which to build Expr object */ u32 staticFlag; /* EP_Static if space not obtained from malloc */ + int nToken = -1; /* Space needed for p->u.zToken. -1 means unknown */ assert( db!=0 ); assert( p ); assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pEdupBuf==0 || dupFlags==EXPRDUP_REDUCE ); /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; + if( pEdupBuf ){ + sEdupBuf.zAlloc = pEdupBuf->zAlloc; +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = pEdupBuf->zEnd; +#endif staticFlag = EP_Static; - assert( zAlloc!=0 ); + assert( sEdupBuf.zAlloc!=0 ); + assert( dupFlags==EXPRDUP_REDUCE ); }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + int nAlloc; + if( dupFlags ){ + nAlloc = dupedExprSize(p); + }else if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30NN(p->u.zToken)+1; + nAlloc = ROUND8(EXPR_FULLSIZE + nToken); + }else{ + nToken = 0; + nAlloc = ROUND8(EXPR_FULLSIZE); + } + assert( nAlloc==ROUND8(nAlloc) ); + sEdupBuf.zAlloc = sqlite3DbMallocRawNN(db, nAlloc); +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = sEdupBuf.zAlloc ? sEdupBuf.zAlloc+nAlloc : 0; +#endif + staticFlag = 0; } - pNew = (Expr *)zAlloc; + pNew = (Expr *)sEdupBuf.zAlloc; + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); if( pNew ){ /* Set nNewSize to the size allocated for the structure pointed to @@ -103503,26 +112224,31 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ ** by the copy of the p->u.zToken string (if any). */ const unsigned nStructSize = dupedExprStructSize(p, dupFlags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; + int nNewSize = nStructSize & 0xfff; + if( nToken<0 ){ + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } } if( dupFlags ){ + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= nNewSize+nToken ); assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); + memcpy(sEdupBuf.zAlloc, p, nNewSize); }else{ u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= + (int)EXPR_FULLSIZE+nToken ); + memcpy(sEdupBuf.zAlloc, p, nSize); if( nSizeflags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; ExprClearVVAProperties(pNew); @@ -103531,44 +112257,50 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + assert( nToken>=0 ); + if( nToken>0 ){ + char *zToken = pNew->u.zToken = (char*)&sEdupBuf.zAlloc[nNewSize]; memcpy(zToken, p->u.zToken, nToken); + nNewSize += nToken; } + sEdupBuf.zAlloc += ROUND8(nNewSize); + + if( ((p->flags|pNew->flags)&(EP_TokenOnly|EP_Leaf))==0 ){ - if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ if( ExprUseXSelect(p) ){ pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, + p->op!=TK_ORDER ? dupFlags : 0); } - } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); - if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ - pNew->pLeft = p->pLeft ? - exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; - pNew->pRight = p->pRight ? - exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; - } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(p, EP_WinFunc) ){ pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); assert( ExprHasProperty(pNew, EP_WinFunc) ); } #endif /* SQLITE_OMIT_WINDOWFUNC */ - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ - if( pNew->op==TK_SELECT_COLUMN ){ + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( dupFlags ){ + if( p->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &sEdupBuf) : 0; + } + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &sEdupBuf) : 0; + }else{ + if( p->op==TK_SELECT_COLUMN ){ pNew->pLeft = p->pLeft; - assert( p->pRight==0 || p->pRight==p->pLeft - || ExprHasProperty(p->pLeft, EP_Subquery) ); + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); }else{ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); } @@ -103576,6 +112308,8 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } } } + if( pEdupBuf ) memcpy(pEdupBuf, &sEdupBuf, sizeof(sEdupBuf)); + assert( sEdupBuf.zAlloc <= sEdupBuf.zEnd ); return pNew; } @@ -103597,6 +112331,7 @@ SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p){ pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + pRet->a[i].eM10d = p->a[i].eM10d; } } } @@ -103697,11 +112432,8 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int } } pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); - pItem->sortFlags = pOldItem->sortFlags; - pItem->eEName = pOldItem->eEName; - pItem->done = 0; - pItem->bNulls = pOldItem->bNulls; - pItem->bSorterRef = pOldItem->bSorterRef; + pItem->fg = pOldItem->fg; + pItem->fg.done = 0; pItem->u = pOldItem->u; } return pNew; @@ -103737,24 +112469,30 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int fla pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; + pNewItem->regResult = pOldItem->regResult; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ pTab->nTabRef++; } pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); - pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); - pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + if( pOldItem->fg.isUsing ){ + assert( pNewItem->fg.isUsing ); + pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); + }else{ + pNewItem->u3.pOn = sqlite3ExprDup(db, pOldItem->u3.pOn, flags); + } pNewItem->colUsed = pOldItem->colUsed; } return pNew; @@ -103764,22 +112502,16 @@ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ int i; assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + assert( p->eU4!=EU4_EXPR ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); - if( pNew->a==0 ){ - sqlite3DbFreeNN(db, pNew); - return 0; - } - /* Note that because the size of the allocation for p->a[] is not - ** necessarily a power of two, sqlite3IdListAppend() may not be called - ** on the duplicate created by this function. */ + pNew->eU4 = p->eU4; for(i=0; inId; i++){ struct IdList_item *pNewItem = &pNew->a[i]; - struct IdList_item *pOldItem = &p->a[i]; + const struct IdList_item *pOldItem = &p->a[i]; pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->idx = pOldItem->idx; + pNewItem->u4 = pOldItem->u4; } return pNew; } @@ -103844,11 +112576,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *p, int flags) ** initially NULL, then create a new expression list. ** ** The pList argument must be either NULL or a pointer to an ExprList -** obtained from a prior call to sqlite3ExprListAppend(). This routine -** may not be used with an ExprList obtained from sqlite3ExprListDup(). -** Reason: This routine assumes that the number of slots in pList->a[] -** is a power of two. That is true for sqlite3ExprListAppend() returns -** but is not necessarily true from the return value of sqlite3ExprListDup(). +** obtained from a prior call to sqlite3ExprListAppend(). ** ** If a memory allocation error occurs, the entire list is freed and ** NULL is returned. If non-NULL is returned, then it is guaranteed @@ -104003,16 +112731,16 @@ SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int ); pItem = &p->a[p->nExpr-1]; - assert( pItem->bNulls==0 ); + assert( pItem->fg.bNulls==0 ); if( iSortOrder==SQLITE_SO_UNDEFINED ){ iSortOrder = SQLITE_SO_ASC; } - pItem->sortFlags = (u8)iSortOrder; + pItem->fg.sortFlags = (u8)iSortOrder; if( eNulls!=SQLITE_SO_UNDEFINED ){ - pItem->bNulls = 1; + pItem->fg.bNulls = 1; if( iSortOrder!=eNulls ){ - pItem->sortFlags |= KEYINFO_ORDER_BIGNULL; + pItem->fg.sortFlags |= KEYINFO_ORDER_BIGNULL; } } } @@ -104038,7 +112766,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( assert( pList->nExpr>0 ); pItem = &pList->a[pList->nExpr-1]; assert( pItem->zEName==0 ); - assert( pItem->eEName==ENAME_NAME ); + assert( pItem->fg.eEName==ENAME_NAME ); pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); if( dequote ){ /* If dequote==0, then pName->z does not point to part of a DDL @@ -104073,7 +112801,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetSpan( assert( pList->nExpr>0 ); if( pItem->zEName==0 ){ pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd); - pItem->eEName = ENAME_SPAN; + pItem->fg.eEName = ENAME_SPAN; } } } @@ -104102,16 +112830,20 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i = pList->nExpr; struct ExprList_item *pItem = pList->a; assert( pList->nExpr>0 ); + assert( db!=0 ); do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zEName); + if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName); pItem++; }while( --i>0 ); - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); } +SQLITE_PRIVATE void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){ + if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -104180,7 +112912,7 @@ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ ** and 0 if it is FALSE. */ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ - pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely((Expr*)pExpr); assert( pExpr->op==TK_TRUEFALSE ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 @@ -104215,6 +112947,54 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -104243,11 +113023,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from - ** the ON or USING clauses of a left join disqualifies the expression + ** the ON or USING clauses of an outer join disqualifies the expression ** from being considered constant. */ - if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_OuterON) ){ pWalker->eCode = 0; return WRC_Abort; } @@ -104262,6 +113043,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -104290,9 +113073,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -104314,15 +113099,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -104334,9 +113119,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -104352,8 +113143,24 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -104361,9 +113168,103 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. +*/ +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Check pExpr to see if it is an constraint on the single data source +** pSrc = &pSrcList->a[iSrc]. In other words, check to see if pExpr +** constrains pSrc but does not depend on any other tables or data +** sources anywhere else in the query. Return true (non-zero) if pExpr +** is a constraint on pSrc only. +** +** This is an optimization. False negatives will perhaps cause slower +** queries, but false positives will yield incorrect answers. So when in +** doubt, return 0. +** +** To be an single-source constraint, the following must be true: +** +** (1) pExpr cannot refer to any table other than pSrc->iCursor. +** +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. +** +** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. +** (Is there some way to relax this constraint?) +** +** (4) If pSrc is the right operand of a LEFT JOIN, then... +** (4a) pExpr must come from an ON clause.. +** (4b) and specifically the ON clause associated with the LEFT JOIN. +** +** (5) If pSrc is not the right operand of a LEFT JOIN or the left +** operand of a RIGHT JOIN, then pExpr must be from the WHERE +** clause, not an ON clause. +** +** (6) Either: +** +** (6a) pExpr does not originate in an ON or USING clause, or +** +** (6b) The ON or USING clause from which pExpr is derived is +** not to the left of a RIGHT JOIN (or FULL JOIN). +** +** Without this restriction, accepting pExpr as a single-table +** constraint might move the the ON/USING filter expression +** from the left side of a RIGHT JOIN over to the right side, +** which leads to incorrect answers. See also restriction (9) +** on push-down. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( + Expr *pExpr, /* The constraint */ + const SrcList *pSrcList, /* Complete FROM clause */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ +){ + const SrcItem *pSrc = &pSrcList->a[iSrc]; + if( pSrc->fg.jointype & JT_LTORJ ){ + return 0; /* rule (3) */ + } + if( pSrc->fg.jointype & JT_LEFT ){ + if( !ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (4a) */ + if( pExpr->w.iJoin!=pSrc->iCursor ) return 0; /* rule (4b) */ + }else{ + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (5) */ + } + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) /* (6a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (6b) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + if( (pSrcList->a[jj].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* restriction (6) */ + } + break; + } + } + } + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -104448,7 +113349,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprLi */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -104538,10 +113439,14 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - p->y.pTab==0 || /* Reference to column of index on expression */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ + && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; @@ -104601,11 +113506,32 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ return 0; } +/* +** Return a pointer to a buffer containing a usable rowid alias for table +** pTab. An alias is usable if there is not an explicit user-defined column +** of the same name. +*/ +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab){ + const char *azOpt[] = {"_ROWID_", "ROWID", "OID"}; + int ii; + assert( VisibleRowid(pTab) ); + for(ii=0; iinCol; iCol++){ + if( sqlite3_stricmp(azOpt[ii], pTab->aCol[iCol].zCnName)==0 ) break; + } + if( iCol==pTab->nCol ){ + return azOpt[ii]; + } + } + return 0; +} + /* ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient +** or if the SELECT statement needs to be materialized into a transient ** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY @@ -104671,13 +113597,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -104693,7 +113619,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** all members of the RHS set, skipping duplicates. ** ** A cursor is opened on the b-tree object that is the RHS of the IN operator -** and pX->iTable is set to the index of that cursor. +** and the *piTab parameter is set to the index of that cursor. ** ** The returned value of this function indicates the b-tree type, as follows: ** @@ -104701,7 +113627,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** populated ephemeral table. ** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be ** implemented as a sequence of comparisons. ** @@ -104713,7 +113639,10 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then ** pX->iTable made to point to the ephemeral table instead of an -** existing table. +** existing table. In this case, the creation and initialization of the +** ephemeral table might be put inside of a subroutine, the EP_Subrtn flag +** will be set on pX and the pX->y.sub fields will be set to show where +** the subroutine is coded. ** ** The inFlags parameter must contain, at a minimum, one of the bits ** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both. If inFlags contains @@ -104723,12 +113652,12 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table will be created unless the selected columns are guaranteed +** An ephemeral table will be created unless the selected columns are guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or due to ** a UNIQUE constraint or index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used -** for fast set membership tests) then an epheremal table must +** for fast set membership tests) then an ephemeral table must ** be used unless is a single INTEGER PRIMARY KEY column or an ** index can be found with the specified as its left-most. ** @@ -104774,12 +113703,13 @@ SQLITE_PRIVATE int sqlite3FindInIndex( ){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ - int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int iTab; /* Cursor of the RHS table */ int mustBeUnique; /* True if RHS must be unique */ Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + iTab = pParse->nTab++; /* If the RHS of this IN(...) operator is a SELECT, and if it matters ** whether or not the SELECT result contains NULL values, check whether @@ -104887,8 +113817,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex( CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); int j; - assert( pReq!=0 || pRhs->iColumn==XN_ROWID - || pParse->nErr || db->mallocFailed ); for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; assert( pIdx->azColl[j] ); @@ -104944,8 +113872,10 @@ SQLITE_PRIVATE int sqlite3FindInIndex( if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) && ExprUseXList(pX) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ + pParse->nTab--; /* Back out the allocation of the unused cursor */ + iTab = -1; /* Cursor is not allocated */ eType = IN_INDEX_NOOP; } @@ -105060,7 +113990,7 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ ** x IN (SELECT a FROM b) -- IN operator with subquery on the right ** ** The pExpr parameter is the IN operator. The cursor number for the -** constructed ephermeral table is returned. The first time the ephemeral +** constructed ephemeral table is returned. The first time the ephemeral ** table is computed, the cursor number is also stored in pExpr->iTable, ** however the cursor number returned might not be the same, as it might ** have been duplicated using OP_OpenDup. @@ -105112,6 +114042,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( assert( ExprUseYSub(pExpr) ); sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, pExpr->y.sub.iAddr); + assert( iTab!=pExpr->iTable ); sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); sqlite3VdbeJumpHere(v, addrOnce); return; @@ -105123,8 +114054,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = - sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; - VdbeComment((v, "return address")); + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } @@ -105225,7 +114155,8 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ + sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); addrOnce = 0; @@ -105243,11 +114174,15 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } if( addrOnce ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iTab); sqlite3VdbeJumpHere(v, addrOnce); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); - sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); - sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn + || pParse->nErr ); + sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr, 1); + VdbeCoverage(v); sqlite3ClearTempRegCache(pParse); } } @@ -105275,6 +114210,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ SelectDest dest; /* How to deal with SELECT result */ int nReg; /* Registers to allocate */ Expr *pLimit; /* New limit expression */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif Vdbe *v = pParse->pVdbe; assert( v!=0 ); @@ -105301,9 +114239,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ExprSetProperty(pExpr, EP_Subrtn); pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = - sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; - VdbeComment((v, "return address")); - + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; /* The evaluation of the EXISTS/SELECT must be repeated every time it ** is encountered if any of the following is true: @@ -105329,8 +114265,9 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** In both cases, the query is augmented with "LIMIT 1". Any ** preexisting limit is discarded in place of the new LIMIT 1. */ - ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", addrOnce?"":"CORRELATED ", pSel->selId)); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, -1); nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); pParse->nMem += nReg; @@ -105355,7 +114292,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pLimit = sqlite3PExpr(pParse, TK_NE, sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); } - sqlite3ExprDelete(db, pSel->pLimit->pLeft); + sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ @@ -105364,10 +114301,8 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ } pSel->iLimit = 0; if( sqlite3Select(pParse, pSel, &dest) ){ - if( pParse->nErr ){ - pExpr->op2 = pExpr->op; - pExpr->op = TK_ERROR; - } + pExpr->op2 = pExpr->op; + pExpr->op = TK_ERROR; return 0; } pExpr->iTable = rReg = dest.iSDParm; @@ -105375,11 +114310,15 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ if( addrOnce ){ sqlite3VdbeJumpHere(v, addrOnce); } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); - sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); - sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn + || pParse->nErr ); + sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr, 1); + VdbeCoverage(v); sqlite3ClearTempRegCache(pParse); return rReg; } @@ -105584,10 +114523,9 @@ static void sqlite3ExprCodeIN( }else{ destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse); } - if( pParse->nErr ) goto sqlite3ExprCodeIN_finished; for(i=0; ipLeft, i); - if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + if( pParse->nErr ) goto sqlite3ExprCodeIN_oom_error; if( sqlite3ExprCanBeNull(p) ){ sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); VdbeCoverage(v); @@ -105725,11 +114663,12 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ c = sqlite3DecOrHexToI64(z, &value); if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){ #ifdef SQLITE_OMIT_FLOATING_POINT - sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); + sqlite3ErrorMsg(pParse, "oversized integer: %s%#T", negFlag?"-":"",pExpr); #else #ifndef SQLITE_OMIT_HEX_INTEGER if( sqlite3_strnicmp(z,"0x",2)==0 ){ - sqlite3ErrorMsg(pParse, "hex literal too big: %s%s", negFlag?"-":"",z); + sqlite3ErrorMsg(pParse, "hex literal too big: %s%#T", + negFlag?"-":"",pExpr); }else #endif { @@ -105780,6 +114719,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( ){ int iAddr; Vdbe *v = pParse->pVdbe; + int nErr = pParse->nErr; assert( v!=0 ); assert( pParse->iSelfTab!=0 ); if( pParse->iSelfTab>0 ){ @@ -105792,6 +114732,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); + if( pParse->nErr>nErr ) pParse->db->errByteOffset = -1; } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ @@ -105807,12 +114748,11 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( ){ Column *pCol; assert( v!=0 ); - if( pTab==0 ){ - sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); - return; - } + assert( pTab!=0 ); + assert( iCol!=XN_EXPR ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + VdbeComment((v, "%s.rowid", pTab->zName)); }else{ int op; int x; @@ -105865,10 +114805,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( u8 p5 /* P5 value for OP_Column + FLAGS */ ){ assert( pParse->pVdbe!=0 ); + assert( (p5 & (OPFLAG_NOCHNG|OPFLAG_TYPEOFARG|OPFLAG_LENGTHARG))==p5 ); + assert( IsVirtual(pTab) || (p5 & OPFLAG_NOCHNG)==0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; + if( pOp->opcode==OP_VColumn ) pOp->p5 = (p5 & OPFLAG_NOCHNG); } return iReg; } @@ -105897,7 +114840,7 @@ static void exprToRegister(Expr *pExpr, int iReg){ /* ** Evaluate an expression (either a vector or a scalar expression) and store -** the result in continguous temporary registers. Return the index of +** the result in contiguous temporary registers. Return the index of ** the first register used to store the result. ** ** If the returned result register is a temporary scalar, then also write @@ -105936,8 +114879,8 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ ** so that a subsequent copy will not be merged into this one. */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ - if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ - sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergeable */ } } @@ -105983,7 +114926,17 @@ static int exprCodeInlineFunction( caseExpr.x.pList = pFarg; return sqlite3ExprCodeTarget(pParse, &caseExpr, target); } - +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + case INLINEFUNC_sqlite_offset: { + Expr *pArg = pFarg->a[0].pExpr; + if( pArg->op==TK_COLUMN && pArg->iTable>=0 ){ + sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } +#endif default: { /* The UNLIKELY() function is a no-op. The result is the value ** of the first argument. @@ -106017,13 +114970,13 @@ static int exprCodeInlineFunction( } case INLINEFUNC_implies_nonnull_row: { - /* REsult of sqlite3ExprImpliesNonNullRow() */ + /* Result of sqlite3ExprImpliesNonNullRow() */ Expr *pA1; assert( nFarg==2 ); pA1 = pFarg->a[1].pExpr; if( pA1->op==TK_COLUMN ){ sqlite3VdbeAddOp2(v, OP_Integer, - sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable,1), target); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); @@ -106036,10 +114989,13 @@ static int exprCodeInlineFunction( ** the type affinity of the argument. This is used for testing of ** the SQLite type logic. */ - const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + const char *azAff[] = { "blob", "text", "numeric", "integer", + "real", "flexnum" }; char aff; assert( nFarg==1 ); aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + assert( aff<=SQLITE_AFF_NONE + || (aff>=SQLITE_AFF_BLOB && aff<=SQLITE_AFF_FLEXNUM) ); sqlite3VdbeLoadString(v, target, (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); break; @@ -106049,6 +115005,99 @@ static int exprCodeInlineFunction( return target; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxEpr; p; p=p->pIENext){ + u8 exprAff; + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + assert( p->aff>=SQLITE_AFF_BLOB && p->aff<=SQLITE_AFF_NUMERIC ); + exprAff = sqlite3ExprAffinity(pExpr); + if( (exprAff<=SQLITE_AFF_BLOB && p->aff!=SQLITE_AFF_BLOB) + || (exprAff==SQLITE_AFF_TEXT && p->aff!=SQLITE_AFF_TEXT) + || (exprAff>=SQLITE_AFF_NUMERIC && p->aff!=SQLITE_AFF_NUMERIC) + ){ + /* Affinity mismatch on a generated column */ + continue; + } + + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxEpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + + +/* +** Expresion pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** function checks the Parse.pIdxPartExpr list to see if this column +** can be replaced with a constant value. If so, it generates code to +** put the constant value in a register (ideally, but not necessarily, +** register iTarget) and returns the register number. +** +** Or, if the TK_COLUMN cannot be replaced by a constant, zero is +** returned. +*/ +static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ + IndexedExpr *p; + for(p=pParse->pIdxPartExpr; p; p=p->pIENext){ + if( pExpr->iColumn==p->iIdxCol && pExpr->iTable==p->iDataCur ){ + Vdbe *v = pParse->pVdbe; + int addr = 0; + int ret; + + if( p->bMaybeNullRow ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNullRow, p->iIdxCur); + } + ret = sqlite3ExprCodeTarget(pParse, p->pExpr, iTarget); + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, ret, 1, 0, + (const char*)&p->aff, 1); + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP3(v, addr, ret); + } + return ret; + } + } + return 0; +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -106077,25 +115126,44 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxEpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; } + assert( op!=TK_ORDER ); switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol; assert( pAggInfo!=0 ); - assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + assert( pExpr->iAgg>=0 ); + if( pExpr->iAgg>=pAggInfo->nColumn ){ + /* Happens when the left table of a RIGHT JOIN is null and + ** is using an expression index */ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); +#ifdef SQLITE_VDBE_COVERAGE + /* Verify that the OP_Null above is exercised by tests + ** tag-20230325-2 */ + sqlite3VdbeAddOp3(v, OP_NotNull, target, 1, 20230325); + VdbeCoverageNeverTaken(v); +#endif + break; + } pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - return pCol->iMem; + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); }else if( pAggInfo->useSortingIdx ){ Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - if( pCol->iColumn<0 ){ + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); }else{ VdbeComment((v,"%s.%s", @@ -106105,6 +115173,11 @@ expr_code_doover: } } return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ /* no break */ deliberate_fall_through @@ -106115,20 +115188,17 @@ expr_code_doover: if( ExprHasProperty(pExpr, EP_FixedCol) ){ /* This COLUMN expression is really a constant due to WHERE clause ** constraints, and that constant is coded by the pExpr->pLeft - ** expresssion. However, make sure the constant has the correct + ** expression. However, make sure the constant has the correct ** datatype by applying the Affinity of the table column to the ** constant. */ int aff; iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); assert( ExprUseYTab(pExpr) ); - if( pExpr->y.pTab ){ - aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); - }else{ - aff = pExpr->affExpr; - } + assert( pExpr->y.pTab!=0 ); + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); if( aff>SQLITE_AFF_BLOB ){ - static const char zAff[] = "B\000C\000D\000E"; + static const char zAff[] = "B\000C\000D\000E\000F"; assert( SQLITE_AFF_BLOB=='A' ); assert( SQLITE_AFF_TEXT=='B' ); sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, @@ -106187,13 +115257,16 @@ expr_code_doover: iTab = pParse->iSelfTab - 1; } } + else if( pParse->pIdxPartExpr + && 0!=(r1 = exprPartidxExprLookup(pParse, pExpr, target)) + ){ + return r1; + } assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); - if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); - } return iReg; } case TK_INTEGER: { @@ -106246,12 +115319,6 @@ expr_code_doover: assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -106260,11 +115327,8 @@ expr_code_doover: #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( inReg==target ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); @@ -106405,9 +115469,9 @@ expr_code_doover: || NEVER(pExpr->iAgg>=pInfo->nFunc) ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - return pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -106428,7 +115492,9 @@ expr_code_doover: } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -106446,10 +115512,10 @@ expr_code_doover: } #endif if( pDef==0 || pDef->xFinalize!=0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); + sqlite3ErrorMsg(pParse, "unknown function: %#T()", pExpr); break; } - if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)!=0 && ALWAYS(pFarg!=0) ){ assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); return exprCodeInlineFunction(pParse, pFarg, @@ -106459,7 +115525,7 @@ expr_code_doover: } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -106475,10 +115541,10 @@ expr_code_doover: r1 = sqlite3GetTempRange(pParse, nFarg); } - /* For length() and typeof() functions with a column argument, + /* For length() and typeof() and octet_length() functions, ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG - ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data - ** loading. + ** or OPFLAG_TYPEOFARG or OPFLAG_BYTELENARG respectively, to avoid + ** unnecessary data loading. */ if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ u8 exprOp; @@ -106488,14 +115554,16 @@ expr_code_doover: if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); - testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); - pFarg->a[0].pExpr->op2 = - pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + assert( SQLITE_FUNC_BYTELEN==OPFLAG_BYTELENARG ); + assert( (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG)==OPFLAG_BYTELENARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG); + pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG; } } - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } @@ -106522,20 +115590,8 @@ expr_code_doover: if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - if( (pDef->funcFlags & SQLITE_FUNC_OFFSET)!=0 && ALWAYS(pFarg!=0) ){ - Expr *pArg = pFarg->a[0].pExpr; - if( pArg->op==TK_COLUMN ){ - sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target); - }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, target); - } - }else -#endif - { - sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, - pDef, pExpr->op2); - } + sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, + pDef, pExpr->op2); if( nFarg ){ if( constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); @@ -106565,16 +115621,18 @@ expr_code_doover: } case TK_SELECT_COLUMN: { int n; - if( pExpr->pLeft->iTable==0 ){ - pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft); + Expr *pLeft = pExpr->pLeft; + if( pLeft->iTable==0 || pParse->withinRJSubrtn > pLeft->op2 ){ + pLeft->iTable = sqlite3CodeSubselect(pParse, pLeft); + pLeft->op2 = pParse->withinRJSubrtn; } - assert( pExpr->pLeft->op==TK_SELECT || pExpr->pLeft->op==TK_ERROR ); - n = sqlite3ExprVectorSize(pExpr->pLeft); + assert( pLeft->op==TK_SELECT || pLeft->op==TK_ERROR ); + n = sqlite3ExprVectorSize(pLeft); if( pExpr->iTable!=n ){ sqlite3ErrorMsg(pParse, "%d columns assigned %d values", pExpr->iTable, n); } - return pExpr->pLeft->iTable + pExpr->iColumn; + return pLeft->iTable + pExpr->iColumn; } case TK_IN: { int destIfFalse = sqlite3VdbeMakeLabel(pParse); @@ -106605,8 +115663,24 @@ expr_code_doover: exprCodeBetween(pParse, pExpr, target, 0, 0); return target; } + case TK_COLLATE: { + if( !ExprHasProperty(pExpr, EP_Collate) ){ + /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called + ** "SOFT-COLLATE" that is added to constraints that are pushed down + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. + */ + assert( pExpr->pLeft ); + sqlite3ExprCode(pParse, pExpr->pLeft, target); + sqlite3VdbeAddOp1(v, OP_ClrSubtype, target); + return target; + }else{ + pExpr = pExpr->pLeft; + goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */ + } + } case TK_SPAN: - case TK_COLLATE: case TK_UPLUS: { pExpr = pExpr->pLeft; goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. OSSFuzz. */ @@ -106686,16 +115760,34 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; u8 okConstFactor = pParse->okConstFactor; - addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); - /* Temporarily disable factoring of constant expressions, since - ** even though expressions may appear to be constant, they are not - ** really constant because they originate from the right-hand side - ** of a LEFT JOIN. */ - pParse->okConstFactor = 0; - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + AggInfo *pAggInfo = pExpr->pAggInfo; + if( pAggInfo ){ + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( !pAggInfo->directMode ){ + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); + break; + } + if( pExpr->pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pAggInfo->aCol[pExpr->iAgg].iSorterColumn, + target); + inReg = target; + break; + } + } + addrINR = sqlite3VdbeAddOp3(v, OP_IfNullRow, pExpr->iTable, 0, target); + /* The OP_IfNullRow opcode above can overwrite the result register with + ** NULL. So we have to ensure that the result register is not a value + ** that is suppose to be a constant. Two defenses are needed: + ** (1) Temporarily disable factoring of constant expressions + ** (2) Make sure the computed value really is stored in register + ** "target" and not someplace else. + */ + pParse->okConstFactor = 0; /* note (1) above */ + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( target==inReg ); pParse->okConstFactor = okConstFactor; sqlite3VdbeJumpHere(v, addrINR); - sqlite3VdbeChangeP3(v, addrINR, inReg); break; } @@ -106827,9 +115919,9 @@ expr_code_doover: ** once. If no functions are involved, then factor the code out and put it at ** the end of the prepared statement in the initialization section. ** -** If regDest>=0 then the result is always stored in that register and the +** If regDest>0 then the result is always stored in that register and the ** result is not reusable. If regDest<0 then this routine is free to -** store the value whereever it wants. The register where the expression +** store the value wherever it wants. The register where the expression ** is stored is returned. When regDest<0, two identical expressions might ** code to the same register, if they do not contain function calls and hence ** are factored out into the initialization section at the end of the @@ -106842,12 +115934,15 @@ SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( ){ ExprList *p; assert( ConstFactorOk(pParse) ); + assert( regDest!=0 ); p = pParse->pConstExpr; if( regDest<0 && p ){ struct ExprList_item *pItem; int i; for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ - if( pItem->reusable && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 ){ + if( pItem->fg.reusable + && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 + ){ return pItem->u.iConstExprReg; } } @@ -106870,7 +115965,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( p = sqlite3ExprListAppend(pParse, p, pExpr); if( p ){ struct ExprList_item *pItem = &p->a[p->nExpr-1]; - pItem->reusable = regDest<0; + pItem->fg.reusable = regDest<0; if( regDest<0 ) regDest = ++pParse->nMem; pItem->u.iConstExprReg = regDest; } @@ -106898,7 +115993,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -106930,7 +116025,11 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){ + Expr *pX = sqlite3ExprSkipCollateAndLikely(pExpr); + testcase( pX!=pExpr ); + if( ALWAYS(pX) + && (ExprHasProperty(pX,EP_Subquery) || pX->op==TK_REGISTER) + ){ op = OP_Copy; }else{ op = OP_SCopy; @@ -106958,7 +116057,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -107004,7 +116103,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( for(pItem=pList->a, i=0; ipExpr; #ifdef SQLITE_ENABLE_SORTER_REFERENCES - if( pItem->bSorterRef ){ + if( pItem->fg.bSorterRef ){ i--; n--; }else @@ -107017,7 +116116,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -107025,7 +116124,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( inReg!=target+i ){ VdbeOp *pOp; if( copyOp==OP_Copy - && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i && pOp->p5==0 /* The do-not-merge flag must be clear */ @@ -107098,8 +116197,8 @@ static void exprCodeBetween( ** so that the sqlite3ExprCodeTarget() routine will not attempt to move ** it into the Parse.pConstExpr list. We should use a new bit for this, ** for clarity, but we are out of bits in the Expr.flags field so we - ** have to reuse the EP_FromJoin bit. Bummer. */ - pDel->flags |= EP_FromJoin; + ** have to reuse the EP_OuterON bit. Bummer. */ + pDel->flags |= EP_OuterON; sqlite3ExprCodeTarget(pParse, &exprAnd, dest); } sqlite3ReleaseTempReg(pParse, regFree1); @@ -107224,6 +116323,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -107398,6 +116498,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -107551,7 +116652,13 @@ SQLITE_PRIVATE int sqlite3ExprCompare( if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } assert( !ExprHasProperty(pA, EP_IntValue) ); assert( !ExprHasProperty(pB, EP_IntValue) ); @@ -107629,7 +116736,7 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB int res; Expr *pExprA = pA->a[i].pExpr; Expr *pExprB = pB->a[i].pExpr; - if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1; + if( pA->a[i].fg.sortFlags!=pB->a[i].fg.sortFlags ) return 1; if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res; } return 0; @@ -107641,8 +116748,8 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB */ SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){ return sqlite3ExprCompare(0, - sqlite3ExprSkipCollateAndLikely(pA), - sqlite3ExprSkipCollateAndLikely(pB), + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), iTab); } @@ -107735,7 +116842,7 @@ static int exprImpliesNotNull( ** pE1: x!=123 pE2: x IS NOT NULL Result: true ** pE1: x!=?1 pE2: x IS NOT NULL Result: true ** pE1: x IS NULL pE2: x IS NOT NULL Result: false -** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** pE1: x IS ?2 pE2: x IS NOT NULL Result: false ** ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has ** Expr.iTable<0 then assume a table number given by iTab. @@ -107772,11 +116879,29 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr( return 0; } +/* This is a helper function to impliesNotNullRow(). In this routine, +** set pWalker->eCode to one only if *both* of the input expressions +** separately have the implies-not-null-row property. +*/ +static void bothImplyNotNullRow(Walker *pWalker, Expr *pE1, Expr *pE2){ + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pE1); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pE2); + } + } +} + /* ** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). ** If the expression node requires that the table at pWalker->iCur ** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. ** +** pWalker->mWFlags is non-zero if this inquiry is being undertaking on +** behalf of a RIGHT JOIN (or FULL JOIN). That makes a difference when +** evaluating terms in the ON clause of an inner join. +** ** This routine controls an optimization. False positives (setting ** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives ** (never setting pWalker->eCode) is a harmless missed optimization. @@ -107784,29 +116909,34 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr( static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); - if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; + if( ExprHasProperty(pExpr, EP_OuterON) ) return WRC_Prune; + if( ExprHasProperty(pExpr, EP_InnerON) && pWalker->mWFlags ){ + /* If iCur is used in an inner-join ON clause to the left of a + ** RIGHT JOIN, that does *not* mean that the table must be non-null. + ** But it is difficult to check for that condition precisely. + ** To keep things simple, any use of iCur from any inner-join is + ** ignored while attempting to simplify a RIGHT JOIN. */ + return WRC_Prune; + } switch( pExpr->op ){ case TK_ISNOT: case TK_ISNULL: case TK_NOTNULL: case TK_IS: - case TK_OR: case TK_VECTOR: - case TK_CASE: - case TK_IN: case TK_FUNCTION: case TK_TRUTH: + case TK_CASE: testcase( pExpr->op==TK_ISNOT ); testcase( pExpr->op==TK_ISNULL ); testcase( pExpr->op==TK_NOTNULL ); testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_OR ); testcase( pExpr->op==TK_VECTOR ); - testcase( pExpr->op==TK_CASE ); - testcase( pExpr->op==TK_IN ); testcase( pExpr->op==TK_FUNCTION ); testcase( pExpr->op==TK_TRUTH ); + testcase( pExpr->op==TK_CASE ); return WRC_Prune; + case TK_COLUMN: if( pWalker->u.iCur==pExpr->iTable ){ pWalker->eCode = 1; @@ -107814,21 +116944,38 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ } return WRC_Prune; + case TK_OR: case TK_AND: - if( pWalker->eCode==0 ){ + /* Both sides of an AND or OR must separately imply non-null-row. + ** Consider these cases: + ** 1. NOT (x AND y) + ** 2. x OR y + ** If only one of x or y is non-null-row, then the overall expression + ** can be true if the other arm is false (case 1) or true (case 2). + */ + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_AND ); + bothImplyNotNullRow(pWalker, pExpr->pLeft, pExpr->pRight); + return WRC_Prune; + + case TK_IN: + /* Beware of "x NOT IN ()" and "x NOT IN (SELECT 1 WHERE false)", + ** both of which can be true. But apart from these cases, if + ** the left-hand side of the IN is NULL then the IN itself will be + ** NULL. */ + if( ExprUseXList(pExpr) && ALWAYS(pExpr->x.pList->nExpr>0) ){ sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( pWalker->eCode ){ - pWalker->eCode = 0; - sqlite3WalkExpr(pWalker, pExpr->pRight); - } } return WRC_Prune; case TK_BETWEEN: - if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ - assert( pWalker->eCode ); - return WRC_Abort; - } + /* In "x NOT BETWEEN y AND z" either x must be non-null-row or else + ** both y and z must be non-null row */ + assert( ExprUseXList(pExpr) ); + assert( pExpr->x.pList->nExpr==2 ); + sqlite3WalkExpr(pWalker, pExpr->pLeft); + bothImplyNotNullRow(pWalker, pExpr->x.pList->a[0].pExpr, + pExpr->x.pList->a[1].pExpr); return WRC_Prune; /* Virtual tables are allowed to use constraints like x=NULL. So @@ -107853,10 +117000,10 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); if( (pLeft->op==TK_COLUMN - && pLeft->y.pTab!=0 + && ALWAYS(pLeft->y.pTab!=0) && IsVirtual(pLeft->y.pTab)) || (pRight->op==TK_COLUMN - && pRight->y.pTab!=0 + && ALWAYS(pRight->y.pTab!=0) && IsVirtual(pRight->y.pTab)) ){ return WRC_Prune; @@ -107881,8 +117028,8 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ** False positives are not allowed, however. A false positive may result ** in an incorrect answer. ** -** Terms of p that are marked with EP_FromJoin (and hence that come from -** the ON or USING clauses of LEFT JOINS) are excluded from the analysis. +** Terms of p that are marked with EP_OuterON (and hence that come from +** the ON or USING clauses of OUTER JOINS) are excluded from the analysis. ** ** This routine is used to check if a LEFT JOIN can be converted into ** an ordinary JOIN. The p argument is the WHERE clause. If the WHERE @@ -107890,7 +117037,7 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ** be non-NULL, then the LEFT JOIN can be safely converted into an ** ordinary join. */ -SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab, int isRJ){ Walker w; p = sqlite3ExprSkipCollateAndLikely(p); if( p==0 ) return 0; @@ -107898,7 +117045,7 @@ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ p = p->pLeft; }else{ while( p->op==TK_AND ){ - if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab, isRJ) ) return 1; p = p->pRight; } } @@ -107906,6 +117053,7 @@ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ w.xSelectCallback = 0; w.xSelectCallback2 = 0; w.eCode = 0; + w.mWFlags = isRJ!=0; w.u.iCur = iTab; sqlite3WalkExpr(&w, p); return w.eCode; @@ -107966,7 +117114,7 @@ SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( } -/* Structure used to pass information throught the Walker in order to +/* Structure used to pass information throughout the Walker in order to ** implement sqlite3ReferencesSrcList(). */ struct RefSrcList { @@ -108061,6 +117209,7 @@ static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ Walker w; struct RefSrcList x; + assert( pParse->db!=0 ); memset(&w, 0, sizeof(w)); memset(&x, 0, sizeof(x)); w.xExprCallback = exprRefToSrcList; @@ -108072,12 +117221,18 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList assert( pExpr->op==TK_AGG_FUNCTION ); assert( ExprUseXList(pExpr) ); sqlite3WalkExprList(&w, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + assert( pExpr->pLeft->x.pList!=0 ); + sqlite3WalkExprList(&w, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(pExpr, EP_WinFunc) ){ sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); } #endif - sqlite3DbFree(pParse->db, x.aiExclude); + if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude); if( w.eCode & 0x01 ){ return 1; }else if( w.eCode ){ @@ -108095,10 +117250,8 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList ** it does, make a copy. This is done because the pExpr argument is ** subject to change. ** -** The copy is stored on pParse->pConstExpr with a register number of 0. -** This will cause the expression to be deleted automatically when the -** Parse object is destroyed, but the zero register number means that it -** will not generate any code in the preamble. +** The copy is scheduled for deletion using the sqlite3ExprDeferredDelete() +** which builds on the sqlite3ParserAddCleanup() mechanism. */ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) @@ -108108,23 +117261,24 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; - assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); - if( pExpr->op==TK_AGG_COLUMN ){ - assert( iAgg>=0 && iAggnColumn ); - if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + assert( iAgg>=0 ); + if( pExpr->op!=TK_AGG_FUNCTION ){ + if( iAggnColumn + && pAggInfo->aCol[iAgg].pCExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ - assert( iAgg>=0 && iAggnFunc ); - if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + assert( pExpr->op==TK_AGG_FUNCTION ); + if( ALWAYS(iAggnFunc) + && pAggInfo->aFunc[iAgg].pFExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } @@ -108175,6 +117329,74 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ return i; } +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an index of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + assert( pAggInfo->iFirstReg==0 ); + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->pCExpr==pExpr ) return; + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( pExpr->pAggInfo==0 || pExpr->pAggInfo==pAggInfo ); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} + /* ** This is the xExprCallback for a tree walker. It is used to ** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates @@ -108188,87 +117410,76 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ AggInfo *pAggInfo = pNC->uNC.pAggInfo; assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + assert( pParse->iSelfTab==0 ); + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; + for(i=0; inSrc; i++){ + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + } + if( i>=pSrcList->nSrc ) break; + if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ + if( pParse->nErr ){ return WRC_Abort; } + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + if( pParse->nErr ){ return WRC_Abort; } + assert( pAggInfo->aCol!=0 ); + assert( tmp.iAggnColumn ); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } + case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_IF_NULL_ROW ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - assert( ExprUseYTab(pExpr) ); - pCol->pTab = pExpr->y.pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pCExpr = pExpr; - if( pAggInfo->pGroupBy ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return WRC_Prune; + return WRC_Continue; } case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 && pWalker->walkerDepth==pExpr->op2 + && pExpr->pAggInfo==0 ){ /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; for(i=0; inFunc; i++, pItem++){ - if( pItem->pFExpr==pExpr ) break; + if( NEVER(pItem->pFExpr==pExpr) ) break; if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } @@ -108279,15 +117490,44 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ + int nArg; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; - pItem->iMem = ++pParse->nMem; assert( ExprUseUToken(pExpr) ); + nArg = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, - pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); - if( pExpr->flags & EP_Distinct ){ + pExpr->u.zToken, nArg, enc, 0); + assert( pItem->bOBUnique==0 ); + if( pExpr->pLeft + && (pItem->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)==0 + ){ + /* The NEEDCOLL test above causes any ORDER BY clause on + ** aggregate min() or max() to be ignored. */ + ExprList *pOBList; + assert( nArg>0 ); + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + pItem->iOBTab = pParse->nTab++; + pOBList = pExpr->pLeft->x.pList; + assert( pOBList->nExpr>0 ); + assert( pItem->bOBUnique==0 ); + if( pOBList->nExpr==1 + && nArg==1 + && sqlite3ExprCompare(0,pOBList->a[0].pExpr, + pExpr->x.pList->a[0].pExpr,0)==0 + ){ + pItem->bOBPayload = 0; + pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); + }else{ + pItem->bOBPayload = 1; + } + pItem->bUseSubtype = + (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; + }else{ + pItem->iOBTab = -1; + } + if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; @@ -108411,6 +117651,37 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } +/* +** Make sure sufficient registers have been allocated so that +** iReg is a valid register number. +*/ +SQLITE_PRIVATE void sqlite3TouchRegister(Parse *pParse, int iReg){ + if( pParse->nMemnMem = iReg; +} + +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +/* +** Return the latest reusable register in the set of all registers. +** The value returned is no less than iMin. If any register iMin or +** greater is in permanent use, then return one more than that last +** permanent register. +*/ +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ + const ExprList *pList = pParse->pConstExpr; + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].u.iConstExprReg>=iMin ){ + iMin = pList->a[i].u.iConstExprReg + 1; + } + } + } + pParse->nTempReg = 0; + pParse->nRangeReg = 0; + return iMin; +} +#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */ + /* ** Validate that no temporary register falls within the range of ** iFirst..iLast, inclusive. This routine is only call from within assert() @@ -108430,6 +117701,14 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ return 0; } } + if( pParse->pConstExpr ){ + ExprList *pList = pParse->pConstExpr; + for(i=0; inExpr; i++){ + int iReg = pList->a[i].u.iConstExprReg; + if( iReg==0 ) continue; + if( iReg>=iFirst && iReg<=iLast ) return 0; + } + } return 1; } #endif /* SQLITE_DEBUG */ @@ -108762,7 +118041,9 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ int r1; /* Temporary registers */ db = pParse->db; - if( pParse->nErr || db->mallocFailed ) return; + assert( db->pParse==pParse ); + if( pParse->nErr ) return; + assert( db->mallocFailed==0 ); pNew = pParse->pNewTable; assert( pNew ); @@ -108882,14 +118163,19 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ /* Verify that constraints are still satisfied */ if( pNew->pCheck!=0 || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) + || (pTab->tabFlags & TF_Strict)!=0 ){ sqlite3NestedParse(pParse, "SELECT CASE WHEN quick_check GLOB 'CHECK*'" " THEN raise(ABORT,'CHECK constraint failed')" + " WHEN quick_check GLOB 'non-* value in*'" + " THEN raise(ABORT,'type mismatch on DEFAULT')" " ELSE raise(ABORT,'NOT NULL constraint failed')" " END" - " FROM pragma_quick_check(\"%w\",\"%w\")" - " WHERE quick_check GLOB 'CHECK*' OR quick_check GLOB 'NULL*'", + " FROM pragma_quick_check(%Q,%Q)" + " WHERE quick_check GLOB 'CHECK*'" + " OR quick_check GLOB 'NULL*'" + " OR quick_check GLOB 'non-* value in*'", zTab, zDb ); } @@ -108978,7 +118264,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->u.tab.pDfltList = sqlite3ExprListDup(db, pTab->u.tab.pDfltList, 0); pNew->pSchema = db->aDb[iDb].pSchema; pNew->u.tab.addColOffset = pTab->u.tab.addColOffset; - pNew->nTabRef = 1; + assert( pNew->nTabRef==1 ); exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); @@ -109067,7 +118353,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; } if( iCol==pTab->nCol ){ - sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld); + sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld); goto exit_rename_column; } @@ -109173,15 +118459,18 @@ struct RenameCtx { ** following a valid object, it may not be used in comparison operations. */ static void renameTokenCheckAll(Parse *pParse, const void *pPtr){ - if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){ + assert( pParse==pParse->db->pParse ); + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + if( pParse->nErr==0 ){ const RenameToken *p; - u8 i = 0; + u32 i = 1; for(p=pParse->pRename; p; p=p->pNext){ if( p->p ){ assert( p->p!=pPtr ); - i += *(u8*)(p->p); + i += *(u8*)(p->p) | 1; } } + assert( i>0 ); } } #else @@ -109292,11 +118581,10 @@ static void unmapColumnIdlistNames( Parse *pParse, const IdList *pIdList ){ - if( pIdList ){ - int ii; - for(ii=0; iinId; ii++){ - sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName); - } + int ii; + assert( pIdList!=0 ); + for(ii=0; iinId; ii++){ + sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName); } } @@ -109315,7 +118603,7 @@ static int renameUnmapSelectCb(Walker *pWalker, Select *p){ if( ALWAYS(p->pEList) ){ ExprList *pList = p->pEList; for(i=0; inExpr; i++){ - if( pList->a[i].zEName && pList->a[i].eEName==ENAME_NAME ){ + if( pList->a[i].zEName && pList->a[i].fg.eEName==ENAME_NAME ){ sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName); } } @@ -109324,8 +118612,11 @@ static int renameUnmapSelectCb(Walker *pWalker, Select *p){ SrcList *pSrc = p->pSrc; for(i=0; inSrc; i++){ sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); - sqlite3WalkExpr(pWalker, pSrc->a[i].pOn); - unmapColumnIdlistNames(pParse, pSrc->a[i].pUsing); + if( pSrc->a[i].fg.isUsing==0 ){ + sqlite3WalkExpr(pWalker, pSrc->a[i].u3.pOn); + }else{ + unmapColumnIdlistNames(pParse, pSrc->a[i].u3.pUsing); + } } } @@ -109361,7 +118652,7 @@ SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ sWalker.xExprCallback = renameUnmapExprCb; sqlite3WalkExprList(&sWalker, pEList); for(i=0; inExpr; i++){ - if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) ){ + if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME) ){ sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName); } } @@ -109478,7 +118769,7 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ } /* -** An error occured while parsing or otherwise processing a database +** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an ** ALTER TABLE RENAME COLUMN program. The error message emitted by the ** sub-routine is currently stored in pParse->zErrMsg. This function @@ -109495,12 +118786,12 @@ static void renameColumnParseError( const char *zN = (const char*)sqlite3_value_text(pObject); char *zErr; - zErr = sqlite3_mprintf("error in %s %s%s%s: %s", + zErr = sqlite3MPrintf(pParse->db, "error in %s %s%s%s: %s", zT, zN, (zWhen[0] ? " " : ""), zWhen, pParse->zErrMsg ); sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); + sqlite3DbFree(pParse->db, zErr); } /* @@ -109519,7 +118810,7 @@ static void renameColumnElistNames( int i; for(i=0; inExpr; i++){ const char *zName = pEList->a[i].zEName; - if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) + if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME) && ALWAYS(zName!=0) && 0==sqlite3_stricmp(zName, zOld) ){ @@ -109564,24 +118855,22 @@ static int renameParseSql( int bTemp /* True if SQL is from temp schema */ ){ int rc; - char *zErr = 0; + sqlite3ParseObjectInit(p, db); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ + return SQLITE_CORRUPT_BKPT; + } db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); - - /* Parse the SQL statement passed as the first argument. If no error - ** occurs and the parse does not result in a new table, index or - ** trigger object, the database must be corrupt. */ - memset(p, 0, sizeof(Parse)); p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; - rc = zSql ? sqlite3RunParser(p, zSql, &zErr) : SQLITE_NOMEM; - assert( p->zErrMsg==0 ); - assert( rc!=SQLITE_OK || zErr==0 ); - p->zErrMsg = zErr; + rc = sqlite3RunParser(p, zSql); if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK - && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) ){ rc = SQLITE_CORRUPT_BKPT; } @@ -109712,6 +119001,19 @@ static int renameEditSql( return rc; } +/* +** Set all pEList->a[].fg.eEName fields in the expression-list to val. +*/ +static void renameSetENames(ExprList *pEList, int val){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); + pEList->a[i].fg.eEName = val; + } + } +} + /* ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming ** it was read from the schema of database zDb. Return SQLITE_OK if @@ -109735,7 +119037,7 @@ static int renameResolveTrigger(Parse *pParse){ /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -109751,27 +119053,43 @@ static int renameResolveTrigger(Parse *pParse){ if( rc==SQLITE_OK && pStep->zTarget ){ SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); if( pSrc ){ - int i; - for(i=0; inSrc && rc==SQLITE_OK; i++){ - SrcItem *p = &pSrc->a[i]; - p->iCursor = pParse->nTab++; - if( p->pSelect ){ - sqlite3SelectPrep(pParse, p->pSelect, 0); - sqlite3ExpandSubquery(pParse, p); - assert( i>0 ); - assert( pStep->pFrom->a[i-1].pSelect ); - sqlite3SelectPrep(pParse, pStep->pFrom->a[i-1].pSelect, 0); - }else{ - p->pTab = sqlite3LocateTableItem(pParse, 0, p); - if( p->pTab==0 ){ - rc = SQLITE_ERROR; - }else{ - p->pTab->nTabRef++; - rc = sqlite3ViewGetColumnNames(pParse, p->pTab); + Select *pSel = sqlite3SelectNew( + pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 + ); + if( pSel==0 ){ + pStep->pExprList = 0; + pSrc = 0; + rc = SQLITE_NOMEM; + }else{ + /* pStep->pExprList contains an expression-list used for an UPDATE + ** statement. So the a[].zEName values are the RHS of the + ** " = " clauses of the UPDATE statement. So, before + ** running SelectPrep(), change all the eEName values in + ** pStep->pExprList to ENAME_SPAN (from their current value of + ** ENAME_NAME). This is to prevent any ids in ON() clauses that are + ** part of pSrc from being incorrectly resolved against the + ** a[].zEName values as if they were column aliases. */ + renameSetENames(pStep->pExprList, ENAME_SPAN); + sqlite3SelectPrep(pParse, pSel, 0); + renameSetENames(pStep->pExprList, ENAME_NAME); + rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK; + assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList ); + assert( pSrc==pSel->pSrc ); + if( pStep->pExprList ) pSel->pEList = 0; + pSel->pSrc = 0; + sqlite3SelectDelete(db, pSel); + } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pFrom->a[i]; + if( p->pSelect ){ + sqlite3SelectPrep(pParse, p->pSelect, 0); } } } - if( rc==SQLITE_OK && db->mallocFailed ){ + + if( db->mallocFailed ){ rc = SQLITE_NOMEM; } sNC.pSrcList = pSrc; @@ -109859,13 +119177,13 @@ static void renameParseCleanup(Parse *pParse){ sqlite3DeleteTrigger(db, pParse->pNewTrigger); sqlite3DbFree(db, pParse->zErrMsg); renameTokenFree(db, pParse->pRename); - sqlite3ParserReset(pParse); + sqlite3ParseObjectReset(pParse); } /* ** SQL function: ** -** sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld) +** sqlite_rename_column(SQL,TYPE,OBJ,DB,TABLE,COL,NEWNAME,QUOTE,TEMP) ** ** 0. zSql: SQL statement to rewrite ** 1. type: Type of object ("table", "view" etc.) @@ -109883,7 +119201,8 @@ static void renameParseCleanup(Parse *pParse){ ** ** This function is used internally by the ALTER TABLE RENAME COLUMN command. ** It is only accessible to SQL created using sqlite3NestedParse(). It is -** not reachable from ordinary SQL passed into sqlite3_prepare(). +** not reachable from ordinary SQL passed into sqlite3_prepare() unless the +** SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test setting is enabled. */ static void renameColumnFunc( sqlite3_context *context, @@ -110032,7 +119351,9 @@ static void renameColumnFunc( renameColumnFunc_done: if( rc!=SQLITE_OK ){ - if( sParse.zErrMsg ){ + if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){ + sqlite3_result_value(context, argv[0]); + }else if( sParse.zErrMsg ){ renameColumnParseError(context, "", argv[1], argv[2], &sParse); }else{ sqlite3_result_error_code(context, rc); @@ -110220,6 +119541,15 @@ static void renameTableFunc( if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ renameTokenFind(&sParse, &sCtx, pStep->zTarget); } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc; i++){ + SrcItem *pItem = &pStep->pFrom->a[i]; + if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ + renameTokenFind(&sParse, &sCtx, pItem->zName); + } + } + } } } } @@ -110231,7 +119561,9 @@ static void renameTableFunc( rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); } if( rc!=SQLITE_OK ){ - if( sParse.zErrMsg ){ + if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){ + sqlite3_result_value(context, argv[3]); + }else if( sParse.zErrMsg ){ renameColumnParseError(context, "", argv[1], argv[2], &sParse); }else{ sqlite3_result_error_code(context, rc); @@ -110256,10 +119588,10 @@ static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } -/* -** The implementation of an SQL scalar function that rewrites DDL statements -** so that any string literals that use double-quotes are modified so that -** they use single quotes. +/* SQL function: sqlite_rename_quotefix(DB,SQL) +** +** Rewrite the DDL statement "SQL" so that any string literals that use +** double-quotes use single quotes instead. ** ** Two arguments must be passed: ** @@ -110278,6 +119610,10 @@ static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){ ** returns the string: ** ** CREATE VIEW v1 AS SELECT "a", 'string' FROM t1 +** +** If there is a error in the input SQL, then raise an error, except +** if PRAGMA writable_schema=ON, then just return the input string +** unmodified following an error. */ static void renameQuotefixFunc( sqlite3_context *context, @@ -110352,7 +119688,11 @@ static void renameQuotefixFunc( renameTokenFree(db, sCtx.pList); } if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(context, rc); + if( sqlite3WritableSchema(db) && rc==SQLITE_ERROR ){ + sqlite3_result_value(context, argv[1]); + }else{ + sqlite3_result_error_code(context, rc); + } } renameParseCleanup(&sParse); } @@ -110364,7 +119704,8 @@ static void renameQuotefixFunc( sqlite3BtreeLeaveAll(db); } -/* +/* Function: sqlite_rename_test(DB,SQL,TYPE,NAME,ISTEMP,WHEN,DQS) +** ** An SQL user function that checks that there are no parse or symbol ** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. ** After an ALTER TABLE .. RENAME operation is performed and the schema @@ -110379,11 +119720,13 @@ static void renameQuotefixFunc( ** 5: "when" part of error message. ** 6: True to disable the DQS quirk when parsing SQL. ** -** Unless it finds an error, this function normally returns NULL. However, it -** returns integer value 1 if: +** The return value is computed as follows: ** -** * the SQL argument creates a trigger, and -** * the table that the trigger is attached to is in database zDb. +** A. If an error is seen and not in PRAGMA writable_schema=ON mode, +** then raise the error. +** B. Else if a trigger is created and the the table that the trigger is +** attached to is in database zDb, then return 1. +** C. Otherwise return NULL. */ static void renameTableTest( sqlite3_context *context, @@ -110428,12 +119771,16 @@ static void renameTableTest( if( rc==SQLITE_OK ){ int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); int i2 = sqlite3FindDbName(db, zDb); - if( i1==i2 ) sqlite3_result_int(context, 1); + if( i1==i2 ){ + /* Handle output case B */ + sqlite3_result_int(context, 1); + } } } } - if( rc!=SQLITE_OK && zWhen ){ + if( rc!=SQLITE_OK && zWhen && !sqlite3WritableSchema(db) ){ + /* Output case A */ renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse); } renameParseCleanup(&sParse); @@ -110549,7 +119896,7 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T } iCol = sqlite3ColumnIndex(pTab, zCol); if( iCol<0 ){ - sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zCol); + sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pName); goto exit_drop_column; } @@ -110573,6 +119920,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDb>=0 ); zDb = db->aDb[iDb].zDbSName; +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ + goto exit_drop_column; + } +#endif renameTestSchema(pParse, zDb, iDb==1, "", 0); renameFixQuotes(pParse, zDb, iDb==1); sqlite3NestedParse(pParse, @@ -110626,7 +119979,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } @@ -110937,9 +120295,9 @@ static void openStatTable( typedef struct StatAccum StatAccum; typedef struct StatSample StatSample; struct StatSample { - tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anDLt; /* sqlite_stat4.nDLt */ #ifdef SQLITE_ENABLE_STAT4 + tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anLt; /* sqlite_stat4.nLt */ union { i64 iRowid; /* Rowid in main table of the key */ @@ -111097,9 +120455,9 @@ static void statInit( /* Allocate the space required for the StatAccum object */ n = sizeof(*p) - + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ - + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ #ifdef SQLITE_ENABLE_STAT4 + n += sizeof(tRowcnt)*nColUp; /* StatAccum.anEq */ if( mxSample ){ n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ @@ -111120,9 +120478,9 @@ static void statInit( p->nKeyCol = nKeyCol; p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; - p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 + p->current.anEq = &p->current.anDLt[nColUp]; p->mxSample = p->nLimit==0 ? mxSample : 0; if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ @@ -111389,7 +120747,9 @@ static void statPush( if( p->nRow==0 ){ /* This is the first call to this function. Do initialization. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; inCol; i++) p->current.anEq[i] = 1; +#endif }else{ /* Second and subsequent calls get processed here */ #ifdef SQLITE_ENABLE_STAT4 @@ -111398,15 +120758,17 @@ static void statPush( /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply ** to the current row of the index. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; icurrent.anEq[i]++; } +#endif for(i=iChng; inCol; i++){ p->current.anDLt[i]++; #ifdef SQLITE_ENABLE_STAT4 if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; -#endif p->current.anEq[i] = 1; +#endif } } @@ -111520,9 +120882,14 @@ static void statGet( ** * "WHERE a=? AND b=?" matches 2 rows. ** ** If D is the count of distinct values and K is the total number of - ** rows, then each estimate is computed as: + ** rows, then each estimate is usually computed as: ** ** I = (K+D-1)/D + ** + ** In other words, I is K/D rounded up to the next whole integer. + ** However, if I is between 1.0 and 1.1 (in other words if I is + ** close to 1.0 but just a little larger) then do not round up but + ** instead keep the I value at 1.0. */ sqlite3_str sStat; /* Text of the constructed "stat" line */ int i; /* Loop counter */ @@ -111533,8 +120900,11 @@ static void statGet( for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); - assert( p->current.anEq[i] ); +#ifdef SQLITE_ENABLE_STAT4 + assert( p->current.anEq[i] || p->nRow==0 ); +#endif } sqlite3ResultStrAccum(context, &sStat); } @@ -111620,6 +120990,7 @@ static void analyzeVdbeCommentIndexWithColumnName( if( NEVER(i==XN_ROWID) ){ VdbeComment((v,"%s.rowid",pIdx->zName)); }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); }else{ VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); @@ -111660,11 +121031,15 @@ static void analyzeOneTable( int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_STAT4 + int doOnce = 1; /* Flag for a one-time computation */ +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK Table *pStat1 = 0; #endif - pParse->nMem = MAX(pParse->nMem, iMem); + sqlite3TouchRegister(pParse, iMem); + assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) ); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -111696,7 +121071,7 @@ static void analyzeOneTable( memcpy(pStat1->zName, "sqlite_stat1", 13); pStat1->nCol = 3; pStat1->iPKey = -1; - sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNBLOB); + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNAMIC); } #endif @@ -111713,7 +121088,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -111737,9 +121112,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -111770,7 +121150,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + sqlite3TouchRegister(pParse, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -111778,41 +121158,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: + /* Implementation of the following: ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -111919,6 +121294,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -111942,7 +121323,42 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol); + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + + if( doOnce ){ + int mxCol = nCol; + Index *pX; + + /* Compute the maximum number of columns in any index */ + for(pX=pTab->pIndex; pX; pX=pX->pNext){ + int nColX; /* Number of columns in pX */ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){ + nColX = pX->nKeyCol; + }else{ + nColX = pX->nColumn; + } + if( nColX>mxCol ) mxCol = nColX; + } + + /* Allocate space to compute results for the largest index */ + sqlite3TouchRegister(pParse, regCol+mxCol); + doOnce = 0; +#ifdef SQLITE_DEBUG + /* Verify that the call to sqlite3ClearTempRegCache() below + ** really is needed. + ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) + */ + testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); +#endif + sqlite3ClearTempRegCache(pParse); /* tag-20230325-1 */ + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); + } + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) ); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); @@ -111966,7 +121382,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -112023,6 +121439,11 @@ static void analyzeDatabase(Parse *pParse, int iDb){ for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); +#ifdef SQLITE_ENABLE_STAT4 + iMem = sqlite3FirstAvailableRegister(pParse, iMem); +#else + assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) ); +#endif } loadAnalysis(pParse, iDb); } @@ -112185,6 +121606,16 @@ static void decodeIntArray( while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } + + /* Set the bLowQual flag if the peak number of rows obtained + ** from a full equality match is so large that a full table scan + ** seems likely to be faster than using the index. + */ + if( aLog[0] > 66 /* Index has more than 100 rows */ + && aLog[0] <= aLog[nOut-1] /* And only a single value seen */ + ){ + pIndex->bLowQual = 1; + } } } @@ -112263,6 +121694,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ + assert( db!=0 ); + assert( pIdx!=0 ); #ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; @@ -112272,7 +121705,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } sqlite3DbFree(db, pIdx->aSample); } - if( db && db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; } @@ -112408,6 +121841,10 @@ static int loadStatTbl( pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); assert( pIdx==0 || pIdx->nSample==0 ); if( pIdx==0 ) continue; + if( pIdx->aSample!=0 ){ + /* The same index appears in sqlite_stat4 under multiple names */ + continue; + } assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ nIdxCol = pIdx->nKeyCol; @@ -112415,6 +121852,7 @@ static int loadStatTbl( nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; + pIdx->mxSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -112454,6 +121892,11 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; + if( pIdx->nSample>=pIdx->mxSample ){ + /* Too many slots used because the same index appears in + ** sqlite_stat4 using multiple names */ + continue; + } /* This next condition is true if data has already been loaded from ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; @@ -112466,14 +121909,15 @@ static int loadStatTbl( decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); - /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + /* Take a copy of the sample. Add 8 extra 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the ** sqlite3VdbeRecordCompare() may read up to two varints past the ** end of the allocated buffer before it realizes it is dealing with - ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a corrupt record. Or it might try to read a large integer from the + ** buffer. In any case, eight 0x00 bytes prevents this from causing ** a buffer overread. */ pSample->n = sqlite3_column_bytes(pStmt, 4); - pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 8); if( pSample->p==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; @@ -112497,11 +121941,12 @@ static int loadStat4(sqlite3 *db, const char *zDb){ const Table *pStat4; assert( db->lookaside.bDisable ); - if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 + if( OptimizationEnabled(db, SQLITE_Stat4) + && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 && IsOrdinaryTable(pStat4) ){ rc = loadStatTbl(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); @@ -112691,7 +122136,7 @@ static void attachFunc( char *zErr = 0; unsigned int flags; Db *aNew; /* New array of Db pointers */ - Db *pNew; /* Db object for the newly attached database */ + Db *pNew = 0; /* Db object for the newly attached database */ char *zErrDyn = 0; sqlite3_vfs *pVfs; @@ -112711,13 +122156,26 @@ static void attachFunc( /* This is not a real ATTACH. Instead, this routine is being called ** from sqlite3_deserialize() to close database db->init.iDb and ** reopen it as a MemDB */ + Btree *pNewBt = 0; pVfs = sqlite3_vfs_find("memdb"); if( pVfs==0 ) return; - pNew = &db->aDb[db->init.iDb]; - if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt); - pNew->pBt = 0; - pNew->pSchema = 0; - rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNewBt, 0, SQLITE_OPEN_MAIN_DB); + if( rc==SQLITE_OK ){ + Schema *pNewSchema = sqlite3SchemaGet(db, pNewBt); + if( pNewSchema ){ + /* Both the Btree and the new Schema were allocated successfully. + ** Close the old db and update the aDb[] slot with the new memdb + ** values. */ + pNew = &db->aDb[db->init.iDb]; + if( ALWAYS(pNew->pBt) ) sqlite3BtreeClose(pNew->pBt); + pNew->pBt = pNewBt; + pNew->pSchema = pNewSchema; + }else{ + sqlite3BtreeClose(pNewBt); + rc = SQLITE_NOMEM; + } + } + if( rc ) goto attach_error; }else{ /* This is a real ATTACH ** @@ -112803,7 +122261,6 @@ static void attachFunc( if( rc==SQLITE_OK && pNew->zDbSName==0 ){ rc = SQLITE_NOMEM_BKPT; } - #ifdef SQLITE_HAS_CODEC if( rc==SQLITE_OK ){ extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); @@ -112837,7 +122294,7 @@ static void attachFunc( break; } } -#endif +#endif /* SQLITE_HAS_CODEC */ sqlite3_free_filename( zPath ); /* If the file was opened successfully, read the schema for the new database. @@ -112865,7 +122322,7 @@ static void attachFunc( } #endif if( rc ){ - if( !REOPEN_AS_MEMDB(db) ){ + if( ALWAYS(!REOPEN_AS_MEMDB(db)) ){ int iDb = db->nDb - 1; assert( iDb>=2 ); if( db->aDb[iDb].pBt ){ @@ -112982,6 +122439,8 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ) goto attach_end; + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -112995,7 +122454,7 @@ static void codeAttach( } #ifndef SQLITE_OMIT_AUTHORIZATION - if( pAuthArg ){ + if( ALWAYS(pAuthArg) ){ char *zAuthArg; if( pAuthArg->op==TK_STRING ){ assert( !ExprHasProperty(pAuthArg, EP_IntValue) ); @@ -113121,7 +122580,11 @@ static int fixSelectCb(Walker *p, Select *pSelect){ pItem->fg.fromDDL = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) - if( sqlite3WalkExpr(&pFix->w, pList->a[i].pOn) ) return WRC_Abort; + if( pList->a[i].fg.isUsing==0 + && sqlite3WalkExpr(&pFix->w, pList->a[i].u3.pOn) + ){ + return WRC_Abort; + } #endif } if( pSelect->pWith ){ @@ -113446,7 +122909,7 @@ SQLITE_PRIVATE int sqlite3AuthCheck( sqlite3 *db = pParse->db; int rc; - /* Don't do any authorization checks if the database is initialising + /* Don't do any authorization checks if the database is initializing ** or if the parser is being invoked from within sqlite3_declare_vtab. */ assert( !IN_RENAME_OBJECT || db->xAuth==0 ); @@ -113653,14 +123116,17 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + int iDb, i; assert( pParse->pToplevel==0 ); db = pParse->db; + assert( db->pParse==pParse ); if( pParse->nested ) return; - if( db->mallocFailed || pParse->nErr ){ - if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + if( pParse->nErr ){ + if( db->mallocFailed ) pParse->rc = SQLITE_NOMEM; return; } + assert( db->mallocFailed==0 ); /* Begin by generating some termination code at the end of the ** vdbe program @@ -113680,12 +123146,10 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ if( pParse->bReturning ){ Returning *pReturning = pParse->u1.pReturning; int addrRewind; - int i; int reg; - if( pReturning->nRetCol==0 ){ - assert( CORRUPT_DB ); - }else{ + if( pReturning->nRetCol ){ + sqlite3VdbeAddOp0(v, OP_FkCheck); addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); VdbeCoverage(v); @@ -113701,7 +123165,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION && !defined(SQLITE_OMIT_SHARED_CACHE) if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelmallocFailed==0 - && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) - ){ - int iDb, i; - assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); - sqlite3VdbeJumpHere(v, 0); - for(iDb=0; iDbnDb; iDb++){ - Schema *pSchema; - if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - pSchema = db->aDb[iDb].pSchema; - sqlite3VdbeAddOp4Int(v, - OP_Transaction, /* Opcode */ - iDb, /* P1 */ - DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pSchema->schema_cookie, /* P3 */ - pSchema->iGeneration /* P4 */ - ); - if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); - } + assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + assert( db->nDb>0 ); + iDb = 0; + do{ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + }while( ++iDbnDb ); #ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); +#ifndef SQLITE_OMIT_SHARED_CACHE + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + if( pParse->nTableLock ) codeTableLocks(pParse); +#endif - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); + /* Initialize any AUTOINCREMENT data structures required. + */ + if( pParse->pAinc ) sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. - */ - if( pParse->pConstExpr ){ - ExprList *pEL = pParse->pConstExpr; - pParse->okConstFactor = 0; - for(i=0; inExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - if( iReg>0 ){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); - } - } + /* Code constant expressions that were factored out of inner loops. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + assert( pEL->a[i].u.iConstExprReg>0 ); + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); } + } - if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; - if( pRet->nRetCol==0 ){ - assert( CORRUPT_DB ); - }else{ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); - } + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + if( pRet->nRetCol ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); } - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); } /* Get the VDBE program ready for execution */ - if( v && pParse->nErr==0 && !db->mallocFailed ){ + assert( v!=0 || pParse->nErr ); + assert( db->mallocFailed==0 || pParse->nErr ); + if( pParse->nErr==0 ){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ assert( pParse->pAinc==0 || pParse->nTab>0 ); @@ -113819,12 +123275,12 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ va_list ap; char *zSql; - char *zErrMsg = 0; sqlite3 *db = pParse->db; u32 savedDbFlags = db->mDbFlags; char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; + if( pParse->eParseMode ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ va_start(ap, zFormat); zSql = sqlite3VMPrintf(db, zFormat, ap); @@ -113841,9 +123297,8 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); db->mDbFlags |= DBFLAG_PreferBuiltin; - sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3RunParser(pParse, zSql); db->mDbFlags = savedDbFlags; - sqlite3DbFree(db, zErrMsg); sqlite3DbFree(db, zSql); memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); pParse->nested--; @@ -113972,7 +123427,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ - if( pParse->disableVtab==0 && db->init.busy==0 ){ + if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){ Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); @@ -113985,7 +123440,7 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( #endif if( flags & LOCATE_NOERR ) return 0; pParse->checkSchema = 1; - }else if( IsVirtual(p) && pParse->disableVtab ){ + }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){ p = 0; } @@ -114241,7 +123696,7 @@ SQLITE_PRIVATE void sqlite3ColumnSetExpr( */ SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ if( pCol->iDflt==0 ) return 0; - if( NEVER(!IsOrdinaryTable(pTab)) ) return 0; + if( !IsOrdinaryTable(pTab) ) return 0; if( NEVER(pTab->u.tab.pDfltList==0) ) return 0; if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; @@ -114273,7 +123728,7 @@ SQLITE_PRIVATE void sqlite3ColumnSetColl( } /* -** Return the collating squence name for a column +** Return the collating sequence name for a column */ SQLITE_PRIVATE const char *sqlite3ColumnColl(Column *pCol){ const char *z; @@ -114294,16 +123749,17 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); + assert( db!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); sqlite3DbFree(db, pCol->zCnName); } - sqlite3DbFree(db, pTable->aCol); + sqlite3DbNNFreeNN(db, pTable->aCol); if( IsOrdinaryTable(pTable) ){ sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); } - if( db==0 || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pTable->aCol = 0; pTable->nCol = 0; if( IsOrdinaryTable(pTable) ){ @@ -114340,7 +123796,8 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ ** a Table object that was going to be marked ephemeral. So do not check ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + assert( db!=0 ); + if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -114350,7 +123807,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); - if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -114364,7 +123821,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ if( IsOrdinaryTable(pTable) ){ sqlite3FkDelete(db, pTable); } -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE else if( IsVirtual(pTable) ){ sqlite3VtabClear(db, pTable); } @@ -114387,10 +123844,14 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ } SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ + assert( db!=0 ); if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } +SQLITE_PRIVATE void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){ + sqlite3DeleteTable(db, (Table*)pTable); +} /* @@ -114410,7 +123871,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK u8 tableType = p->eTabType; -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ sqlite3DeleteTable(db, p); db->mDbFlags |= DBFLAG_SchemaChange; #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK @@ -114425,7 +123886,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char memcpy(db->mDropSchemaName, db->aDb[iDb].zDbSName, strlen(db->aDb[iDb].zDbSName) + 1); } } -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ } /* @@ -114819,7 +124280,8 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable = sqlite3FindTable(db, zName, zDb); if( pTable ){ if( !noErr ){ - sqlite3ErrorMsg(pParse, "table %T already exists", pName); + sqlite3ErrorMsg(pParse, "%s %T already exists", + (IsView(pTable)? "view" : "table"), pName); }else{ assert( !db->init.busy || CORRUPT_DB ); sqlite3CodeVerifySchema(pParse, iDb); @@ -114940,20 +124402,14 @@ SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ } #endif -/* -** Name of the special TEMP trigger used to implement RETURNING. The -** name begins with "sqlite_" so that it is guaranteed not to collide -** with any application-generated triggers. -*/ -#define RETURNING_TRIGGER_NAME "sqlite_returning" - /* ** Clean up the data structures associated with the RETURNING clause. */ -static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ +static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){ + Returning *pRet = (Returning*)pArg; Hash *pHash; pHash = &(db->aDb[1].pSchema->trigHash); - sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, 0); + sqlite3HashInsert(pHash, pRet->zName, 0); sqlite3ExprListDelete(db, pRet->pReturnEL); sqlite3DbFree(db, pRet); } @@ -114981,7 +124437,7 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ if( pParse->pNewTrigger ){ sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); }else{ - assert( pParse->bReturning==0 ); + assert( pParse->bReturning==0 || pParse->ifNotExists ); } pParse->bReturning = 1; pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); @@ -114992,11 +124448,12 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pParse->u1.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); + sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); testcase( pParse->earlyCleanup ); if( db->mallocFailed ) return; - pRet->retTrig.zName = RETURNING_TRIGGER_NAME; + sqlite3_snprintf(sizeof(pRet->zName), pRet->zName, + "sqlite_returning_%p", pParse); + pRet->retTrig.zName = pRet->zName; pRet->retTrig.op = TK_RETURNING; pRet->retTrig.tr_tm = TRIGGER_AFTER; pRet->retTrig.bReturning = 1; @@ -115007,8 +124464,9 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); - if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) + assert( sqlite3HashFind(pHash, pRet->zName)==0 + || pParse->nErr || pParse->ifNotExists ); + if( sqlite3HashInsert(pHash, pRet->zName, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); } @@ -115042,7 +124500,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ } if( !IN_RENAME_OBJECT ) sqlite3DequoteToken(&sName); - /* Because keywords GENERATE ALWAYS can be converted into indentifiers + /* Because keywords GENERATE ALWAYS can be converted into identifiers ** by the parser, we can sometimes end up with a typename that ends ** with "generated always". Check for this case and omit the surplus ** text. */ @@ -115189,7 +124647,8 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, Column *pCol){ assert( zIn!=0 ); while( zIn[0] ){ - h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + u8 x = *(u8*)zIn; + h = (h<<8) + sqlite3UpperToLower[x]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ aff = SQLITE_AFF_TEXT; @@ -115263,7 +124722,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The parsed expression of the default value */ const char *zStart, /* Start of the default value text */ - const char *zEnd /* First character past end of defaut value text */ + const char *zEnd /* First character past end of default value text */ ){ Table *p; Column *pCol; @@ -115415,7 +124874,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); pTab->tabFlags |= autoInc*TF_Autoincrement; - if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags; + if( pList ) pParse->iPkSortOrder = pList->a[0].fg.sortFlags; (void)sqlite3HasExplicitNulls(pParse, pList); }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT @@ -115535,6 +124994,14 @@ SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType if( pCol->colFlags & COLFLAG_PRIMKEY ){ makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ } + if( ALWAYS(pExpr) && pExpr->op==TK_ID ){ + /* The value of a generated column needs to be a real expression, not + ** just a reference to another column, in order for covering index + ** optimizations to work correctly. So if the value is not an expression, + ** turn it into one by adding a unary "+" operator. */ + pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); + } + if( pExpr && pExpr->op!=TK_RAISE ) pExpr->affExpr = pCol->affinity; sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; @@ -115603,7 +125070,7 @@ static int identLength(const char *z){ ** to the specified offset in the buffer and updates *pIdx to refer ** to the first byte after the last byte written before returning. ** -** If the string zSignedIdent consists entirely of alpha-numeric +** If the string zSignedIdent consists entirely of alphanumeric ** characters, does not begin with a digit and is not an SQL keyword, ** then it is copied to the output buffer exactly as it is. Otherwise, ** it is quoted using double-quotes. @@ -115671,7 +125138,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", - /* SQLITE_AFF_REAL */ " REAL" + /* SQLITE_AFF_REAL */ " REAL", + /* SQLITE_AFF_FLEXNUM */ " NUM", }; int len; const char *zType; @@ -115687,10 +125155,12 @@ static char *createTableStmt(sqlite3 *db, Table *p){ testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); + testcase( pCol->affinity==SQLITE_AFF_FLEXNUM ); zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -115752,7 +125222,7 @@ static void estimateIndexWidth(Index *pIdx){ for(i=0; inColumn; i++){ i16 x = pIdx->aiColumn[i]; assert( xpTable->nCol ); - wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + wIndex += x<0 ? 1 : aCol[x].szEst; } pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } @@ -115807,7 +125277,8 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ /* Recompute the colNotIdxed field of the Index. ** ** colNotIdxed is a bitmask that has a 0 bit representing each indexed -** columns that are within the first 63 columns of the table. The +** columns that are within the first 63 columns of the table and a 1 for +** all other bits (all columns that are not in the index). The ** high-order bit of colNotIdxed is always 1. All unindexed columns ** of the table have a 1. ** @@ -115835,7 +125306,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){ } } pIdx->colNotIdxed = ~m; - assert( (pIdx->colNotIdxed>>63)==1 ); + assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */ } /* @@ -115909,15 +125380,16 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); } - pList->a[0].sortFlags = pParse->iPkSortOrder; + pList->a[0].fg.sortFlags = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); pTab->iPKey = -1; sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, SQLITE_IDXTYPE_PRIMARYKEY); - if( db->mallocFailed || pParse->nErr ){ + if( pParse->nErr ){ pTab->tabFlags &= ~TF_WithoutRowid; return; } + assert( db->mallocFailed==0 ); pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk->nKeyCol==1 ); }else{ @@ -116103,6 +125575,7 @@ SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ ** not pass them into code generator routines by mistake. */ static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + (void)pWalker; ExprSetVVAProperty(pExpr, EP_Immutable); return WRC_Continue; } @@ -116347,15 +125820,20 @@ SQLITE_PRIVATE void sqlite3EndTable( int regRowid; /* Rowid of the next row to insert */ int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + int iCsr; /* Write cursor on the new table */ + if( IN_SPECIAL_PARSE ){ + pParse->rc = SQLITE_ERROR; + pParse->nErr++; + return; + } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -116376,11 +125854,11 @@ SQLITE_PRIVATE void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -116433,6 +125911,14 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + + /* Test for cycles in generated columns and illegal expressions + ** in CHECK constraints and in DEFAULT clauses. */ + if( p->tabFlags & TF_HasGenerated ){ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, + sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", + db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); + } } /* Add the table to the in-memory representation of the database. @@ -116509,9 +125995,12 @@ SQLITE_PRIVATE void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -116567,14 +126056,14 @@ create_view_fail: #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ -SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ +static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ - int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ #ifndef SQLITE_OMIT_VIRTUALTABLE int rc; @@ -116596,9 +126085,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #ifndef SQLITE_OMIT_VIEW /* A positive nCol means the columns names for this view are - ** already known. + ** already known. This routine is not called unless either the + ** table is virtual or nCol is zero. */ - if( pTable->nCol>0 ) return 0; + assert( pTable->nCol<=0 ); /* A negative nCol is a special marker meaning that we are currently ** trying to compute the column names. If we enter this routine with @@ -116632,8 +126122,9 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pSel = sqlite3SelectDup(db, pTable->u.view.pSelect, 0); if( pSel ){ u8 eParseMode = pParse->eParseMode; + int nTab = pParse->nTab; + int nSelect = pParse->nSelect; pParse->eParseMode = PARSE_MODE_NORMAL; - n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; DisableLookaside; @@ -116645,7 +126136,8 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #else pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); #endif - pParse->nTab = n; + pParse->nTab = nTab; + pParse->nSelect = nSelect; if( pSelTab==0 ){ pTable->nCol = 0; nErr++; @@ -116658,12 +126150,11 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ */ sqlite3ColumnsFromExprList(pParse, pTable->pCheck, &pTable->nCol, &pTable->aCol); - if( db->mallocFailed==0 - && pParse->nErr==0 + if( pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr ){ - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, - SQLITE_AFF_NONE); + assert( db->mallocFailed==0 ); + sqlite3SubqueryColumnTypes(pParse, pTable, pSel, SQLITE_AFF_NONE); } }else{ /* CREATE VIEW name AS... without an argument list. Construct @@ -116690,7 +126181,12 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; +} +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + assert( pTable!=0 ); + if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; + return viewGetColumnNames(pParse, pTable); } #endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ @@ -117280,7 +126776,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + assert( pKey!=0 || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; @@ -117390,8 +126886,8 @@ SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ if( pList ){ int i; for(i=0; inExpr; i++){ - if( pList->a[i].bNulls ){ - u8 sf = pList->a[i].sortFlags; + if( pList->a[i].fg.bNulls ){ + u8 sf = pList->a[i].fg.sortFlags; sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", (sf==0 || sf==3) ? "FIRST" : "LAST" ); @@ -117444,9 +126940,11 @@ SQLITE_PRIVATE void sqlite3CreateIndex( char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - if( db->mallocFailed || pParse->nErr>0 ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto exit_create_index; } + assert( db->mallocFailed==0 ); if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ goto exit_create_index; } @@ -117474,7 +126972,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the table ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. + ** if initializing a database schema. */ if( !db->init.busy ){ pTab = sqlite3SrcListLookup(pParse, pTblName); @@ -117510,7 +127008,6 @@ SQLITE_PRIVATE void sqlite3CreateIndex( pDb = &db->aDb[iDb]; assert( pTab!=0 ); - assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 && pTblName!=0 @@ -117556,7 +127053,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( !IN_RENAME_OBJECT ){ if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ + if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){ sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); goto exit_create_index; } @@ -117709,6 +127206,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); @@ -117720,6 +127218,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ pIndex->bHasVCol = 1; + pIndex->bHasExpr = 1; } } pIndex->aiColumn[i] = (i16)j; @@ -117743,7 +127242,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } pIndex->azColl[i] = zColl; - requestedSortOrder = pListItem->sortFlags & sortOrderMask; + requestedSortOrder = pListItem->fg.sortFlags & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; } @@ -117934,13 +127433,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( /* Add an entry in sqlite_schema for this index */ sqlite3NestedParse(pParse, - "INSERT INTO %Q." LEGACY_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zDbSName, - pIndex->zName, - pTab->zName, - iMem, - zStmt - ); + "INSERT INTO %Q." LEGACY_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); sqlite3DbFree(db, zStmt); /* Fill the index with data and reparse the schema. Code an OP_Expire @@ -118074,10 +127573,10 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists sqlite3 *db = pParse->db; int iDb; - assert( pParse->nErr==0 ); /* Never called with prior errors */ if( db->mallocFailed ){ goto exit_drop_index; } + assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; @@ -118186,18 +127685,17 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token * if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); if( pList==0 ) return 0; + }else{ + IdList *pNew; + pNew = sqlite3DbRealloc(db, pList, + sizeof(IdList) + pList->nId*sizeof(pList->a)); + if( pNew==0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList = pNew; } - pList->a = sqlite3ArrayAllocate( - db, - pList->a, - sizeof(pList->a[0]), - &pList->nId, - &i - ); - if( i<0 ){ - sqlite3IdListDelete(db, pList); - return 0; - } + i = pList->nId++; pList->a[i].zName = sqlite3NameFromToken(db, pToken); if( IN_RENAME_OBJECT && pList->a[i].zName ){ sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); @@ -118210,12 +127708,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token * */ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; + assert( db!=0 ); if( pList==0 ) return; + assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } - sqlite3DbFree(db, pList->a); - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -118224,7 +127723,7 @@ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ */ SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ int i; - if( pList==0 ) return -1; + assert( pList!=0 ); for(i=0; inId; i++){ if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; } @@ -118418,19 +127917,23 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; SrcItem *pItem; + assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); sqlite3DeleteTable(db, pItem->pTab); if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); - if( pItem->pOn ) sqlite3ExprDelete(db, pItem->pOn); - if( pItem->pUsing ) sqlite3IdListDelete(db, pItem->pUsing); + if( pItem->fg.isUsing ){ + sqlite3IdListDelete(db, pItem->u3.pUsing); + }else if( pItem->u3.pOn ){ + sqlite3ExprDelete(db, pItem->u3.pOn); + } } - sqlite3DbFreeNN(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -118456,14 +127959,13 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( Token *pDatabase, /* Name of the database containing pTable */ Token *pAlias, /* The right-hand side of the AS subexpression */ Select *pSubquery, /* A subquery used in place of a table name */ - Expr *pOn, /* The ON clause of a join */ - IdList *pUsing /* The USING clause of a join */ + OnOrUsing *pOnUsing /* Either the ON clause or the USING clause */ ){ SrcItem *pItem; sqlite3 *db = pParse->db; - if( !p && (pOn || pUsing) ){ + if( !p && pOnUsing!=0 && (pOnUsing->pOn || pOnUsing->pUsing) ){ sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", - (pOn ? "ON" : "USING") + (pOnUsing->pOn ? "ON" : "USING") ); goto append_from_error; } @@ -118483,15 +127985,27 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } - pItem->pSelect = pSubquery; - pItem->pOn = pOn; - pItem->pUsing = pUsing; + if( pSubquery ){ + pItem->pSelect = pSubquery; + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } + } + assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); + assert( pItem->fg.isUsing==0 ); + if( pOnUsing==0 ){ + pItem->u3.pOn = 0; + }else if( pOnUsing->pUsing ){ + pItem->fg.isUsing = 1; + pItem->u3.pUsing = pOnUsing->pUsing; + }else{ + pItem->u3.pOn = pOnUsing->pOn; + } return p; - append_from_error: +append_from_error: assert( p==0 ); - sqlite3ExprDelete(db, pOn); - sqlite3IdListDelete(db, pUsing); + sqlite3ClearOnOrUsing(db, pOnUsing); sqlite3SelectDelete(db, pSubquery); return 0; } @@ -118536,6 +128050,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, Src p1 = pNew; memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); sqlite3DbFree(pParse->db, p2); + p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); } } return p1; @@ -118572,14 +128087,34 @@ SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList * ** The operator is "natural cross join". The A and B operands are stored ** in p->a[0] and p->a[1], respectively. The parser initially stores the ** operator with A. This routine shifts that operator over to B. +** +** Additional changes: +** +** * All tables to the left of the right-most RIGHT JOIN are tagged with +** JT_LTORJ (mnemonic: Left Table Of Right Join) so that the +** code generator can easily tell that the table is part of +** the left operand of at least one RIGHT JOIN. */ -SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ - if( p ){ - int i; - for(i=p->nSrc-1; i>0; i--){ - p->a[i].fg.jointype = p->a[i-1].fg.jointype; - } +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(Parse *pParse, SrcList *p){ + (void)pParse; + if( p && p->nSrc>1 ){ + int i = p->nSrc-1; + u8 allFlags = 0; + do{ + allFlags |= p->a[i].fg.jointype = p->a[i-1].fg.jointype; + }while( (--i)>0 ); p->a[0].fg.jointype = 0; + + /* All terms to the left of a RIGHT JOIN should be tagged with the + ** JT_LTORJ flags */ + if( allFlags & JT_RIGHT ){ + for(i=p->nSrc-1; ALWAYS(i>0) && (p->a[i].fg.jointype&JT_RIGHT)==0; i--){} + i--; + assert( i>=0 ); + do{ + p->a[i].fg.jointype |= JT_LTORJ; + }while( (--i)>=0 ); + } } } @@ -118979,7 +128514,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -118989,6 +128524,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); if( pIndex ){ + iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3RefillIndex(pParse, pIndex, -1); return; @@ -119094,7 +128630,7 @@ SQLITE_PRIVATE void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ /* ** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. The CTE described by teh third argument is added to +** WITH clause. The CTE described by the third argument is added to ** the WITH clause of the second argument. If the second argument is ** NULL, then a new WITH argument is created. */ @@ -119154,6 +128690,9 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ sqlite3DbFree(db, pWith); } } +SQLITE_PRIVATE void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){ + sqlite3WithDelete(db, (With*)pWith); +} #endif /* !defined(SQLITE_OMIT_CTE) */ /************** End of build.c ***********************************************/ @@ -119345,6 +128884,7 @@ SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ ** strings is BINARY. */ db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); + sqlite3ExpirePreparedStatements(db, 1); } /* @@ -119518,7 +129058,6 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( const char *zName = aDef[i].zName; int nName = sqlite3Strlen30(zName); int h = SQLITE_FUNC_HASH(zName[0], nName); - assert( zName[0]>='a' && zName[0]<='z' ); assert( aDef[i].funcFlags & SQLITE_FUNC_BUILTIN ); pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ @@ -119651,19 +129190,21 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ Hash temp2; HashElem *pElem; Schema *pSchema = (Schema *)p; + sqlite3 xdb; + memset(&xdb, 0, sizeof(xdb)); temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; sqlite3HashInit(&pSchema->trigHash); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); + sqlite3DeleteTable(&xdb, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); @@ -119734,8 +129275,9 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - sqlite3DeleteTable(pParse->db, pItem->pTab); + if( pItem->pTab ) sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; + pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ @@ -119745,6 +129287,16 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ return pTab; } +/* Generate byte-code that will report the number of rows modified +** by a DELETE, INSERT, or UPDATE statement. +*/ +SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ + sqlite3VdbeAddOp0(v, OP_FkCheck); + sqlite3VdbeAddOp2(v, OP_ResultRow, regCounter, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zColName, SQLITE_STATIC); +} + /* Return true if table pTab is read-only. ** ** A table is read-only if any of the following are true: @@ -119752,18 +129304,42 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ ** 1) It is a virtual table and no implementation of the xUpdate method ** has been provided ** -** 2) It is a system table (i.e. sqlite_schema), this call is not +** 2) A trigger is currently being coded and the table is a virtual table +** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and +** the table is not SQLITE_VTAB_INNOCUOUS. +** +** 3) It is a system table (i.e. sqlite_schema), this call is not ** part of a nested parse and writable_schema pragma has not ** been specified ** -** 3) The table is a shadow table, the database connection is in +** 4) The table is a shadow table, the database connection is in ** defensive mode, and the current sqlite3_prepare() ** is for a top-level SQL statement. */ +static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ + return 1; + } + + /* Within triggers: + ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY + ** virtual tables + ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS + ** virtual tables if PRAGMA trusted_schema=ON. + */ + if( pParse->pToplevel!=0 + && pTab->u.vtab.p->eVtabRisk > + ((pParse->db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + return 0; +} static int tabIsReadOnly(Parse *pParse, Table *pTab){ sqlite3 *db; if( IsVirtual(pTab) ){ - return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + return vtabIsReadOnly(pParse, pTab); } if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; db = pParse->db; @@ -119775,17 +129351,21 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ } /* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; +** Check to make sure the given table is writable. +** +** If pTab is not writable -> generate an error message and return 1. +** If pTab is writable but other errors have occurred -> return 1. +** If pTab is writable and no prior errors -> return 0; */ -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ if( tabIsReadOnly(pParse, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); return 1; } #ifndef SQLITE_OMIT_VIEW - if( !viewOk && IsView(pTab) ){ + if( IsView(pTab) + && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) + ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); return 1; } @@ -119819,8 +129399,8 @@ SQLITE_PRIVATE void sqlite3MaterializeView( assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); + assert( pFrom->a[0].fg.isUsing==0 ); + assert( pFrom->a[0].u3.pOn==0 ); } pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, SF_IncludeHidden, pLimit); @@ -119850,7 +129430,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( sqlite3 *db = pParse->db; Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + ExprList *pEList = NULL; /* Expression list containing only pSelectRowid*/ SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ Select *pSelect = NULL; /* Complete SELECT tree */ Table *pTab; @@ -119888,14 +129468,20 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( ); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol>=1 ); if( pPk->nKeyCol==1 ){ - const char *zName = pTab->aCol[pPk->aiColumn[0]].zCnName; + const char *zName; + assert( pPk->aiColumn[0]>=0 && pPk->aiColumn[0]nCol ); + zName = pTab->aCol[pPk->aiColumn[0]].zCnName; pLhs = sqlite3Expr(db, TK_ID, zName); pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); }else{ int i; for(i=0; inKeyCol; i++){ - Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); + Expr *p; + assert( pPk->aiColumn[i]>=0 && pPk->aiColumn[i]nCol ); + p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); pEList = sqlite3ExprListAppend(pParse, pEList, p); } pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -119924,7 +129510,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( pOrderBy,0,pLimit ); - /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + /* now generate the new WHERE rowid IN clause for the DELETE/UPDATE */ pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; @@ -119984,12 +129570,13 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( memset(&sContext, 0, sizeof(sContext)); db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto delete_from_cleanup; } + assert( db->mallocFailed==0 ); assert( pTabList->nSrc==1 ); - /* Locate the table which we want to delete. This table has to be ** put in an SrcList structure because some of the subroutines we ** will be calling are designed to work with multiple tables and expect @@ -120014,6 +129601,14 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( # define isView 0 #endif +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Delete() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewDelete(pParse->pWith, pTabList, pWhere, + pOrderBy, pLimit, pTrigger); + } +#endif + #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT if( !isView ){ pWhere = sqlite3LimitWhere( @@ -120030,7 +129625,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( goto delete_from_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto delete_from_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -120129,21 +129724,22 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1); + sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1); + }else{ + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } } }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + if( sNC.ncFlags & NC_Subquery ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; - nPk = 1; + assert( nPk==1 ); iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ @@ -120167,11 +129763,12 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0,0,wcf,iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF + || OptimizationDisabled(db, SQLITE_OnePass) ); if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); @@ -120320,9 +129917,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** invoke the callback function. */ if( memCnt ){ - sqlite3VdbeAddOp2(v, OP_ChngCntRow, memCnt, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); + sqlite3CodeChangeCount(v, memCnt, "rows deleted"); } delete_from_cleanup: @@ -120333,7 +129928,7 @@ delete_from_cleanup: sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); #endif - sqlite3DbFree(db, aToOpen); + if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -120510,9 +130105,11 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); /* Invoke AFTER DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel - ); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + } /* Jump here if the row had already been deleted before any BEFORE ** trigger programs were invoked. Or if a trigger program throws a @@ -120773,6 +130370,18 @@ static void typeofFunc( sqlite3_result_text(context, azType[i], -1, SQLITE_STATIC); } +/* subtype(X) +** +** Return the subtype of X +*/ +static void subtypeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + sqlite3_result_int(context, sqlite3_value_subtype(argv[0])); +} /* ** Implementation of the length() function @@ -120813,6 +130422,42 @@ static void lengthFunc( } } +/* +** Implementation of the octet_length() function +*/ +static void bytelengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + i64 m = sqlite3_context_db_handle(context)->enc<=SQLITE_UTF8 ? 1 : 2; + sqlite3_result_int64(context, sqlite3_value_bytes(argv[0])*m); + break; + } + case SQLITE_TEXT: { + if( sqlite3_value_encoding(argv[0])<=SQLITE_UTF8 ){ + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + }else{ + sqlite3_result_int(context, sqlite3_value_bytes16(argv[0])); + } + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + /* ** Implementation of the abs() function. ** @@ -120934,7 +130579,7 @@ endInstrOOM: } /* -** Implementation of the printf() function. +** Implementation of the printf() (a.k.a. format()) SQL function. */ static void printfFunc( sqlite3_context *context, @@ -121089,7 +130734,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ }else if( n==0 ){ r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ - zBuf = sqlite3_mprintf("%.*f",n,r); + zBuf = sqlite3_mprintf("%!.*f",n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; @@ -121289,7 +130934,7 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, provde the Utf8Read() +** character is exactly one byte in size. Also, provide the Utf8Read() ** macro for fast reading of the next character in the common case where ** the next character is ASCII. */ @@ -121404,7 +131049,7 @@ static int patternCompare( ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ + if( c<0x80 ){ char zStop[3]; int bMatch; if( noCase ){ @@ -121487,7 +131132,13 @@ static int patternCompare( ** non-zero if there is no match. */ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + if( zString==0 ){ + return zGlobPattern!=0; + }else if( zGlobPattern==0 ){ + return 1; + }else { + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + } } /* @@ -121495,7 +131146,13 @@ SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ ** a miss - like strcmp(). */ SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + if( zStr==0 ){ + return zPattern!=0; + }else if( zPattern==0 ){ + return 1; + }else{ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + } } /* @@ -121510,7 +131167,7 @@ SQLITE_API int sqlite3_like_count = 0; /* ** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the +** the built-in LIKE operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: ** ** A LIKE B @@ -121703,39 +131360,42 @@ static const char hexdigits[] = { }; /* -** Implementation of the QUOTE() function. This function takes a single -** argument. If the argument is numeric, the return value is the same as -** the argument. If the argument is NULL, the return value is the string -** "NULL". Otherwise, the argument is enclosed in single quotes with -** single-quote escapes. +** Append to pStr text that is the SQL literal representation of the +** value contained in pValue. */ -static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - assert( argc==1 ); - UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ + /* As currently implemented, the string must be initially empty. + ** we might relax this requirement in the future, but that will + ** require enhancements to the implementation. */ + assert( pStr!=0 && pStr->nChar==0 ); + + switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { double r1, r2; - char zBuf[50]; - r1 = sqlite3_value_double(argv[0]); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); - sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); - if( r1!=r2 ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + const char *zVal; + r1 = sqlite3_value_double(pValue); + sqlite3_str_appendf(pStr, "%!0.15g", r1); + zVal = sqlite3_str_value(pStr); + if( zVal ){ + sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_str_reset(pStr); + sqlite3_str_appendf(pStr, "%!0.20e", r1); + } } - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); break; } case SQLITE_INTEGER: { - sqlite3_result_value(context, argv[0]); + sqlite3_str_appendf(pStr, "%lld", sqlite3_value_int64(pValue)); break; } case SQLITE_BLOB: { - char *zText = 0; - char const *zBlob = sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); - assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); - if( zText ){ + char const *zBlob = sqlite3_value_blob(pValue); + i64 nBlob = sqlite3_value_bytes(pValue); + assert( zBlob==sqlite3_value_blob(pValue) ); /* No encoding change */ + sqlite3StrAccumEnlarge(pStr, nBlob*2 + 4); + if( pStr->accError==0 ){ + char *zText = pStr->zText; int i; for(i=0; i>4)&0x0F]; @@ -121745,42 +131405,48 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ zText[(nBlob*2)+3] = '\0'; zText[0] = 'X'; zText[1] = '\''; - sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); - sqlite3_free(zText); + pStr->nChar = nBlob*2 + 3; } break; } case SQLITE_TEXT: { - int i,j; - u64 n; - const unsigned char *zArg = sqlite3_value_text(argv[0]); - char *z; - - if( zArg==0 ) return; - for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } - z = contextMalloc(context, ((i64)i)+((i64)n)+3); - if( z ){ - z[0] = '\''; - for(i=0, j=1; zArg[i]; i++){ - z[j++] = zArg[i]; - if( zArg[i]=='\'' ){ - z[j++] = '\''; - } - } - z[j++] = '\''; - z[j] = 0; - sqlite3_result_text(context, z, j, sqlite3_free); - } + const unsigned char *zArg = sqlite3_value_text(pValue); + sqlite3_str_appendf(pStr, "%Q", zArg); break; } default: { - assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); - sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + assert( sqlite3_value_type(pValue)==SQLITE_NULL ); + sqlite3_str_append(pStr, "NULL", 4); break; } } } +/* +** Implementation of the QUOTE() function. +** +** The quote(X) function returns the text of an SQL literal which is the +** value of its argument suitable for inclusion into an SQL statement. +** Strings are surrounded by single-quotes with escapes on interior quotes +** as needed. BLOBs are encoded as hexadecimal literals. Strings with +** embedded NUL characters cannot be represented as string literals in SQL +** and hence the returned string literal is truncated prior to the first NUL. +*/ +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + sqlite3_str str; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( argc==1 ); + UNUSED_PARAMETER(argc); + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + sqlite3QuoteValue(&str,argv[0]); + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar, + SQLITE_DYNAMIC); + if( str.accError!=SQLITE_OK ){ + sqlite3_result_null(context); + sqlite3_result_error_code(context, str.accError); + } +} + /* ** The unicode() function. Return the integer unicode code-point value ** for the first character of the input string. @@ -121834,6 +131500,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } + *zOut = 0; sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } @@ -121862,10 +131529,101 @@ static void hexFunc( *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); + sqlite3_result_text64(context, zHex, (u64)(z-zHex), + sqlite3_free, SQLITE_UTF8); } } +/* +** Buffer zStr contains nStr bytes of utf-8 encoded text. Return 1 if zStr +** contains character ch, or 0 if it does not. +*/ +static int strContainsChar(const u8 *zStr, int nStr, u32 ch){ + const u8 *zEnd = &zStr[nStr]; + const u8 *z = zStr; + while( z0 ){ + const char *v = (const char*)sqlite3_value_text(argv[i]); + if( v!=0 ){ + if( j>0 && nSep>0 ){ + memcpy(&z[j], zSep, nSep); + j += nSep; + } + memcpy(&z[j], v, k); + j += k; + } + } + } + z[j] = 0; + assert( j<=n ); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); +} + +/* +** The CONCAT(...) function. Generate a string result that is the +** concatentation of all non-null arguments. +*/ +static void concatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + concatFuncCore(context, argc, argv, 0, ""); +} + +/* +** The CONCAT_WS(separator, ...) function. +** +** Generate a string that is the concatenation of 2nd through the Nth +** argument. Use the first argument (which must be non-NULL) as the +** separator. +*/ +static void concatwsFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nSep = sqlite3_value_bytes(argv[0]); + const char *zSep = (const char*)sqlite3_value_text(argv[0]); + if( zSep==0 ) return; + concatFuncCore(context, argc-1, argv+1, nSep, zSep); +} + #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION /* ** The "unknown" function is automatically substituted in place of ** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN -** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** when the SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION compile-time option is used. ** When the "sqlite3" command-line shell is built using this functionality, ** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries ** involving application-defined functions to be examined in a generic @@ -122083,6 +131916,9 @@ static void unknownFunc( sqlite3_value **argv ){ /* no-op */ + (void)context; + (void)argc; + (void)argv; } #endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ @@ -122184,13 +132020,68 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ */ typedef struct SumCtx SumCtx; struct SumCtx { - double rSum; /* Floating point sum */ - i64 iSum; /* Integer sum */ + double rSum; /* Running sum as as a double */ + double rErr; /* Error term for Kahan-Babushka-Neumaier summation */ + i64 iSum; /* Running sum as a signed integer */ i64 cnt; /* Number of elements summed */ - u8 overflow; /* True if integer overflow seen */ - u8 approx; /* True if non-integer value was input to the sum */ + u8 approx; /* True if any non-integer value was input to the sum */ + u8 ovrfl; /* Integer overflow seen */ }; +/* +** Do one step of the Kahan-Babushka-Neumaier summation. +** +** https://en.wikipedia.org/wiki/Kahan_summation_algorithm +** +** Variables are marked "volatile" to defeat c89 x86 floating point +** optimizations can mess up this algorithm. +*/ +static void kahanBabuskaNeumaierStep( + volatile SumCtx *pSum, + volatile double r +){ + volatile double s = pSum->rSum; + volatile double t = s + r; + if( fabs(s) > fabs(r) ){ + pSum->rErr += (s - t) + r; + }else{ + pSum->rErr += (r - t) + s; + } + pSum->rSum = t; +} + +/* +** Add a (possibly large) integer to the running sum. +*/ +static void kahanBabuskaNeumaierStepInt64(volatile SumCtx *pSum, i64 iVal){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iBig, iSm; + iSm = iVal % 16384; + iBig = iVal - iSm; + kahanBabuskaNeumaierStep(pSum, iBig); + kahanBabuskaNeumaierStep(pSum, iSm); + }else{ + kahanBabuskaNeumaierStep(pSum, (double)iVal); + } +} + +/* +** Initialize the Kahan-Babaska-Neumaier sum from a 64-bit integer +*/ +static void kahanBabuskaNeumaierInit( + volatile SumCtx *p, + i64 iVal +){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iSm = iVal % 16384; + p->rSum = (double)(iVal - iSm); + p->rErr = (double)iSm; + }else{ + p->rSum = (double)iVal; + p->rErr = 0.0; + } +} + /* ** Routines used to compute the sum, average, and total. ** @@ -122210,15 +132101,29 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ type = sqlite3_value_numeric_type(argv[0]); if( p && type!=SQLITE_NULL ){ p->cnt++; - if( type==SQLITE_INTEGER ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum += v; - if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->approx = p->overflow = 1; + if( p->approx==0 ){ + if( type!=SQLITE_INTEGER ){ + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + }else{ + i64 x = p->iSum; + if( sqlite3AddInt64(&x, sqlite3_value_int64(argv[0]))==0 ){ + p->iSum = x; + }else{ + p->ovrfl = 1; + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + } } }else{ - p->rSum += sqlite3_value_double(argv[0]); - p->approx = 1; + if( type==SQLITE_INTEGER ){ + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + }else{ + p->ovrfl = 0; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + } } } } @@ -122235,13 +132140,18 @@ static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ if( ALWAYS(p) && type!=SQLITE_NULL ){ assert( p->cnt>0 ); p->cnt--; - assert( type==SQLITE_INTEGER || p->approx ); - if( type==SQLITE_INTEGER && p->approx==0 ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum -= v; - p->iSum -= v; + if( !p->approx ){ + p->iSum -= sqlite3_value_int64(argv[0]); + }else if( type==SQLITE_INTEGER ){ + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal!=SMALLEST_INT64 ){ + kahanBabuskaNeumaierStepInt64(p, -iVal); + }else{ + kahanBabuskaNeumaierStepInt64(p, LARGEST_INT64); + kahanBabuskaNeumaierStepInt64(p, 1); + } }else{ - p->rSum -= sqlite3_value_double(argv[0]); + kahanBabuskaNeumaierStep(p, -sqlite3_value_double(argv[0])); } } } @@ -122252,10 +132162,14 @@ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - if( p->overflow ){ - sqlite3_result_error(context,"integer overflow",-1); - }else if( p->approx ){ - sqlite3_result_double(context, p->rSum); + if( p->approx ){ + if( p->ovrfl ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( !sqlite3IsOverflow(p->rErr) ){ + sqlite3_result_double(context, p->rSum+p->rErr); + }else{ + sqlite3_result_double(context, p->rSum); + } }else{ sqlite3_result_int64(context, p->iSum); } @@ -122265,14 +132179,29 @@ static void avgFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - sqlite3_result_double(context, p->rSum/(double)p->cnt); + double r; + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + sqlite3_result_double(context, r/(double)p->cnt); } } static void totalFinalize(sqlite3_context *context){ SumCtx *p; + double r = 0.0; p = sqlite3_aggregate_context(context, 0); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - sqlite3_result_double(context, p ? p->rSum : (double)0); + if( p ){ + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + } + sqlite3_result_double(context, r); } /* @@ -122391,6 +132320,7 @@ static void minMaxFinalize(sqlite3_context *context){ /* ** group_concat(EXPR, ?SEPARATOR?) +** string_agg(EXPR, SEPARATOR) ** ** The SEPARATOR goes before the EXPR string. This is tragic. The ** groupConcatInverse() implementation would have been easier if the @@ -122494,7 +132424,7 @@ static void groupConcatInverse( if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); /* pGCC is always non-NULL since groupConcatStep() will have always - ** run frist to initialize it */ + ** run first to initialize it */ if( ALWAYS(pGCC) ){ int nVS; /* Must call sqlite3_value_text() to convert the argument into text prior @@ -122549,6 +132479,8 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); @@ -122566,20 +132498,16 @@ static void groupConcatValue(sqlite3_context *context){ */ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); -/* BEGIN CODEC */ #ifdef SQLITE_HAS_CODEC extern void sqlite3CodecExportData(sqlite3_context *, int, sqlite3_value **); -#endif -/* END CODEC */ +#endif /* SQLITE_HAS_CODEC */ assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); } -/* BEGIN CODEC */ #ifdef SQLITE_HAS_CODEC sqlite3CreateFunc(db, "export_database", 1, SQLITE_TEXT, 0, sqlite3CodecExportData, 0, 0, 0, 0, 0); -#endif -/* END CODEC */ +#endif /* SQLITE_HAS_CODEC */ } /* @@ -122588,8 +132516,10 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ ** sensitive. */ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + FuncDef *pDef; struct compareInfo *pInfo; int flags; + int nArg; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; @@ -122597,10 +132527,13 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) pInfo = (struct compareInfo*)&likeInfoNorm; flags = SQLITE_FUNC_LIKE; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; - sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; + for(nArg=2; nArg<=3; nArg++){ + sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, + 0, 0, 0, 0, 0); + pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + pDef->funcFlags |= flags; + pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; + } } /* @@ -122721,6 +132654,18 @@ static void ceilingFunc( static double xCeil(double x){ return ceil(x); } static double xFloor(double x){ return floor(x); } +/* +** Some systems do not have log2() and log10() in their standard math +** libraries. +*/ +#if defined(HAVE_LOG10) && HAVE_LOG10==0 +# define log10(X) (0.4342944819032517867*log(X)) +#endif +#if defined(HAVE_LOG2) && HAVE_LOG2==0 +# define log2(X) (1.442695040888963456*log(X)) +#endif + + /* ** Implementation of SQL functions: ** @@ -122759,17 +132704,15 @@ static void logFunc( } ans = log(x)/b; }else{ - ans = log(x); switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ case 1: - /* Convert from natural logarithm to log base 10 */ - ans *= 1.0/M_LN10; + ans = log10(x); break; case 2: - /* Convert from natural logarithm to log base 2 */ - ans *= 1.0/M_LN2; + ans = log2(x); break; default: + ans = log(x); break; } } @@ -122838,6 +132781,7 @@ static void piFunc( sqlite3_value **argv ){ assert( argc==0 ); + (void)argv; sqlite3_result_double(context, M_PI); } @@ -122861,6 +132805,37 @@ static void signFunc( sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); } +#ifdef SQLITE_DEBUG +/* +** Implementation of fpdecode(x,y,z) function. +** +** x is a real number that is to be decoded. y is the precision. +** z is the maximum real precision. +*/ +static void fpdecodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FpDecode s; + double x; + int y, z; + char zBuf[100]; + UNUSED_PARAMETER(argc); + assert( argc==3 ); + x = sqlite3_value_double(argv[0]); + y = sqlite3_value_int(argv[1]); + z = sqlite3_value_int(argv[2]); + sqlite3FpDecode(&s, x, y, z); + if( s.isSpecial==2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%c%.*s/%d", s.sign, s.n, s.z, s.iDP); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); +} +#endif /* SQLITE_DEBUG */ + /* ** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as @@ -122906,8 +132881,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ INLINE_FUNC(likelihood, 2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), INLINE_FUNC(likely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET| - SQLITE_FUNC_TYPEOF), + INLINE_FUNC(sqlite_offset, 1, INLINEFUNC_sqlite_offset, 0 ), #endif FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), @@ -122924,12 +132898,18 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(subtype, 1, 0, 0, subtypeFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION2(octet_length, 1, 0, 0, bytelengthFunc,SQLITE_FUNC_BYTELEN), FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), + FUNCTION(format, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), +#ifdef SQLITE_DEBUG + FUNCTION(fpdecode, 3, 0, 0, fpdecodeFunc ), +#endif #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), FUNCTION(round, 2, 0, 0, roundFunc ), @@ -122937,6 +132917,13 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(hex, 1, 0, 0, hexFunc ), + FUNCTION(unhex, 1, 0, 0, unhexFunc ), + FUNCTION(unhex, 2, 0, 0, unhexFunc ), + FUNCTION(concat, -1, 0, 0, concatFunc ), + FUNCTION(concat, 0, 0, 0, 0 ), + FUNCTION(concat_ws, -1, 0, 0, concatwsFunc ), + FUNCTION(concat_ws, 0, 0, 0, 0 ), + FUNCTION(concat_ws, 1, 0, 0, 0 ), INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), @@ -122966,6 +132953,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(string_agg, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), #ifdef SQLITE_CASE_SENSITIVE_LIKE @@ -123025,6 +133014,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #endif sqlite3WindowFunctions(); sqlite3RegisterDateTimeFunctions(); + sqlite3RegisterJsonFunctions(); sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); #if 0 /* Enable to print out how the built-in functions are hashed */ @@ -123439,7 +133429,6 @@ static void fkLookupParent( }else{ int nCol = pFKey->nCol; int regTemp = sqlite3GetTempRange(pParse, nCol); - int regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); @@ -123479,11 +133468,10 @@ static void fkLookupParent( sqlite3VdbeGoto(v, iOk); } - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + sqlite3VdbeAddOp4(v, OP_Affinity, regTemp, nCol, 0, sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); - sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); - - sqlite3ReleaseTempReg(pParse, regRec); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regTemp, nCol); + VdbeCoverage(v); sqlite3ReleaseTempRange(pParse, regTemp, nCol); } } @@ -123585,14 +133573,10 @@ static Expr *exprTableColumn( ** Operation | FK type | Action taken ** -------------------------------------------------------------------------- ** DELETE immediate Increment the "immediate constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT immediate Decrement the "immediate constraint counter". ** ** DELETE deferred Increment the "deferred constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT deferred Decrement the "deferred constraint counter". ** @@ -123700,7 +133684,7 @@ static void fkScanChildren( ** clause. For each row found, increment either the deferred or immediate ** foreign key constraint counter. */ if( pParse->nErr==0 ){ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); @@ -123913,6 +133897,7 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) ){ + assert( (pTop->db->flags & SQLITE_FkNoAction)==0 ); return 1; } } @@ -124107,6 +134092,8 @@ SQLITE_PRIVATE void sqlite3FkCheck( } if( regOld!=0 ){ int eAction = pFKey->aAction[aChange!=0]; + if( (db->flags & SQLITE_FkNoAction) ) eAction = OE_None; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); /* If this is a deferred FK constraint, or a CASCADE or SET NULL ** action applies, then any foreign key violations caused by @@ -124222,7 +134209,11 @@ SQLITE_PRIVATE int sqlite3FkRequired( /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ - if( p->aAction[1]!=OE_None ) return 2; + if( (pParse->db->flags & SQLITE_FkNoAction)==0 + && p->aAction[1]!=OE_None + ){ + return 2; + } bHaveFK = 1; } } @@ -124240,9 +134231,9 @@ SQLITE_PRIVATE int sqlite3FkRequired( ** ** It returns a pointer to a Trigger structure containing a trigger ** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. -** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is -** returned (these actions require no special handling by the triggers -** sub-system, code for them is created by fkScanChildren()). +** If the action is "NO ACTION" then a NULL pointer is returned (these actions +** require no special handling by the triggers sub-system, code for them is +** created by fkScanChildren()). ** ** For example, if pFKey is the foreign key and pTab is table "p" in ** the following schema: @@ -124272,6 +134263,7 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( (db->flags & SQLITE_FkNoAction) ) action = OE_None; if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ return 0; } @@ -124371,18 +134363,23 @@ static Trigger *fkActionTrigger( nFrom = sqlite3Strlen30(zFrom); if( action==OE_Restrict ){ - Token tFrom; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + SrcList *pSrc; Expr *pRaise; - tFrom.z = zFrom; - tFrom.n = nFrom; pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ pRaise->affExpr = OE_Abort; } + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + assert( pSrc->nSrc==1 ); + pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); + pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(pParse, 0, &tFrom, 0), + pSrc, pWhere, 0, 0, 0, 0, 0 ); @@ -124489,17 +134486,17 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pNext; /* Copy of pFKey->pNextFrom */ assert( IsOrdinaryTable(pTab) ); + assert( db!=0 ); for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ - void *p = (void *)pFKey->pNextTo; - const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + const char *z = (pFKey->pNextTo ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, pFKey->pNextTo); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; @@ -124562,8 +134559,10 @@ SQLITE_PRIVATE void sqlite3OpenTable( assert( pParse->pVdbe!=0 ); v = pParse->pVdbe; assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(pParse, iDb, pTab->tnum, - (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( !pParse->db->noSharedCache ){ + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + } if( HasRowid(pTab) ){ sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); @@ -124597,43 +134596,68 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); - if( !pIdx->zColAff ){ - sqlite3OomFault(db); - return 0; + sqlite3OomFault(db); + return 0; + } + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } - for(n=0; nnColumn; n++){ - i16 x = pIdx->aiColumn[n]; - char aff; - if( x>=0 ){ - aff = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - aff = SQLITE_AFF_INTEGER; - }else{ - assert( x==XN_EXPR ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; + return pIdx->zColAff; +} +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx); + return pIdx->zColAff; +} + + +/* +** Compute an affinity string for a table. Space is obtained +** from sqlite3DbMalloc(). The caller is responsible for freeing +** the space when done. +*/ +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ + char *zColAff; + zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1); + if( zColAff ){ + int i, j; + for(i=j=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; } - if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; - pIdx->zColAff[n] = aff; } - pIdx->zColAff[n] = 0; + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); } - - return pIdx->zColAff; + return zColAff; } /* @@ -124667,7 +134691,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** For STRICT tables: ** ------------------ ** -** Generate an appropropriate OP_TypeCheck opcode that will verify the +** Generate an appropriate OP_TypeCheck opcode that will verify the ** datatypes against the column definitions in pTab. If iReg==0, that ** means an OP_MakeRecord opcode has already been generated and should be ** the last opcode generated. The new OP_TypeCheck needs to be inserted @@ -124677,7 +134701,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** Apply the type checking to that array of registers. */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i, j; + int i; char *zColAff; if( pTab->tabFlags & TF_Strict ){ if( iReg==0 ){ @@ -124686,7 +134710,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** OP_MakeRecord is found */ VdbeOp *pPrev; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); - pPrev = sqlite3VdbeGetOp(v, -1); + pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; @@ -124700,22 +134724,11 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ } zColAff = pTab->zColAff; if( zColAff==0 ){ - sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + zColAff = sqlite3TableAffinityStr(0, pTab); if( !zColAff ){ - sqlite3OomFault(db); + sqlite3OomFault(sqlite3VdbeDb(v)); return; } - - for(i=j=0; inCol; i++){ - assert( pTab->aCol[i].affinity!=0 ); - if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ - zColAff[j++] = pTab->aCol[i].affinity; - } - } - do{ - zColAff[j--] = 0; - }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); @@ -124724,7 +134737,7 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); }else{ - assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord + assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); sqlite3VdbeChangeP4(v, -1, zColAff, i); } @@ -124810,7 +134823,7 @@ SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( */ sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); if( (pTab->tabFlags & TF_HasStored)!=0 ){ - pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Affinity ){ /* Change the OP_Affinity argument to '@' (NONE) for all stored ** columns. '@' is the no-op affinity and those columns have not @@ -125088,6 +135101,196 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->addrFillSub - 1); + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->pSelect = pLeft; + p->fg.viaCoroutine = 1; + p->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + p->regReturn = ++pParse->nMem; + p->iCursor = -1; + p->u1.nRow = 2; + sqlite3VdbeAddOp3(v,OP_InitCoroutine,p->regReturn,0,p->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, p->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + p->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + assert( p!=0 ); + if( p->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, p->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, p->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -125242,9 +135445,11 @@ SQLITE_PRIVATE void sqlite3Insert( #endif db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto insert_cleanup; } + assert( db->mallocFailed==0 ); dest.iSDParm = 0; /* Suppress a harmless compiler warning */ /* If the Select object is really just a simple VALUES() list with a @@ -125290,6 +135495,14 @@ SQLITE_PRIVATE void sqlite3Insert( #endif assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Insert() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewInsert(pParse->pWith, pTabList, pColumn, pSelect, pList, + onError, pUpsert, pTrigger); + } +#endif + /* If pTab is really a view, make sure it has been initialized. ** ViewGetColumnNames() is a no-op if pTab is not a view. */ @@ -125299,7 +135512,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* Cannot insert into a read-only table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto insert_cleanup; } @@ -125320,7 +135533,11 @@ SQLITE_PRIVATE void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 + && pSelect!=0 + && pTrigger==0 + && xferOptimization(pParse, pTab, pSelect, onError, iDb) + ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; @@ -125364,13 +135581,15 @@ SQLITE_PRIVATE void sqlite3Insert( */ bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ + assert( pColumn->eU4!=EU4_EXPR ); + pColumn->eU4 = EU4_IDX; for(i=0; inId; i++){ - pColumn->a[i].idx = -1; + pColumn->a[i].u4.idx = -1; } for(i=0; inId; i++){ for(j=0; jnCol; j++){ if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){ - pColumn->a[i].idx = j; + pColumn->a[i].u4.idx = j; if( i!=j ) bIdListInOrder = 0; if( j==pTab->iPKey ){ ipkColumn = i; assert( !withoutRowid ); @@ -125408,23 +135627,40 @@ SQLITE_PRIVATE void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + dest.iSDParm = pItem->regReturn; + regFromSelect = pItem->regResult; + nColumn = pItem->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -125579,7 +135815,7 @@ SQLITE_PRIVATE void sqlite3Insert( pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } @@ -125670,7 +135906,8 @@ SQLITE_PRIVATE void sqlite3Insert( } } if( pColumn ){ - for(j=0; jnId && pColumn->a[j].idx!=i; j++){} + assert( pColumn->eU4==EU4_IDX ); + for(j=0; jnId && pColumn->a[j].u4.idx!=i; j++){} if( j>=pColumn->nId ){ /* A column not named in the insert column list gets its ** default value */ @@ -125697,7 +135934,12 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); } }else{ - sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + Expr *pX = pList->a[k].pExpr; + int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore); + if( y!=iRegStore ){ + sqlite3VdbeAddOp2(v, + ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore); + } } } @@ -125732,7 +135974,7 @@ SQLITE_PRIVATE void sqlite3Insert( } /* Copy the new data already generated. */ - assert( pTab->nNVCol>0 ); + assert( pTab->nNVCol>0 || pParse->nErr>0 ); sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); #ifndef SQLITE_OMIT_GENERATED_COLUMNS @@ -125834,7 +136076,9 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert ); - sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + if( db->flags & SQLITE_ForeignKeys ){ + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + } /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE ** constraints or (b) there are no triggers and this table is not a @@ -125909,9 +136153,7 @@ insert_end: ** invoke the callback function. */ if( regRowCount ){ - sqlite3VdbeAddOp2(v, OP_ChngCntRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + sqlite3CodeChangeCount(v, regRowCount, "rows inserted"); } insert_cleanup: @@ -125920,7 +136162,7 @@ insert_cleanup: sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -125946,7 +136188,7 @@ insert_cleanup: /* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). * Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this ** expression node references any of the -** columns that are being modifed by an UPDATE statement. +** columns that are being modified by an UPDATE statement. */ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN ){ @@ -126169,7 +136411,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int *aiChng, /* column i is unchanged if aiChng[i]<0 */ Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ ){ - Vdbe *v; /* VDBE under constrution */ + Vdbe *v; /* VDBE under construction */ Index *pIdx; /* Pointer to one of the indices */ Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ sqlite3 *db; /* Database connection */ @@ -126284,6 +136526,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pCol->zCnName); + testcase( zMsg==0 && db->mallocFailed==0 ); sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, iReg); sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); @@ -126651,7 +136894,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pIdx; pIdx = indexIteratorNext(&sIdxIter, &ix) ){ - int regIdx; /* Range of registers hold conent for pIdx */ + int regIdx; /* Range of registers holding content for pIdx */ int regR; /* Range of registers holding conflicting PK */ int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ @@ -126803,7 +137046,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( isUpdate ){ /* If currently processing the PRIMARY KEY of a WITHOUT ROWID ** table, only conflict if the new PRIMARY KEY values are actually - ** different from the old. + ** different from the old. See TH3 withoutrowid04.test. ** ** For a UNIQUE index, only conflict if the PRIMARY KEY values ** of the matched index row are different from the original PRIMARY @@ -127146,6 +137389,8 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); assert( op==OP_OpenWrite || p5==0 ); + assert( piDataCur!=0 ); + assert( piIdxCur!=0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur set to illegal cursor numbers @@ -127158,18 +137403,18 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( assert( v!=0 ); if( iBase<0 ) iBase = pParse->nTab; iDataCur = iBase++; - if( piDataCur ) *piDataCur = iDataCur; + *piDataCur = iDataCur; if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); - }else{ + }else if( pParse->db->noSharedCache==0 ){ sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); } - if( piIdxCur ) *piIdxCur = iBase; + *piIdxCur = iBase; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; + *piDataCur = iIdxCur; p5 = 0; } if( aToOpen==0 || aToOpen[i+1] ){ @@ -127291,18 +137536,13 @@ static int xferOptimization( int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ int regData, regRowid; /* Registers holding data and rowid */ - if( pSelect==0 ){ - return 0; /* Must be of the form INSERT INTO ... SELECT ... */ - } + assert( pSelect!=0 ); if( pParse->pWith || pSelect->pWith ){ /* Do not attempt to process this query if there are an WITH clauses ** attached to it. Proceeding may generate a false "no such table: xxx" ** error if pSelect reads from a CTE named "xxx". */ return 0; } - if( sqlite3TriggerList(pParse, pDest) ){ - return 0; /* tab1 must not have triggers */ - } #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ @@ -127467,12 +137707,15 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY - /* Disallow the transfer optimization if the destination table constains + /* Disallow the transfer optimization if the destination table contains ** any foreign key constraints. This is more restrictive than necessary. ** But the main beneficiary of the transfer optimization is the VACUUM ** command, and the VACUUM command disables foreign key constraints. So @@ -128152,9 +138395,9 @@ struct sqlite3_api_routines { const char *(*filename_journal)(const char*); const char *(*filename_wal)(const char*); /* Version 3.32.0 and later */ - char *(*create_filename)(const char*,const char*,const char*, + const char *(*create_filename)(const char*,const char*,const char*, int,const char**); - void (*free_filename)(char*); + void (*free_filename)(const char*); sqlite3_file *(*database_file_object)(const char*); /* Version 3.34.0 and later */ int (*txn_state)(sqlite3*,const char*); @@ -128165,9 +138408,32 @@ struct sqlite3_api_routines { int (*autovacuum_pages)(sqlite3*, unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int), void*, void(*)(void*)); + /* Version 3.38.0 and later */ + int (*error_offset)(sqlite3*); + int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**); + int (*vtab_distinct)(sqlite3_index_info*); + int (*vtab_in)(sqlite3_index_info*,int,int); + int (*vtab_in_first)(sqlite3_value*,sqlite3_value**); + int (*vtab_in_next)(sqlite3_value*,sqlite3_value**); + /* Version 3.39.0 and later */ + int (*deserialize)(sqlite3*,const char*,unsigned char*, + sqlite3_int64,sqlite3_int64,unsigned); + unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, + unsigned int); + const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); + /* Version 3.41.0 and later */ + int (*is_interrupted)(sqlite3*); + /* Version 3.43.0 and later */ + int (*stmt_explain)(sqlite3_stmt*,int); + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK + /* handle after drop table done */ int (*set_droptable_handle)(sqlite3*,void(*)(sqlite3*,const char*,const char*)); -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ }; /* @@ -128479,11 +138745,32 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_total_changes64 sqlite3_api->total_changes64 /* Version 3.37.0 and later */ #define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages - +/* Version 3.38.0 and later */ +#define sqlite3_error_offset sqlite3_api->error_offset +#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value +#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct +#define sqlite3_vtab_in sqlite3_api->vtab_in +#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first +#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next +/* Version 3.39.0 and later */ +#ifndef SQLITE_OMIT_DESERIALIZE +#define sqlite3_deserialize sqlite3_api->deserialize +#define sqlite3_serialize sqlite3_api->serialize +#endif +#define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding +/* Version 3.41.0 and later */ +#define sqlite3_is_interrupted sqlite3_api->is_interrupted +/* Version 3.43.0 and later */ +#define sqlite3_stmt_explain sqlite3_api->stmt_explain +/* Version 3.44.0 and later */ +#define sqlite3_get_clientdata sqlite3_api->get_clientdata +#define sqlite3_set_clientdata sqlite3_api->set_clientdata #ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK -#define sqlite3_set_droptable_handle sqlite3_api->set_droptable_handle -#endif - +/* handle after drop table done */ +#define sqlite3_set_droptable_handle sqlite3_api->set_droptable_handle +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -128500,6 +138787,7 @@ typedef int (*sqlite3_loadext_entry)( # define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ # define SQLITE_EXTENSION_INIT3 /*no-op*/ #endif + #endif /* SQLITE3EXT_H */ /************** End of sqlite3ext.h ******************************************/ @@ -128520,7 +138808,7 @@ typedef int (*sqlite3_loadext_entry)( # define sqlite3_column_origin_name 0 # define sqlite3_column_origin_name16 0 #endif -#endif +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) */ #ifndef SQLITE_OMIT_LOAD_EXTENSION #ifdef SQLITE_OMIT_AUTHORIZATION @@ -128603,7 +138891,8 @@ typedef int (*sqlite3_loadext_entry)( #if defined(SQLITE_OMIT_TRACE) # define sqlite3_trace_v2 0 #endif -#endif +#endif /* !SQLITE_OMIT_LOAD_EXTENSION */ + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -128893,7 +139182,7 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_load_extension, #else 0, -#endif +#endif /* !SQLITE_OMIT_LOAD_EXTENSION */ sqlite3_malloc64, sqlite3_msize, sqlite3_realloc64, @@ -128979,9 +139268,44 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_total_changes64, /* Version 3.37.0 and later */ sqlite3_autovacuum_pages, -#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK - sqlite3_set_droptable_handle, + /* Version 3.38.0 and later */ + sqlite3_error_offset, +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_vtab_rhs_value, + sqlite3_vtab_distinct, + sqlite3_vtab_in, + sqlite3_vtab_in_first, + sqlite3_vtab_in_next, +#else + 0, + 0, + 0, + 0, + 0, #endif + /* Version 3.39.0 and later */ +#ifndef SQLITE_OMIT_DESERIALIZE + sqlite3_deserialize, + sqlite3_serialize, +#else + 0, + 0, +#endif + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_encoding, + /* Version 3.41.0 and later */ + sqlite3_is_interrupted, + /* Version 3.43.0 and later */ + sqlite3_stmt_explain, + /* Version 3.44.0 and later */ + sqlite3_get_clientdata, + sqlite3_set_clientdata, +#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK + sqlite3_set_droptable_handle +#else + 0 +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ }; /* True if x is the directory separator character @@ -128991,7 +139315,9 @@ static const sqlite3_api_routines sqlite3Apis = { #else # define DirSep(X) ((X)=='/') #endif -#endif + +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) || defined(SQLITE_EXPORT_SYMBOLS) */ + #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Attempt to load an SQLite extension library contained in the file @@ -129055,15 +139381,25 @@ static int sqlite3LoadExtension( /* tag-20210611-1. Some dlopen() implementations will segfault if given ** an oversize filename. Most filesystems have a pathname limit of 4K, ** so limit the extension filename length to about twice that. - ** https://sqlite.org/forum/forumpost/08a0d6d9bf */ + ** https://sqlite.org/forum/forumpost/08a0d6d9bf + ** + ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. + ** See https://sqlite.org/forum/forumpost/24083b579d. + */ if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; + /* Do not allow sqlite3_load_extension() to link to a copy of the + ** running application, by passing in an empty filename. */ + if( nMsg==0 ) goto extension_not_found; + handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; iimutex); if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; @@ -129237,6 +139576,9 @@ SQLITE_API int sqlite3_auto_extension( void (*xInit)(void) ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return SQLITE_MISUSE_BKPT; +#endif #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ){ @@ -129289,6 +139631,9 @@ SQLITE_API int sqlite3_cancel_auto_extension( int i; int n = 0; wsdAutoextInit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return 0; +#endif sqlite3_mutex_enter(mutex); for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ @@ -129453,9 +139798,9 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ #define PragTyp_WAL_CHECKPOINT 43 #define PragTyp_LOCK_STATUS 44 #define PragTyp_STATS 45 -#if defined(SQLITE_HAS_CODEC) +#ifdef SQLITE_HAS_CODEC #define PragTyp_KEY 255 -#endif +#endif /* SQLITE_HAS_CODEC */ /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -129656,7 +140001,7 @@ static const PragmaName aPragmaName[] = { #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, + /* ePragFlg: */ PragFlg_Result0, /* ColNames: */ 47, 3, /* iArg: */ 0 }, #endif @@ -130107,6 +140452,34 @@ static const PragmaName aPragmaName[] = { /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or @@ -130384,15 +140757,16 @@ static void pragmaFunclistLine( int isBuiltin, /* True if this is a built-in function */ int showInternFuncs /* True if showing internal functions */ ){ + u32 mask = + SQLITE_DETERMINISTIC | + SQLITE_DIRECTONLY | + SQLITE_SUBTYPE | + SQLITE_INNOCUOUS | + SQLITE_FUNC_INTERNAL + ; + if( showInternFuncs ) mask = 0xffffffff; for(; p; p=p->pNext){ const char *zType; - static const u32 mask = - SQLITE_DETERMINISTIC | - SQLITE_DIRECTONLY | - SQLITE_SUBTYPE | - SQLITE_INNOCUOUS | - SQLITE_FUNC_INTERNAL - ; static const char *azEnc[] = { 0, "utf8", "utf16le", "utf16be" }; assert( SQLITE_FUNC_ENCMASK==0x3 ); @@ -130473,11 +140847,9 @@ SQLITE_PRIVATE void sqlite3Pragma( Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ const PragmaName *pPragma; /* The pragma */ -/* BEGIN CODEC */ #ifdef SQLITE_HAS_CODEC extern int sqlite3CodecPragma(sqlite3*, int, Parse *, const char *, const char *); -#endif -/* END CODEC */ +#endif /* SQLITE_HAS_CODEC */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -130548,7 +140920,6 @@ SQLITE_PRIVATE void sqlite3Pragma( goto pragma_out; } -/* BEGIN CODEC */ #ifdef SQLITE_HAS_CODEC if(sqlite3CodecPragma(db, iDb, pParse, zLeft, zRight)) { /* sqlite3CodecPragma executes internal */ @@ -130556,7 +140927,12 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* END CODEC */ - +#ifdef SQLITE_META_DWR + if(PragmaMetaDoubleWrie(db, iDb, pParse, zLeft, zRight)) { + /* PragmaMetaDoubleWrie executes internal */ + goto pragma_out; + } +#endif /* Locate the pragma in the lookup table */ pPragma = pragmaLocate(zLeft); if( pPragma==0 ){ @@ -130899,7 +141275,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ #ifndef SQLITE_OMIT_AUTOVACUUM case PragTyp_INCREMENTAL_VACUUM: { - int iLimit, addr; + int iLimit = 0, addr; if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ iLimit = 0x7fffffff; } @@ -130945,7 +141321,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The first form reports the current local setting for the ** page cache spill size. The second form turns cache spill on - ** or off. When turnning cache spill on, the size is set to the + ** or off. When turning cache spill on, the size is set to the ** current cache_size. The third form sets a spill size that ** may be different form the cache size. ** If N is positive then that is the @@ -131056,6 +141432,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_TEMP_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_temp_directory); }else{ @@ -131065,6 +141442,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -131082,6 +141460,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } @@ -131100,6 +141479,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_DATA_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_data_directory); }else{ @@ -131109,6 +141489,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -131120,6 +141501,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } #endif @@ -131209,7 +141591,11 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif if( sqlite3GetBoolean(zRight, 0) ){ - db->flags |= mask; + if( (mask & SQLITE_WriteSchema)==0 + || (db->flags & SQLITE_Defensive)==0 + ){ + db->flags |= mask; + } }else{ db->flags &= ~mask; if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; @@ -131342,6 +141728,10 @@ SQLITE_PRIVATE void sqlite3Pragma( (void)sqlite3_finalize(pDummy); sqlite3DbFree(db, zSql); } + if( db->mallocFailed ){ + sqlite3ErrorMsg(db->pParse, "out of memory"); + db->pParse->rc = SQLITE_NOMEM_BKPT; + } pHash = &db->aDb[ii].pSchema->tblHash; break; } @@ -131583,7 +141973,6 @@ SQLITE_PRIVATE void sqlite3Pragma( HashElem *k; /* Loop counter: Next table in schema */ int x; /* result variable */ int regResult; /* 3 registers to hold a result row */ - int regKey; /* Register to hold key for checking the FK */ int regRow; /* Registers to hold a row from pTab */ int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ @@ -131591,7 +141980,6 @@ SQLITE_PRIVATE void sqlite3Pragma( regResult = pParse->nMem+1; pParse->nMem += 4; - regKey = ++pParse->nMem; regRow = ++pParse->nMem; k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -131607,7 +141995,7 @@ SQLITE_PRIVATE void sqlite3Pragma( zDb = db->aDb[iDb].zDbSName; sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3TouchRegister(pParse, pTab->nCol+regRow); sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); assert( IsOrdinaryTable(pTab) ); @@ -131648,7 +142036,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** regRow..regRow+n. If any of the child key values are NULL, this ** row cannot cause an FK violation. Jump directly to addrOk in ** this case. */ - if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol; + sqlite3TouchRegister(pParse, regRow + pFK->nCol); for(j=0; jnCol; j++){ int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); @@ -131658,9 +142046,9 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Generate code to query the parent index for a matching parent ** key. If a match is found, jump to addrOk. */ if( pIdx ){ - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3VdbeAddOp4(v, OP_Affinity, regRow, pFK->nCol, 0, sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); - sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regRow, pFK->nCol); VdbeCoverage(v); }else if( pParent ){ int jmp = sqlite3VdbeCurrentAddr(v)+2; @@ -131715,9 +142103,9 @@ SQLITE_PRIVATE void sqlite3Pragma( ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption ** without the overhead of cross-checking indexes. Quick_check - ** is linear time wherease integrity_check is O(NlogN). + ** is linear time whereas integrity_check is O(NlogN). ** - ** The maximum nubmer of errors is 100 by default. A different default + ** The maximum number of errors is 100 by default. A different default ** can be specified using a numeric parameter N. ** ** Or, the parameter N can be the name of a table. In that case, only @@ -131754,7 +142142,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -131771,12 +142159,12 @@ SQLITE_PRIVATE void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); + pParse->okConstFactor = 0; /* tag-20230327-1 */ /* Do an integrity check of the B-Tree ** @@ -131792,7 +142180,6 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pObjTab && pObjTab!=pTab ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -131812,11 +142199,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3TouchRegister(pParse, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, @@ -131826,20 +142213,59 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; - Index *pPrior = 0; + Index *pPrior = 0; /* Previous index */ int loopTop; int iDataCur, iIdxCur; int r1 = -1; - int bStrict; + int bStrict; /* True for a STRICT table */ + int r2; /* Previous key for WITHOUT ROWID tables */ + int mxCol; /* Maximum non-virtual column number */ - if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; - pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + if( !IsOrdinaryTable(pTab) ) continue; + if( isQuick || HasRowid(pTab) ){ + pPk = 0; + r2 = 0; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol); + sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -131853,52 +142279,181 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + + /* Fetch the right-most column from the table. This will cause + ** the entire record header to be parsed and sanity checked. It + ** will also prepopulate the cursor column cache that is used + ** by the OP_IsType code, so it is a required step. + */ + assert( !IsVirtual(pTab) ); + if( HasRowid(pTab) ){ + mxCol = -1; + for(j=0; jnCol; j++){ + if( (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)==0 ) mxCol++; + } + if( mxCol==pTab->iPKey ) mxCol--; + }else{ + /* COLFLAG_VIRTUAL columns are not included in the WITHOUT ROWID + ** PK index column-count, so there is no need to account for them + ** in this case. */ + mxCol = sqlite3PrimaryKeyIndex(pTab)->nColumn-1; + } + if( mxCol>=0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, mxCol, 3); + sqlite3VdbeTypeofColumn(v, 3); + } + if( !isQuick ){ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "(right-most column)")); + if( pPk ){ + /* Verify WITHOUT ROWID keys are in ascending order */ + int a1; + char *zErr; + a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, + "row not in PRIMARY KEY order for %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + sqlite3VdbeJumpHere(v, a1+1); + for(j=0; jnKeyCol; j++){ + sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j); + } + } } - /* Verify that all NOT NULL columns really are NOT NULL. At the - ** same time verify the type of the content of STRICT tables */ + /* Verify datatypes for all columns: + ** + ** (1) NOT NULL columns may not contain a NULL + ** (2) Datatype must be exact for non-ANY columns in STRICT tables + ** (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. + ** (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be losslessly converted to numeric. + */ bStrict = (pTab->tabFlags & TF_Strict)!=0; for(j=0; jnCol; j++){ char *zErr; - Column *pCol = pTab->aCol + j; - int doError, jmp2; + Column *pCol = pTab->aCol + j; /* The column to be checked */ + int labelError; /* Jump here to report an error */ + int labelOk; /* Jump here if all looks ok */ + int p1, p3, p4; /* Operands to the OP_IsType opcode */ + int doTypeCheck; /* Check datatypes (besides NOT NULL) */ + if( j==pTab->iPKey ) continue; - if( pCol->notNull==0 && !bStrict ) continue; - doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + if( bStrict ){ + doTypeCheck = pCol->eCType>COLTYPE_ANY; + }else{ + doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB; + } + if( pCol->notNull==0 && !doTypeCheck ) continue; + + /* Compute the operands that will be needed for OP_IsType */ + p4 = SQLITE_NULL; + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + p1 = -1; + p3 = 3; + }else{ + if( pCol->iDflt ){ + sqlite3_value *pDfltValue = 0; + sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), + pCol->affinity, &pDfltValue); + if( pDfltValue ){ + p4 = sqlite3_value_type(pDfltValue); + sqlite3ValueFree(pDfltValue); + } + } + p1 = iDataCur; + if( !HasRowid(pTab) ){ + testcase( j!=sqlite3TableColumnToStorage(pTab, j) ); + p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j); + }else{ + p3 = sqlite3TableColumnToStorage(pTab,j); + testcase( p3!=j); + } } + + labelError = sqlite3VdbeMakeLabel(pParse); + labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + /* (1) NOT NULL columns may not contain a NULL */ + int jmp3; + int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + VdbeCoverage(v); + if( p1<0 ){ + sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */ + jmp3 = jmp2; + }else{ + sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */ + /* OP_IsType does not detect NaN values in the database file + ** which should be treated as a NULL. So if the header type + ** is REAL, we have to load the actual data using OP_Column + ** to reliably determine if the value is a NULL. */ + sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); + jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); + VdbeCoverage(v); + } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - if( bStrict && pCol->eCType!=COLTYPE_ANY ){ - sqlite3VdbeGoto(v, doError); + if( doTypeCheck ){ + sqlite3VdbeGoto(v, labelError); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); }else{ - integrityCheckResultRow(v); + /* VDBE byte code will fall thru */ } - sqlite3VdbeJumpHere(v, jmp2); } - if( (pTab->tabFlags & TF_Strict)!=0 - && pCol->eCType!=COLTYPE_ANY - ){ - jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, - sqlite3StdTypeMap[pCol->eCType-1]); + if( bStrict && doTypeCheck ){ + /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/ + static unsigned char aStdTypeMask[] = { + 0x1f, /* ANY */ + 0x18, /* BLOB */ + 0x11, /* INT */ + 0x11, /* INTEGER */ + 0x13, /* REAL */ + 0x14 /* TEXT */ + }; + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) ); + sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]); VdbeCoverage(v); zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", sqlite3StdType[pCol->eCType-1], pTab->zName, pTab->aCol[j].zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeResolveLabel(v, doError); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){ + /* (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){ + /* (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be converted to numeric. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */ + VdbeCoverage(v); + if( p1>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + } + sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC); + sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "TEXT value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); } + sqlite3VdbeResolveLabel(v, labelError); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, labelOk); } /* Verify CHECK constraints */ if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ @@ -131927,7 +142482,8 @@ SQLITE_PRIVATE void sqlite3Pragma( if( !isQuick ){ /* Omit the remaining tests for quick_check */ /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2, jmp3, jmp4, jmp5; + int jmp2, jmp3, jmp4, jmp5, label6; + int kk; int ckUniq = sqlite3VdbeMakeLabel(pParse); if( pPk==pIdx ) continue; r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, @@ -131945,13 +142501,49 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp4 = integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, jmp2); + + /* The OP_IdxRowid opcode is an optimized version of OP_Column + ** that extracts the rowid off the end of the index record. + ** But it only works correctly if index record does not have + ** any extra bytes at the end. Verify that this is the case. */ + if( HasRowid(pTab) ){ + int jmp7; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); + jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 3, + "rowid not at end-of-record for row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " of index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp7); + } + + /* Any indexed columns with non-BINARY collations must still hold + ** the exact same text value as the table. */ + label6 = 0; + for(kk=0; kknKeyCol; kk++){ + if( pIdx->azColl[kk]==sqlite3StrBINARY ) continue; + if( label6==0 ) label6 = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+j, kk, 3); + sqlite3VdbeAddOp3(v, OP_Ne, 3, label6, r1+kk); VdbeCoverage(v); + } + if( label6 ){ + int jmp6 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeResolveLabel(v, label6); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " values differ from index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp6); + } + /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL ** or (2) the next entry has a different key */ if( IsUniqueIndex(pIdx) ){ int uniqOk = sqlite3VdbeMakeLabel(pParse); int jmp6; - int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; assert( iCol!=XN_ROWID && iColnCol ); @@ -131974,20 +142566,43 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); @@ -132136,6 +142751,11 @@ SQLITE_PRIVATE void sqlite3Pragma( aOp[1].p2 = iCookie; aOp[1].p3 = sqlite3Atoi(zRight); aOp[1].p5 = 1; + if( iCookie==BTREE_SCHEMA_VERSION && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow the use of PRAGMA schema_version=VALUE in defensive + ** mode. Change the OP_SetCookie opcode into a no-op. */ + aOp[1].opcode = OP_Noop; + } }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -132246,44 +142866,63 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. + ** + ** (2) The table is an ordinary table, not a virtual table or view. + ** + ** (3) The table name does not begin with "sqlite_". ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -132292,9 +142931,13 @@ SQLITE_PRIVATE void sqlite3Pragma( Schema *pSchema; /* The current schema */ Table *pTab; /* A table in the schema */ Index *pIdx; /* An index of the table */ - LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -132302,6 +142945,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -132310,23 +142961,61 @@ SQLITE_PRIVATE void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -132336,11 +143025,27 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddrnConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -132660,12 +143366,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -132685,6 +143392,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; iazArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; @@ -132825,7 +143533,8 @@ static const sqlite3_module pragmaVtabModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; /* @@ -133157,7 +143866,14 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl #else encoding = SQLITE_UTF8; #endif - sqlite3SetTextEncoding(db, encoding); + if( db->nVdbeActive>0 && encoding!=ENC(db) + && (db->mDbFlags & DBFLAG_Vacuum)==0 + ){ + rc = SQLITE_LOCKED; + goto initone_error_out; + }else{ + sqlite3SetTextEncoding(db, encoding); + } }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ @@ -133240,7 +143956,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl sqlite3ResetAllSchemasOfConnection(db); pDb = &db->aDb[iDb]; }else - if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + if( rc==SQLITE_OK || ((db->flags&SQLITE_NoSchemaError) && rc!=SQLITE_NOMEM)){ /* Hack: If the SQLITE_NoSchemaError flag is set, then consider ** the schema loaded, even if errors (other than OOM) occurred. In ** this situation the current sqlite3_prepare() operation will fail, @@ -133371,8 +144087,8 @@ static void schemaIsValid(Parse *pParse){ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; sqlite3ResetOneSchema(db, iDb); - pParse->rc = SQLITE_SCHEMA; } /* Close the transaction, if one was opened. */ @@ -133419,24 +144135,29 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ /* ** Free all memory allocations in the pParse object */ -SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ +SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ sqlite3 *db = pParse->db; + assert( db!=0 ); + assert( db->pParse==pParse ); + assert( pParse->nested==0 ); +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock); +#endif while( pParse->pCleanup ){ ParseCleanup *pCleanup = pParse->pCleanup; pParse->pCleanup = pCleanup->pNext; pCleanup->xCleanup(db, pCleanup->pPtr); - sqlite3DbFreeNN(db, pCleanup); + sqlite3DbNNFreeNN(db, pCleanup); } - sqlite3DbFree(db, pParse->aLabel); + if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel); if( pParse->pConstExpr ){ sqlite3ExprListDelete(db, pParse->pConstExpr); } - if( db ){ - assert( db->lookaside.bDisable >= pParse->disableLookaside ); - db->lookaside.bDisable -= pParse->disableLookaside; - db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; - } - pParse->disableLookaside = 0; + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; + assert( pParse->db->pParse==pParse ); + db->pParse = pParse->pOuterParse; } /* @@ -133445,10 +144166,10 @@ SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ ** immediately. ** ** Use this mechanism for uncommon cleanups. There is a higher setup -** cost for this mechansim (an extra malloc), so it should not be used +** cost for this mechanism (an extra malloc), so it should not be used ** for common cleanups that happen on most calls. But for less ** common cleanups, we save a single NULL-pointer comparison in -** sqlite3ParserReset(), which reduces the total CPU cycle count. +** sqlite3ParseObjectReset(), which reduces the total CPU cycle count. ** ** If a memory allocation error occurs, then the cleanup happens immediately. ** When either SQLITE_DEBUG or SQLITE_COVERAGE_TEST are defined, the @@ -133472,7 +144193,13 @@ SQLITE_PRIVATE void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; @@ -133488,6 +144215,33 @@ SQLITE_PRIVATE void *sqlite3ParserAddCleanup( return pPtr; } +/* +** Turn bulk memory into a valid Parse object and link that Parse object +** into database connection db. +** +** Call sqlite3ParseObjectReset() to undo this operation. +** +** Caution: Do not confuse this routine with sqlite3ParseObjectInit() which +** is generated by Lemon. +*/ +SQLITE_PRIVATE void sqlite3ParseObjectInit(Parse *pParse, sqlite3 *db){ + memset(PARSE_HDR(pParse), 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + assert( db->pParse!=pParse ); + pParse->pOuterParse = db->pParse; + db->pParse = pParse; + pParse->db = db; + if( db->mallocFailed ) sqlite3ErrorMsg(pParse, "out of memory"); +} + +/* +** Maximum number of times that we will try again to prepare a statement +** that returns SQLITE_ERROR_RETRY. +*/ +#ifndef SQLITE_MAX_PREPARE_RETRY +# define SQLITE_MAX_PREPARE_RETRY 25 +#endif + /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ @@ -133500,16 +144254,28 @@ static int sqlite3Prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ Parse sParse; /* Parsing context */ - memset(&sParse, 0, PARSE_HDR_SZ); + /* sqlite3ParseObjectInit(&sParse, db); // inlined for performance */ + memset(PARSE_HDR(&sParse), 0, PARSE_HDR_SZ); memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); - sParse.pReprepare = pReprepare; + sParse.pOuterParse = db->pParse; + db->pParse = &sParse; + sParse.db = db; + if( pReprepare ){ + sParse.pReprepare = pReprepare; + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); + }else{ + assert( sParse.pReprepare==0 ); + } assert( ppStmt && *ppStmt==0 ); - /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ + if( db->mallocFailed ){ + sqlite3ErrorMsg(&sParse, "out of memory"); + db->errCode = rc = SQLITE_NOMEM; + goto end_prepare; + } assert( sqlite3_mutex_held(db->mutex) ); /* For a long-term use prepared statement avoid the use of @@ -133519,7 +144285,7 @@ static int sqlite3Prepare( sParse.disableLookaside++; DisableLookaside; } - sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + sParse.prepFlags = prepFlags & 0xff; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that @@ -133560,9 +144326,10 @@ static int sqlite3Prepare( } } - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( db->pDisconnect ) sqlite3VtabUnlockList(db); +#endif - sParse.db = db; if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -133575,14 +144342,14 @@ static int sqlite3Prepare( } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sqlite3RunParser(&sParse, zSqlCopy); sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; sqlite3DbFree(db, zSqlCopy); }else{ sParse.zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(&sParse, zSql, &zErrMsg); + sqlite3RunParser(&sParse, zSql); } assert( 0==sParse.nQueryLoop ); @@ -133606,14 +144373,14 @@ static int sqlite3Prepare( } assert( 0==(*ppStmt) ); rc = sParse.rc; - if( zErrMsg ){ - sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); - sqlite3DbFree(db, zErrMsg); + if( sParse.zErrMsg ){ + sqlite3ErrorWithMsg(db, rc, "%s", sParse.zErrMsg); + sqlite3DbFree(db, sParse.zErrMsg); }else{ sqlite3Error(db, rc); } }else{ - assert( zErrMsg==0 ); + assert( sParse.zErrMsg==0 ); *ppStmt = (sqlite3_stmt*)sParse.pVdbe; rc = SQLITE_OK; sqlite3ErrorClear(db); @@ -133629,7 +144396,7 @@ static int sqlite3Prepare( end_prepare: - sqlite3ParserReset(&sParse); + sqlite3ParseObjectReset(&sParse); return rc; } static int sqlite3LockAndPrepare( @@ -133660,13 +144427,14 @@ static int sqlite3LockAndPrepare( rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); assert( rc==SQLITE_OK || *ppStmt==0 ); if( rc==SQLITE_OK || db->mallocFailed ) break; - }while( rc==SQLITE_ERROR_RETRY + }while( (rc==SQLITE_ERROR_RETRY && (cnt++)errMask)==rc ); db->busyHandler.nBusy = 0; sqlite3_mutex_leave(db->mutex); + assert( rc==SQLITE_OK || (*ppStmt)==0 ); return rc; } @@ -133901,7 +144669,7 @@ SQLITE_API int sqlite3_prepare16_v3( */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -133945,6 +144713,10 @@ struct SortCtx { } aDefer[4]; #endif struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrPush; /* First instruction to push data into sorter */ + int addrPushEnd; /* Last instruction that pushes data into sorter */ +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -133956,6 +144728,7 @@ struct SortCtx { ** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ + assert( db!=0 ); while( p ){ Select *pPrior = p->pPrior; sqlite3ExprListDelete(db, p->pEList); @@ -133975,7 +144748,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3WindowUnlinkFromSelect(p->pWin); } #endif - if( bFree ) sqlite3DbFreeNN(db, p); + if( bFree ) sqlite3DbNNFreeNN(db, p); p = pPrior; bFree = 1; } @@ -134059,6 +144832,9 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } +SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1); +} /* ** Return a pointer to the right-most SELECT statement in a compound. @@ -134084,6 +144860,52 @@ static Select *findRightmost(Select *p){ ** ** If an illegal or unsupported join type is seen, then still return ** a join type, but put an error in the pParse structure. +** +** These are the valid join types: +** +** +** pA pB pC Return Value +** ------- ----- ----- ------------ +** CROSS - - JT_CROSS +** INNER - - JT_INNER +** LEFT - - JT_LEFT|JT_OUTER +** LEFT OUTER - JT_LEFT|JT_OUTER +** RIGHT - - JT_RIGHT|JT_OUTER +** RIGHT OUTER - JT_RIGHT|JT_OUTER +** FULL - - JT_LEFT|JT_RIGHT|JT_OUTER +** FULL OUTER - JT_LEFT|JT_RIGHT|JT_OUTER +** NATURAL INNER - JT_NATURAL|JT_INNER +** NATURAL LEFT - JT_NATURAL|JT_LEFT|JT_OUTER +** NATURAL LEFT OUTER JT_NATURAL|JT_LEFT|JT_OUTER +** NATURAL RIGHT - JT_NATURAL|JT_RIGHT|JT_OUTER +** NATURAL RIGHT OUTER JT_NATURAL|JT_RIGHT|JT_OUTER +** NATURAL FULL - JT_NATURAL|JT_LEFT|JT_RIGHT +** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT +** +** To preserve historical compatibly, SQLite also accepts a variety +** of other non-standard and in many cases nonsensical join types. +** This routine makes as much sense at it can from the nonsense join +** type and returns a result. Examples of accepted nonsense join types +** include but are not limited to: +** +** INNER CROSS JOIN -> same as JOIN +** NATURAL CROSS JOIN -> same as NATURAL JOIN +** OUTER LEFT JOIN -> same as LEFT JOIN +** LEFT NATURAL JOIN -> same as NATURAL LEFT JOIN +** LEFT RIGHT JOIN -> same as FULL JOIN +** RIGHT OUTER FULL JOIN -> same as FULL JOIN +** CROSS CROSS CROSS JOIN -> same as JOIN +** +** The only restrictions on the join type name are: +** +** * "INNER" cannot appear together with "OUTER", "LEFT", "RIGHT", +** or "FULL". +** +** * "CROSS" cannot appear together with "OUTER", "LEFT", "RIGHT, +** or "FULL". +** +** * If "OUTER" is present then there must also be one of +** "LEFT", "RIGHT", or "FULL" */ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ int jointype = 0; @@ -134096,13 +144918,13 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p u8 nChar; /* Length of the keyword in characters */ u8 code; /* Join type mask */ } aKeyword[] = { - /* natural */ { 0, 7, JT_NATURAL }, - /* left */ { 6, 4, JT_LEFT|JT_OUTER }, - /* outer */ { 10, 5, JT_OUTER }, - /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, - /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, - /* inner */ { 23, 5, JT_INNER }, - /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + /* (0) natural */ { 0, 7, JT_NATURAL }, + /* (1) left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* (2) outer */ { 10, 5, JT_OUTER }, + /* (3) right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* (4) full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* (5) inner */ { 23, 5, JT_INNER }, + /* (6) cross */ { 28, 5, JT_INNER|JT_CROSS }, }; int i, j; apAll[0] = pA; @@ -134125,18 +144947,15 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p } if( (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || - (jointype & JT_ERROR)!=0 + (jointype & JT_ERROR)!=0 || + (jointype & (JT_OUTER|JT_LEFT|JT_RIGHT))==JT_OUTER ){ - const char *zSp = " "; - assert( pB!=0 ); - if( pC==0 ){ zSp++; } - sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " - "%T %T%s%T", pA, pB, zSp, pC); - jointype = JT_INNER; - }else if( (jointype & JT_OUTER)!=0 - && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ - sqlite3ErrorMsg(pParse, - "RIGHT and FULL OUTER JOINs are not currently supported"); + const char *zSp1 = " "; + const char *zSp2 = " "; + if( pB==0 ){ zSp1++; } + if( pC==0 ){ zSp2++; } + sqlite3ErrorMsg(pParse, "unknown join type: " + "%T%s%T%s%T", pA, zSp1, pB, zSp2, pC); jointype = JT_INNER; } return jointype; @@ -134157,8 +144976,25 @@ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ } /* -** Search the first N tables in pSrc, from left to right, looking for a -** table that has a column named zCol. +** Mark a subquery result column as having been used. +*/ +SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ + assert( pItem!=0 ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + if( pItem->fg.isNestedFrom ){ + ExprList *pResults; + assert( pItem->pSelect!=0 ); + pResults = pItem->pSelect->pEList; + assert( pResults!=0 ); + assert( iCol>=0 && iColnExpr ); + pResults->a[iCol].fg.bUsed = 1; + } +} + +/* +** Search the tables iStart..iEnd (inclusive) in pSrc, looking for a +** table that has a column named zCol. The search is left-to-right. +** The first match found is returned. ** ** When found, set *piTab and *piCol to the table index and column index ** of the matching column and return TRUE. @@ -134167,22 +145003,27 @@ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ static int tableAndColumnIndex( SrcList *pSrc, /* Array of tables to search */ - int N, /* Number of tables in pSrc->a[] to search */ + int iStart, /* First member of pSrc->a[] to check */ + int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ - int bIgnoreHidden /* True to ignore hidden columns */ + int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ int iCol; /* Index of column matching zCol */ + assert( iEndnSrc ); + assert( iStart>=0 ); assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ - for(i=0; ia[i].pTab, zCol); if( iCol>=0 && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) ){ if( piTab ){ + sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); *piTab = i; *piCol = iCol; } @@ -134193,66 +145034,19 @@ static int tableAndColumnIndex( } /* -** This function is used to add terms implied by JOIN syntax to the -** WHERE clause expression of a SELECT statement. The new term, which -** is ANDed with the existing WHERE clause, is of the form: -** -** (tab1.col1 = tab2.col2) -** -** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the -** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is -** column iColRight of tab2. -*/ -static void addWhereTerm( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* List of tables in FROM clause */ - int iLeft, /* Index of first table to join in pSrc */ - int iColLeft, /* Index of column in first table */ - int iRight, /* Index of second table in pSrc */ - int iColRight, /* Index of column in second table */ - int isOuterJoin, /* True if this is an OUTER join */ - Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ -){ - sqlite3 *db = pParse->db; - Expr *pE1; - Expr *pE2; - Expr *pEq; - - assert( iLeftnSrc>iRight ); - assert( pSrc->a[iLeft].pTab ); - assert( pSrc->a[iRight].pTab ); - - pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); - pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); - - pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); - assert( pE2!=0 || pEq==0 ); /* Due to db->mallocFailed test - ** in sqlite3DbMallocRawNN() called from - ** sqlite3PExpr(). */ - if( pEq && isOuterJoin ){ - ExprSetProperty(pEq, EP_FromJoin); - assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); - ExprSetVVAProperty(pEq, EP_NoReduce); - pEq->iRightJoinTable = pE2->iTable; - } - *ppWhere = sqlite3ExprAnd(pParse, *ppWhere, pEq); -} - -/* -** Set the EP_FromJoin property on all terms of the given expression. -** And set the Expr.iRightJoinTable to iTable for every term in the +** Set the EP_OuterON property on all terms of the given expression. +** And set the Expr.w.iJoin to iTable for every term in the ** expression. ** -** The EP_FromJoin property is used on terms of an expression to tell -** the LEFT OUTER JOIN processing logic that this term is part of the +** The EP_OuterON property is used on terms of an expression to tell +** the OUTER JOIN processing logic that this term is part of the ** join restriction specified in the ON or USING clause and not a part ** of the more general WHERE clause. These terms are moved over to the ** WHERE clause during join processing but we need to remember that they ** originated in the ON or USING clause. ** -** The Expr.iRightJoinTable tells the WHERE clause processing that the -** expression depends on table iRightJoinTable even if that table is not +** The Expr.w.iJoin tells the WHERE clause processing that the +** expression depends on table w.iJoin even if that table is not ** explicitly mentioned in the expression. That information is needed ** for cases like this: ** @@ -134265,70 +145059,87 @@ static void addWhereTerm( ** after the t1 loop and rows with t1.x!=5 will never appear in ** the output, which is incorrect. */ -SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable){ +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ + assert( joinFlag==EP_OuterON || joinFlag==EP_InnerON ); while( p ){ - ExprSetProperty(p, EP_FromJoin); + ExprSetProperty(p, joinFlag); assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); - p->iRightJoinTable = iTable; + p->w.iJoin = iTable; if( p->op==TK_FUNCTION ){ assert( ExprUseXList(p) ); if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ - sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable); + sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable, joinFlag); } } } - sqlite3SetJoinExpr(p->pLeft, iTable); + sqlite3SetJoinExpr(p->pLeft, iTable, joinFlag); p = p->pRight; } } -/* Undo the work of sqlite3SetJoinExpr(). In the expression p, convert every -** term that is marked with EP_FromJoin and iRightJoinTable==iTable into -** an ordinary term that omits the EP_FromJoin mark. +/* Undo the work of sqlite3SetJoinExpr(). This is used when a LEFT JOIN +** is simplified into an ordinary JOIN, and when an ON expression is +** "pushed down" into the WHERE clause of a subquery. +** +** Convert every term that is marked with EP_OuterON and w.iJoin==iTable into +** an ordinary term that omits the EP_OuterON mark. Or if iTable<0, then +** just clear every EP_OuterON and EP_InnerON mark from the expression tree. ** -** This happens when a LEFT JOIN is simplified into an ordinary JOIN. +** If nullable is true, that means that Expr p might evaluate to NULL even +** if it is a reference to a NOT NULL column. This can happen, for example, +** if the table that p references is on the left side of a RIGHT JOIN. +** If nullable is true, then take care to not remove the EP_CanBeNull bit. +** See forum thread https://sqlite.org/forum/forumpost/b40696f50145d21c */ -static void unsetJoinExpr(Expr *p, int iTable){ +static void unsetJoinExpr(Expr *p, int iTable, int nullable){ while( p ){ - if( ExprHasProperty(p, EP_FromJoin) - && (iTable<0 || p->iRightJoinTable==iTable) ){ - ExprClearProperty(p, EP_FromJoin); + if( iTable<0 || (ExprHasProperty(p, EP_OuterON) && p->w.iJoin==iTable) ){ + ExprClearProperty(p, EP_OuterON|EP_InnerON); + if( iTable>=0 ) ExprSetProperty(p, EP_InnerON); } - if( p->op==TK_COLUMN && p->iTable==iTable ){ + if( p->op==TK_COLUMN && p->iTable==iTable && !nullable ){ ExprClearProperty(p, EP_CanBeNull); } if( p->op==TK_FUNCTION ){ assert( ExprUseXList(p) ); + assert( p->pLeft==0 ); if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ - unsetJoinExpr(p->x.pList->a[i].pExpr, iTable); + unsetJoinExpr(p->x.pList->a[i].pExpr, iTable, nullable); } } } - unsetJoinExpr(p->pLeft, iTable); + unsetJoinExpr(p->pLeft, iTable, nullable); p = p->pRight; } } /* ** This routine processes the join information for a SELECT statement. -** ON and USING clauses are converted into extra terms of the WHERE clause. -** NATURAL joins also create extra WHERE clause terms. +** +** * A NATURAL join is converted into a USING join. After that, we +** do not need to be concerned with NATURAL joins and we only have +** think about USING joins. +** +** * ON and USING clauses result in extra terms being added to the +** WHERE clause to enforce the specified constraints. The extra +** WHERE clause terms will be tagged with EP_OuterON or +** EP_InnerON so that we know that they originated in ON/USING. ** ** The terms of a FROM clause are contained in the Select.pSrc structure. ** The left most table is the first entry in Select.pSrc. The right-most ** table is the last entry. The join operator is held in the entry to -** the left. Thus entry 0 contains the join operator for the join between +** the right. Thus entry 1 contains the join operator for the join between ** entries 0 and 1. Any ON or USING clauses associated with the join are -** also attached to the left entry. +** also attached to the right entry. ** ** This routine returns the number of errors encountered. */ -static int sqliteProcessJoin(Parse *pParse, Select *p){ +static int sqlite3ProcessJoin(Parse *pParse, Select *p){ SrcList *pSrc; /* All tables in the FROM clause */ int i, j; /* Loop counters */ SrcItem *pLeft; /* Left table being joined */ @@ -134339,49 +145150,41 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ pRight = &pLeft[1]; for(i=0; inSrc-1; i++, pRight++, pLeft++){ Table *pRightTab = pRight->pTab; - int isOuter; + u32 joinType; if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->fg.jointype & JT_OUTER)!=0; + joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; - /* When the NATURAL keyword is present, add WHERE clause terms for - ** every column that the two tables have in common. + /* If this is a NATURAL join, synthesize an appropriate USING clause + ** to specify which columns should be joined. */ if( pRight->fg.jointype & JT_NATURAL ){ - if( pRight->pOn || pRight->pUsing ){ + IdList *pUsing = 0; + if( pRight->fg.isUsing || pRight->u3.pOn ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); return 1; } for(j=0; jnCol; j++){ char *zName; /* Name of column in the right table */ - int iLeft; /* Matching left table */ - int iLeftCol; /* Matching column in the left table */ if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue; zName = pRightTab->aCol[j].zCnName; - if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 1) ){ - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, - isOuter, &p->pWhere); + if( tableAndColumnIndex(pSrc, 0, i, zName, 0, 0, 1) ){ + pUsing = sqlite3IdListAppend(pParse, pUsing, 0); + if( pUsing ){ + assert( pUsing->nId>0 ); + assert( pUsing->a[pUsing->nId-1].zName==0 ); + pUsing->a[pUsing->nId-1].zName = sqlite3DbStrDup(pParse->db, zName); + } } } - } - - /* Disallow both ON and USING clauses in the same join - */ - if( pRight->pOn && pRight->pUsing ){ - sqlite3ErrorMsg(pParse, "cannot have both ON and USING " - "clauses in the same join"); - return 1; - } - - /* Add the ON clause to the end of the WHERE clause, connected by - ** an AND operator. - */ - if( pRight->pOn ){ - if( isOuter ) sqlite3SetJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->pOn); - pRight->pOn = 0; + if( pUsing ){ + pRight->fg.isUsing = 1; + pRight->fg.isSynthUsing = 1; + pRight->u3.pUsing = pUsing; + } + if( pParse->nErr ) return 1; } /* Create extra terms on the WHERE clause for each column named @@ -134391,27 +145194,88 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ ** Report an error if any column mentioned in the USING clause is ** not contained in both tables to be joined. */ - if( pRight->pUsing ){ - IdList *pList = pRight->pUsing; + if( pRight->fg.isUsing ){ + IdList *pList = pRight->u3.pUsing; + sqlite3 *db = pParse->db; + assert( pList!=0 ); for(j=0; jnId; j++){ char *zName; /* Name of the term in the USING clause */ int iLeft; /* Table on the left with matching column name */ int iLeftCol; /* Column number of matching column on the left */ int iRightCol; /* Column number of matching column on the right */ + Expr *pE1; /* Reference to the column on the LEFT of the join */ + Expr *pE2; /* Reference to the column on the RIGHT of the join */ + Expr *pEq; /* Equality constraint. pE1 == pE2 */ zName = pList->a[j].zName; iRightCol = sqlite3ColumnIndex(pRightTab, zName); if( iRightCol<0 - || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 0) + || tableAndColumnIndex(pSrc, 0, i, zName, &iLeft, &iLeftCol, + pRight->fg.isSynthUsing)==0 ){ sqlite3ErrorMsg(pParse, "cannot join using column %s - column " "not present in both tables", zName); return 1; } - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, - isOuter, &p->pWhere); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); + sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* This branch runs if the query contains one or more RIGHT or FULL + ** JOINs. If only a single table on the left side of this join + ** contains the zName column, then this branch is a no-op. + ** But if there are two or more tables on the left side + ** of the join, construct a coalesce() function that gathers all + ** such tables. Raise an error if more than one of those references + ** to zName is not also within a prior USING clause. + ** + ** We really ought to raise an error if there are two or more + ** non-USING references to zName on the left of an INNER or LEFT + ** JOIN. But older versions of SQLite do not do that, so we avoid + ** adding a new error so as to not break legacy applications. + */ + ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ + static const Token tkCoalesce = { "coalesce", 8 }; + while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, + pRight->fg.isSynthUsing)!=0 ){ + if( pSrc->a[iLeft].fg.isUsing==0 + || sqlite3IdListIndex(pSrc->a[iLeft].u3.pUsing, zName)<0 + ){ + sqlite3ErrorMsg(pParse, "ambiguous reference to %s in USING()", + zName); + break; + } + pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); + sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); + } + if( pFuncArgs ){ + pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); + pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + } + } + pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); + sqlite3SrcItemColumnUsed(pRight, iRightCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); + assert( pE2!=0 || pEq==0 ); + if( pEq ){ + ExprSetProperty(pEq, joinType); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); + pEq->w.iJoin = pE2->iTable; + } + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pEq); } } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + else if( pRight->u3.pOn ){ + sqlite3SetJoinExpr(pRight->u3.pOn, pRight->iCursor, joinType); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn); + pRight->u3.pOn = 0; + pRight->fg.isOn = 1; + } } return 0; } @@ -134505,14 +145369,18 @@ static void pushOntoSorter( ** (2) All output columns are included in the sort record. In that ** case regData==regOrigData. ** (3) Some output columns are omitted from the sort record due to - ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the ** SQLITE_ECEL_OMITREF optimization, or due to the - ** SortCtx.pDeferredRowLoad optimiation. In any of these cases + ** SortCtx.pDeferredRowLoad optimization. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy ** values that might not yet exist. */ assert( nData==1 || regData==regOrigData || regOrigData==0 ); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPush = sqlite3VdbeCurrentAddr(v); +#endif + if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nPrefixReg; @@ -134559,7 +145427,7 @@ static void pushOntoSorter( testcase( pKI->nAllField > pKI->nKeyField+2 ); pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, pKI->nAllField-pKI->nKeyField-1); - pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ + pOp = 0; /* Ensure pOp not used after sqlite3VdbeAddOp3() */ addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); @@ -134613,6 +145481,9 @@ static void pushOntoSorter( sqlite3VdbeChangeP2(v, iSkip, pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushEnd = sqlite3VdbeCurrentAddr(v)-1; +#endif } /* @@ -134650,7 +145521,7 @@ static void codeOffset( ** The returned value in this case is a copy of parameter iTab. ** ** WHERE_DISTINCT_ORDERED: -** In this case rows are being delivered sorted order. The ephermal +** In this case rows are being delivered sorted order. The ephemeral ** table is not required. Instead, the current set of values ** is compared against previous row. If they match, the new row ** is not distinct and control jumps to VM address addrRepeat. Otherwise, @@ -134800,7 +145671,7 @@ static void fixDistinctOpenEph( ** retrieved directly from table t1. If the values are very large, this ** can be more efficient than storing them directly in the sorter records. ** -** The ExprList_item.bSorterRef flag is set for each expression in pEList +** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList ** for which the sorter-reference optimization should be enabled. ** Additionally, the pSort->aDefer[] array is populated with entries ** for all cursors required to evaluate all selected expressions. Finally. @@ -134860,7 +145731,7 @@ static void selectExprDefer( nDefer++; } } - pItem->bSorterRef = 1; + pItem->fg.bSorterRef = 1; } } } @@ -134909,14 +145780,12 @@ static void selectInnerLoop( assert( p->pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pSort && pSort->pOrderBy==0 ) pSort = 0; - #ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION if( hasDistinct && (pDistinct->eTnctType==WHERE_DISTINCT_UNIQUE) ){ hasDistinct = WHERE_DISTINCT_NOOP; sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); } -#endif - +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ if( pSort==0 && !hasDistinct ){ assert( iContinue!=0 ); codeOffset(v, p->iOffset, iContinue); @@ -134929,7 +145798,7 @@ static void selectInnerLoop( sqlite3VdbeAddOp2(v, OP_Jump, 0, iContinue); sqlite3VdbeChangeP5(v, 128); } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ } /* Pull the requested columns. @@ -135009,7 +145878,7 @@ static void selectInnerLoop( for(i=0; inExpr; i++){ if( pEList->a[i].u.x.iOrderByCol>0 #ifdef SQLITE_ENABLE_SORTER_REFERENCES - || pEList->a[i].bSorterRef + || pEList->a[i].fg.bSorterRef #endif ){ nResultCol--; @@ -135067,7 +145936,7 @@ static void selectInnerLoop( sqlite3VdbeAddOp2(v, OP_Jump, 0, iContinue); sqlite3VdbeChangeP5(v, 128); } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ } } @@ -135107,6 +145976,16 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -135312,7 +146191,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->nRef = 1; memset(&p[1], 0, nExtra); }else{ - sqlite3OomFault(db); + return (KeyInfo*)sqlite3OomFault(db); } return p; } @@ -135322,9 +146201,10 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ */ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ + assert( p->db!=0 ); assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p); } } @@ -135381,7 +146261,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; iaColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); - pInfo->aSortFlags[i-iStart] = pItem->sortFlags; + pInfo->aSortFlags[i-iStart] = pItem->fg.sortFlags; } } return pInfo; @@ -135463,6 +146343,23 @@ static void generateSortTail( int bSeq; /* True if sorter record includes seq. no. */ int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif + + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } + sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); + assert( addrBreak<0 ); if( pSort->labelBkOut ){ @@ -135483,6 +146380,9 @@ static void generateSortTail( iTab = pSort->iECursor; if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ + if( eDest==SRT_Mem && p->iOffset ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pDest->iSdst); + } regRowid = 0; regRow = pDest->iSdst; }else{ @@ -135494,7 +146394,6 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; @@ -135506,13 +146405,13 @@ static void generateSortTail( if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); #ifdef SQLITE_SHARED_BLOCK_OPTIMIZATION if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_Jump, 0, addrContinue); sqlite3VdbeChangeP5(v, 128); } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -135523,13 +146422,16 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_Jump, 0, addrContinue); sqlite3VdbeChangeP5(v, 128); } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } for(i=0, iCol=nKey+bSeq-1; i=0; i--){ #ifdef SQLITE_ENABLE_SORTER_REFERENCES - if( aOutEx[i].bSorterRef ){ + if( aOutEx[i].fg.bSorterRef ){ sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); }else #endif @@ -135581,6 +146483,7 @@ static void generateSortTail( VdbeComment((v, "%s", aOutEx[i].zEName)); } } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -135642,6 +146545,7 @@ static void generateSortTail( }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + sqlite3VdbeScanStatusRange(v, addrExplain, sqlite3VdbeCurrentAddr(v)-1, -1); if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); } @@ -135650,9 +146554,6 @@ static void generateSortTail( ** Return a pointer to a string containing the 'declaration type' of the ** expression pExpr. The string may be treated as static by the caller. ** -** Also try to estimate the size of the returned value and return that -** result in *pEstWidth. -** ** The declaration type is the exact datatype definition extracted from the ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression @@ -135744,11 +146645,7 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see @@ -135906,17 +146803,10 @@ SQLITE_PRIVATE void sqlite3GenerateColumnNames( int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; - } -#endif - if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; - SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); pTabList = pSelect->pSrc; pEList = pSelect->pEList; assert( v!=0 ); @@ -135932,7 +146822,7 @@ SQLITE_PRIVATE void sqlite3GenerateColumnNames( assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ assert( p->op!=TK_COLUMN || (ExprUseYTab(p) && p->y.pTab!=0) ); /* Covering idx not yet coded */ - if( pEList->a[i].zEName && pEList->a[i].eEName==ENAME_NAME ){ + if( pEList->a[i].zEName && pEList->a[i].fg.eEName==ENAME_NAME ){ /* An AS clause always takes first priority */ char *zName = pEList->a[i].zEName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); @@ -136016,23 +146906,26 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + for(i=0, pCol=aCol; inErr; i++, pCol++){ + struct ExprList_item *pX = &pEList->a[i]; + struct ExprList_item *pCollide; /* Get an appropriate name for the column */ - if( (zName = pEList->a[i].zEName)!=0 && pEList->a[i].eEName==ENAME_NAME ){ + if( (zName = pX->zEName)!=0 && pX->fg.eEName==ENAME_NAME ){ /* If the column contains an "AS " phrase, use as the name */ }else{ - Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr); + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pX->pExpr); while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } if( pColExpr->op==TK_COLUMN && ALWAYS( ExprUseYTab(pColExpr) ) - && (pTab = pColExpr->y.pTab)!=0 + && ALWAYS( pColExpr->y.pTab!=0 ) ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; + pTab = pColExpr->y.pTab; if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zCnName : "rowid"; }else if( pColExpr->op==TK_ID ){ @@ -136040,7 +146933,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( zName = pColExpr->u.zToken; }else{ /* Use the original text of the column expression as its name */ - zName = pEList->a[i].zEName; + assert( zName==pX->zEName ); /* pointer comparison intended */ } } if( zName && !sqlite3IsTrueOrFalse(zName) ){ @@ -136053,88 +146946,135 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( ** append an integer to the name so that it becomes unique. */ cnt = 0; - while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + while( zName && (pCollide = sqlite3HashFind(&ht, zName))!=0 ){ + if( pCollide->fg.bUsingTerm ){ + pCol->colFlags |= COLFLAG_NOEXPAND; + } nName = sqlite3Strlen30(zName); if( nName>0 ){ for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + sqlite3ProgressCheck(pParse); + if( cnt>3 ){ + sqlite3_randomness(sizeof(cnt), &cnt); + } } pCol->zCnName = zName; pCol->hName = sqlite3StrIHash(zName); + if( pX->fg.bNoExpand ){ + pCol->colFlags |= COLFLAG_NOEXPAND; + } sqlite3ColumnPropertiesFromName(0, pCol); - if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + if( zName && sqlite3HashInsert(&ht, zName, pX)==pX ){ sqlite3OomFault(db); } } sqlite3HashClear(&ht); - if( db->mallocFailed ){ + if( pParse->nErr ){ for(j=0; jrc; } return SQLITE_OK; } /* -** Add type and collation information to a column list based on -** a SELECT statement. +** pTab is a transient Table object that represents a subquery of some +** kind (maybe a parenthesized subquery in the FROM clause of a larger +** query, or a VIEW, or a CTE). This routine computes type information +** for that Table object based on the Select object that implements the +** subquery. For the purposes of this routine, "type information" means: ** -** The column list presumably came from selectColumnNamesFromExprList(). -** The column list has only names, not types or collations. This -** routine goes through and adds the types and collations. -** -** This routine requires that all identifiers in the SELECT -** statement be resolved. +** * The datatype name, as it might appear in a CREATE TABLE statement +** * Which collating sequence to use for the column +** * The affinity of the column */ -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( - Parse *pParse, /* Parsing contexts */ - Table *pTab, /* Add column type information to this table */ - Select *pSelect, /* SELECT used to determine types and collations */ - char aff /* Default affinity for columns */ +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( + Parse *pParse, /* Parsing contexts */ + Table *pTab, /* Add column type information to this table */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity. */ ){ sqlite3 *db = pParse->db; - NameContext sNC; Column *pCol; CollSeq *pColl; - int i; + int i,j; Expr *p; struct ExprList_item *a; + NameContext sNC; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); - if( db->mallocFailed ) return; + assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); + assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); + if( db->mallocFailed || IN_RENAME_OBJECT ) return; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + a = pSelect->pEList->a; memset(&sNC, 0, sizeof(sNC)); sNC.pSrcList = pSelect->pSrc; - a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; - i64 n, m; + i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; - zType = columnType(&sNC, p, 0, 0, 0); /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity<=SQLITE_AFF_NONE ){ + pCol->affinity = aff; + } + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + }else + if( pCol->affinity>=SQLITE_AFF_NUMERIC && (m&0x02)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ + pCol->affinity = SQLITE_AFF_FLEXNUM; + } + } + zType = columnType(&sNC, p, 0, 0, 0); + if( zType==0 || pCol->affinity!=sqlite3AffinityType(zType, 0) ){ + if( pCol->affinity==SQLITE_AFF_NUMERIC + || pCol->affinity==SQLITE_AFF_FLEXNUM + ){ + zType = "NUM"; + }else{ + zType = 0; + for(j=1; jaffinity ){ + zType = sqlite3StdType[j]; + break; + } + } + } + } if( zType ){ - m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); + pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; - }else{ - testcase( pCol->colFlags & COLFLAG_HASTYPE ); - pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); } } - if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl ){ assert( pTab->pIndex==0 ); @@ -136168,7 +147108,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, c pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); + sqlite3SubqueryColumnTypes(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -136322,7 +147262,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ } assert( sqlite3KeyInfoIsWriteable(pRet) ); pRet->aColl[i] = pColl; - pRet->aSortFlags[i] = pOrderBy->a[i].sortFlags; + pRet->aSortFlags[i] = pOrderBy->a[i].fg.sortFlags; } } @@ -136383,7 +147323,7 @@ static void generateWithRecursiveQuery( int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ int eDest = SRT_Fifo; /* How to write to Queue */ - SelectDest destQueue; /* SelectDest targetting the Queue table */ + SelectDest destQueue; /* SelectDest targeting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ @@ -136540,7 +147480,7 @@ static int multiSelectOrderBy( ** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES ** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))"). ** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case. -** Since the limit is exactly 1, we only need to evalutes the left-most VALUES. +** Since the limit is exactly 1, we only need to evaluate the left-most VALUES. */ static int multiSelectValues( Parse *pParse, /* Parsing context */ @@ -136693,7 +147633,7 @@ static int multiSelect( pPrior->iLimit = p->iLimit; pPrior->iOffset = p->iOffset; pPrior->pLimit = p->pLimit; - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); rc = sqlite3Select(pParse, pPrior, &dest); pPrior->pLimit = 0; if( rc ){ @@ -136711,7 +147651,7 @@ static int multiSelect( } } ExplainQueryPlan((pParse, 1, "UNION ALL")); - SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -136764,7 +147704,7 @@ static int multiSelect( */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; @@ -136784,7 +147724,7 @@ static int multiSelect( uniondest.eDest = op; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); @@ -136845,7 +147785,7 @@ static int multiSelect( /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; @@ -136862,7 +147802,7 @@ static int multiSelect( intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); - SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -136959,9 +147899,7 @@ multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; if( pDelete ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3SelectDelete, - pDelete); + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } return rc; } @@ -136983,7 +147921,7 @@ SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ /* ** Code an output subroutine for a coroutine implementation of a -** SELECT statment. +** SELECT statement. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should @@ -137205,7 +148143,7 @@ static int generateOutputSubroutine( ** ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop -** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** until all data is exhausted then jump to the "end" label. AltB, AeqB, ** and AgtB jump to either L2 or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -137216,6 +148154,8 @@ static int multiSelectOrderBy( ){ int i, j; /* Loop counters */ Select *pPrior; /* Another SELECT immediately to our left */ + Select *pSplit; /* Left-most SELECT in the right-hand group */ + int nSelect; /* Number of SELECT statements in the compound */ Vdbe *v; /* Generate code to this VDBE */ SelectDest destA; /* Destination for coroutine A */ SelectDest destB; /* Destination for coroutine B */ @@ -137240,7 +148180,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int addr1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargeted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -137261,8 +148201,7 @@ static int multiSelectOrderBy( /* Patch up the ORDER BY clause */ op = p->op; - pPrior = p->pPrior; - assert( pPrior->pOrderBy==0 ); + assert( p->pPrior->pOrderBy==0 ); pOrderBy = p->pOrderBy; assert( pOrderBy ); nOrderBy = pOrderBy->nExpr; @@ -137312,11 +148251,6 @@ static int multiSelectOrderBy( pKeyMerge = 0; } - /* Reattach the ORDER BY clause to the query. - */ - p->pOrderBy = pOrderBy; - pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); - /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). @@ -137341,12 +148275,30 @@ static int multiSelectOrderBy( /* Separate the left and the right query from one another */ - p->pPrior = 0; + nSelect = 1; + if( (op==TK_ALL || op==TK_UNION) + && OptimizationEnabled(db, SQLITE_BalancedMerge) + ){ + for(pSplit=p; pSplit->pPrior!=0 && pSplit->op==op; pSplit=pSplit->pPrior){ + nSelect++; + assert( pSplit->pPrior->pNext==pSplit ); + } + } + if( nSelect<=3 ){ + pSplit = p; + }else{ + pSplit = p; + for(i=2; ipPrior; } + } + pPrior = pSplit->pPrior; + assert( pPrior!=0 ); + pSplit->pPrior = 0; pPrior->pNext = 0; + assert( p->pOrderBy == pOrderBy ); + assert( pOrderBy!=0 || db->mallocFailed ); + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); - if( pPrior->pPrior==0 ){ - sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); - } + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); /* Compute the limit registers */ computeLimitRegisters(pParse, p, labelEnd); @@ -137495,14 +148447,13 @@ static int multiSelectOrderBy( */ sqlite3VdbeResolveLabel(v, labelEnd); - /* Reassembly the compound query so that it will be freed correctly - ** by the calling function */ - if( p->pPrior ){ - sqlite3SelectDelete(db, p->pPrior); + /* Make arrangements to free the 2nd and subsequent arms of the compound + ** after the parse has finished */ + if( pSplit->pPrior ){ + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior); } - p->pPrior = pPrior; - pPrior->pNext = p; - + pSplit->pPrior = pPrior; + pPrior->pNext = pSplit; sqlite3ExprListDelete(db, pPrior->pOrderBy); pPrior->pOrderBy = 0; @@ -137520,13 +148471,42 @@ static int multiSelectOrderBy( ** ** All references to columns in table iTable are to be replaced by corresponding ** expressions in pEList. +** +** ## About "isOuterJoin": +** +** The isOuterJoin column indicates that the replacement will occur into a +** position in the parent that NULL-able due to an OUTER JOIN. Either the +** target slot in the parent is the right operand of a LEFT JOIN, or one of +** the left operands of a RIGHT JOIN. In either case, we need to potentially +** bypass the substituted expression with OP_IfNullRow. +** +** Suppose the original expression is an integer constant. Even though the table +** has the nullRow flag set, because the expression is an integer constant, +** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode +** that checks to see if the nullRow flag is set on the table. If the nullRow +** flag is set, then the value in the register is set to NULL and the original +** expression is bypassed. If the nullRow flag is not set, then the original +** expression runs to populate the register. +** +** Example where this is needed: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); +** CREATE TABLE t2(x INT UNIQUE); +** +** SELECT a,b,m,x FROM t1 LEFT JOIN (SELECT 59 AS m,x FROM t2) ON b=x; +** +** When the subquery on the right side of the LEFT JOIN is flattened, we +** have to add OP_IfNullRow in front of the OP_Integer that implements the +** "m" value of the subquery so that a NULL will be loaded instead of 59 +** when processing a non-matched row of the left. */ typedef struct SubstContext { Parse *pParse; /* The parsing context */ int iTable; /* Replace references to this table */ int iNewTable; /* New table number */ - int isLeftJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; /* Forward Declarations */ @@ -137551,10 +148531,11 @@ static Expr *substExpr( Expr *pExpr /* Expr in which substitution occurs */ ){ if( pExpr==0 ) return 0; - if( ExprHasProperty(pExpr, EP_FromJoin) - && pExpr->iRightJoinTable==pSubst->iTable + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) + && pExpr->w.iJoin==pSubst->iTable ){ - pExpr->iRightJoinTable = pSubst->iNewTable; + testcase( ExprHasProperty(pExpr, EP_InnerON) ); + pExpr->w.iJoin = pSubst->iNewTable; } if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable @@ -137567,19 +148548,26 @@ static Expr *substExpr( #endif { Expr *pNew; - Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + int iColumn; + Expr *pCopy; Expr ifNullRow; - assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + iColumn = pExpr->iColumn; + assert( iColumn>=0 ); + assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); + pCopy = pSubst->pEList->a[iColumn].pExpr; if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ sqlite3 *db = pSubst->pParse->db; - if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){ + if( pSubst->isOuterJoin + && (pCopy->op!=TK_COLUMN || pCopy->iTable!=pSubst->iNewTable) + ){ memset(&ifNullRow, 0, sizeof(ifNullRow)); ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.iColumn = -99; ifNullRow.flags = EP_IfNullRow; pCopy = &ifNullRow; } @@ -137589,22 +148577,33 @@ static Expr *substExpr( sqlite3ExprDelete(db, pNew); return pExpr; } - if( pSubst->isLeftJoin ){ + if( pSubst->isOuterJoin ){ ExprSetProperty(pNew, EP_CanBeNull); } - if( ExprHasProperty(pExpr,EP_FromJoin) ){ - sqlite3SetJoinExpr(pNew, pExpr->iRightJoinTable); + if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ + sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, + pExpr->flags & (EP_OuterON|EP_InnerON)); } sqlite3ExprDelete(db, pExpr); pExpr = pNew; + if( pExpr->op==TK_TRUEFALSE ){ + pExpr->u.iValue = sqlite3ExprTruthValue(pExpr); + pExpr->op = TK_INTEGER; + ExprSetProperty(pExpr, EP_IntValue); + } /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ - if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ - CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, - (pColl ? pColl->zName : "BINARY") + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } } ExprClearProperty(pExpr, EP_Collate); } @@ -137757,8 +148756,8 @@ static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){ renumberCursorDoMapping(pWalker, &pExpr->iTable); } - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - renumberCursorDoMapping(pWalker, &pExpr->iRightJoinTable); + if( ExprHasProperty(pExpr, EP_OuterON) ){ + renumberCursorDoMapping(pWalker, &pExpr->w.iJoin); } return WRC_Continue; } @@ -137797,6 +148796,46 @@ static void renumberCursors( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -137841,8 +148880,10 @@ static void renumberCursors( ** (3a) the subquery may not be a join and ** (3b) the FROM clause of the subquery may not contain a virtual ** table and -** (3c) the outer query may not be an aggregate. +** (**) Was: "The outer query may not have a GROUP BY." This case +** is now managed correctly ** (3d) the outer query may not be DISTINCT. +** See also (26) for restrictions on RIGHT JOIN. ** ** (4) The subquery can not be DISTINCT. ** @@ -137863,7 +148904,7 @@ static void renumberCursors( ** (9) If the subquery uses LIMIT then the outer query may not be aggregate. ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we -** accidently carried the comment forward until 2014-09-15. Original +** accidentally carried the comment forward until 2014-09-15. Original ** constraint: "If the subquery is aggregate then the outer query ** may not use LIMIT." ** @@ -137894,6 +148935,11 @@ static void renumberCursors( ** (17d2) DISTINCT ** (17e) the subquery may not contain window functions, and ** (17f) the subquery must not be the RHS of a LEFT JOIN. +** (17g) either the subquery is the first element of the outer +** query or there are no RIGHT or FULL JOINs in any arm +** of the subquery. (This is a duplicate of condition (27b).) +** (17h) The corresponding result set expressions in all arms of the +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -137941,6 +148987,18 @@ static void renumberCursors( ** function in the select list or ORDER BY clause, flattening ** is not attempted. ** +** (26) The subquery may not be the right operand of a RIGHT JOIN. +** See also (3) for restrictions on LEFT JOIN. +** +** (27) The subquery may not contain a FULL or RIGHT JOIN unless it +** is the first element of the parent query. Two subcases: +** (27a) the subquery is not a compound query. +** (27b) the subquery is a compound query and the RIGHT JOIN occurs +** in any arm of the compound query. (See also (17g).) +** +** (28) The subquery is not a MATERIALIZED CTE. (This is handled +** in the caller before ever reaching this routine.) +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -137966,7 +149024,7 @@ static int flattenSubquery( SrcList *pSubSrc; /* The FROM clause of the subquery */ int iParent; /* VDBE cursor number of the pSub result set temp table */ int iNewParent = -1;/* Replacement table for iParent */ - int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ + int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ SrcItem *pSubitem; /* The subquery */ @@ -138032,32 +149090,26 @@ static int flattenSubquery( ** ** which is not at all the same thing. ** - ** If the subquery is the right operand of a LEFT JOIN, then the outer - ** query cannot be an aggregate. (3c) This is an artifact of the way - ** aggregates are processed - there is no mechanism to determine if - ** the LEFT JOIN table should be all-NULL. - ** ** See also tickets #306, #350, and #3300. */ - if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ - isLeftJoin = 1; - if( pSubSrc->nSrc>1 /* (3a) */ - || isAgg /* (3b) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3c) */ - || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ + if( pSubSrc->nSrc>1 /* (3a) */ + || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ return 0; } + isOuterJoin = 1; } -#ifdef SQLITE_EXTRA_IFNULLROW - else if( iFrom>0 && !isAgg ){ - /* Setting isLeftJoin to -1 causes OP_IfNullRow opcodes to be generated for - ** every reference to any result column from subquery in a join, even - ** though they are not necessary. This will stress-test the OP_IfNullRow - ** opcode. */ - isLeftJoin = -1; + + assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ + if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* Restriction (27a) */ } -#endif + + /* Condition (28) is blocked by the caller */ + assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes ); /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries @@ -138065,10 +149117,11 @@ static int flattenSubquery( ** queries. */ if( pSub->pPrior ){ + int ii; if( pSub->pOrderBy ){ return 0; /* Restriction (20) */ } - if( isAgg || (p->selFlags & SF_Distinct)!=0 || isLeftJoin>0 ){ + if( isAgg || (p->selFlags & SF_Distinct)!=0 || isOuterJoin>0 ){ return 0; /* (17d1), (17d2), or (17f) */ } for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ @@ -138086,12 +149139,17 @@ static int flattenSubquery( ){ return 0; } + if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* Without this restriction, the JT_LTORJ flag would end up being + ** omitted on left-hand tables of the right join that is being + ** flattened. */ + return 0; /* Restrictions (17g), (27b) */ + } testcase( pSub1->pSrc->nSrc>1 ); } /* Restriction (18). */ if( p->pOrderBy ){ - int ii; for(ii=0; iipOrderBy->nExpr; ii++){ if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } @@ -138100,15 +149158,19 @@ static int flattenSubquery( /* Restriction (23) */ if( (p->selFlags & SF_Recursive) ) return 0; + /* Restriction (17h) */ + if( compoundHasDifferentAffinities(pSub) ) return 0; + if( pSrc->nSrc>1 ){ if( pParse->nSelect>500 ) return 0; + if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; aCsrMap = sqlite3DbMallocZero(db, ((i64)pParse->nTab+1)*sizeof(int)); if( aCsrMap ) aCsrMap[0] = pParse->nTab; } } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", pSub->selId, pSub, iFrom)); /* Authorize the subquery */ @@ -138117,7 +149179,7 @@ static int flattenSubquery( testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; - /* Delete the transient structures associated with thesubquery */ + /* Delete the transient structures associated with the subquery */ pSub1 = pSubitem->pSelect; sqlite3DbFree(db, pSubitem->zDatabase); sqlite3DbFree(db, pSubitem->zName); @@ -138126,7 +149188,7 @@ static int flattenSubquery( pSubitem->zName = 0; pSubitem->zAlias = 0; pSubitem->pSelect = 0; - assert( pSubitem->pOn==0 ); + assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions ** 17 and 18 above) it must be a UNION ALL and the parent query must @@ -138187,7 +149249,7 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p,("compound-subquery flattener" + TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } assert( pSubitem->pSelect==0 ); @@ -138209,9 +149271,7 @@ static int flattenSubquery( Table *pTabToDel = pSubitem->pTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3ParserAddCleanup(pToplevel, - (void(*)(sqlite3*,void*))sqlite3DeleteTable, - pTabToDel); + sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); testcase( pToplevel->earlyCleanup ); }else{ pTabToDel->nTabRef--; @@ -138236,6 +149296,7 @@ static int flattenSubquery( for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; u8 jointype = 0; + u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ; assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ @@ -138270,13 +149331,16 @@ static int flattenSubquery( ** outer query. */ for(i=0; ia[i+iFrom].pUsing); - assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); - pSrc->a[i+iFrom] = pSubSrc->a[i]; + SrcItem *pItem = &pSrc->a[i+iFrom]; + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); + assert( pItem->fg.isTabFunc==0 ); + *pItem = pSubSrc->a[i]; + pItem->fg.jointype |= ltorj; iNewParent = pSubSrc->a[i].iCursor; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype = jointype; + pSrc->a[iFrom].fg.jointype &= JT_LTORJ; + pSrc->a[iFrom].fg.jointype |= jointype | ltorj; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -138295,7 +149359,7 @@ static int flattenSubquery( ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, - ** zero them before transfering the ORDER BY clause. + ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this ** function attempts to flatten a compound sub-query into pParent @@ -138311,8 +149375,8 @@ static int flattenSubquery( } pWhere = pSub->pWhere; pSub->pWhere = 0; - if( isLeftJoin>0 ){ - sqlite3SetJoinExpr(pWhere, iNewParent); + if( isOuterJoin>0 ){ + sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON); } if( pWhere ){ if( pParent->pWhere ){ @@ -138326,8 +149390,9 @@ static int flattenSubquery( x.pParse = pParse; x.iTable = iParent; x.iNewTable = iNewParent; - x.isLeftJoin = isLeftJoin; + x.isOuterJoin = isOuterJoin; x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } @@ -138347,23 +149412,22 @@ static int flattenSubquery( pSub->pLimit = 0; } - /* Recompute the SrcList_item.colUsed masks for the flattened + /* Recompute the SrcItem.colUsed masks for the flattened ** tables. */ for(i=0; ia[i+iFrom]); } } - /* Finially, delete what is left of the subquery and return - ** success. + /* Finally, delete what is left of the subquery and return success. */ sqlite3AggInfoPersistWalkerInit(&w, pParse); sqlite3WalkSelect(&w,pSub1); sqlite3SelectDelete(db, pSub1); -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -138383,12 +149447,14 @@ struct WhereConst { int nConst; /* Number for COLUMN=CONSTANT terms */ int nChng; /* Number of times a constant is propagated */ int bHasAffBlob; /* At least one column in apExpr[] as affinity BLOB */ + u32 mExcludeOn; /* Which ON expressions to exclude from considertion. + ** Either EP_OuterON or EP_InnerON|EP_OuterON */ Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ }; /* ** Add a new entry to the pConst object. Except, do not add duplicate -** pColumn entires. Also, do not add if doing so would not be appropriate. +** pColumn entries. Also, do not add if doing so would not be appropriate. ** ** The caller guarantees the pColumn is a column and pValue is a constant. ** This routine has to do some additional checks before completing the @@ -138402,7 +149468,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -138445,7 +149511,11 @@ static void constInsert( static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ Expr *pRight, *pLeft; if( NEVER(pExpr==0) ) return; - if( ExprHasProperty(pExpr, EP_FromJoin) ) return; + if( ExprHasProperty(pExpr, pConst->mExcludeOn) ){ + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + testcase( ExprHasProperty(pExpr, EP_InnerON) ); + return; + } if( pExpr->op==TK_AND ){ findConstInWhere(pConst, pExpr->pRight); findConstInWhere(pConst, pExpr->pLeft); @@ -138456,10 +149526,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -138481,9 +149551,10 @@ static int propagateConstantExprRewriteOne( int i; if( pConst->pOomFault[0] ) return WRC_Prune; if( pExpr->op!=TK_COLUMN ) return WRC_Continue; - if( ExprHasProperty(pExpr, EP_FixedCol|EP_FromJoin) ){ + if( ExprHasProperty(pExpr, EP_FixedCol|pConst->mExcludeOn) ){ testcase( ExprHasProperty(pExpr, EP_FixedCol) ); - testcase( ExprHasProperty(pExpr, EP_FromJoin) ); + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + testcase( ExprHasProperty(pExpr, EP_InnerON) ); return WRC_Continue; } for(i=0; inConst; i++){ @@ -138569,7 +149640,7 @@ static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ ** SELECT * FROM t1 WHERE a=123 AND b=123; ** ** The two SELECT statements above should return different answers. b=a -** is alway true because the comparison uses numeric affinity, but b=123 +** is always true because the comparison uses numeric affinity, but b=123 ** is false because it uses text affinity and '0123' is not the same as '123'. ** To work around this, the expression tree is not actually changed from ** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol @@ -138607,6 +149678,17 @@ static int propagateConstants( x.nChng = 0; x.apExpr = 0; x.bHasAffBlob = 0; + if( ALWAYS(p->pSrc!=0) + && p->pSrc->nSrc>0 + && (p->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 + ){ + /* Do not propagate constants on any ON clause if there is a + ** RIGHT JOIN anywhere in the query */ + x.mExcludeOn = EP_InnerON | EP_OuterON; + }else{ + /* Do not propagate constants through the ON clause of a LEFT JOIN */ + x.mExcludeOn = EP_OuterON; + } findConstInWhere(&x, p->pWhere); if( x.nConst ){ memset(&w, 0, sizeof(w)); @@ -138642,7 +149724,7 @@ static int propagateConstants( ** At the time this function is called it is guaranteed that ** ** * the sub-query uses only one distinct window frame, and -** * that the window frame has a PARTITION BY clase. +** * that the window frame has a PARTITION BY clause. */ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ assert( pSubq->pWin->pPartition ); @@ -138668,6 +149750,18 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -138719,6 +149813,33 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** be materialized. (This restriction is implemented in the calling ** routine.) ** +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. +** +** (9) All three of the following are true: +** +** (9a) The WHERE clause expression originates in the ON or USING clause +** of a join (either an INNER or an OUTER join), and +** +** (9b) The subquery is to the right of the ON/USING clause +** +** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING +** clause and the subquery. +** +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** +** (10) The inner query is not the right-hand table of a RIGHT JOIN. +** +** (11) The subquery is not a VALUES clause +** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -138726,24 +149847,56 @@ static int pushDownWhereTerms( Parse *pParse, /* Parse context (for malloc() and error reporting) */ Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ Expr *pWhere, /* The WHERE clause of the outer query */ - int iCursor, /* Cursor number of the subquery */ - int isLeftJoin /* True if pSubq is the right term of a LEFT JOIN */ + SrcList *pSrcList, /* The complete from clause of the outer query */ + int iSrc /* Which FROM clause term to try to push into */ ){ Expr *pNew; + SrcItem *pSrc; /* The subquery FROM term into which WHERE is pushed */ int nChng = 0; + pSrc = &pSrcList->a[iSrc]; if( pWhere==0 ) return 0; - if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ){ + return 0; /* restrictions (2) and (11) */ + } + if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ){ + return 0; /* restrictions (10) */ + } -#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pPrior ){ Select *pSel; + int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + u8 op = pSel->op; + assert( op==TK_ALL || op==TK_SELECT + || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif + } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; - } #endif + } #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make @@ -138762,31 +149915,66 @@ static int pushDownWhereTerms( return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, - iCursor, isLeftJoin); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrcList, iSrc); pWhere = pWhere->pLeft; } + +#if 0 /* These checks now done by sqlite3ExprIsSingleTableConstraint() */ + if( ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) /* (9a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (9c) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + /* If we reach this point, both (9a) and (9b) are satisfied. + ** The following loop checks (9c): + */ + for(jj++; jja[jj].fg.jointype & JT_RIGHT)!=0 ){ + return 0; /* restriction (9) */ + } + } + } + } + } if( isLeftJoin - && (ExprHasProperty(pWhere,EP_FromJoin)==0 - || pWhere->iRightJoinTable!=iCursor) + && (ExprHasProperty(pWhere,EP_OuterON)==0 + || pWhere->w.iJoin!=iCursor) ){ return 0; /* restriction (4) */ } - if( ExprHasProperty(pWhere,EP_FromJoin) && pWhere->iRightJoinTable!=iCursor ){ + if( ExprHasProperty(pWhere,EP_OuterON) + && pWhere->w.iJoin!=iCursor + ){ return 0; /* restriction (5) */ } - if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ +#endif + +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ SubstContext x; pNew = sqlite3ExprDup(pParse->db, pWhere, 0); - unsetJoinExpr(pNew, -1); + unsetJoinExpr(pNew, -1, 1); x.pParse = pParse; - x.iTable = iCursor; - x.iNewTable = iCursor; - x.isLeftJoin = 0; + x.iTable = pSrc->iCursor; + x.iNewTable = pSrc->iCursor; + x.isOuterJoin = 0; x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ @@ -138808,6 +149996,78 @@ static int pushDownWhereTerms( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ + + assert( pItem!=0 ); + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ + return 0; + } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + for(pX=pSub; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pX->selFlags & SF_Distinct ); + testcase( pX->selFlags & SF_Aggregate ); + return 0; + } + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } +#endif + } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + nCol = pTab->nCol; + for(j=0; jpPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + ExprClearProperty(pY, EP_Skip|EP_Unlikely); + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + /* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. @@ -138858,7 +150118,7 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ } *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); assert( pOrderBy!=0 || db->mallocFailed ); - if( pOrderBy ) pOrderBy->a[0].sortFlags = sortFlags; + if( pOrderBy ) pOrderBy->a[0].fg.sortFlags = sortFlags; return eRet; } @@ -138890,6 +150150,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } @@ -138994,7 +150255,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); - pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); if( pNewSrc==0 ) return WRC_Abort; *pNew = *p; p->pSrc = pNewSrc; @@ -139085,8 +150346,7 @@ static struct Cte *searchWith( SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ if( pWith ){ if( bFree ){ - pWith = (With*)sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3WithDelete, + pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric, pWith); if( pWith==0 ) return 0; } @@ -139198,9 +150458,6 @@ static int resolveFromTermToCte( pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; - if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){ - pCteUse->eM10d = M10d_Yes; - } /* Check if this is a recursive CTE. */ pRecTerm = pSel = pFrom->pSelect; @@ -139310,9 +150567,9 @@ SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ #endif /* -** The SrcList_item structure passed as the second argument represents a +** The SrcItem structure passed as the second argument represents a ** sub-query in the FROM clause of a SELECT statement. This function -** allocates and populates the SrcList_item.pTab object. If successful, +** allocates and populates the SrcItem.pTab object. If successful, ** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, ** SQLITE_NOMEM. */ @@ -139327,23 +150584,49 @@ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ if( pFrom->zAlias ){ pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); }else{ - pTab->zName = sqlite3MPrintf(pParse->db, "subquery_%u", pSel->selId); + pTab->zName = sqlite3MPrintf(pParse->db, "%!S", pFrom); } while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +} - return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +/* +** Check the N SrcItem objects to the right of pBase. (N might be zero!) +** If any of those SrcItem objects have a USING clause containing zName +** then return true. +** +** If N is zero, or none of the N SrcItem objects to the right of pBase +** contains a USING clause, or if none of the USING clauses contain zName, +** then return false. +*/ +static int inAnyUsingClause( + const char *zName, /* Name we are looking for */ + SrcItem *pBase, /* The base SrcItem. Looking at pBase[1] and following */ + int N /* How many SrcItems to check */ +){ + while( N>0 ){ + N--; + pBase++; + if( pBase->fg.isUsing==0 ) continue; + if( NEVER(pBase->u3.pUsing==0) ) continue; + if( sqlite3IdListIndex(pBase->u3.pUsing, zName)>=0 ) return 1; + } + return 0; } + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -139492,7 +150775,8 @@ static int selectExpander(Walker *pWalker, Select *p){ /* Process NATURAL keywords, and ON and USING clauses of joins. */ - if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + assert( db->mallocFailed==0 || pParse->nErr!=0 ); + if( pParse->nErr || sqlite3ProcessJoin(pParse, p) ){ return WRC_Abort; } @@ -139540,7 +150824,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); if( pNew ){ pNew->a[pNew->nExpr-1].zEName = a[k].zEName; - pNew->a[pNew->nExpr-1].eEName = a[k].eEName; + pNew->a[pNew->nExpr-1].fg.eEName = a[k].fg.eEName; a[k].zEName = 0; } a[k].pExpr = 0; @@ -139549,102 +150833,176 @@ static int selectExpander(Walker *pWalker, Select *p){ ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ char *zTName = 0; /* text of name of TABLE */ + int iErrOfst; if( pE->op==TK_DOT ){ + assert( (selFlags & SF_NestedFrom)==0 ); assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; + assert( ExprUseWOfst(pE->pLeft) ); + iErrOfst = pE->pRight->w.iOfst; + }else{ + assert( ExprUseWOfst(pE) ); + iErrOfst = pE->w.iOfst; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - Select *pSub = pFrom->pSelect; - char *zTabName = pFrom->zAlias; - const char *zSchemaName = 0; - int iDb; - if( zTabName==0 ){ + int nAdd; /* Number of cols including rowid */ + Table *pTab = pFrom->pTab; /* Table for this data source */ + ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ + char *zTabName; /* AS name for this data source */ + const char *zSchemaName = 0; /* Schema name for this data source */ + int iDb; /* Schema index for this data src */ + IdList *pUsing; /* USING clause for pFrom[1] */ + + if( (zTabName = pFrom->zAlias)==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ - pSub = 0; + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + if( pFrom->fg.isNestedFrom ){ + assert( pFrom->pSelect!=0 ); + pNestedFrom = pFrom->pSelect->pEList; + assert( pNestedFrom!=0 ); + assert( pNestedFrom->nExpr==pTab->nCol ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); + }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; } + pNestedFrom = 0; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; } - for(j=0; jnCol; j++){ - char *zName = pTab->aCol[j].zCnName; - char *zColname; /* The computed column name */ - char *zToFree; /* Malloced string that needs to be freed */ - Token sColname; /* Computed column name as a token */ - - assert( zName ); - if( zTName && pSub - && sqlite3MatchEName(&pSub->pEList->a[j], 0, zTName, 0)==0 - ){ - continue; + if( i+1nSrc + && pFrom[1].fg.isUsing + && (selFlags & SF_NestedFrom)!=0 + ){ + int ii; + pUsing = pFrom[1].u3.pUsing; + for(ii=0; iinId; ii++){ + const char *zUName = pUsing->a[ii].zName; + pRight = sqlite3Expr(db, TK_ID, zUName); + sqlite3ExprSetErrorOffset(pRight, iErrOfst); + pNew = sqlite3ExprListAppend(pParse, pNew, pRight); + if( pNew ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + assert( pX->zEName==0 ); + pX->zEName = sqlite3MPrintf(db,"..%s", zUName); + pX->fg.eEName = ENAME_TAB; + pX->fg.bUsingTerm = 1; + } } + }else{ + pUsing = 0; + } - /* If a column is marked as 'hidden', omit it from the expanded - ** result-set list unless the SELECT has the SF_IncludeHidden - ** bit set. - */ - if( (p->selFlags & SF_IncludeHidden)==0 - && IsHiddenColumn(&pTab->aCol[j]) - ){ - continue; - } - tableSeen = 1; + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; + for(j=0; jnCol ){ + zName = sqlite3RowidAlias(pTab); + if( zName==0 ) continue; + }else{ + zName = pTab->aCol[j].zCnName; - if( i>0 && zTName==0 ){ - if( (pFrom->fg.jointype & JT_NATURAL)!=0 - && tableAndColumnIndex(pTabList, i, zName, 0, 0, 1) + /* If pTab is actually an SF_NestedFrom sub-select, do not + ** expand any ENAME_ROWID columns. */ + if( pNestedFrom && pNestedFrom->a[j].fg.eEName==ENAME_ROWID ){ + continue; + } + + if( zTName + && pNestedFrom + && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + && zTName==0 + && (selFlags & (SF_NestedFrom))==0 ){ - /* In a NATURAL join, omit the join columns from the - ** table to the right of the join */ continue; } - if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + } + assert( zName ); + tableSeen = 1; + + if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){ + if( pFrom->fg.isUsing + && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0 + ){ /* In a join with a USING clause, omit columns in the ** using clause from the table on the right. */ continue; } } pRight = sqlite3Expr(db, TK_ID, zName); - zColname = zName; - zToFree = 0; - if( longNames || pTabList->nSrc>1 ){ + if( (pTabList->nSrc>1 + && ( (pFrom->fg.jointype & JT_LTORJ)==0 + || (selFlags & SF_NestedFrom)!=0 + || !inAnyUsingClause(zName,pFrom,pTabList->nSrc-i-1) + ) + ) + || IN_RENAME_OBJECT + ){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + if( IN_RENAME_OBJECT && pE->pLeft ){ + sqlite3RenameTokenRemap(pParse, pLeft, pE->pLeft); + } if( zSchemaName ){ pLeft = sqlite3Expr(db, TK_ID, zSchemaName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr); } - if( longNames ){ - zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); - zToFree = zColname; - } }else{ pExpr = pRight; } + sqlite3ExprSetErrorOffset(pExpr, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sqlite3TokenInit(&sColname, zColname); - sqlite3ExprListSetName(pParse, pNew, &sColname, 0); - if( pNew && (p->selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; - sqlite3DbFree(db, pX->zEName); - if( pSub ){ - pX->zEName = sqlite3DbStrDup(db, pSub->pEList->a[j].zEName); + if( pNew==0 ){ + break; /* OOM */ + } + pX = &pNew->a[pNew->nExpr-1]; + assert( pX->zEName==0 ); + if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); + pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", - zSchemaName, zTabName, zColname); + zSchemaName, zTabName, zName); testcase( pX->zEName==0 ); } - pX->eEName = ENAME_TAB; + pX->fg.eEName = (j==pTab->nCol ? ENAME_ROWID : ENAME_TAB); + if( (pFrom->fg.isUsing + && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0) + || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0) + || (jnCol && (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)) + ){ + pX->fg.bNoExpand = 1; + } + }else if( longNames ){ + pX->zEName = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + pX->fg.eEName = ENAME_NAME; + }else{ + pX->zEName = sqlite3DbStrDup(db, zName); + pX->fg.eEName = ENAME_NAME; } - sqlite3DbFree(db, zToFree); } } if( !tableSeen ){ @@ -139668,6 +151026,12 @@ static int selectExpander(Walker *pWalker, Select *p){ p->selFlags |= SF_ComplexResult; } } +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif return WRC_Continue; } @@ -139715,14 +151079,14 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ ** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() ** interface. ** -** For each FROM-clause subquery, add Column.zType and Column.zColl -** information to the Table structure that represents the result set -** of that subquery. +** For each FROM-clause subquery, add Column.zType, Column.zColl, and +** Column.affinity information to the Table structure that represents +** the result set of that subquery. ** ** The Table structure that represents the result set was constructed -** by selectExpander() but the type and collation information was omitted -** at that point because identifiers had not yet been resolved. This -** routine is called after identifier resolution. +** by selectExpander() but the type and collation and affinity information +** was omitted at that point because identifiers had not yet been resolved. +** This routine is called after identifier resolution. */ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; @@ -139730,10 +151094,10 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ SrcList *pTabList; SrcItem *pFrom; - assert( p->selFlags & SF_Resolved ); if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; @@ -139742,9 +151106,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, - SQLITE_AFF_NONE); + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -139789,15 +151151,196 @@ SQLITE_PRIVATE void sqlite3SelectPrep( NameContext *pOuterNC /* Name context for container */ ){ assert( p!=0 || pParse->db->mallocFailed ); + assert( pParse->db->pParse==pParse ); if( pParse->db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); - if( pParse->nErr || pParse->db->mallocFailed ) return; + if( pParse->nErr ) return; sqlite3ResolveSelectNames(pParse, p, pOuterNC); - if( pParse->nErr || pParse->db->mallocFailed ) return; + if( pParse->nErr ) return; sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->iFirstReg+pAggInfo->nColumn+ii); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr->op==TK_FUNCTION || pExpr->op==TK_AGG_FUNCTION ); + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->pLeft->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ + assert( pAggInfo->iFirstReg==0 ); + assert( pSelect!=0 ); + assert( pSelect->pGroupBy!=0 ); + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ + int mx = pSelect->pGroupBy->nExpr - 1; + int j, k; + for(j=0; jnColumn; j++){ + k = pAggInfo->aCol[j].iSorterColumn; + if( k>mx ) mx = k; + } + pAggInfo->nSortingColumn = mx+1; + } + analyzeAggFuncArgs(pAggInfo, pNC); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + IndexedExpr *pIEpr; + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); +#endif +} + +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue; + assert( pExpr->iAgg>=0 ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + ExprClearProperty(pExpr, EP_Skip|EP_Collate|EP_Unlikely); + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After calling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; +} + /* ** Reset the aggregate accumulator. ** @@ -139811,22 +151354,13 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); + assert( pParse->db->pParse==pParse ); + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; - if( pParse->nErr || pParse->db->mallocFailed ) return; -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + if( pParse->nErr ) return; + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pFExpr; @@ -139843,6 +151377,36 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ pFunc->pFunc->zName)); } } + if( pFunc->iOBTab>=0 ){ + ExprList *pOBList; + KeyInfo *pKeyInfo; + int nExtra = 0; + assert( pFunc->pFExpr->pLeft!=0 ); + assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + assert( pFunc->pFunc!=0 ); + pOBList = pFunc->pFExpr->pLeft->x.pList; + if( !pFunc->bOBUnique ){ + nExtra++; /* One extra column for the OP_Sequence */ + } + if( pFunc->bOBPayload ){ + /* extra columns for the function arguments */ + assert( ExprUseXList(pFunc->pFExpr) ); + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + if( pFunc->bUseSubtype ){ + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); + if( !pFunc->bOBUnique && pParse->nErr==0 ){ + pKeyInfo->nKeyField++; + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iOBTab, pOBList->nExpr+nExtra, 0, + (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(ORDER BY)", + pFunc->pFunc->zName)); + } } } @@ -139858,20 +151422,71 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; - sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + if( pF->iOBTab>=0 ){ + /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs + ** were stored in emphermal table pF->iOBTab. Here, we extract those + ** inputs (in ORDER BY order) and make all calls to OP_AggStep + ** before doing the OP_AggFinal call. */ + int iTop; /* Start of loop for extracting columns */ + int nArg; /* Number of columns to extract */ + int nKey; /* Key columns to be skipped */ + int regAgg; /* Extract into this array */ + int j; /* Loop counter */ + + assert( pF->pFunc!=0 ); + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + + if( pF->bOBPayload==0 ){ + nKey = 0; + }else{ + assert( pF->pFExpr->pLeft!=0 ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + assert( pF->pFExpr->pLeft->x.pList!=0 ); + nKey = pF->pFExpr->pLeft->x.pList->nExpr; + if( ALWAYS(!pF->bOBUnique) ) nKey++; + } + iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); + } + if( pF->bUseSubtype ){ + int regSubtype = sqlite3GetTempReg(pParse); + int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); + sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); + } + sqlite3ReleaseTempReg(pParse, regSubtype); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, iTop); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + } + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } - /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. ** ** If regAcc is non-zero and there are no min() or max() aggregates ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator ** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. +** +** For an ORDER BY aggregate, the actual accumulator memory cell update +** is deferred until after all input rows have been received, so that they +** can be run in the requested order. In that case, instead of invoking +** OP_AggStep to update the accumulator, just add the arguments that would +** have been passed into OP_AggStep into the sorting ephemeral table +** (along with the appropriate sort key). */ static void updateAccumulator( Parse *pParse, @@ -139886,14 +151501,19 @@ static void updateAccumulator( struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; int addrNext = 0; int regAgg; + int regAggSz = 0; + int regDistinct = 0; ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); + assert( pF->pFunc!=0 ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; @@ -139917,9 +151537,55 @@ static void updateAccumulator( addrNext = sqlite3VdbeMakeLabel(pParse); sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); } - if( pList ){ + if( pF->iOBTab>=0 ){ + /* Instead of invoking AggStep, we must push the arguments that would + ** have been passed to AggStep onto the sorting table. */ + int jj; /* Registered used so far in building the record */ + ExprList *pOBList; /* The ORDER BY clause */ + assert( pList!=0 ); + nArg = pList->nExpr; + assert( nArg>0 ); + assert( pF->pFExpr->pLeft!=0 ); + assert( pF->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + pOBList = pF->pFExpr->pLeft->x.pList; + assert( pOBList!=0 ); + assert( pOBList->nExpr>0 ); + regAggSz = pOBList->nExpr; + if( !pF->bOBUnique ){ + regAggSz++; /* One register for OP_Sequence */ + } + if( pF->bOBPayload ){ + regAggSz += nArg; + } + if( pF->bUseSubtype ){ + regAggSz += nArg; + } + regAggSz++; /* One extra register to hold result of MakeRecord */ + regAgg = sqlite3GetTempRange(pParse, regAggSz); + regDistinct = regAgg; + sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); + jj = pOBList->nExpr; + if( !pF->bOBUnique ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); + jj++; + } + if( pF->bOBPayload ){ + regDistinct = regAgg+jj; + sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + jj += nArg; + } + if( pF->bUseSubtype ){ + int kk; + int regBase = pF->bOBPayload ? regDistinct : regAgg; + for(kk=0; kknExpr; regAgg = sqlite3GetTempRange(pParse, nArg); + regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; @@ -139930,26 +151596,37 @@ static void updateAccumulator( addrNext = sqlite3VdbeMakeLabel(pParse); } pF->iDistinct = codeDistinct(pParse, eDistinctType, - pF->iDistinct, addrNext, pList, regAgg); - } - if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - CollSeq *pColl = 0; - struct ExprList_item *pItem; - int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ - for(j=0, pItem=pList->a; !pColl && jpExpr); - } - if( !pColl ){ - pColl = pParse->db->pDfltColl; + pF->iDistinct, addrNext, pList, regDistinct); + } + if( pF->iOBTab>=0 ){ + /* Insert a new record into the ORDER BY table */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regAgg, regAggSz-1, + regAgg+regAggSz-1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pF->iOBTab, regAgg+regAggSz-1, + regAgg, regAggSz-1); + sqlite3ReleaseTempRange(pParse, regAgg, regAggSz); + }else{ + /* Invoke the AggStep function */ + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, + (char *)pColl, P4_COLLSEQ); } - if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); - sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } @@ -139961,7 +151638,7 @@ static void updateAccumulator( addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); } pAggInfo->directMode = 0; @@ -140055,28 +151732,33 @@ static void havingToWhere(Parse *pParse, Select *p){ sWalker.xExprCallback = havingToWhereExprCb; sWalker.u.pSelect = p; sqlite3WalkExpr(&sWalker, p->pHaving); -#if SELECTTRACE_ENABLED - if( sWalker.eCode && (sqlite3SelectTrace & 0x100)!=0 ){ - SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); +#if TREETRACE_ENABLED + if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif } /* -** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, -** then return 0. +** Check to see if the pThis entry of pTabList is a self-join of another view. +** Search FROM-clause entries in the range of iFirst..iEnd, including iFirst +** but stopping before iEnd. +** +** If pThis is a self-join, then return the SrcItem for the first other +** instance of that view found. If pThis is not a self-join then return 0. */ static SrcItem *isSelfJoinView( SrcList *pTabList, /* Search for self-joins in this FROM clause */ - SrcItem *pThis /* Search for prior reference to this subquery */ + SrcItem *pThis, /* Search for prior reference to this subquery */ + int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; assert( pThis->pSelect!=0 ); if( pThis->pSelect->selFlags & SF_PushDown ) return 0; - for(pItem = pTabList->a; pItema[iFirst++]; if( pItem->pSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; @@ -140103,13 +151785,13 @@ static SrcItem *isSelfJoinView( /* ** Deallocate a single AggInfo object */ -static void agginfoFree(sqlite3 *db, AggInfo *p){ +static void agginfoFree(sqlite3 *db, void *pArg){ + AggInfo *p = (AggInfo*)pArg; sqlite3DbFree(db, p->aCol); sqlite3DbFree(db, p->aFunc); sqlite3DbFreeNN(db, p); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION /* ** Attempt to transform a query of the form ** @@ -140137,7 +151819,9 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; + if( p->pHaving ) return 0; if( p->pGroupBy ) return 0; + if( p->pOrderBy ) return 0; pExpr = p->pEList->a[0].pExpr; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ assert( ExprUseUToken(pExpr) ); @@ -140145,15 +151829,18 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ assert( ExprUseXList(pExpr) ); if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ + if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ pSub = p->pSrc->a[0].pSelect; if( pSub==0 ) return 0; /* The FROM is a subquery */ - if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */ + if( pSub->pPrior==0 ) return 0; /* Must be a compound */ + if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ - pSub = pSub->pPrior; /* Repeat over compound */ + assert( pSub->pHaving==0 ); /* Due to the previous */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -140173,7 +151860,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub->selFlags |= SF_Aggregate; pSub->selFlags &= ~SF_Compound; pSub->nSelectRow = 0; - sqlite3ExprListDelete(db, pSub->pEList); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); @@ -140188,15 +151875,99 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ p->pEList->a[0].pExpr = pExpr; p->selFlags &= ~SF_Aggregate; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif return 1; } -#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ + +/* +** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same +** as pSrcItem but has the same alias as p0, then return true. +** Otherwise return false. +*/ +static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ + int i; + for(i=0; inSrc; i++){ + SrcItem *p1 = &pSrc->a[i]; + if( p1==p0 ) continue; + if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + return 1; + } + if( p1->pSelect + && (p1->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->pSelect->pSrc) + ){ + return 1; + } + } + return 0; +} + +/* +** Return TRUE (non-zero) if the i-th entry in the pTabList SrcList can +** be implemented as a co-routine. The i-th entry is guaranteed to be +** a subquery. +** +** The subquery is implemented as a co-routine if all of the following are +** true: +** +** (1) The subquery will likely be implemented in the outer loop of +** the query. This will be the case if any one of the following +** conditions hold: +** (a) The subquery is the only term in the FROM clause +** (b) The subquery is the left-most term and a CROSS JOIN or similar +** requires it to be the outer loop +** (c) All of the following are true: +** (i) The subquery is the left-most subquery in the FROM clause +** (ii) There is nothing that would prevent the subquery from +** being used as the outer loop if the sqlite3WhereBegin() +** routine nominates it to that position. +** (iii) The query is not a UPDATE ... FROM +** (2) The subquery is not a CTE that should be materialized because +** (a) the AS MATERIALIZED keyword is used, or +** (b) the CTE is used multiple times and does not have the +** NOT MATERIALIZED keyword +** (3) The subquery is not part of a left operand for a RIGHT JOIN +** (4) The SQLITE_Coroutine optimization disable flag is not set +** (5) The subquery is not self-joined +*/ +static int fromClauseTermCanBeCoroutine( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* FROM clause */ + int i, /* Which term of the FROM clause holds the subquery */ + int selFlags /* Flags on the SELECT statement */ +){ + SrcItem *pItem = &pTabList->a[i]; + if( pItem->fg.isCte ){ + const CteUse *pCteUse = pItem->u2.pCteUse; + if( pCteUse->eM10d==M10d_Yes ) return 0; /* (2a) */ + if( pCteUse->nUse>=2 && pCteUse->eM10d!=M10d_No ) return 0; /* (2b) */ + } + if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0; /* (3) */ + if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4) */ + if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){ + return 0; /* (5) */ + } + if( i==0 ){ + if( pTabList->nSrc==1 ) return 1; /* (1a) */ + if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1; /* (1b) */ + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + return 1; + } + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + while( 1 /*exit-by-break*/ ){ + if( pItem->fg.jointype & (JT_OUTER|JT_CROSS) ) return 0; /* (1c-ii) */ + if( i==0 ) break; + i--; + pItem--; + if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + } + return 1; +} /* ** Generate code for the SELECT statement given in the p argument. @@ -140235,15 +152006,21 @@ SQLITE_PRIVATE int sqlite3Select( u8 minMaxFlag; /* Flag for min/max queries */ db = pParse->db; + assert( pParse==db->pParse ); v = sqlite3GetVdbe(pParse); - if( p==0 || db->mallocFailed || pParse->nErr ){ + if( p==0 || pParse->nErr ){ return 1; } + assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; -#if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); - if( sqlite3SelectTrace & 0x100 ){ - sqlite3TreeViewSelect(0, p, 0); +#if TREETRACE_ENABLED + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ + if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", + __FILE__, __LINE__); + } + sqlite3ShowSelect(p); } #endif @@ -140257,15 +152034,14 @@ SQLITE_PRIVATE int sqlite3Select( pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ -#if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); - if( sqlite3SelectTrace & 0x100 ){ +#if TREETRACE_ENABLED + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - p->pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + p->pOrderBy); testcase( pParse->earlyCleanup ); p->pOrderBy = 0; } @@ -140273,13 +152049,14 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); - if( pParse->nErr || db->mallocFailed ){ + if( pParse->nErr ){ goto select_end; } + assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x104 ){ - SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140295,15 +152072,12 @@ SQLITE_PRIVATE int sqlite3Select( ** disallow it altogether. */ if( p->selFlags & SF_UFSrcCheck ){ SrcItem *p0 = &p->pSrc->a[0]; - for(i=1; ipSrc->nSrc; i++){ - SrcItem *p1 = &p->pSrc->a[i]; - if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ - sqlite3ErrorMsg(pParse, - "target object/alias may not appear in FROM clause: %s", - p0->zAlias ? p0->zAlias : p0->pTab->zName - ); - goto select_end; - } + if( sameSrcAlias(p0, p->pSrc) ){ + sqlite3ErrorMsg(pParse, + "target object/alias may not appear in FROM clause: %s", + p0->zAlias ? p0->zAlias : p0->pTab->zName + ); + goto select_end; } /* Clear the SF_UFSrcCheck flag. The check has already been performed, @@ -140319,12 +152093,12 @@ SQLITE_PRIVATE int sqlite3Select( #ifndef SQLITE_OMIT_WINDOWFUNC if( sqlite3WindowRewrite(pParse, p) ){ - assert( db->mallocFailed || pParse->nErr>0 ); + assert( pParse->nErr ); goto select_end; } -#if SELECTTRACE_ENABLED - if( p->pWin && (sqlite3SelectTrace & 0x108)!=0 ){ - SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); +#if TREETRACE_ENABLED + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140348,20 +152122,58 @@ SQLITE_PRIVATE int sqlite3Select( ** to a real table */ assert( pTab!=0 ); - /* Convert LEFT JOIN into JOIN if there are terms of the right table - ** of the LEFT JOIN used in the WHERE clause. + /* Try to simplify joins: + ** + ** LEFT JOIN -> JOIN + ** RIGHT JOIN -> JOIN + ** FULL JOIN -> RIGHT JOIN + ** + ** If terms of the i-th table are used in the WHERE clause in such a + ** way that the i-th table cannot be the NULL row of a join, then + ** perform the appropriate simplification. This is called + ** "OUTER JOIN strength reduction" in the SQLite documentation. */ - if( (pItem->fg.jointype & JT_LEFT)!=0 - && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, + pItem->fg.jointype & JT_LTORJ) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - SELECTTRACE(0x100,pParse,p, - ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); - pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); - unsetJoinExpr(p->pWhere, pItem->iCursor); + if( pItem->fg.jointype & JT_LEFT ){ + if( pItem->fg.jointype & JT_RIGHT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to RIGHT-JOIN on term %d\n",i)); + pItem->fg.jointype &= ~JT_LEFT; + }else{ + TREETRACE(0x1000,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor, 0); + } + } + if( pItem->fg.jointype & JT_LTORJ ){ + for(j=i+1; jnSrc; j++){ + SrcItem *pI2 = &pTabList->a[j]; + if( pI2->fg.jointype & JT_RIGHT ){ + if( pI2->fg.jointype & JT_LEFT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to LEFT-JOIN on term %d\n",j)); + pI2->fg.jointype &= ~JT_RIGHT; + }else{ + TREETRACE(0x1000,pParse,p, + ("RIGHT-JOIN simplifies to JOIN on term %d\n",j)); + pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER); + unsetJoinExpr(p->pWhere, pI2->iCursor, 1); + } + } + } + for(j=pTabList->nSrc-1; j>=0; j--){ + pTabList->a[j].fg.jointype &= ~JT_LTORJ; + if( pTabList->a[j].fg.jointype & JT_RIGHT ) break; + } + } } - /* No futher action if this term of the FROM clause is no a subquery */ + /* No further action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -140372,6 +152184,14 @@ SQLITE_PRIVATE int sqlite3Select( goto select_end; } + /* Do not attempt the usual optimizations (flattening and ORDER BY + ** elimination) on a MATERIALIZED common table expression because + ** a MATERIALIZED common table expression is an optimization fence. + */ + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){ + continue; + } + /* Do not try to flatten an aggregate subquery. ** ** Flattening an aggregate subquery is only possible if the outer query @@ -140401,6 +152221,8 @@ SQLITE_PRIVATE int sqlite3Select( ** (a) The outer query has a different ORDER BY clause ** (b) The subquery is part of a join ** See forum post 062d576715d277c8 + ** + ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. */ if( pSub->pOrderBy!=0 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ @@ -140409,9 +152231,10 @@ SQLITE_PRIVATE int sqlite3Select( && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ - SELECTTRACE(0x100,pParse,p, + TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); - sqlite3ExprListDelete(db, pSub->pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + pSub->pOrderBy); pSub->pOrderBy = 0; } @@ -140437,7 +152260,7 @@ SQLITE_PRIVATE int sqlite3Select( && i==0 && (p->selFlags & SF_ComplexResult)!=0 && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) + || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) ){ continue; } @@ -140461,9 +152284,9 @@ SQLITE_PRIVATE int sqlite3Select( */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); -#if SELECTTRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); - if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ +#if TREETRACE_ENABLED + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140482,25 +152305,22 @@ SQLITE_PRIVATE int sqlite3Select( && OptimizationEnabled(db, SQLITE_PropagateConst) && propagateConstants(pParse, p) ){ -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif }else{ - SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ if( db->mallocFailed ) goto select_end; - pEList = p->pEList; pTabList = p->pSrc; } -#endif /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables @@ -140539,7 +152359,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Generate code for all sub-queries in the FROM clause */ pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pSub==0 || pItem->addrFillSub!=0 ) continue; /* The code for a subquery should only be generated once. */ assert( pItem->addrFillSub==0 ); @@ -140559,39 +152379,42 @@ SQLITE_PRIVATE int sqlite3Select( if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) - && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor, - (pItem->fg.jointype & JT_OUTER)!=0) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i) ){ -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); }else{ - SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); + } + + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif } zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if: - ** (1) the subquery is guaranteed to be the outer loop (so that - ** it does not need to be computed more than once), and - ** (2) the subquery is not a CTE that should be materialized - ** - ** TODO: Are there other reasons beside (1) and (2) to use a co-routine - ** implementation? */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ - ){ + if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ @@ -140613,7 +152436,7 @@ SQLITE_PRIVATE int sqlite3Select( }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemerial table that + ** the make the pItem->iCursor be a copy of the ephemeral table that ** holds the result of the materialization. */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); @@ -140622,7 +152445,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "%!S", pItem)); } pSub->nSelectRow = pCteUse->nRowEst; - }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){ + }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in ** this same FROM clause. Reuse it. */ if( pPrior->addrFillSub ){ @@ -140636,11 +152459,14 @@ SQLITE_PRIVATE int sqlite3Select( ** the same view can reuse the materialization. */ int topAddr; int onceAddr = 0; - int retAddr; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif pItem->regReturn = ++pParse->nMem; - topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + topAddr = sqlite3VdbeAddOp0(v, OP_Goto); pItem->addrFillSub = topAddr+1; + pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery @@ -140651,13 +152477,15 @@ SQLITE_PRIVATE int sqlite3Select( VdbeNoopComment((v, "materialize %!S", pItem)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + + ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); - sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); + sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; @@ -140681,9 +152509,9 @@ SQLITE_PRIVATE int sqlite3Select( pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140716,10 +152544,11 @@ SQLITE_PRIVATE int sqlite3Select( ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + sDistinct.isTnct = 2; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -140751,6 +152580,18 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); + if( p->selFlags & SF_NestedFrom ){ + /* Delete or NULL-out result columns that will never be used */ + int ii; + for(ii=pEList->nExpr-1; ii>0 && pEList->a[ii].fg.bUsed==0; ii--){ + sqlite3ExprDelete(db, pEList->a[ii].pExpr); + sqlite3DbFree(db, pEList->a[ii].zEName); + pEList->nExpr--; + } + for(ii=0; iinExpr; ii++){ + if( pEList->a[ii].fg.bUsed==0 ) pEList->a[ii].pExpr->op = TK_NULL; + } + } } /* Set the limiter. @@ -140759,7 +152600,7 @@ SQLITE_PRIVATE int sqlite3Select( if( (p->selFlags & SF_FixedLimit)==0 ){ p->nSelectRow = 320; /* 4 billion rows */ } - computeLimitRegisters(pParse, p, iEnd); + if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; @@ -140793,9 +152634,9 @@ SQLITE_PRIVATE int sqlite3Select( /* Begin the database scan. */ - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, p->nSelectRow); + p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -140810,7 +152651,7 @@ SQLITE_PRIVATE int sqlite3Select( sSort.pOrderBy = 0; } } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -140849,7 +152690,7 @@ SQLITE_PRIVATE int sqlite3Select( /* End the database scan loop. */ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ @@ -140900,8 +152741,9 @@ SQLITE_PRIVATE int sqlite3Select( ** ORDER BY to maximize the chances of rows being delivered in an ** order that makes the ORDER BY redundant. */ for(ii=0; iinExpr; ii++){ - u8 sortFlags = sSort.pOrderBy->a[ii].sortFlags & KEYINFO_ORDER_DESC; - pGroupBy->a[ii].sortFlags = sortFlags; + u8 sortFlags; + sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].fg.sortFlags = sortFlags; } if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ orderByGrp = 1; @@ -140921,20 +152763,21 @@ SQLITE_PRIVATE int sqlite3Select( */ pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); if( pAggInfo ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo); testcase( pParse->earlyCleanup ); } if( db->mallocFailed ){ goto select_end; } pAggInfo->selId = p->selId; +#ifdef SQLITE_DEBUG + pAggInfo->pSelect = p; +#endif memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - pAggInfo->mnReg = pParse->nMem+1; pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); @@ -140955,40 +152798,17 @@ SQLITE_PRIVATE int sqlite3Select( }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ - Expr *pExpr = pAggInfo->aFunc[i].pFExpr; - assert( ExprUseXList(pExpr) ); - sNC.ncFlags |= NC_InAggFunc; - sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); -#ifndef SQLITE_OMIT_WINDOWFUNC - assert( !IsWindowFunc(pExpr) ); - if( ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); - } -#endif - sNC.ncFlags &= ~NC_InAggFunc; - } - pAggInfo->mxReg = pParse->nMem; + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - int ii; - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); if( minMaxFlag ){ sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); } - for(ii=0; iinColumn; ii++){ - sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", - ii, pAggInfo->aCol[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); - } - for(ii=0; iinFunc; ii++){ - sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, pAggInfo->aFunc[ii].iMem); - sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); - } + printAggInfo(pAggInfo); } #endif @@ -140998,7 +152818,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int addr1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparison jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -141057,16 +152877,21 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDistinct); goto select_end; } + if( pParse->pIdxEpr ){ + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); + } + assignAggregateRegisters(pParse, pAggInfo); eDist = sqlite3WhereIsDistinct(pWInfo); - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be @@ -141084,9 +152909,13 @@ SQLITE_PRIVATE int sqlite3Select( int nCol; int nGroupBy; - explainTempTable(pParse, +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp; /* Address of OP_Explain instruction */ +#endif + ExplainQueryPlan2(addrExp, (pParse, 0, "USE TEMP B-TREE FOR %s", (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? - "DISTINCT" : "GROUP BY"); + "DISTINCT" : "GROUP BY" + )); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -141101,28 +152930,50 @@ SQLITE_PRIVATE int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; + pAggInfo->directMode = 1; for(i=0; inColumn; i++){ struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ - int r1 = j + regBase; - sqlite3ExprCodeGetColumnOfTable(v, - pCol->pTab, pCol->iTable, pCol->iColumn, r1); + sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase); j++; } } + pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, 0, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3VdbeScanStatusRange(v, addrExp, sqlite3VdbeCurrentAddr(v)-2, -1); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, sqlite3VdbeCurrentAddr(v), 0); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); pAggInfo->useSortingIdx = 1; + sqlite3VdbeScanStatusRange(v, addrExp, -1, sortPTab); + sqlite3VdbeScanStatusRange(v, addrExp, -1, pAggInfo->sortingIdx); + } + + /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif } /* If the index or temporary table used by the GROUP BY sort @@ -141193,7 +153044,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } @@ -141241,7 +153092,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); - if( eDist!=WHERE_DISTINCT_NOOP ){ + if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ struct AggInfo_func *pF = &pAggInfo->aFunc[0]; fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); } @@ -141303,7 +153154,8 @@ SQLITE_PRIVATE int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + assignAggregateRegisters(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ @@ -141339,6 +153191,7 @@ SQLITE_PRIVATE int sqlite3Select( pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } + assignAggregateRegisters(pParse, pAggInfo); /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row @@ -141355,25 +153208,27 @@ SQLITE_PRIVATE int sqlite3Select( assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); - SELECTTRACE(1,pParse,p,("WhereBegin\n")); + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, - pDistinct, minMaxFlag|distFlag, 0); + pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ goto select_end; } - SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); eDist = sqlite3WhereIsDistinct(pWInfo); updateAccumulator(pParse, regAcc, pAggInfo, eDist); if( eDist!=WHERE_DISTINCT_NOOP ){ - struct AggInfo_func *pF = &pAggInfo->aFunc[0]; - fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + struct AggInfo_func *pF = pAggInfo->aFunc; + if( pF ){ + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } } if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } - SELECTTRACE(1,pParse,p,("WhereEnd\n")); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } @@ -141395,8 +153250,6 @@ SQLITE_PRIVATE int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -141414,13 +153267,19 @@ SQLITE_PRIVATE int sqlite3Select( */ select_end: assert( db->mallocFailed==0 || db->mallocFailed==1 ); - pParse->nErr += db->mallocFailed; + assert( db->mallocFailed==0 || pParse->nErr!=0 ); sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; - assert( pExpr!=0 ); + if( pExpr==0 ) continue; assert( pExpr->pAggInfo==pAggInfo ); assert( pExpr->iAgg==i ); } @@ -141433,9 +153292,9 @@ select_end: } #endif -#if SELECTTRACE_ENABLED - SELECTTRACE(0x1,pParse,p,("end processing\n")); - if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ +#if TREETRACE_ENABLED + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ sqlite3TreeViewSelect(0, p, 0); } #endif @@ -141700,9 +153559,7 @@ SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ Trigger *pList; /* List of triggers to return */ HashElem *p; /* Loop variable for TEMP triggers */ - if( pParse->disableTriggers ){ - return 0; - } + assert( pParse->disableTriggers==0 ); pTmpSchema = pParse->db->aDb[1].pSchema; p = sqliteHashFirst(&pTmpSchema->trigHash); pList = pTab->pTrigger; @@ -141711,15 +153568,14 @@ SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; - }else if( pTrig->op==TK_RETURNING + }else if( pTrig->op==TK_RETURNING ){ #ifndef SQLITE_OMIT_VIRTUALTABLE - && pParse->db->pVtabCtx==0 + assert( pParse->db->pVtabCtx==0 ); #endif - ){ assert( pParse->bReturning ); assert( &(pParse->u1.pReturning->retTrig) == pTrig ); pTrig->table = pTab->zName; @@ -141834,6 +153690,10 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); goto trigger_orphan_error; } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on shadow tables"); + goto trigger_orphan_error; + } /* Check that the trigger name is not reserved and that no trigger of the ** specified name exists */ @@ -141853,6 +153713,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( }else{ assert( !db->init.busy ); sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) } goto trigger_cleanup; } @@ -142002,6 +153863,23 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Vdbe *v; char *z; + /* If this is a new CREATE TABLE statement, and if shadow tables + ** are read-only, and the trigger makes a change to a shadow table, + ** then raise an error - do not allow the trigger to be created. */ + if( sqlite3ReadOnlyShadowTables(db) ){ + TriggerStep *pStep; + for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) + ){ + sqlite3ErrorMsg(pParse, + "trigger \"%s\" may not write to shadow table \"%s\"", + pTrig->zName, pStep->zTarget); + goto triggerfinish_cleanup; + } + } + } + /* Make an entry in the sqlite_schema table */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto triggerfinish_cleanup; @@ -142094,6 +153972,7 @@ static TriggerStep *triggerStepAllocate( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; + if( pParse->nErr ) return 0; pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); if( pTriggerStep ){ char *z = (char*)&pTriggerStep[1]; @@ -142164,7 +154043,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table to be updated */ - SrcList *pFrom, + SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ u8 orconf, /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ @@ -142377,13 +154256,22 @@ static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ return 0; } +/* +** Return true if any TEMP triggers exist +*/ +static int tempTriggersExist(sqlite3 *db){ + if( NEVER(db->aDb[1].pSchema==0) ) return 0; + if( sqliteHashFirst(&db->aDb[1].pSchema->trigHash)==0 ) return 0; + return 1; +} + /* ** Return a list of all triggers on table pTab if there exists at least ** one trigger that must be fired when an operation of type 'op' is ** performed on the table, and, if that operation is an UPDATE, if at ** least one of the columns in pChanges is being modified. */ -SQLITE_PRIVATE Trigger *sqlite3TriggersExist( +static SQLITE_NOINLINE Trigger *triggersReallyExist( Parse *pParse, /* Parse context */ Table *pTab, /* The table the contains the triggers */ int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ @@ -142446,6 +154334,22 @@ exit_triggers_exist: } return (mask ? pList : 0); } +SQLITE_PRIVATE Trigger *sqlite3TriggersExist( + Parse *pParse, /* Parse context */ + Table *pTab, /* The table the contains the triggers */ + int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ + ExprList *pChanges, /* Columns that change in an UPDATE statement */ + int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +){ + assert( pTab!=0 ); + if( (pTab->pTrigger==0 && !tempTriggersExist(pParse->db)) + || pParse->disableTriggers + ){ + if( pMask ) *pMask = 0; + return 0; + } + return triggersReallyExist(pParse,pTab,op,pChanges,pMask); +} /* ** Convert the pStep->zTarget string into a SrcList and return a pointer @@ -142475,6 +154379,14 @@ SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc( } if( pStep->pFrom ){ SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); + if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ + Select *pSubquery; + Token as; + pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); + as.n = 0; + as.z = 0; + pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); + } pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); } }else{ @@ -142530,7 +154442,7 @@ static ExprList *sqlite3ExpandReturning( if( !db->mallocFailed ){ struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; pItem->zEName = sqlite3DbStrDup(db, pTab->aCol[jj].zCnName); - pItem->eEName = ENAME_NAME; + pItem->fg.eEName = ENAME_NAME; } } }else{ @@ -142539,13 +154451,79 @@ static ExprList *sqlite3ExpandReturning( if( !db->mallocFailed && ALWAYS(pList->a[i].zEName!=0) ){ struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; pItem->zEName = sqlite3DbStrDup(db, pList->a[i].zEName); - pItem->eEName = pList->a[i].eEName; + pItem->fg.eEName = pList->a[i].fg.eEName; } } } return pNew; } +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; inSrc; i++){ + if( pSrc->a[i].pTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + /* ** Generate code for the RETURNING trigger. Unlike other triggers ** that invoke a subprogram in the bytecode, the code for RETURNING @@ -142565,23 +154543,33 @@ static void codeReturningTrigger( SrcList sFrom; assert( v!=0 ); - assert( pParse->bReturning ); + if( !pParse->bReturning ){ + /* This RETURNING trigger must be for a different statement as + ** this statement lacks a RETURNING clause. */ + return; + } + assert( db->pParse==pParse ); pReturning = pParse->u1.pReturning; - assert( pTrigger == &(pReturning->retTrig) ); + if( pTrigger != &(pReturning->retTrig) ){ + /* This RETURNING trigger is for a different statement */ + return; + } memset(&sSelect, 0, sizeof(sSelect)); memset(&sFrom, 0, sizeof(sFrom)); sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = &sFrom; sFrom.nSrc = 1; sFrom.a[0].pTab = pTab; + sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ sFrom.a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); - if( db->mallocFailed==0 && pParse->nErr==0 ){ + if( pParse->nErr==0 ){ + assert( db->mallocFailed==0 ); sqlite3GenerateColumnNames(pParse, &sSelect); } sqlite3ExprListDelete(db, sSelect.pEList); pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); - if( !db->mallocFailed ){ + if( pParse->nErr==0 ){ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); if( pReturning->nRetCol==0 ){ @@ -142594,17 +154582,21 @@ static void codeReturningTrigger( pParse->eTriggerOp = pTrigger->op; pParse->pTriggerTab = pTab; if( sqlite3ResolveExprListNames(&sNC, pNew)==SQLITE_OK - && !db->mallocFailed + && ALWAYS(!db->mallocFailed) ){ int i; int nCol = pNew->nExpr; int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); pParse->nMem += nCol+2; pReturning->iRetReg = reg; for(i=0; ia[i].pExpr; assert( pCol!=0 ); /* Due to !db->mallocFailed ~9 lines above */ sqlite3ExprCodeFactorable(pParse, pCol, reg+i); + if( sqlite3ExprAffinity(pCol)==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, reg+i); + } } sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, i, reg+i); sqlite3VdbeAddOp2(v, OP_NewRowid, pReturning->iRetCur, reg+i+1); @@ -142755,8 +154747,8 @@ static TriggerPrg *codeRowTrigger( Vdbe *v; /* Temporary VM */ NameContext sNC; /* Name context for sub-vdbe */ SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ - Parse *pSubParse; /* Parse context for sub-vdbe */ int iEndTrigger = 0; /* Label to jump to if WHEN is false */ + Parse sSubParse; /* Parse context for sub-vdbe */ assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); assert( pTop->pVdbe ); @@ -142778,19 +154770,17 @@ static TriggerPrg *codeRowTrigger( /* Allocate and populate a new Parse context to use for coding the ** trigger sub-program. */ - pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); - if( !pSubParse ) return 0; + sqlite3ParseObjectInit(&sSubParse, db); memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pSubParse; - pSubParse->db = db; - pSubParse->pTriggerTab = pTab; - pSubParse->pToplevel = pTop; - pSubParse->zAuthContext = pTrigger->zName; - pSubParse->eTriggerOp = pTrigger->op; - pSubParse->nQueryLoop = pParse->nQueryLoop; - pSubParse->disableVtab = pParse->disableVtab; - - v = sqlite3GetVdbe(pSubParse); + sNC.pParse = &sSubParse; + sSubParse.pTriggerTab = pTab; + sSubParse.pToplevel = pTop; + sSubParse.zAuthContext = pTrigger->zName; + sSubParse.eTriggerOp = pTrigger->op; + sSubParse.nQueryLoop = pParse->nQueryLoop; + sSubParse.prepFlags = pParse->prepFlags; + + v = sqlite3GetVdbe(&sSubParse); if( v ){ VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", pTrigger->zName, onErrorText(orconf), @@ -142816,14 +154806,14 @@ static TriggerPrg *codeRowTrigger( if( db->mallocFailed==0 && SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) ){ - iEndTrigger = sqlite3VdbeMakeLabel(pSubParse); - sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); + iEndTrigger = sqlite3VdbeMakeLabel(&sSubParse); + sqlite3ExprIfFalse(&sSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); } sqlite3ExprDelete(db, pWhen); } /* Code the trigger program into the sub-vdbe. */ - codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); + codeTriggerProgram(&sSubParse, pTrigger->step_list, orconf); /* Insert an OP_Halt at the end of the sub-program. */ if( iEndTrigger ){ @@ -142831,23 +154821,24 @@ static TriggerPrg *codeRowTrigger( } sqlite3VdbeAddOp0(v, OP_Halt); VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); + transferParseError(pParse, &sSubParse); - transferParseError(pParse, pSubParse); - if( db->mallocFailed==0 && pParse->nErr==0 ){ + if( pParse->nErr==0 ){ + assert( db->mallocFailed==0 ); pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); } - pProgram->nMem = pSubParse->nMem; - pProgram->nCsr = pSubParse->nTab; + pProgram->nMem = sSubParse.nMem; + pProgram->nCsr = sSubParse.nTab; pProgram->token = (void *)pTrigger; - pPrg->aColmask[0] = pSubParse->oldmask; - pPrg->aColmask[1] = pSubParse->newmask; + pPrg->aColmask[0] = sSubParse.oldmask; + pPrg->aColmask[1] = sSubParse.newmask; sqlite3VdbeDelete(v); + }else{ + transferParseError(pParse, &sSubParse); } - assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); - sqlite3ParserReset(pSubParse); - sqlite3StackFree(db, pSubParse); - + assert( !sSubParse.pTriggerPrg && !sSubParse.nMaxArg ); + sqlite3ParseObjectReset(&sSubParse); return pPrg; } @@ -142880,6 +154871,7 @@ static TriggerPrg *getRowTrigger( /* If an existing TriggerPrg could not be located, create a new one. */ if( !pPrg ){ pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + pParse->db->errByteOffset = -1; } return pPrg; @@ -142902,7 +154894,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ TriggerPrg *pPrg; pPrg = getRowTrigger(pParse, p, pTab, orconf); - assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); + assert( pPrg || pParse->nErr ); /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ @@ -143047,6 +155039,9 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ if( p->op==op && (tr_tm&p->tr_tm) @@ -143132,11 +155127,14 @@ static void updateVirtualTable( ** it has been converted into REAL. */ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + Column *pCol; assert( pTab!=0 ); - if( !IsView(pTab) ){ + assert( pTab->nCol>i ); + pCol = &pTab->aCol[i]; + if( pCol->iDflt ){ sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; + assert( !IsView(pTab) ); VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); assert( inCol ); sqlite3ValueFromExpr(sqlite3VdbeDb(v), @@ -143147,7 +155145,7 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ } } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif @@ -143294,7 +155292,7 @@ static void updateFromSelect( assert( pTabList->nSrc>1 ); if( pSrc ){ - pSrc->a[0].fg.notCte = 1; + assert( pSrc->a[0].fg.notCte ); pSrc->a[0].iCursor = -1; pSrc->a[0].pTab->nTabRef--; pSrc->a[0].pTab = 0; @@ -143333,7 +155331,8 @@ static void updateFromSelect( } } pSelect = sqlite3SelectNew(pParse, pList, - pSrc, pWhere2, pGrp, 0, pOrderBy2, SF_UFSrcCheck|SF_IncludeHidden, pLimit2 + pSrc, pWhere2, pGrp, 0, pOrderBy2, + SF_UFSrcCheck|SF_IncludeHidden|SF_UpdateFrom, pLimit2 ); if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; sqlite3SelectDestInit(&dest, eDest, iEph); @@ -143420,9 +155419,11 @@ SQLITE_PRIVATE void sqlite3Update( memset(&sContext, 0, sizeof(sContext)); db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto update_cleanup; } + assert( db->mallocFailed==0 ); /* Locate the table which we want to update. */ @@ -143447,6 +155448,14 @@ SQLITE_PRIVATE void sqlite3Update( # define isView 0 #endif +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Update() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewUpdate(pParse->pWith, pTabList, pChanges, pWhere, + onError, pOrderBy, pLimit, pUpsert, pTrigger); + } +#endif + /* If there was a FROM clause, set nChangeFrom to the number of expressions ** in the change-list. Otherwise, set it to 0. There cannot be a FROM ** clause if this function is being called to generate code for part of @@ -143467,7 +155476,7 @@ SQLITE_PRIVATE void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } @@ -143786,15 +155795,25 @@ SQLITE_PRIVATE void sqlite3Update( /* Begin the database scan. ** ** Do not consider a single-pass strategy for a multi-row update if - ** there are any triggers or foreign keys to process, or rows may - ** be deleted as a result of REPLACE conflict handling. Any of these - ** things might disturb a cursor being used to scan through the table - ** or index, causing a single-pass approach to malfunction. */ + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ flags = WHERE_ONEPASS_DESIRED; - if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery)) + ){ flags |= WHERE_ONEPASS_MULTIROW; } - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags,iIdxCur); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); if( pWInfo==0 ) goto update_cleanup; /* A one-pass strategy that might update more than one row may not @@ -143862,6 +155881,8 @@ SQLITE_PRIVATE void sqlite3Update( if( !isView ){ int addrOnce = 0; + int iNotUsed1 = 0; + int iNotUsed2 = 0; /* Open every index that needs updating. */ if( eOnePass!=ONEPASS_OFF ){ @@ -143873,7 +155894,7 @@ SQLITE_PRIVATE void sqlite3Update( addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, - aToOpen, 0, 0); + aToOpen, &iNotUsed1, &iNotUsed2); if( addrOnce ){ sqlite3VdbeJumpHereOrPopInst(v, addrOnce); } @@ -143968,6 +155989,9 @@ SQLITE_PRIVATE void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } @@ -144091,7 +156115,7 @@ SQLITE_PRIVATE void sqlite3Update( }else{ sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); } - VdbeCoverageNeverTaken(v); + VdbeCoverage(v); } /* Do FK constraint checks. */ @@ -144164,8 +156188,10 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } - sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + } /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. @@ -144194,9 +156220,7 @@ SQLITE_PRIVATE void sqlite3Update( ** that information. */ if( regRowCount ){ - sqlite3VdbeAddOp2(v, OP_ChngCntRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + sqlite3CodeChangeCount(v, regRowCount, "rows updated"); } update_cleanup: @@ -144262,7 +156286,7 @@ static void updateVirtualTable( int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ int regArg; /* First register in VUpdate arg array */ int regRec; /* Register in which to assemble record */ - int regRowid; /* Register for ephem table rowid */ + int regRowid; /* Register for ephemeral table rowid */ int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ int eOnePass; /* True to use onepass strategy */ @@ -144306,7 +156330,9 @@ static void updateVirtualTable( sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0) ); }else{ - pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i)); + Expr *pRowExpr = exprRowColumn(pParse, i); + if( pRowExpr ) pRowExpr->op2 = OPFLAG_NOCHNG; + pList = sqlite3ExprListAppend(pParse, pList, pRowExpr); } } @@ -144318,7 +156344,9 @@ static void updateVirtualTable( regRowid = ++pParse->nMem; /* Start scanning the virtual table */ - pWInfo = sqlite3WhereBegin(pParse, pSrc,pWhere,0,0,WHERE_ONEPASS_DESIRED,0); + pWInfo = sqlite3WhereBegin( + pParse, pSrc, pWhere, 0, 0, 0, WHERE_ONEPASS_DESIRED, 0 + ); if( pWInfo==0 ) return; /* Populate the argument registers. */ @@ -144374,14 +156402,13 @@ static void updateVirtualTable( } } - if( eOnePass==ONEPASS_OFF ){ /* End the virtual table scan */ if( pSrc->nSrc==1 ){ sqlite3WhereEnd(pWInfo); } - /* Begin scannning through the ephemeral table. */ + /* Begin scanning through the ephemeral table. */ addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); /* Extract arguments from the current row of the ephemeral table and @@ -144501,7 +156528,8 @@ SQLITE_PRIVATE Upsert *sqlite3UpsertNew( SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( Parse *pParse, /* The parsing context */ SrcList *pTabList, /* Table into which we are inserting */ - Upsert *pUpsert /* The ON CONFLICT clauses */ + Upsert *pUpsert, /* The ON CONFLICT clauses */ + Upsert *pAll /* Complete list of all ON CONFLICT clauses */ ){ Table *pTab; /* That table into which we are inserting */ int rc; /* Result code */ @@ -144577,6 +156605,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( if( pIdx->aiColumn[ii]==XN_EXPR ){ assert( pIdx->aColExpr!=0 ); assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); pExpr = pIdx->aColExpr->a[ii].pExpr; if( pExpr->op!=TK_COLLATE ){ sCol[0].pLeft = pExpr; @@ -144588,7 +156617,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( pExpr = &sCol[0]; } for(jj=0; jja[jj].pExpr,pExpr,iCursor)<2 ){ + if( sqlite3ExprCompare(0,pTarget->a[jj].pExpr,pExpr,iCursor)<2 ){ break; /* Column ii of the index matches column jj of target */ } } @@ -144603,6 +156632,14 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( continue; } pUpsert->pUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } break; } if( pUpsert->pUpsertIdx==0 ){ @@ -144629,9 +156666,13 @@ SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ Upsert *pNext; if( NEVER(pUpsert==0) ) return 0; pNext = pUpsert->pNextUpsert; - if( pNext==0 ) return 1; - if( pNext->pUpsertTarget==0 ) return 1; - if( pNext->pUpsertIdx==0 ) return 1; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } return 0; } @@ -144890,6 +156931,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ + u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -144936,7 +156978,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** - ** An optimisation would be to use a non-journaled pager. + ** An optimization would be to use a non-journaled pager. ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially @@ -144961,10 +157003,16 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( goto end_of_vacuum; } db->mDbFlags |= DBFLAG_VacuumInto; + + /* For a VACUUM INTO, the pager-flags are set to the same values as + ** they are for the database being vacuumed, except that PAGER_CACHESPILL + ** is always set. */ + pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); } nRes = sqlite3BtreeGetRequestedReserve(pMain); - /* A VACUUM cannot change the pagesize of an encrypted database. */ + #ifdef SQLITE_HAS_CODEC + /* A VACUUM cannot change the pagesize of an encrypted database. */ if( db->nextPagesize ){ extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); int nKey; @@ -144972,11 +157020,11 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( sqlite3CodecGetKey(db, iDb, (void**)&zKey, &nKey); if( nKey ) db->nextPagesize = 0; } -#endif +#endif /* SQLITE_HAS_CODEC */ sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); - sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, @@ -145107,6 +157155,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( assert( rc==SQLITE_OK ); if( pOut==0 ){ + nRes = sqlite3BtreeGetRequestedReserve(pTemp); rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1); } @@ -145359,10 +157408,10 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ pVTab->nRef--; if( pVTab->nRef==0 ){ sqlite3_vtab *p = pVTab->pVtab; - sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); if( p ){ p->pModule->xDisconnect(p); } + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); sqlite3DbFree(db, pVTab); } } @@ -145463,7 +157512,6 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ if( p ){ db->pDisconnect = 0; - sqlite3ExpirePreparedStatements(db, 0); do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); @@ -145488,7 +157536,8 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ */ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ assert( IsVirtual(p) ); - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + assert( db!=0 ); + if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->u.vtab.azArg ){ int i; for(i=0; iu.vtab.nArg; i++){ @@ -145628,7 +157677,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ ** the information we've collected. ** ** The VM register number pParse->regRowid holds the rowid of an - ** entry in the sqlite_schema table tht was created for this vtab + ** entry in the sqlite_schema table that was created for this vtab ** by sqlite3StartTable(). */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -145757,7 +157806,11 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); @@ -145779,7 +157832,7 @@ static int vtabCallConstructor( pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -145955,28 +158008,46 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ VtabCtx *pCtx; int rc = SQLITE_OK; Table *pTab; - char *zErr = 0; Parse sParse; int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ z += sqlite3GetToken(z, &tokenType); }while( tokenType==TK_SPACE ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ - sqlite3Error(db, SQLITE_MISUSE); + sqlite3Error(db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( IsVirtual(pTab) ); - memset(&sParse, 0, sizeof(sParse)); + sqlite3ParseObjectInit(&sParse, db); sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; - sParse.db = db; + sParse.disableTriggers = 1; /* We should never be able to reach this point while loading the ** schema. Nevertheless, defend against that (turn off db->init.busy) ** in case a bug arises. */ @@ -145984,11 +158055,11 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ initBusy = db->init.busy; db->init.busy = 0; sParse.nQueryLoop = 1; - if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable, &zErr) - && sParse.pNewTable - && !db->mallocFailed - && IsOrdinaryTable(sParse.pNewTable) - ){ + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); + assert( sParse.zErrMsg==0 ); if( !pTab->aCol ){ Table *pNew = sParse.pNewTable; Index *pIdx; @@ -146018,8 +158089,9 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ } pCtx->bDeclared = 1; }else{ - sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, + (sParse.zErrMsg ? "%s" : 0), sParse.zErrMsg); + sqlite3DbFree(db, sParse.zErrMsg); rc = SQLITE_ERROR; } sParse.eParseMode = PARSE_MODE_NORMAL; @@ -146028,7 +158100,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3VdbeFinalize(sParse.pVdbe); } sqlite3DeleteTable(db, sParse.pNewTable); - sqlite3ParserReset(&sParse); + sqlite3ParseObjectReset(&sParse); db->init.busy = initBusy; assert( (rc&0xff)==rc ); @@ -146246,7 +158318,10 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ + u64 savedFlags = (db->flags & SQLITE_Defensive); + db->flags &= ~(u64)SQLITE_Defensive; rc = xMethod(pVTab->pVtab, iSavepoint); + db->flags |= savedFlags; } sqlite3VtabUnlock(pVTab); } @@ -146287,7 +158362,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( if( pExpr->op!=TK_COLUMN ) return pDef; assert( ExprUseYTab(pExpr) ); pTab = pExpr->y.pTab; - if( pTab==0 ) return pDef; + if( NEVER(pTab==0) ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); @@ -146366,7 +158441,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ ** ** An eponymous virtual table instance is one that is named after its ** module, and more importantly, does not require a CREATE VIRTUAL TABLE -** statement in order to come into existance. Eponymous virtual table +** statement in order to come into existence. Eponymous virtual table ** instances always exist. They cannot be DROP-ed. ** ** Any virtual table module for which xConnect and xCreate are the same @@ -146475,6 +158550,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; break; } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } default: { rc = SQLITE_MISUSE_BKPT; break; @@ -146548,6 +158627,28 @@ typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; typedef struct WhereOrCost WhereOrCost; typedef struct WhereOrSet WhereOrSet; +typedef struct WhereMemBlock WhereMemBlock; +typedef struct WhereRightJoin WhereRightJoin; + +/* +** This object is a header on a block of allocated memory that will be +** automatically freed when its WInfo object is destructed. +*/ +struct WhereMemBlock { + WhereMemBlock *pNext; /* Next block in the chain */ + u64 sz; /* Bytes of space */ +}; + +/* +** Extra information attached to a WhereLevel that is a RIGHT JOIN. +*/ +struct WhereRightJoin { + int iMatch; /* Cursor used to determine prior matched rows */ + int regBloom; /* Bloom filter for iRJMatch */ + int regReturn; /* Return register for the interior subroutine */ + int addrSubrtn; /* Starting address for the interior subroutine */ + int endSubrtn; /* The last opcode in the interior subroutine */ +}; /* ** This object contains information needed to implement a single nested @@ -146580,6 +158681,8 @@ struct WhereLevel { u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ int addrLikeRep; /* LIKE range processing address */ #endif + int regFilter; /* Bloom filter */ + WhereRightJoin *pRJ; /* Extra information for RIGHT JOIN */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to end the loop */ @@ -146590,7 +158693,7 @@ struct WhereLevel { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ int iBase; /* Base register of multi-key index record */ - int nPrefix; /* Number of prior entires in the key */ + int nPrefix; /* Number of prior entries in the key */ u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ @@ -146638,10 +158741,12 @@ struct WhereLoop { } btree; struct { /* Information for virtual tables */ int idxNum; /* Index number */ - u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + u32 needFree : 1; /* True if sqlite3_free(idxStr) is needed */ + u32 bOmitOffset : 1; /* True to let virtual table handle offset */ i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ + u32 mHandleIn; /* Terms to handle as IN(...) instead of == */ } vtab; } u; u32 wsFlags; /* WHERE_* flags describing the plan */ @@ -146785,7 +158890,7 @@ struct WhereTerm { #define TERM_COPIED 0x0008 /* Has a child */ #define TERM_ORINFO 0x0010 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x0020 /* Need to free the WhereTerm.u.pAndInfo obj */ -#define TERM_OR_OK 0x0040 /* Used during OR-clause processing */ +#define TERM_OK 0x0040 /* Used during OR-clause processing */ #define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ #define TERM_LIKEOPT 0x0100 /* Virtual terms from the LIKE optimization */ #define TERM_LIKECOND 0x0200 /* Conditionally this LIKE operator term */ @@ -146798,6 +158903,7 @@ struct WhereTerm { #else # define TERM_HIGHTRUTH 0 /* Only used with STAT4 */ #endif +#define TERM_SLICE 0x8000 /* One slice of a row-value/vector comparison */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -146808,11 +158914,11 @@ struct WhereScan { WhereClause *pWC; /* WhereClause currently being scanned */ const char *zCollName; /* Required collating sequence, if not NULL */ Expr *pIdxExpr; /* Search for this index expression */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + u32 opMask; /* Acceptable operators */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char iEquiv; /* Current slot in aiCur[] and aiColumn[] */ unsigned char nEquiv; /* Number of entries in aiCur[] and aiColumn[] */ - unsigned char iEquiv; /* Next unused slot in aiCur[] and aiColumn[] */ - u32 opMask; /* Acceptable operators */ - int k; /* Resume scanning at this->pWC->a[this->k] */ int aiCur[11]; /* Cursors in the equivalence class */ i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; @@ -146836,7 +158942,8 @@ struct WhereClause { u8 hasOr; /* True if any a[].eOperator is WO_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ - WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ + int nBase; /* Number of terms through the last non-Virtual */ + WhereTerm *a; /* Each a[] describes a term of the WHERE clause */ #if defined(SQLITE_SMALL_STACK) WhereTerm aStatic[1]; /* Initial static space for a[] */ #else @@ -146866,7 +158973,7 @@ struct WhereAndInfo { ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** ** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** SrcItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping @@ -146893,11 +159000,6 @@ struct WhereMaskSet { int ix[BMS]; /* Cursor assigned to each bit */ }; -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 - /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -146905,7 +159007,6 @@ struct WhereMaskSet { struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ WhereClause *pWC; /* WHERE clause terms */ - ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ #ifdef SQLITE_ENABLE_STAT4 @@ -146943,20 +159044,6 @@ struct WhereLoopBuilder { # define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 #endif -/* -** Each instance of this object records a change to a single node -** in an expression tree to cause that node to point to a column -** of an index rather than an expression or a virtual column. All -** such transformations need to be undone at the end of WHERE clause -** processing. -*/ -typedef struct WhereExprMod WhereExprMod; -struct WhereExprMod { - WhereExprMod *pNext; /* Next translation on a list of them all */ - Expr *pExpr; /* The Expr node that was transformed */ - Expr orig; /* Original value of the Expr node */ -}; - /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second @@ -146972,7 +159059,10 @@ struct WhereInfo { SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ +#if WHERETRACE_ENABLED Expr *pWhere; /* The complete WHERE clause */ +#endif + Select *pSelect; /* The entire SELECT statement containing WHERE */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ @@ -146991,7 +159081,7 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ WhereLoop *pLoops; /* List of all WhereLoop objects */ - WhereExprMod *pExprMods; /* Expression modifications */ + WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ @@ -147007,7 +159097,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); #ifdef WHERETRACE_ENABLED SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); -SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC); +SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC); #endif SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -147017,6 +159107,8 @@ SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ); +SQLITE_PRIVATE void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte); +SQLITE_PRIVATE void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte); /* wherecode.c: */ #ifndef SQLITE_OMIT_EXPLAIN @@ -147026,8 +159118,14 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ); +SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( + const Parse *pParse, /* Parse context */ + const WhereInfo *pWInfo, /* WHERE clause */ + const WhereLevel *pLevel /* Bloom filter on this level */ +); #else # define sqlite3WhereExplainOneScan(u,v,w,x) 0 +# define sqlite3WhereExplainBloomFilter(u,v,w) 0 #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS SQLITE_PRIVATE void sqlite3WhereAddScanStatus( @@ -147047,11 +159145,17 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( WhereLevel *pLevel, /* The current level pointer */ Bitmask notReady /* Which tables are currently available */ ); +SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( + WhereInfo *pWInfo, + int iLevel, + WhereLevel *pLevel +); /* whereexpr.c: */ SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause*, Select*); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); @@ -147088,8 +159192,9 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WO_AND 0x0400 /* Two or more AND-connected terms */ #define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ #define WO_NOOP 0x1000 /* This term does not restrict search space */ +#define WO_ROWVAL 0x2000 /* A row-value term */ -#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_ALL 0x3fff /* Mask of all possible WO_* values */ #define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ /* @@ -147120,6 +159225,11 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */ #define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */ #define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */ +#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ +#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ +#define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ + /* 0x02000000 -- available for reuse */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ @@ -147217,9 +159327,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -147231,8 +159341,8 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( sqlite3ParseToplevel(pParse)->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { SrcItem *pItem = &pTabList->a[pLevel->iFrom]; @@ -147282,19 +159392,27 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( explainIndexRange(&str, pLoop); } }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRangeOp; + char cRangeOp; +#if 0 /* Better output, but breaks many tests */ + const Table *pTab = pItem->pTab; + const char *zRowid = pTab->iPKey>=0 ? pTab->aCol[pTab->iPKey].zCnName: + "rowid"; +#else + const char *zRowid = "rowid"; +#endif + sqlite3_str_appendf(&str, " USING INTEGER PRIMARY KEY (%s", zRowid); if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRangeOp = "="; + cRangeOp = '='; }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRangeOp = ">? AND rowid<"; + sqlite3_str_appendf(&str, ">? AND %s", zRowid); + cRangeOp = '<'; }else if( flags&WHERE_BTM_LIMIT ){ - zRangeOp = ">"; + cRangeOp = '>'; }else{ assert( flags&WHERE_TOP_LIMIT); - zRangeOp = "<"; + cRangeOp = '<'; } - sqlite3_str_appendf(&str, - " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); + sqlite3_str_appendf(&str, "%c?)", cRangeOp); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ @@ -147302,6 +159420,9 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&str, " LEFT-JOIN"); + } #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS if( pLoop->nOut>=10 ){ sqlite3_str_appendf(&str, " (~%llu rows)", @@ -147317,6 +159438,58 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( } return ret; } + +/* +** Add a single OP_Explain opcode that describes a Bloom filter. +** +** Or if not processing EXPLAIN QUERY PLAN and not in a SQLITE_DEBUG and/or +** SQLITE_ENABLE_STMT_SCANSTATUS build, then OP_Explain opcodes are not +** required and this routine is a no-op. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. +*/ +SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( + const Parse *pParse, /* Parse context */ + const WhereInfo *pWInfo, /* WHERE clause */ + const WhereLevel *pLevel /* Bloom filter on this level */ +){ + int ret = 0; + SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + int i; /* Loop counter */ + WhereLoop *pLoop; /* The where loop */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ + + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem); + pLoop = pLevel->pWLoop; + if( pLoop->wsFlags & WHERE_IPK ){ + const Table *pTab = pItem->pTab; + if( pTab->iPKey>=0 ){ + sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName); + }else{ + sqlite3_str_appendf(&str, "rowid=?"); + } + }else{ + for(i=pLoop->nSkip; iu.btree.nEq; i++){ + const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i); + if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5); + sqlite3_str_appendf(&str, "%s=?", z); + } + } + sqlite3_str_append(&str, ")", 1); + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), + pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + + sqlite3VdbeScanStatus(v, sqlite3VdbeCurrentAddr(v)-1, 0, 0, 0, 0); + return ret; +} #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -147335,16 +159508,37 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } + }else{ + int addr = pSrclist->a[pLvl->iFrom].addrFillSub; + VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); + sqlite3VdbeScanStatusRange(v, addrExplain, addr, pOp->p2-1); + } } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); } #endif @@ -147395,7 +159589,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ int nLoop = 0; assert( pTerm!=0 ); while( (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_OuterON)) && (pLevel->notReady & pTerm->prereqAll)==0 ){ if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ @@ -147404,7 +159598,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ pTerm->wtFlags |= TERM_CODED; } #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("DISABLE-"); sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); } @@ -147519,68 +159713,75 @@ static Expr *removeUnindexableInClauseTerms( Expr *pX /* The IN expression to be reduced */ ){ sqlite3 *db = pParse->db; + Select *pSelect; /* Pointer to the SELECT on the RHS */ Expr *pNew; pNew = sqlite3ExprDup(db, pX, 0); if( db->mallocFailed==0 ){ - ExprList *pOrigRhs; /* Original unmodified RHS */ - ExprList *pOrigLhs; /* Original unmodified LHS */ - ExprList *pRhs = 0; /* New RHS after modifications */ - ExprList *pLhs = 0; /* New LHS after mods */ - int i; /* Loop counter */ - Select *pSelect; /* Pointer to the SELECT on the RHS */ - - assert( ExprUseXSelect(pNew) ); - pOrigRhs = pNew->x.pSelect->pEList; - assert( pNew->pLeft!=0 ); - assert( ExprUseXList(pNew->pLeft) ); - pOrigLhs = pNew->pLeft->x.pList; - for(i=iEq; inLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iField; - assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); - iField = pLoop->aLTerm[i]->u.x.iField - 1; - if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ - pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); - pOrigRhs->a[iField].pExpr = 0; - assert( pOrigLhs->a[iField].pExpr!=0 ); - pLhs = sqlite3ExprListAppend(pParse, pLhs, pOrigLhs->a[iField].pExpr); - pOrigLhs->a[iField].pExpr = 0; - } - } - sqlite3ExprListDelete(db, pOrigRhs); - sqlite3ExprListDelete(db, pOrigLhs); - pNew->pLeft->x.pList = pLhs; - pNew->x.pSelect->pEList = pRhs; - if( pLhs && pLhs->nExpr==1 ){ - /* Take care here not to generate a TK_VECTOR containing only a - ** single value. Since the parser never creates such a vector, some - ** of the subroutines do not handle this case. */ - Expr *p = pLhs->a[0].pExpr; - pLhs->a[0].pExpr = 0; - sqlite3ExprDelete(db, pNew->pLeft); - pNew->pLeft = p; - } - pSelect = pNew->x.pSelect; - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; inExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; + for(pSelect=pNew->x.pSelect; pSelect; pSelect=pSelect->pPrior){ + ExprList *pOrigRhs; /* Original unmodified RHS */ + ExprList *pOrigLhs = 0; /* Original unmodified LHS */ + ExprList *pRhs = 0; /* New RHS after modifications */ + ExprList *pLhs = 0; /* New LHS after mods */ + int i; /* Loop counter */ + + assert( ExprUseXSelect(pNew) ); + pOrigRhs = pSelect->pEList; + assert( pNew->pLeft!=0 ); + assert( ExprUseXList(pNew->pLeft) ); + if( pSelect==pNew->x.pSelect ){ + pOrigLhs = pNew->pLeft->x.pList; + } + for(i=iEq; inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField; + assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); + iField = pLoop->aLTerm[i]->u.x.iField - 1; + if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); + pOrigRhs->a[iField].pExpr = 0; + if( pOrigLhs ){ + assert( pOrigLhs->a[iField].pExpr!=0 ); + pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); + pOrigLhs->a[iField].pExpr = 0; + } + } + } + sqlite3ExprListDelete(db, pOrigRhs); + if( pOrigLhs ){ + sqlite3ExprListDelete(db, pOrigLhs); + pNew->pLeft->x.pList = pLhs; + } + pSelect->pEList = pRhs; + if( pLhs && pLhs->nExpr==1 ){ + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + Expr *p = pLhs->a[0].pExpr; + pLhs->a[0].pExpr = 0; + sqlite3ExprDelete(db, pNew->pLeft); + pNew->pLeft = p; + } + if( pSelect->pOrderBy ){ + /* If the SELECT statement has an ORDER BY clause, zero the + ** iOrderByCol variables. These are set to non-zero when an + ** ORDER BY term exactly matches one of the terms of the + ** result-set. Since the result-set of the SELECT statement may + ** have been modified or reordered, these variables are no longer + ** set correctly. Since setting them is just an optimization, + ** it's easiest just to zero them here. */ + ExprList *pOrderBy = pSelect->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } } - } #if 0 - printf("For indexing, change the IN expr:\n"); - sqlite3TreeViewExpr(0, pX, 0); - printf("Into:\n"); - sqlite3TreeViewExpr(0, pNew, 0); + printf("For indexing, change the IN expr:\n"); + sqlite3TreeViewExpr(0, pX, 0); + printf("Into:\n"); + sqlite3TreeViewExpr(0, pNew, 0); #endif + } } return pNew; } @@ -147656,16 +159857,22 @@ static int codeEqualityTerm( if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); }else{ - sqlite3 *db = pParse->db; - pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); - - if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + Expr *pExpr = pTerm->pExpr; + if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ + sqlite3 *db = pParse->db; + pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); + pExpr->iTable = iTab; + } + sqlite3ExprDelete(db, pX); + }else{ + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); - pTerm->pExpr->iTable = iTab; } - sqlite3ExprDelete(db, pX); - pX = pTerm->pExpr; + pX = pExpr; } if( eType==IN_INDEX_INDEX_DESC ){ @@ -147688,8 +159895,9 @@ static int codeEqualityTerm( i = pLevel->u.in.nIn; pLevel->u.in.nIn += nEq; pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + sqlite3WhereRealloc(pTerm->pWC->pWInfo, + pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); pIn = pLevel->u.in.aInLoop; if( pIn ){ int iMap = 0; /* Index in aiMap[] */ @@ -147826,7 +160034,7 @@ static int codeAllEqualityTerms( /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; + nReg = nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); @@ -147840,6 +160048,7 @@ static int codeAllEqualityTerms( VdbeCoverageIf(v, bRev!=0); VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); + assert( pLevel->addrSkip==0 ); pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), iIdxCur, 0, regBase, nSkip); VdbeCoverageIf(v, bRev==0); @@ -147886,7 +160095,8 @@ static int codeAllEqualityTerms( sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); VdbeCoverage(v); } - if( pParse->db->mallocFailed==0 && pParse->nErr==0 ){ + if( pParse->nErr==0 ){ + assert( pParse->db->mallocFailed==0 ); if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ zAff[j] = SQLITE_AFF_BLOB; } @@ -147926,7 +160136,7 @@ static void whereLikeOptimizationStringFixup( if( pTerm->wtFlags & TERM_LIKEOPT ){ VdbeOp *pOp; assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); + pOp = sqlite3VdbeGetLastOp(v); assert( pOp!=0 ); assert( pOp->opcode==OP_String8 || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); @@ -148013,18 +160223,19 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ ** 2) transform the expression node to a TK_REGISTER node that reads ** from the newly populated register. ** -** Also, if the node is a TK_COLUMN that does access the table idenified +** Also, if the node is a TK_COLUMN that does access the table identified ** by pCCurHint.iTabCur, and an index is being used (which we will ** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into ** an access of the index rather than the original table. */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCode(pWalker->pParse, pExpr, reg); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -148032,15 +160243,15 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pExpr->op==TK_TRUEFALSE ){ + /* Do not walk disabled expressions. tag-20230504-1 */ + return WRC_Prune; } return rc; } @@ -148076,7 +160287,7 @@ static void codeCursorHint( sWalker.pParse = pParse; sWalker.u.pCCurHint = &sHint; pWC = &pWInfo->sWC; - for(i=0; inTerm; i++){ + for(i=0; inBase; i++){ pTerm = &pWC->a[i]; if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( pTerm->prereqAll & pLevel->notReady ) continue; @@ -148105,8 +160316,8 @@ static void codeCursorHint( */ if( pTabItem->fg.jointype & JT_LEFT ){ Expr *pExpr = pTerm->pExpr; - if( !ExprHasProperty(pExpr, EP_FromJoin) - || pExpr->iRightJoinTable!=pTabItem->iCursor + if( !ExprHasProperty(pExpr, EP_OuterON) + || pExpr->w.iJoin!=pTabItem->iCursor ){ sWalker.eCode = 0; sWalker.xExprCallback = codeCursorHintIsOrFunction; @@ -148114,7 +160325,7 @@ static void codeCursorHint( if( sWalker.eCode ) continue; } }else{ - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) continue; } /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize @@ -148142,7 +160353,7 @@ static void codeCursorHint( } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); @@ -148162,13 +160373,21 @@ static void codeCursorHint( ** ** OP_DeferredSeek $iCur $iRowid ** +** Which causes a seek on $iCur to the row with rowid $iRowid. +** ** However, if the scan currently being coded is a branch of an OR-loop and -** the statement currently being coded is a SELECT, then P3 of OP_DeferredSeek -** is set to iIdxCur and P4 is set to point to an array of integers -** containing one entry for each column of the table cursor iCur is open -** on. For each table column, if the column is the i'th column of the -** index, then the corresponding array entry is set to (i+1). If the column -** does not appear in the index at all, the array entry is set to 0. +** the statement currently being coded is a SELECT, then additional information +** is added that might allow OP_Column to omit the seek and instead do its +** lookup on the index, thus avoiding an expensive seek operation. To +** enable this optimization, the P3 of OP_DeferredSeek is set to iIdxCur +** and P4 is set to an array of integers containing one entry for each column +** in the table. For each table column, if the column is the i'th +** column of the index, then the corresponding array entry is set to (i+1). +** If the column does not appear in the index at all, the array entry is set +** to 0. The OP_Column opcode can check this array to see if the column it +** wants is in the index and if it is, it will substitute the index cursor +** and column number and continue with those new values, rather than seeking +** the table cursor. */ static void codeDeferredSeek( WhereInfo *pWInfo, /* Where clause context */ @@ -148184,7 +160403,7 @@ static void codeDeferredSeek( pWInfo->bDeferredSeek = 1; sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); - if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) + if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN)) && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) ){ int i; @@ -148242,143 +160461,6 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ } } -/* An instance of the IdxExprTrans object carries information about a -** mapping from an expression on table columns into a column in an index -** down through the Walker. -*/ -typedef struct IdxExprTrans { - Expr *pIdxExpr; /* The index expression */ - int iTabCur; /* The cursor of the corresponding table */ - int iIdxCur; /* The cursor for the index */ - int iIdxCol; /* The column for the index */ - int iTabCol; /* The column for the table */ - WhereInfo *pWInfo; /* Complete WHERE clause information */ - sqlite3 *db; /* Database connection (for malloc()) */ -} IdxExprTrans; - -/* -** Preserve pExpr on the WhereETrans list of the WhereInfo. -*/ -static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ - WhereExprMod *pNew; - pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); - if( pNew==0 ) return; - pNew->pNext = pTrans->pWInfo->pExprMods; - pTrans->pWInfo->pExprMods = pNew; - pNew->pExpr = pExpr; - memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); -} - -/* The walker node callback used to transform matching expressions into -** a reference to an index column for an index on an expression. -** -** If pExpr matches, then transform it into a reference to the index column -** that contains the value of pExpr. -*/ -static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3ExprAffinity(pExpr); - pExpr->op = TK_COLUMN; - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - testcase( ExprHasProperty(pExpr, EP_Skip) ); - testcase( ExprHasProperty(pExpr, EP_Unlikely) ); - ExprClearProperty(pExpr, EP_Skip|EP_Unlikely|EP_WinFunc|EP_Subrtn); - pExpr->y.pTab = 0; - return WRC_Prune; - }else{ - return WRC_Continue; - } -} - -#ifndef SQLITE_OMIT_GENERATED_COLUMNS -/* A walker node callback that translates a column reference to a table -** into a corresponding column reference of an index. -*/ -static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ - if( pExpr->op==TK_COLUMN ){ - IdxExprTrans *pX = p->u.pIdxTrans; - if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ - assert( ExprUseYTab(pExpr) && pExpr->y.pTab!=0 ); - preserveExpr(pX, pExpr); - pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); - pExpr->iTable = pX->iIdxCur; - pExpr->iColumn = pX->iIdxCol; - pExpr->y.pTab = 0; - } - } - return WRC_Continue; -} -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - -/* -** For an indexes on expression X, locate every instance of expression X -** in pExpr and change that subexpression into a reference to the appropriate -** column of the index. -** -** 2019-10-24: Updated to also translate references to a VIRTUAL column in -** the table into references to the corresponding (stored) column of the -** index. -*/ -static void whereIndexExprTrans( - Index *pIdx, /* The Index */ - int iTabCur, /* Cursor of the table that is being indexed */ - int iIdxCur, /* Cursor of the index itself */ - WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ -){ - int iIdxCol; /* Column number of the index */ - ExprList *aColExpr; /* Expressions that are indexed */ - Table *pTab; - Walker w; - IdxExprTrans x; - aColExpr = pIdx->aColExpr; - if( aColExpr==0 && !pIdx->bHasVCol ){ - /* The index does not reference any expressions or virtual columns - ** so no translations are needed. */ - return; - } - pTab = pIdx->pTable; - memset(&w, 0, sizeof(w)); - w.u.pIdxTrans = &x; - x.iTabCur = iTabCur; - x.iIdxCur = iIdxCur; - x.pWInfo = pWInfo; - x.db = pWInfo->pParse->db; - for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ - i16 iRef = pIdx->aiColumn[iIdxCol]; - if( iRef==XN_EXPR ){ - assert( aColExpr!=0 && aColExpr->a[iIdxCol].pExpr!=0 ); - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; - if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; - w.xExprCallback = whereIndexExprTransNode; -#ifndef SQLITE_OMIT_GENERATED_COLUMNS - }else if( iRef>=0 - && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 - && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0 - || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]), - sqlite3StrBINARY)==0) - ){ - /* Check to see if there are direct references to generated columns - ** that are contained in the index. Pulling the generated column - ** out of the index is an optimization only - the main table is always - ** available if the index cannot be used. To avoid unnecessary - ** complication, omit this optimization if the collating sequence for - ** the column is non-standard */ - x.iTabCol = iRef; - w.xExprCallback = whereIndexExprTransColumn; -#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ - }else{ - continue; - } - x.iIdxCol = iIdxCol; - sqlite3WalkExpr(&w, pWInfo->pWhere); - sqlite3WalkExprList(&w, pWInfo->pOrderBy); - sqlite3WalkExprList(&w, pWInfo->pResultSet); - } -} - /* ** The pTruth expression is always true because it is the WHERE clause ** a partial index that is driving a query loop. Look through all of the @@ -148407,6 +160489,91 @@ static void whereApplyPartialIndexConstraints( } } +/* +** This routine is called right after An OP_Filter has been generated and +** before the corresponding index search has been performed. This routine +** checks to see if there are additional Bloom filters in inner loops that +** can be checked prior to doing the index lookup. If there are available +** inner-loop Bloom filters, then evaluate those filters now, before the +** index lookup. The idea is that a Bloom filter check is way faster than +** an index lookup, and the Bloom filter might return false, meaning that +** the index lookup can be skipped. +** +** We know that an inner loop uses a Bloom filter because it has the +** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked, +** then clear the WhereLevel.regFilter value to prevent the Bloom filter +** from being checked a second time when the inner loop is evaluated. +*/ +static SQLITE_NOINLINE void filterPullDown( + Parse *pParse, /* Parsing context */ + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + int addrNxt, /* Jump here to bypass inner loops */ + Bitmask notReady /* Loops that are not ready */ +){ + while( ++iLevel < pWInfo->nLevel ){ + WhereLevel *pLevel = &pWInfo->a[iLevel]; + WhereLoop *pLoop = pLevel->pWLoop; + if( pLevel->regFilter==0 ) continue; + if( pLevel->pWLoop->nSkip ) continue; + /* ,--- Because sqlite3ConstructBloomFilter() has will not have set + ** vvvvv--' pLevel->regFilter if this were true. */ + if( NEVER(pLoop->prereq & notReady) ) continue; + assert( pLevel->addrBrk==0 ); + pLevel->addrBrk = addrNxt; + if( pLoop->wsFlags & WHERE_IPK ){ + WhereTerm *pTerm = pLoop->aLTerm[0]; + int regRowid; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + regRowid = sqlite3GetTempReg(pParse); + regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt); + VdbeCoverage(pParse->pVdbe); + sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, + addrNxt, regRowid, 1); + VdbeCoverage(pParse->pVdbe); + }else{ + u16 nEq = pLoop->u.btree.nEq; + int r1; + char *zStartAff; + + assert( pLoop->wsFlags & WHERE_INDEXED ); + assert( (pLoop->wsFlags & WHERE_COLUMN_IN)==0 ); + r1 = codeAllEqualityTerms(pParse,pLevel,0,0,&zStartAff); + codeApplyAffinity(pParse, r1, nEq, zStartAff); + sqlite3DbFree(pParse->db, zStartAff); + sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, + addrNxt, r1, nEq); + VdbeCoverage(pParse->pVdbe); + } + pLevel->regFilter = 0; + pLevel->addrBrk = 0; + } +} + +/* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; iiu.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -148444,13 +160611,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x800 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); - sqlite3WhereLoopPrint(pLoop, pWC); + if( sqlite3WhereTrace & 0x1000 ){ + sqlite3WhereLoopPrint(pLoop, pWC); + } } - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ if( iLevel==0 ){ sqlite3DebugPrintf("WHERE clause being coded:\n"); sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); @@ -148477,18 +160646,21 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** initialize a memory cell that records if this table matches any ** row of the left table of the join. */ - assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) + assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN)) || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0 ); if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } /* Compute a safe address to jump to if we discover that the table for ** this loop is empty and can never contribute content. */ - for(j=iLevel; j>0 && pWInfo->a[j].iLeftJoin==0; j--){} + for(j=iLevel; j>0; j--){ + if( pWInfo->a[j].iLeftJoin ) break; + if( pWInfo->a[j].pRJ ) break; + } addrHalt = pWInfo->a[j].addrBrk; /* Special case of a FROM clause subquery implemented as a co-routine */ @@ -148509,7 +160681,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; int nConstraint = pLoop->nLTerm; - int iIn; /* Counter for IN constraints */ iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; @@ -148518,11 +160689,27 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pTerm = pLoop->aLTerm[j]; if( NEVER(pTerm==0) ) continue; if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; + if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ + int iTab = pParse->nTab++; + int iCache = ++pParse->nMem; + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); + sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); + }else{ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + } }else{ Expr *pRight = pTerm->pExpr->pRight; codeExprOrVector(pParse, pRight, iTarget, 1); + if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET + && pLoop->u.vtab.bOmitOffset + ){ + assert( pTerm->eOperator==WO_AUX ); + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->iOffset>0 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset); + VdbeComment((v,"Zero OFFSET counter")); + } } } sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); @@ -148539,44 +160726,54 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; pLevel->p2 = sqlite3VdbeCurrentAddr(v); assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - if( pLoop->wsFlags & WHERE_IN_ABLE ){ - iIn = pLevel->u.in.nIn; - }else{ - iIn = 0; - } - for(j=nConstraint-1; j>=0; j--){ + + for(j=0; jaLTerm[j]; - if( (pTerm->eOperator & WO_IN)!=0 ) iIn--; if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ disableTerm(pLevel, pTerm); - }else if( (pTerm->eOperator & WO_IN)!=0 - && sqlite3ExprVectorSize(pTerm->pExpr->pLeft)==1 + continue; + } + if( (pTerm->eOperator & WO_IN)!=0 + && (SMASKBIT32(j) & pLoop->u.vtab.mHandleIn)==0 + && !db->mallocFailed ){ Expr *pCompare; /* The comparison operator */ Expr *pRight; /* RHS of the comparison */ VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + int iIn; /* IN loop corresponding to the j-th constraint */ /* Reload the constraint value into reg[iReg+j+2]. The same value ** was loaded into the same register prior to the OP_VFilter, but ** the xFilter implementation might have changed the datatype or - ** encoding of the value in the register, so it *must* be reloaded. */ - assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); - if( !db->mallocFailed ){ - assert( iIn>=0 && iInu.in.nIn ); + ** encoding of the value in the register, so it *must* be reloaded. + */ + for(iIn=0; ALWAYS(iInu.in.nIn); iIn++){ pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[iIn].addrInTop); - assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); - assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); - assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); - testcase( pOp->opcode==OP_Rowid ); - sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + if( (pOp->opcode==OP_Column && pOp->p3==iReg+j+2) + || (pOp->opcode==OP_Rowid && pOp->p2==iReg+j+2) + ){ + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + break; + } } /* Generate code that will continue to the next row if - ** the IN constraint is not satisfied */ + ** the IN constraint is not satisfied + */ pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0); - assert( pCompare!=0 || db->mallocFailed ); - if( pCompare ){ - pCompare->pLeft = pTerm->pExpr->pLeft; + if( !db->mallocFailed ){ + int iFld = pTerm->u.x.iField; + Expr *pLeft = pTerm->pExpr->pLeft; + assert( pLeft!=0 ); + if( iFld>0 ){ + assert( pLeft->op==TK_VECTOR ); + assert( ExprUseXList(pLeft) ); + assert( iFld<=pLeft->x.pList->nExpr ); + pCompare->pLeft = pLeft->x.pList->a[iFld-1].pExpr; + }else{ + pCompare->pLeft = pLeft; + } pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); if( pRight ){ pRight->iTable = iReg+j+2; @@ -148585,11 +160782,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ); } pCompare->pLeft = 0; - sqlite3ExprDelete(db, pCompare); } + sqlite3ExprDelete(db, pCompare); } } - assert( iIn==0 || db->mallocFailed ); + /* These registers need to be preserved in case there is an IN operator ** loop. So we could deallocate the registers here (and potentially ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems @@ -148617,6 +160814,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; + if( pLevel->regFilter ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + iRowidReg, 1); + VdbeCoverage(v); + filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); + } sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); VdbeCoverage(v); pLevel->op = OP_Noop; @@ -148657,7 +160862,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + assert( TK_GE==TK_GT+3 ); /* ... is correct. */ assert( (pStart->wtFlags & TERM_VNULL)==0 ); testcase( pStart->wtFlags & TERM_VIRTUAL ); @@ -148942,6 +161147,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull); VdbeComment((v, "NULL-scan pass ctr")); } + if( pLevel->regFilter ){ + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + regBase, nEq); + VdbeCoverage(v); + filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); + } op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); @@ -148957,6 +161168,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); + if( pRangeStart || pRangeEnd ){ + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); + addrSeekScan = 0; + } VdbeCoverage(v); } sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -148993,16 +161209,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - if( addrSeekScan ){ - /* For a seek-scan that has a range on the lowest term of the index, - ** we have to make the top of the loop be code that sets the end - ** condition of the range. Otherwise, the OP_SeekScan might jump - ** over that initialization, leaving the range-end value set to the - ** range-start value, resulting in a wrong answer. - ** See ticket 5981a8c041a3c2f3 (2021-11-02). - */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - } + assert( addrSeekScan==0 ); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -149032,11 +161239,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } nConstraint++; } - sqlite3DbFree(db, zStartAff); - sqlite3DbFree(db, zEndAff); + if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff); + if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ - if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ @@ -149078,7 +161285,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Seek the table cursor, if required */ omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; + && (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0; if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ @@ -149095,27 +161302,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } if( pLevel->iLeftJoin==0 ){ - /* If pIdx is an index on one or more expressions, then look through - ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. Also attempt to translate references - ** to virtual columns in the table into references to (stored) columns - ** of the index. - ** - ** Do not do this for the RHS of a LEFT JOIN. This is because the - ** expression may be evaluated after OP_NullRow has been executed on - ** the cursor. In this case it is important to do the full evaluation, - ** as the result of the expression may not be NULL, even if all table - ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a - ** - ** Also, do not do this when processing one index an a multi-index - ** OR clause, since the transformation will become invalid once we - ** move forward to the next index. - ** https://sqlite.org/src/info/4e8e4857d32d401f - */ - if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ - whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); - } - /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of ** the partial index. @@ -149131,11 +161317,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* The following assert() is not a requirement, merely an observation: ** The OR-optimization doesn't work for the right hand table of ** a LEFT JOIN: */ - assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ); + assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ); } /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -149228,7 +161416,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, + pOrTab = sqlite3DbMallocRawNN(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); @@ -149269,7 +161457,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z + ** Then for every term xN, evaluate as the subexpression: xN AND y ** That way, terms in y that are factored into the disjunction will ** be picked up by the recursive calls to sqlite3WhereBegin() below. ** @@ -149281,6 +161469,20 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** This optimization also only applies if the (x1 OR x2 OR ...) term ** is not contained in the ON clause of a LEFT JOIN. ** See ticket http://www.sqlite.org/src/info/f2369304e4 + ** + ** 2022-02-04: Do not push down slices of a row-value comparison. + ** In other words, "w" or "y" may not be a slice of a vector. Otherwise, + ** the initialization of the right-hand operand of the vector comparison + ** might not occur, or might occur only in an OR branch that is not + ** taken. dbsqlfuzz 80a9fade844b4fb43564efc972bcb2c68270f5d1. + ** + ** 2022-03-03: Do not push down expressions that involve subqueries. + ** The subquery might get coded as a subroutine. Any table-references + ** in the subquery might be resolved to index-references for the index on + ** the OR branch in which the subroutine is coded. But if the subroutine + ** is invoked from a different OR branch that uses a different index, such + ** index-references will not work. tag-20220303a + ** https://sqlite.org/forum/forumpost/36937b197273d403 */ if( pWC->nTerm>1 ){ int iTerm; @@ -149289,9 +161491,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( &pWC->a[iTerm] == pTerm ) continue; testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); - if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_SLICE ); + if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED|TERM_SLICE))!=0 ){ + continue; + } if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + if( ExprHasProperty(pExpr, EP_Subquery) ) continue; /* tag-20220303a */ pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } @@ -149318,7 +161523,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( Expr *pDelete; /* Local copy of OR clause term */ int jmp1 = 0; /* Address of jump operation */ testcase( (pTabItem[0].fg.jointype & JT_LEFT)!=0 - && !ExprHasProperty(pOrExpr, EP_FromJoin) + && !ExprHasProperty(pOrExpr, EP_OuterON) ); /* See TH3 vtab25.400 and ticket 614b25314c766238 */ pDelete = pOrExpr = sqlite3ExprDup(db, pOrExpr, 0); if( db->mallocFailed ){ @@ -149331,10 +161536,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + WHERETRACE(0xffffffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, 0, WHERE_OR_SUBCLAUSE, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + assert( pSubWInfo || pParse->nErr ); if( pSubWInfo ){ WhereLoop *pSubLoop; int addrExplain = sqlite3WhereExplainOneScan( @@ -149456,7 +161661,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeGoto(v, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } + /* Set the P2 operand of the OP_Return opcode that will end the current + ** loop to point to this spot, which is the top of the next containing + ** loop. The byte-code formatter will use that P2 value as a hint to + ** indent everything in between the this point and the final OP_Return. + ** See tag-20220407a in vdbe.c and shell.c */ + assert( pLevel->op==OP_Return ); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -149500,6 +161713,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ @@ -149518,10 +161737,22 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } pE = pTerm->pExpr; assert( pE!=0 ); - if( (pTabItem->fg.jointype&JT_LEFT) && !ExprHasProperty(pE,EP_FromJoin) ){ - continue; + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ + if( !ExprHasProperty(pE,EP_OuterON|EP_InnerON) ){ + /* Defer processing WHERE clause constraints until after outer + ** join processing. tag-20220513a */ + continue; + }else if( (pTabItem->fg.jointype & JT_LEFT)==JT_LEFT + && !ExprHasProperty(pE,EP_OuterON) ){ + continue; + }else{ + Bitmask m = sqlite3WhereGetMask(&pWInfo->sMaskSet, pE->w.iJoin); + if( m & pLevel->notReady ){ + /* An ON clause that is not ripe */ + continue; + } + } } - if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ iNext = 2; continue; @@ -149548,12 +161779,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } #endif } -#ifdef WHERETRACE_ENABLED /* 0xffff */ +#ifdef WHERETRACE_ENABLED /* 0xffffffff */ if( sqlite3WhereTrace ){ VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", pWC->nTerm-j, pTerm, iLoop)); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("Coding auxiliary constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } @@ -149573,22 +161804,22 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** then we cannot use the "t1.a=t2.b" constraint, but we can code ** the implied "t1.a=123" constraint. */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + for(pTerm=pWC->a, j=pWC->nBase; j>0; j--, pTerm++){ Expr *pE, sEAlt; WhereTerm *pAlt; if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; if( pTerm->leftCursor!=iCur ) continue; - if( pTabItem->fg.jointype & JT_LEFT ) continue; + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ) continue; pE = pTerm->pExpr; -#ifdef WHERETRACE_ENABLED /* 0x800 */ - if( sqlite3WhereTrace & 0x800 ){ +#ifdef WHERETRACE_ENABLED /* 0x4001 */ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ sqlite3DebugPrintf("Coding transitive constraint:\n"); sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); } #endif - assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( !ExprHasProperty(pE, EP_OuterON) ); assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.x.leftColumn, notReady, @@ -149611,6 +161842,47 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pAlt->wtFlags |= TERM_CODED; } + /* For a RIGHT OUTER JOIN, record the fact that the current row has + ** been matched at least once. + */ + if( pLevel->pRJ ){ + Table *pTab; + int nPk; + int r; + int jmp1 = 0; + WhereRightJoin *pRJ = pLevel->pRJ; + + /* pTab is the right-hand table of the RIGHT JOIN. Generate code that + ** will record that the current row of that table has been matched at + ** least once. This is accomplished by storing the PK for the row in + ** both the iMatch index and the regBloom Bloom filter. + */ + pTab = pWInfo->pTabList->a[pLevel->iFrom].pTab; + if( HasRowid(pTab) ){ + r = sqlite3GetTempRange(pParse, 2); + sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1); + nPk = 1; + }else{ + int iPk; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + nPk = pPk->nKeyCol; + r = sqlite3GetTempRange(pParse, nPk+1); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+1+iPk); + } + } + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, pRJ->iMatch, 0, r+1, nPk); + VdbeCoverage(v); + VdbeComment((v, "match against %s", pTab->zName)); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r+1, nPk, r); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pRJ->iMatch, r, r+1, nPk); + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pRJ->regBloom, 0, r+1, nPk); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3VdbeJumpHere(v, jmp1); + sqlite3ReleaseTempRange(pParse, r, nPk+1); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -149618,7 +161890,31 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); VdbeComment((v, "record LEFT JOIN hit")); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + if( pLevel->pRJ==0 ){ + goto code_outer_join_constraints; /* WHERE clause constraints */ + } + } + + if( pLevel->pRJ ){ + /* Create a subroutine used to process all interior loops and code + ** of the RIGHT JOIN. During normal operation, the subroutine will + ** be in-line with the rest of the code. But at the end, a separate + ** loop will run that invokes this subroutine for unmatched rows + ** of pTab, with all tables to left begin set to NULL. + */ + WhereRightJoin *pRJ = pLevel->pRJ; + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pRJ->regReturn); + pRJ->addrSubrtn = sqlite3VdbeCurrentAddr(v); + assert( pParse->withinRJSubrtn < 255 ); + pParse->withinRJSubrtn++; + + /* WHERE clause constraints must be deferred until after outer join + ** row elimination has completed, since WHERE clause constraints apply + ** to the results of the OUTER JOIN. The following loop generates the + ** appropriate WHERE clause constraint checks. tag-20220513a. + */ + code_outer_join_constraints: + for(pTerm=pWC->a, j=0; jnBase; j++, pTerm++){ testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; @@ -149626,19 +161922,20 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pWInfo->untestedTerms ); continue; } + if( pTabItem->fg.jointype & JT_LTORJ ) continue; assert( pTerm->pExpr ); sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); pTerm->wtFlags |= TERM_CODED; } } -#if WHERETRACE_ENABLED /* 0x20800 */ - if( sqlite3WhereTrace & 0x20000 ){ +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x4000 ){ sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", iLevel); sqlite3WhereClausePrint(pWC); } - if( sqlite3WhereTrace & 0x800 ){ + if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", iLevel, (u64)pLevel->notReady); } @@ -149646,6 +161943,105 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( return pLevel->notReady; } +/* +** Generate the code for the loop that finds all non-matched terms +** for a RIGHT JOIN. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( + WhereInfo *pWInfo, + int iLevel, + WhereLevel *pLevel +){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; + WhereRightJoin *pRJ = pLevel->pRJ; + Expr *pSubWhere = 0; + WhereClause *pWC = &pWInfo->sWC; + WhereInfo *pSubWInfo; + WhereLoop *pLoop = pLevel->pWLoop; + SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + SrcList sFrom; + Bitmask mAll = 0; + int k; + + ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pTab->zName)); + sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn, + pRJ->regReturn); + for(k=0; ka[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; + mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + sqlite3VdbeAddOp3( + v, OP_Null, 0, pRight->regResult, + pRight->regResult + pRight->pSelect->pEList->nExpr-1 + ); + } + sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); + iIdxCur = pWInfo->a[k].iIdxCur; + if( iIdxCur ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); + } + } + if( (pTabItem->fg.jointype & JT_LTORJ)==0 ){ + mAll |= pLoop->maskSelf; + for(k=0; knTerm; k++){ + WhereTerm *pTerm = &pWC->a[k]; + if( (pTerm->wtFlags & (TERM_VIRTUAL|TERM_SLICE))!=0 + && pTerm->eOperator!=WO_ROWVAL + ){ + break; + } + if( pTerm->prereqAll & ~mAll ) continue; + if( ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) ) continue; + pSubWhere = sqlite3ExprAnd(pParse, pSubWhere, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } + } + sFrom.nSrc = 1; + sFrom.nAlloc = 1; + memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem)); + sFrom.a[0].fg.jointype = 0; + assert( pParse->withinRJSubrtn < 100 ); + pParse->withinRJSubrtn++; + pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0, + WHERE_RIGHT_JOIN, 0); + if( pSubWInfo ){ + int iCur = pLevel->iTabCur; + int r = ++pParse->nMem; + int nPk; + int jmp; + int addrCont = sqlite3WhereContinueLabel(pSubWInfo); + Table *pTab = pTabItem->pTab; + if( HasRowid(pTab) ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); + nPk = 1; + }else{ + int iPk; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + nPk = pPk->nKeyCol; + pParse->nMem += nPk - 1; + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); + } + } + jmp = sqlite3VdbeAddOp4Int(v, OP_Filter, pRJ->regBloom, 0, r, nPk); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, pRJ->iMatch, addrCont, r, nPk); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, jmp); + sqlite3VdbeAddOp2(v, OP_Gosub, pRJ->regReturn, pRJ->addrSubrtn); + sqlite3WhereEnd(pSubWInfo); + } + sqlite3ExprDelete(pParse->db, pSubWhere); + ExplainQueryPlanPop(pParse); + assert( pParse->withinRJSubrtn>0 ); + pParse->withinRJSubrtn--; +} + /************** End of wherecode.c *******************************************/ /************** Begin file whereexpr.c ***************************************/ /* @@ -149663,7 +162059,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** the WHERE clause of SQL statements. ** ** This file was originally part of where.c but was split out to improve -** readability and editabiliity. This file contains utility routines for +** readability and editability. This file contains utility routines for ** analyzing Expr objects in the WHERE clause. */ /* #include "sqliteInt.h" */ @@ -149714,7 +162110,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + pWC->a = sqlite3WhereMalloc(pWC->pWInfo, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, p); @@ -149723,12 +162119,10 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ return 0; } memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); - } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + pWC->nSlot = pWC->nSlot*2; } pTerm = &pWC->a[idx = pWC->nTerm++]; + if( (wtFlags & TERM_VIRTUAL)==0 ) pWC->nBase = pWC->nTerm; if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ @@ -149881,7 +162275,7 @@ static int isLikeOrGlob( ** range search. The third is because the caller assumes that the pattern ** consists of at least one character after all escapes have been ** removed. */ - if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){ + if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ @@ -149918,7 +162312,7 @@ static int isLikeOrGlob( if( pLeft->op!=TK_COLUMN || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT || (ALWAYS( ExprUseYTab(pLeft) ) - && pLeft->y.pTab + && ALWAYS(pLeft->y.pTab) && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ ){ int isNum; @@ -150035,8 +162429,7 @@ static int isAuxiliaryVtabOperator( ** MATCH(expression,vtab_column) */ pCol = pList->a[1].pExpr; - assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ for(i=0; ia[0].pExpr; assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); - testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); if( ExprIsVtab(pCol) ){ sqlite3_vtab *pVtab; sqlite3_module *pMod; @@ -150071,7 +162464,7 @@ static int isAuxiliaryVtabOperator( assert( pVtab!=0 ); assert( pVtab->pModule!=0 ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pMod = (sqlite3_module *)pVtab->pModule; + pMod = (sqlite3_module *)pVtab->pModule; if( pMod->xFindFunction!=0 ){ i = pMod->xFindFunction(pVtab,2, pExpr->u.zToken, &xNotUsed, &pNotUsed); if( i>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ @@ -150086,13 +162479,12 @@ static int isAuxiliaryVtabOperator( int res = 0; Expr *pLeft = pExpr->pLeft; Expr *pRight = pExpr->pRight; - assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); - testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) ); if( ExprIsVtab(pLeft) ){ res++; } - assert( pRight==0 || pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); - testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + assert( pRight==0 || pRight->op!=TK_COLUMN + || (ExprUseYTab(pRight) && pRight->y.pTab!=0) ); if( pRight && ExprIsVtab(pRight) ){ res++; SWAP(Expr*, pLeft, pRight); @@ -150113,9 +162505,9 @@ static int isAuxiliaryVtabOperator( ** a join, then transfer the appropriate markings over to derived. */ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - if( pDerived ){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; + if( pDerived && ExprHasProperty(pBase, EP_OuterON|EP_InnerON) ){ + pDerived->flags |= pBase->flags & (EP_OuterON|EP_InnerON); + pDerived->w.iJoin = pBase->w.iJoin; } } @@ -150445,7 +162837,7 @@ static void exprAnalyzeOrTerm( pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ assert( pOrTerm->eOperator & WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; + pOrTerm->wtFlags &= ~TERM_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and ** current term is from the first iteration. So skip this term. */ @@ -150456,7 +162848,7 @@ static void exprAnalyzeOrTerm( pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** or followed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); @@ -150486,7 +162878,7 @@ static void exprAnalyzeOrTerm( assert( pOrTerm->eOperator & WO_EQ ); assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 ); if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; + pOrTerm->wtFlags &= ~TERM_OK; }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1) )){ @@ -150502,7 +162894,7 @@ static void exprAnalyzeOrTerm( if( affRight!=0 && affRight!=affLeft ){ okToChngToIN = 0; }else{ - pOrTerm->wtFlags |= TERM_OR_OK; + pOrTerm->wtFlags |= TERM_OK; } } } @@ -150519,7 +162911,7 @@ static void exprAnalyzeOrTerm( Expr *pNew; /* The complete IN operator */ for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + if( (pOrTerm->wtFlags & TERM_OK)==0 ) continue; assert( pOrTerm->eOperator & WO_EQ ); assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 ); assert( pOrTerm->leftCursor==iCursor ); @@ -150569,7 +162961,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){ CollSeq *pColl; if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; - if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; aff1 = sqlite3ExprAffinity(pExpr->pLeft); aff2 = sqlite3ExprAffinity(pExpr->pRight); if( aff1!=aff2 @@ -150600,7 +162992,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ int i; for(i=0; inSrc; i++){ mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + if( pSrc->a[i].fg.isUsing==0 ){ + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); + } if( pSrc->a[i].fg.isTabFunc ){ mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg); } @@ -150626,35 +163020,40 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ */ static SQLITE_NOINLINE int exprMightBeIndexed2( SrcList *pFrom, /* The FROM clause */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ int *aiCurCol, /* Write the referenced table cursor and column here */ - Expr *pExpr /* An operand of a comparison operator */ + Expr *pExpr, /* An operand of a comparison operator */ + int j /* Start looking with the j-th pFrom entry */ ){ Index *pIdx; int i; int iCur; - for(i=0; mPrereq>1; i++, mPrereq>>=1){} - iCur = pFrom->a[i].iCursor; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->aColExpr==0 ) continue; - for(i=0; inKeyCol; i++){ - if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ - aiCurCol[0] = iCur; - aiCurCol[1] = XN_EXPR; - return 1; + do{ + iCur = pFrom->a[j].iCursor; + for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); + if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) + ){ + aiCurCol[0] = iCur; + aiCurCol[1] = XN_EXPR; + return 1; + } } } - } + }while( ++j < pFrom->nSrc ); return 0; } static int exprMightBeIndexed( SrcList *pFrom, /* The FROM clause */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ int *aiCurCol, /* Write the referenced table cursor & column here */ Expr *pExpr, /* An operand of a comparison operator */ int op /* The specific comparison operator */ ){ + int i; + /* If this expression is a vector to the left or right of a ** inequality constraint (>, <, >= or <=), perform the processing ** on the first element of the vector. */ @@ -150664,7 +163063,6 @@ static int exprMightBeIndexed( if( pExpr->op==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ assert( ExprUseXList(pExpr) ); pExpr = pExpr->x.pList->a[0].pExpr; - } if( pExpr->op==TK_COLUMN ){ @@ -150672,9 +163070,16 @@ static int exprMightBeIndexed( aiCurCol[1] = pExpr->iColumn; return 1; } - if( mPrereq==0 ) return 0; /* No table references */ - if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ - return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr); + + for(i=0; inSrc; i++){ + Index *pIdx; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr ){ + return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); + } + } + } + return 0; } @@ -150705,8 +163110,8 @@ static void exprAnalyze( WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask prereqLeft; /* Prerequisites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequisites of pExpr */ Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ @@ -150720,10 +163125,13 @@ static void exprAnalyze( if( db->mallocFailed ){ return; } + assert( pWC->nTerm > idxTerm ); pTerm = &pWC->a[idxTerm]; pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; + assert( pExpr!=0 ); /* Because malloc() has not failed */ assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + pMaskSet->bVarSelect = 0; prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ @@ -150734,22 +163142,50 @@ static void exprAnalyze( }else{ pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; + prereqAll = prereqLeft | pTerm->prereqRight; }else{ pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); + if( pExpr->pLeft==0 + || ExprHasProperty(pExpr, EP_xIsSelect|EP_IfNullRow) + || pExpr->x.pList!=0 + ){ + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); + }else{ + prereqAll = prereqLeft | pTerm->prereqRight; + } } - pMaskSet->bVarSelect = 0; - prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ - if( (prereqAll>>1)>=x ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; + +#ifdef SQLITE_DEBUG + if( prereqAll!=sqlite3WhereExprUsageNN(pMaskSet, pExpr) ){ + printf("\n*** Incorrect prereqAll computed for:\n"); + sqlite3TreeViewExpr(0,pExpr,0); + assert( 0 ); + } +#endif + + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) ){ + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->w.iJoin); + if( ExprHasProperty(pExpr, EP_OuterON) ){ + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + if( (prereqAll>>1)>=x ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } + }else if( (prereqAll>>1)>=x ){ + /* The ON clause of an INNER JOIN references a table to its right. + ** Most other SQL database engines raise an error. But SQLite versions + ** 3.0 through 3.38 just put the ON clause constraint into the WHERE + ** clause and carried on. Beginning with 3.39, raise an error only + ** if there is a RIGHT or FULL JOIN in the query. This makes SQLite + ** more like other systems, and also preserves legacy. */ + if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } + ExprClearProperty(pExpr, EP_InnerON); } } pTerm->prereqAll = prereqAll; @@ -150769,7 +163205,7 @@ static void exprAnalyze( pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr; } - if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){ + if( exprMightBeIndexed(pSrc, aiCurCol, pLeft, op) ){ pTerm->leftCursor = aiCurCol[0]; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); pTerm->u.x.leftColumn = aiCurCol[1]; @@ -150777,7 +163213,7 @@ static void exprAnalyze( } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; if( pRight - && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op) + && exprMightBeIndexed(pSrc, aiCurCol, pRight, op) && !ExprHasProperty(pRight, EP_FixedCol) ){ WhereTerm *pNew; @@ -150817,11 +163253,11 @@ static void exprAnalyze( pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; }else if( op==TK_ISNULL - && !ExprHasProperty(pExpr,EP_FromJoin) + && !ExprHasProperty(pExpr,EP_OuterON) && 0==sqlite3ExprCanBeNull(pLeft) ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->op = TK_TRUEFALSE; + pExpr->op = TK_TRUEFALSE; /* See tag-20230504-1 */ pExpr->u.zToken = "false"; ExprSetProperty(pExpr, EP_IsFalse); pTerm->prereqAll = 0; @@ -150888,7 +163324,7 @@ static void exprAnalyze( else if( pExpr->op==TK_NOTNULL ){ if( pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && !ExprHasProperty(pExpr, EP_FromJoin) + && !ExprHasProperty(pExpr, EP_OuterON) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -150988,7 +163424,6 @@ static void exprAnalyze( transferJoinMarkings(pNewExpr1, pExpr); idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), @@ -150996,6 +163431,7 @@ static void exprAnalyze( transferJoinMarkings(pNewExpr2, pExpr); idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew1); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ @@ -151011,7 +163447,10 @@ static void exprAnalyze( ** no longer used. ** ** This is only required if at least one side of the comparison operation - ** is not a sub-select. */ + ** is not a sub-select. + ** + ** tag-20220128a + */ if( (pExpr->op==TK_EQ || pExpr->op==TK_IS) && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 && sqlite3ExprVectorSize(pExpr->pRight)==nLeft @@ -151028,12 +163467,12 @@ static void exprAnalyze( pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight); transferJoinMarkings(pNew, pExpr); - idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); + idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC|TERM_SLICE); exprAnalyze(pSrc, pWC, idxNew); } pTerm = &pWC->a[idxTerm]; pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ - pTerm->eOperator = 0; + pTerm->eOperator = WO_ROWVAL; } /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create @@ -151049,7 +163488,7 @@ static void exprAnalyze( && pTerm->u.x.iField==0 && pExpr->pLeft->op==TK_VECTOR && ALWAYS( ExprUseXSelect(pExpr) ) - && pExpr->x.pSelect->pPrior==0 + && (pExpr->x.pSelect->pPrior==0 || (pExpr->x.pSelect->selFlags & SF_Values)) #ifndef SQLITE_OMIT_WINDOWFUNC && pExpr->x.pSelect->pWin==0 #endif @@ -151058,7 +163497,7 @@ static void exprAnalyze( int i; for(i=0; ipLeft); i++){ int idxNew; - idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); + idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL|TERM_SLICE); pWC->a[idxNew].u.x.iField = i+1; exprAnalyze(pSrc, pWC, idxNew); markTermAsChild(pWC, idxNew, idxTerm); @@ -151089,9 +163528,9 @@ static void exprAnalyze( Expr *pNewExpr; pNewExpr = sqlite3PExpr(pParse, TK_MATCH, 0, sqlite3ExprDup(db, pRight, 0)); - if( ExprHasProperty(pExpr, EP_FromJoin) && pNewExpr ){ - ExprSetProperty(pNewExpr, EP_FromJoin); - pNewExpr->iRightJoinTable = pExpr->iRightJoinTable; + if( ExprHasProperty(pExpr, EP_OuterON) && pNewExpr ){ + ExprSetProperty(pNewExpr, EP_OuterON); + pNewExpr->w.iJoin = pExpr->w.iJoin; } idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); @@ -151154,6 +163593,123 @@ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ } } +/* +** Add either a LIMIT (if eMatchOp==SQLITE_INDEX_CONSTRAINT_LIMIT) or +** OFFSET (if eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET) term to the +** where-clause passed as the first argument. The value for the term +** is found in register iReg. +** +** In the common case where the value is a simple integer +** (example: "LIMIT 5 OFFSET 10") then the expression codes as a +** TK_INTEGER so that it will be available to sqlite3_vtab_rhs_value(). +** If not, then it codes as a TK_REGISTER expression. +*/ +static void whereAddLimitExpr( + WhereClause *pWC, /* Add the constraint to this WHERE clause */ + int iReg, /* Register that will hold value of the limit/offset */ + Expr *pExpr, /* Expression that defines the limit/offset */ + int iCsr, /* Cursor to which the constraint applies */ + int eMatchOp /* SQLITE_INDEX_CONSTRAINT_LIMIT or _OFFSET */ +){ + Parse *pParse = pWC->pWInfo->pParse; + sqlite3 *db = pParse->db; + Expr *pNew; + int iVal = 0; + + if( sqlite3ExprIsInteger(pExpr, &iVal) && iVal>=0 ){ + Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); + if( pVal==0 ) return; + ExprSetProperty(pVal, EP_IntValue); + pVal->u.iValue = iVal; + pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); + }else{ + Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); + if( pVal==0 ) return; + pVal->iTable = iReg; + pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); + } + if( pNew ){ + WhereTerm *pTerm; + int idx; + idx = whereClauseInsert(pWC, pNew, TERM_DYNAMIC|TERM_VIRTUAL); + pTerm = &pWC->a[idx]; + pTerm->leftCursor = iCsr; + pTerm->eOperator = WO_AUX; + pTerm->eMatchOp = eMatchOp; + } +} + +/* +** Possibly add terms corresponding to the LIMIT and OFFSET clauses of the +** SELECT statement passed as the second argument. These terms are only +** added if: +** +** 1. The SELECT statement has a LIMIT clause, and +** 2. The SELECT statement is not an aggregate or DISTINCT query, and +** 3. The SELECT statement has exactly one object in its from clause, and +** that object is a virtual table, and +** 4. There are no terms in the WHERE clause that will not be passed +** to the virtual table xBestIndex method. +** 5. The ORDER BY clause, if any, will be made available to the xBestIndex +** method. +** +** LIMIT and OFFSET terms are ignored by most of the planner code. They +** exist only so that they may be passed to the xBestIndex method of the +** single virtual table in the FROM clause of the SELECT. +*/ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ + assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ + if( p->pGroupBy==0 + && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ + && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ + ){ + ExprList *pOrderBy = p->pOrderBy; + int iCsr = p->pSrc->a[0].iCursor; + int ii; + + /* Check condition (4). Return early if it is not met. */ + for(ii=0; iinTerm; ii++){ + if( pWC->a[ii].wtFlags & TERM_CODED ){ + /* This term is a vector operation that has been decomposed into + ** other, subsequent terms. It can be ignored. See tag-20220128a */ + assert( pWC->a[ii].wtFlags & TERM_VIRTUAL ); + assert( pWC->a[ii].eOperator==WO_ROWVAL ); + continue; + } + if( pWC->a[ii].nChild ){ + /* If this term has child terms, then they are also part of the + ** pWC->a[] array. So this term can be ignored, as a LIMIT clause + ** will only be added if each of the child terms passes the + ** (leftCursor==iCsr) test below. */ + continue; + } + if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].prereqRight!=0 ) return; + } + + /* Check condition (5). Return early if it is not met. */ + if( pOrderBy ){ + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pOrderBy->a[ii].pExpr; + if( pExpr->op!=TK_COLUMN ) return; + if( pExpr->iTable!=iCsr ) return; + if( pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) return; + } + } + + /* All conditions are met. Add the terms to the where-clause object. */ + assert( p->pLimit->op==TK_LIMIT ); + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, + iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); + } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } + } +} + /* ** Initialize a preallocated WhereClause structure. */ @@ -151165,6 +163721,7 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( pWC->hasOr = 0; pWC->pOuter = 0; pWC->nTerm = 0; + pWC->nBase = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; } @@ -151175,22 +163732,36 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( ** sqlite3WhereClauseInit(). */ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); + assert( pWC->nTerm>=pWC->nBase ); + if( pWC->nTerm>0 ){ + WhereTerm *a = pWC->a; + WhereTerm *aLast = &pWC->a[pWC->nTerm-1]; +#ifdef SQLITE_DEBUG + int i; + /* Verify that every term past pWC->nBase is virtual */ + for(i=pWC->nBase; inTerm; i++){ + assert( (pWC->a[i].wtFlags & TERM_VIRTUAL)!=0 ); } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); +#endif + while(1){ + assert( a->eMatchOp==0 || a->eOperator==WO_AUX ); + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & (TERM_ORINFO|TERM_ANDINFO) ){ + if( a->wtFlags & TERM_ORINFO ){ + assert( (a->wtFlags & TERM_ANDINFO)==0 ); + whereOrInfoDelete(db, a->u.pOrInfo); + }else{ + assert( (a->wtFlags & TERM_ANDINFO)!=0 ); + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( a==aLast ) break; + a++; } } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } } @@ -151198,15 +163769,38 @@ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ ** These routines walk (recursively) an expression tree and generate ** a bitmask indicating which tables are used in that expression ** tree. +** +** sqlite3WhereExprUsage(MaskSet, Expr) -> +** +** Return a Bitmask of all tables referenced by Expr. Expr can be +** be NULL, in which case 0 is returned. +** +** sqlite3WhereExprUsageNN(MaskSet, Expr) -> +** +** Same as sqlite3WhereExprUsage() except that Expr must not be +** NULL. The "NN" suffix on the name stands for "Not Null". +** +** sqlite3WhereExprListUsage(MaskSet, ExprList) -> +** +** Return a Bitmask of all tables referenced by every expression +** in the expression list ExprList. ExprList can be NULL, in which +** case 0 is returned. +** +** sqlite3WhereExprUsageFull(MaskSet, ExprList) -> +** +** Internal use only. Called only by sqlite3WhereExprUsageNN() for +** complex expressions that require pushing register values onto +** the stack. Many calls to sqlite3WhereExprUsageNN() do not need +** the more complex analysis done by this routine. Hence, the +** computations done by this routine are broken out into a separate +** "no-inline" function to avoid the stack push overhead in the +** common case where it is not needed. */ -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ +static SQLITE_NOINLINE Bitmask sqlite3WhereExprUsageFull( + WhereMaskSet *pMaskSet, + Expr *p +){ Bitmask mask; - if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ - return sqlite3WhereGetMask(pMaskSet, p->iTable); - }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ - assert( p->op!=TK_IF_NULL_ROW ); - return 0; - } mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); if( p->pRight ){ @@ -151228,6 +163822,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ #endif return mask; } +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ + if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ + return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; + } + return sqlite3WhereExprUsageFull(pMaskSet, p); +} SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; } @@ -151285,6 +163888,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( if( pArgs==0 ) return; for(j=k=0; jnExpr; j++){ Expr *pRhs; + u32 joinType; while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} if( k>=pTab->nCol ){ sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", @@ -151297,12 +163901,19 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( pColRef->iColumn = k++; assert( ExprUseYTab(pColRef) ); pColRef->y.pTab = pTab; + pItem->colUsed |= sqlite3ExprColUsed(pColRef); pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3SetJoinExpr(pTerm, pItem->iCursor); + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ + joinType = EP_OuterON; + }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ + joinType = EP_InnerON; } + sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); } } @@ -151341,8 +163952,14 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( */ typedef struct HiddenIndexInfo HiddenIndexInfo; struct HiddenIndexInfo { - WhereClause *pWC; /* The Where clause being analyzed */ - Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The Where clause being analyzed */ + Parse *pParse; /* The parsing context */ + int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ + u32 mIn; /* Mask of terms that are IN (...) */ + u32 mHandleIn; /* Terms that vtab will handle as IN (...) */ + sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST + ** because extra space is allocated to hold up + ** to nTerm such values */ }; /* Forward declaration of methods */ @@ -151372,7 +163989,7 @@ SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** block sorting is required. */ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; } /* @@ -151407,7 +164024,7 @@ SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ } pInner = &pWInfo->a[pWInfo->nLevel-1]; assert( pInner->addrNxt!=0 ); - return pInner->addrNxt; + return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt; } /* @@ -151545,7 +164162,12 @@ whereOrInsert_done: SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ int i; assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ + assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 ); + assert( iCursor>=-1 ); + if( pMaskSet->ix[0]==iCursor ){ + return 1; + } + for(i=1; in; i++){ if( pMaskSet->ix[i]==iCursor ){ return MASKBIT(i); } @@ -151553,6 +164175,30 @@ SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ return 0; } +/* Allocate memory that is automatically freed when pWInfo is freed. +*/ +SQLITE_PRIVATE void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){ + WhereMemBlock *pBlock; + pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock)); + if( pBlock ){ + pBlock->pNext = pWInfo->pMemToFree; + pBlock->sz = nByte; + pWInfo->pMemToFree = pBlock; + pBlock++; + } + return (void*)pBlock; +} +SQLITE_PRIVATE void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){ + void *pNew = sqlite3WhereMalloc(pWInfo, nByte); + if( pNew && pOld ){ + WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld; + pOldBlk--; + assert( pOldBlk->szsz); + } + return pNew; +} + /* ** Create a new mask for cursor iCursor. ** @@ -151578,6 +164224,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -151606,7 +164288,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ && (iColumn!=XN_EXPR || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, pScan->pIdxExpr,iCur)==0) - && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) @@ -151628,16 +164310,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } @@ -151730,16 +164420,16 @@ static WhereTerm *whereScanInit( if( pIdx ){ int j = iColumn; iColumn = pIdx->aiColumn[j]; - if( iColumn==XN_EXPR ){ - pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; - pScan->zCollName = pIdx->azColl[j]; - pScan->aiColumn[0] = XN_EXPR; - return whereScanInitIndexExpr(pScan); - }else if( iColumn==pIdx->pTable->iPKey ){ + if( iColumn==pIdx->pTable->iPKey ){ iColumn = XN_ROWID; }else if( iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; pScan->zCollName = pIdx->azColl[j]; + }else if( iColumn==XN_EXPR ){ + pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + pScan->zCollName = pIdx->azColl[j]; + pScan->aiColumn[0] = XN_EXPR; + return whereScanInitIndexExpr(pScan); } }else if( iColumn==XN_EXPR ){ return 0; @@ -151954,11 +164644,22 @@ static void translateColumnToCopy( for(; iStartp1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart); + } +#endif pOp->opcode = OP_Copy; pOp->p1 = pOp->p2 + iRegister; pOp->p2 = pOp->p3; pOp->p3 = 0; + pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */ }else if( pOp->opcode==OP_Rowid ){ +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart); + } +#endif pOp->opcode = OP_Sequence; pOp->p1 = iAutoidxCur; #ifdef SQLITE_ALLOW_ROWID_IN_VIEW @@ -151978,16 +164679,22 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + sqlite3DebugPrintf( + " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", i, p->aConstraint[i].iColumn, p->aConstraint[i].iTermOffset, p->aConstraint[i].op, - p->aConstraint[i].usable); + p->aConstraint[i].usable, + sqlite3_vtab_collation(p,i)); } for(i=0; inOrderBy; i++){ sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", @@ -151996,9 +164703,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -152012,10 +164723,47 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define whereTraceIndexInfoInputs(A) -#define whereTraceIndexInfoOutputs(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif +/* +** We know that pSrc is an operand of an outer join. Return true if +** pTerm is a constraint that is compatible with that join. +** +** pTerm must be EP_OuterON if pSrc is the right operand of an +** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc +** is the left operand of a RIGHT join. +** +** See https://sqlite.org/forum/forumpost/206d99a16dd9212f +** for an example of a WHERE clause constraints that may not be used on +** the right table of a RIGHT JOIN because the constraint implies a +** not-NULL condition on the left table of the RIGHT JOIN. +*/ +static int constraintCompatibleWithOuterJoin( + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc /* Table we are trying to access */ +){ + assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); + testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) + testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) + || pTerm->pExpr->w.iJoin != pSrc->iCursor + ){ + return 0; + } + if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + ){ + return 0; + } + return 1; +} + + + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -152023,21 +164771,18 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ ** index existed. */ static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - SrcItem *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc, /* Table we are trying to access */ + const Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; - if( (pSrc->fg.jointype & JT_LEFT) - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - && (pTerm->eOperator & WO_IS) + assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) ){ - /* Cannot use an IS term from the WHERE clause as an index driver for - ** the RHS of a LEFT JOIN. Such a term can only be used if it is from - ** the ON clause. */ - return 0; + return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); @@ -152051,16 +164796,66 @@ static int termCanDriveIndex( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON () [WHERE ] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + sqlite3_str *pStr = sqlite3_str_new(pParse->db); + sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + sqlite3_str_appendf(pStr, "%s%s", zSep, zName); + zSep = ", "; + } + zText = sqlite3_str_finish(pStr); + if( zText==0 ){ + sqlite3OomFault(pParse->db); + }else{ + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE " : "") + ); + sqlite3_free(zText); + } + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator ** makes use of the automatic index. */ -static void constructAutomaticIndex( +static SQLITE_NOINLINE void constructAutomaticIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ - SrcItem *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ + const Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ int nKeyCol; /* Number of columns in the constructed index */ @@ -152080,12 +164875,17 @@ static void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ - SrcItem *pTabItem; /* FROM clause term being indexed */ + SrcList *pTabList; /* The complete FROM clause */ + SrcItem *pSrc; /* The FROM clause term to get the next index */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -152096,19 +164896,20 @@ static void constructAutomaticIndex( /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ nKeyCol = 0; + pTabList = pWC->pWInfo->pTabList; + pSrc = &pTabList->a[pLevel->iFrom]; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermpExpr; - assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ - || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ - || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ - if( pLoop->prereq==0 - && (pTerm->wtFlags & TERM_VIRTUAL)==0 - && !ExprHasProperty(pExpr, EP_FromJoin) - && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + /* Make the automatic index a partial index if there are terms in the + ** WHERE clause (or the ON clause of a LEFT join) that constrain which + ** rows of the target table (pSrc) that can be used. */ + if( (pTerm->wtFlags & TERM_VIRTUAL)==0 + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) + ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); } @@ -152148,7 +164949,11 @@ static void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS & ~idxCols; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -152184,6 +164989,16 @@ static void constructAutomaticIndex( assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -152210,21 +165025,27 @@ static void constructAutomaticIndex( pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); + pLevel->regFilter = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); + } /* Fill the automatic index with content */ - pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; + assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); + if( pSrc->fg.viaCoroutine ){ + int regYield = pSrc->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -152237,17 +165058,22 @@ static void constructAutomaticIndex( regBase = sqlite3GenerateIndexKey( pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 ); + if( pLevel->regFilter ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, + regBase, pLoop->u.btree.nEq); + } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->fg.viaCoroutine ){ + if( pSrc->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pTabItem->regResult, pLevel->iIdxCur); + pSrc->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); - pTabItem->fg.viaCoroutine = 0; + pSrc->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); @@ -152257,28 +165083,169 @@ static void constructAutomaticIndex( /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ +/* +** Generate bytecode that will initialize a Bloom filter that is appropriate +** for pLevel. +** +** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER +** flag set, initialize a Bloomfilter for them as well. Except don't do +** this recursive initialization if the SQLITE_BloomPulldown optimization has +** been turned off. +** +** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared +** from the loop, but the regFilter value is set to a register that implements +** the Bloom filter. When regFilter is positive, the +** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter +** and skip the subsequence B-Tree seek if the Bloom filter indicates that +** no matching rows exist. +** +** This routine may only be called if it has previously been determined that +** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit +** is set. +*/ +static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( + WhereInfo *pWInfo, /* The WHERE clause */ + int iLevel, /* Index in pWInfo->a[] that is pLevel */ + WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */ + Bitmask notReady /* Loops that are not ready */ +){ + int addrOnce; /* Address of opening OP_Once */ + int addrTop; /* Address of OP_Rewind */ + int addrCont; /* Jump here to skip a row */ + const WhereTerm *pTerm; /* For looping over WHERE clause terms */ + const WhereTerm *pWCEnd; /* Last WHERE clause term */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ + int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + saved_pIdxPartExpr = pParse->pIdxPartExpr; + pParse->pIdxEpr = 0; + pParse->pIdxPartExpr = 0; + + assert( pLoop!=0 ); + assert( v!=0 ); + assert( pLoop->wsFlags & WHERE_BLOOMFILTER ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + do{ + const SrcList *pTabList; + const SrcItem *pItem; + const Table *pTab; + u64 sz; + int iSrc; + sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); + addrCont = sqlite3VdbeMakeLabel(pParse); + iCur = pLevel->iTabCur; + pLevel->regFilter = ++pParse->nMem; + + /* The Bloom filter is a Blob held in a register. Initialize it + ** to zero-filled blob of at least 80K bits, but maybe more if the + ** estimated size of the table is larger. We could actually + ** measure the size of the table at run-time using OP_Count with + ** P3==1 and use that value to initialize the blob. But that makes + ** testing complicated. By basing the blob size on the value in the + ** sqlite_stat1 table, testing is much easier. + */ + pTabList = pWInfo->pTabList; + iSrc = pLevel->iFrom; + pItem = &pTabList->a[iSrc]; + assert( pItem!=0 ); + pTab = pItem->pTab; + assert( pTab!=0 ); + sz = sqlite3LogEstToInt(pTab->nRowLogEst); + if( sz<10000 ){ + sz = 10000; + }else if( sz>10000000 ){ + sz = 10000000; + } + sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter); + + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm]; + for(pTerm=pWInfo->sWC.a; pTermpExpr; + if( (pTerm->wtFlags & TERM_VIRTUAL)==0 + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) + ){ + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + } + } + if( pLoop->wsFlags & WHERE_IPK ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1); + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1); + sqlite3ReleaseTempReg(pParse, r1); + }else{ + Index *pIdx = pLoop->u.btree.pIndex; + int n = pLoop->u.btree.nEq; + int r1 = sqlite3GetTempRange(pParse, n); + int jj; + for(jj=0; jjpTable==pItem->pTab ); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); + } + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); + sqlite3ReleaseTempRange(pParse, r1, n); + } + sqlite3VdbeResolveLabel(v, addrCont); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + pLoop->wsFlags &= ~WHERE_BLOOMFILTER; + if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; + while( ++iLevel < pWInfo->nLevel ){ + const SrcItem *pTabItem; + pLevel = &pWInfo->a[iLevel]; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue; + pLoop = pLevel->pWLoop; + if( NEVER(pLoop==0) ) continue; + if( pLoop->prereq & notReady ) continue; + if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN)) + ==WHERE_BLOOMFILTER + ){ + /* This is a candidate for bloom-filter pull-down (early evaluation). + ** The test that WHERE_COLUMN_IN is omitted is important, as we are + ** not able to do early evaluation of bloom filters that make use of + ** the IN operator */ + break; + } + } + }while( iLevel < pWInfo->nLevel ); + sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; + pParse->pIdxPartExpr = saved_pIdxPartExpr; +} + + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Allocate and populate an sqlite3_index_info structure. It is the ** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). +** by passing the pointer returned by this function to freeIndexInfo(). */ static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, /* The parsing context */ + WhereInfo *pWInfo, /* The WHERE clause */ WhereClause *pWC, /* The WHERE clause being analyzed */ Bitmask mUnusable, /* Ignore terms with these prereqs */ SrcItem *pSrc, /* The FROM clause term that is the vtab */ - ExprList *pOrderBy, /* The ORDER BY clause */ u16 *pmNoOmit /* Mask of terms not to omit */ ){ int i, j; int nTerm; + Parse *pParse = pWInfo->pParse; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_orderby *pIdxOrderBy; struct sqlite3_index_constraint_usage *pUsage; @@ -152287,10 +165254,21 @@ static sqlite3_index_info *allocateIndexInfo( int nOrderBy; sqlite3_index_info *pIdxInfo; u16 mNoOmit = 0; + const Table *pTab; + int eDistinct = 0; + ExprList *pOrderBy = pWInfo->pOrderBy; + + assert( pSrc!=0 ); + pTab = pSrc->pTab; + assert( pTab!=0 ); + assert( IsVirtual(pTab) ); - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ + /* Find all WHERE clause constraints referring to this virtual table. + ** Mark each term with the TERM_OK flag. Set nTerm to the number of + ** terms found. + */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + pTerm->wtFlags &= ~TERM_OK; if( pTerm->leftCursor != pSrc->iCursor ) continue; if( pTerm->prereqRight & mUnusable ) continue; assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); @@ -152300,9 +165278,17 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - assert( pTerm->u.x.leftColumn>=(-1) ); + assert( pTerm->u.x.leftColumn>=XN_ROWID ); + assert( pTerm->u.x.leftColumnnCol ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; + } nTerm++; + pTerm->wtFlags |= TERM_OK; } /* If the ORDER BY clause contains only columns in the current @@ -152314,11 +165300,49 @@ static sqlite3_index_info *allocateIndexInfo( int n = pOrderBy->nExpr; for(i=0; ia[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; - if( pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL ) break; + Expr *pE2; + + /* Skip over constant terms in the ORDER BY clause */ + if( sqlite3ExprIsConstant(0, pExpr) ){ + continue; + } + + /* Virtual tables are unable to deal with NULLS FIRST */ + if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; + + /* First case - a direct column references without a COLLATE operator */ + if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){ + assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumnnCol ); + continue; + } + + /* 2nd case - a column reference with a COLLATE operator. Only match + ** of the COLLATE operator matches the collation of the column. */ + if( pExpr->op==TK_COLLATE + && (pE2 = pExpr->pLeft)->op==TK_COLUMN + && pE2->iTable==pSrc->iCursor + ){ + const char *zColl; /* The collating sequence name */ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pE2->iColumn>=XN_ROWID && pE2->iColumnnCol ); + pExpr->iColumn = pE2->iColumn; + if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */ + zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]); + if( zColl==0 ) zColl = sqlite3StrBINARY; + if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue; + } + + /* No matches cause a break out of the loop */ + break; } - if( i==n){ + if( i==n ){ nOrderBy = n; + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ + eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ + eDistinct = 1; + } } } @@ -152326,47 +165350,35 @@ static sqlite3_index_info *allocateIndexInfo( */ pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) ); + + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) + + sizeof(sqlite3_value*)*nTerm ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); return 0; } pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; - pIdxCons = (struct sqlite3_index_constraint*)&pHidden[1]; + pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm]; pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - pIdxInfo->nOrderBy = nOrderBy; pIdxInfo->aConstraint = pIdxCons; pIdxInfo->aOrderBy = pIdxOrderBy; pIdxInfo->aConstraintUsage = pUsage; pHidden->pWC = pWC; pHidden->pParse = pParse; + pHidden->eDistinct = eDistinct; + pHidden->mIn = 0; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ u16 op; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - - /* tag-20191211-002: WHERE-clause constraints are not useful to the - ** right-hand table of a LEFT JOIN. See tag-20191211-001 for the - ** equivalent restriction for ordinary tables. */ - if( (pSrc->fg.jointype & JT_LEFT)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - ){ - continue; - } - assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - assert( pTerm->u.x.leftColumn>=(-1) ); + if( (pTerm->wtFlags & TERM_OK)==0 ) continue; pIdxCons[j].iColumn = pTerm->u.x.leftColumn; pIdxCons[j].iTermOffset = i; op = pTerm->eOperator & WO_ALL; - if( op==WO_IN ) op = WO_EQ; + if( op==WO_IN ){ + if( (pTerm->wtFlags & TERM_SLICE)==0 ){ + pHidden->mIn |= SMASKBIT32(j); + } + op = WO_EQ; + } if( op==WO_AUX ){ pIdxCons[j].op = pTerm->eMatchOp; }else if( op & (WO_ISNULL|WO_IS) ){ @@ -152399,17 +165411,42 @@ static sqlite3_index_info *allocateIndexInfo( j++; } + assert( j==nTerm ); pIdxInfo->nConstraint = j; - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; + assert( pExpr->op==TK_COLUMN + || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN + && pExpr->iColumn==pExpr->pLeft->iColumn) ); + pIdxOrderBy[j].iColumn = pExpr->iColumn; + pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC; + j++; } + pIdxInfo->nOrderBy = j; *pmNoOmit = mNoOmit; return pIdxInfo; } +/* +** Free an sqlite3_index_info structure allocated by allocateIndexInfo() +** and possibly modified by xBestIndex methods. +*/ +static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden; + int i; + assert( pIdxInfo!=0 ); + pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + assert( pHidden->pParse!=0 ); + assert( pHidden->pParse->db==db ); + for(i=0; inConstraint; i++){ + sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ + pHidden->aRhs[i] = 0; + } + sqlite3DbFree(db, pIdxInfo); +} + /* ** The table object reference passed as the second argument to this function ** must represent a virtual table. This function invokes the xBestIndex() @@ -152430,9 +165467,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; - whereTraceIndexInfoInputs(p); + whereTraceIndexInfoInputs(p, pTab); + pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); - whereTraceIndexInfoOutputs(p); + pParse->db->nSchemaLock--; + whereTraceIndexInfoOutputs(p, pTab); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ @@ -152443,6 +165482,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; return rc; @@ -152487,6 +165529,7 @@ static int whereKeyStats( assert( pIdx->nSample>0 ); assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search ** is simply the aSample[] array. If the samples in aSample[] contain more @@ -152531,7 +165574,12 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = MIN(pRec->nField, pIdx->nSample); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -152597,12 +165645,12 @@ static int whereKeyStats( if( iCol>0 ){ pRec->nField = iCol; assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } if( i>0 ){ pRec->nField = nField; assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } } } @@ -152619,7 +165667,7 @@ static int whereKeyStats( ** is larger than all samples in the array. */ tRowcnt iUpper, iGap; if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = pIdx->nRowEst0; }else{ iUpper = aSample[i].anLt[iCol]; } @@ -152694,7 +165742,7 @@ SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCo ** Value pLoop->nOut is currently set to the estimated number of rows ** visited for scanning (a=? AND b=?). This function reduces that estimate ** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple +** on the stat4 data for the index. this scan will be performed multiple ** times (once for each (a,b) combination that matches a=?) is dealt with ** by the caller. ** @@ -152775,7 +165823,7 @@ static int whereRangeSkipScanEst( int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); pLoop->nOut -= nAdjust; *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", nLower, nUpper, nAdjust*-1, pLoop->nOut)); } @@ -152946,14 +165994,15 @@ static int whereRangeScanEst( ** sample, then assume they are 4x more selective. This brings ** the estimated selectivity more in line with what it would be ** if estimated without the use of STAT4 tables. */ - if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + if( iLwrIdx==iUprIdx ){ nNew -= 20; } + assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } if( nNewwtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); @@ -152986,7 +166035,7 @@ static int whereRangeScanEst( if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", pLoop->nOut, nOut)); } #endif @@ -153051,7 +166100,7 @@ static int whereEqualScanEst( pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; @@ -153099,9 +166148,9 @@ static int whereInScanEst( } if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; + if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; @@ -153122,7 +166171,7 @@ SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ memcpy(zType, "....", 5); if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L'; if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; if( pTerm->eOperator & WO_SINGLE ){ assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); @@ -153170,17 +166219,34 @@ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes -*/ -SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); +** +** Format example: +** +** .--- Position in WHERE clause rSetup, rRun, nOut ---. +** | | +** | .--- selfMask nTerm ------. | +** | | | | +** | | .-- prereq Idx wsFlags----. | | +** | | | Name | | | +** | | | __|__ nEq ---. ___|__ | __|__ +** | / \ / \ / \ | / \ / \ / \ +** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 +*/ +SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ + if( pWC ){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; + SrcItem *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + }else{ + sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", + p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); + } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ const char *zName; if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ @@ -153205,18 +166271,27 @@ SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; inLTerm; i++){ sqlite3WhereTermPrint(p->aLTerm[i], i); } } } +SQLITE_PRIVATE void sqlite3ShowWhereLoop(const WhereLoop *p){ + if( p ) sqlite3WhereLoopPrint(p, 0); +} +SQLITE_PRIVATE void sqlite3ShowWhereLoopList(const WhereLoop *p){ + while( p ){ + sqlite3ShowWhereLoop(p); + p = p->pNextLoop; + } +} #endif /* @@ -153248,12 +166323,18 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ } /* -** Deallocate internal memory used by a WhereLoop object +** Deallocate internal memory used by a WhereLoop object. Leave the +** object in an initialized state, as if it had been newly allocated. */ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ){ + sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = p->aLTermSpace; + p->nLSlot = ArraySize(p->aLTermSpace); + } whereLoopClearUnion(db, p); - whereLoopInit(p); + p->nLTerm = 0; + p->wsFlags = 0; } /* @@ -153277,7 +166358,9 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ */ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + if( pFrom->nLTerm > pTo->nLSlot + && whereLoopResize(db, pTo, pFrom->nLTerm) + ){ memset(pTo, 0, WHERE_LOOP_XFER_SZ); return SQLITE_NOMEM_BKPT; } @@ -153295,85 +166378,86 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + assert( db!=0 ); whereLoopClear(db, p); - sqlite3DbFreeNN(db, p); + sqlite3DbNNFreeNN(db, p); } /* ** Free a WhereInfo structure */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - int i; assert( pWInfo!=0 ); - for(i=0; inLevel; i++){ - WhereLevel *pLevel = &pWInfo->a[i]; - if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE)!=0 ){ - assert( (pLevel->pWLoop->wsFlags & WHERE_MULTI_OR)==0 ); - sqlite3DbFree(db, pLevel->u.in.aInLoop); - } - } + assert( db!=0 ); sqlite3WhereClauseClear(&pWInfo->sWC); while( pWInfo->pLoops ){ WhereLoop *p = pWInfo->pLoops; pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); } - assert( pWInfo->pExprMods==0 ); - sqlite3DbFreeNN(db, pWInfo); -} - -/* Undo all Expr node modifications -*/ -static void whereUndoExprMods(WhereInfo *pWInfo){ - while( pWInfo->pExprMods ){ - WhereExprMod *p = pWInfo->pExprMods; - pWInfo->pExprMods = p->pNext; - memcpy(p->pExpr, &p->orig, sizeof(p->orig)); - sqlite3DbFree(pWInfo->pParse->db, p); + while( pWInfo->pMemToFree ){ + WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; + sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); + pWInfo->pMemToFree = pNext; } + sqlite3DbNNFreeNN(db, pWInfo); } /* -** Return TRUE if all of the following are true: +** Return TRUE if X is a proper subset of Y but is of equal or less cost. +** In other words, return true if all constraints of X are also part of Y +** and Y has additional constraints that might speed the search that X lacks +** but the cost of running X is not more than the cost of running Y. +** +** In other words, return true if the cost relationwship between X and Y +** is inverted and needs to be adjusted. +** +** Case 1: ** -** (1) X has the same or lower cost, or returns the same or fewer rows, -** than Y. -** (2) X uses fewer WHERE clause terms than Y -** (3) Every WHERE clause term used by X is also used by Y -** (4) X skips at least as many columns as Y -** (5) If X is a covering index, than Y is too +** (1a) X and Y use the same index. +** (1b) X has fewer == terms than Y +** (1c) Neither X nor Y use skip-scan +** (1d) X does not have a a greater cost than Y ** -** Conditions (2) and (3) mean that X is a "proper subset" of Y. -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. Constraint (4) -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. Constraint (5) -** was added because a covering index probably deserves to have a lower cost -** than a non-covering index even if it is a proper subset. +** Case 2: +** +** (2a) X has the same or lower cost, or returns the same or fewer rows, +** than Y. +** (2b) X uses fewer WHERE clause terms than Y +** (2c) Every WHERE clause term used by X is also used by Y +** (2d) X skips at least as many columns as Y +** (2e) If X is a covering index, than Y is too */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; + if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */ + assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */ + && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */ + && pX->nSkip==0 && pY->nSkip==0 /* (1c) */ + ){ + return 1; /* Case 1 is true */ + } if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ + return 0; /* (2b) */ } - if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; - if( pY->nSkip > pX->nSkip ) return 0; + if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */ for(i=pX->nLTerm-1; i>=0; i--){ if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + if( j<0 ) return 0; /* (2c) */ } if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ - return 0; /* Constraint (5) */ + return 0; /* (2e) */ } - return 1; /* All conditions meet */ + return 1; /* Case 2 is true */ } /* @@ -153454,7 +166538,7 @@ static WhereLoop **whereLoopFindLesser( ** rSetup. Call this SETUP-INVARIANT */ assert( p->rSetup>=pTemplate->rSetup ); - /* Any loop using an appliation-defined index (or PRIMARY KEY or + /* Any loop using an application-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 @@ -153481,7 +166565,7 @@ static WhereLoop **whereLoopFindLesser( /* If pTemplate is always better than p, then cause p to be overwritten ** with pTemplate. pTemplate is better than p if: - ** (1) pTemplate has no more dependences than p, and + ** (1) pTemplate has no more dependencies than p, and ** (2) pTemplate has an equal or lower cost than p. */ if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ @@ -153599,7 +166683,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ }else{ /* We will be overwriting WhereLoop p[]. But before we do, first ** go through the rest of the list and delete any other entries besides - ** p[] that are also supplated by pTemplate */ + ** p[] that are also supplanted by pTemplate */ WhereLoop **ppTail = &p->pNextLoop; WhereLoop *pToDel; while( *ppTail ){ @@ -153667,11 +166751,11 @@ static void whereLoopOutputAdjust( LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ assert( pTerm!=0 ); - if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; - if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue; for(j=pLoop->nLTerm-1; j>=0; j--){ pX = pLoop->aLTerm[j]; if( pX==0 ) continue; @@ -153679,6 +166763,24 @@ static void whereLoopOutputAdjust( if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ + sqlite3ProgressCheck(pWC->pWInfo->pParse); + if( pLoop->maskSelf==pTerm->prereqAll ){ + /* If there are extra terms in the WHERE clause not used by an index + ** that depend only on the table being scanned, and that will tend to + ** cause many rows to be omitted, then mark that table as + ** "self-culling". + ** + ** 2022-03-24: Self-culling only applies if either the extra terms + ** are straight comparison operators that are non-true with NULL + ** operand, or if the loop is not an OUTER JOIN. + */ + if( (pTerm->eOperator & 0x3f)!=0 + || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype + & (JT_LEFT|JT_LTORJ))==0 + ){ + pLoop->wsFlags |= WHERE_SELFCULL; + } + } if( pTerm->truthProb<=0 ){ /* If a truth probability is specified using the likelihood() hints, ** then use the probability provided by the application. */ @@ -153706,7 +166808,9 @@ static void whereLoopOutputAdjust( } } } - if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; + if( pLoop->nOut > nRow-iReduce ){ + pLoop->nOut = nRow - iReduce; + } } /* @@ -153779,7 +166883,7 @@ static int whereRangeVectorLen( } /* -** Adjust the cost C by the costMult facter T. This only occurs if +** Adjust the cost C by the costMult factor T. This only occurs if ** compiled with -DSQLITE_ENABLE_COSTMULT */ #ifdef SQLITE_ENABLE_COSTMULT @@ -153806,7 +166910,7 @@ static int whereLoopAddBtreeIndex( Index *pProbe, /* An index on pSrc */ LogEst nInMul /* log(Number of iterations due to IN) */ ){ - WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection malloc context */ WhereLoop *pNew; /* Template WhereLoop under construction */ @@ -153827,7 +166931,10 @@ static int whereLoopAddBtreeIndex( WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + assert( db->mallocFailed==0 || pParse->nErr>0 ); + if( pParse->nErr ){ + return pParse->rc; + } WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", pProbe->pTable->zName,pProbe->zName, pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); @@ -153840,7 +166947,12 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nBtm==0 ); opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bUnordered || pProbe->bLowQual ){ + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } + } assert( pNew->u.btree.nEqnColumn ); assert( pNew->u.btree.nEqnKeyCol @@ -153878,15 +166990,11 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - /* tag-20191211-001: Do not allow constraints from the WHERE clause to - ** be used by the right table of a LEFT JOIN. Only constraints in the - ** ON clause are allowed. See tag-20191211-002 for the vtab equivalent. */ - if( (pSrc->fg.jointype & JT_LEFT)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) ){ continue; } - if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; }else{ @@ -153897,7 +167005,11 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nBtm = saved_nBtm; pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + if( pNew->nLTerm>=pNew->nLSlot + && whereLoopResize(db, pNew, pNew->nLTerm+1) + ){ + break; /* OOM while trying to enlarge the pNew->aLTerm array */ + } pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; @@ -153990,38 +167102,39 @@ static int whereLoopAddBtreeIndex( if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->u.btree.nBtm = whereRangeVectorLen( - pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm - ); - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range constraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - pNew->u.btree.nTop = 1; - } - }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->u.btree.nTop = whereRangeVectorLen( + }else{ + int nVecLen = whereRangeVectorLen( pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm ); - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; + if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = nVecLen; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = nVecLen; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } } /* At this point pNew->nOut is set to the number of rows expected to @@ -154073,7 +167186,7 @@ static int whereLoopAddBtreeIndex( && pNew->nOut+10 > pProbe->aiRowLogEst[0] ){ #if WHERETRACE_ENABLED /* 0x01 */ - if( sqlite3WhereTrace & 0x01 ){ + if( sqlite3WhereTrace & 0x20 ){ sqlite3DebugPrintf( "STAT4 determines term has low selectivity:\n"); sqlite3WhereTermPrint(pTerm, 999); @@ -154105,14 +167218,33 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ assert( pSrc->pTab->szTabRow>0 ); - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* The pProbe->szIdxRow is low for an IPK table since the interior + ** pages are small. Thus szIdxRow gives a good estimate of seek cost. + ** But the leaf pages are full-size, so pProbe->szIdxRow would badly + ** under-estimate the scanning cost. */ + rCostIdx = pNew->nOut + 16; + }else{ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + } + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -154134,6 +167266,9 @@ static int whereLoopAddBtreeIndex( && (pNew->u.btree.nEqnKeyCol || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ + if( pNew->u.btree.nEq>3 ){ + sqlite3ProgressCheck(pParse); + } whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; @@ -154214,7 +167349,9 @@ static int indexMightHelpWithOrderBy( for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( NEVER(pExpr==0) ) continue; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -154236,23 +167373,26 @@ static int indexMightHelpWithOrderBy( */ static int whereUsablePartialIndex( int iTab, /* The table for which we want an index */ - int isLeft, /* True if iTab is the right table of a LEFT JOIN */ + u8 jointype, /* The JT_* flags on the join */ WhereClause *pWC, /* The WHERE clause of the query */ Expr *pWhere /* The WHERE clause from the partial index */ ){ int i; WhereTerm *pTerm; - Parse *pParse = pWC->pWInfo->pParse; + Parse *pParse; + + if( jointype & JT_LTORJ ) return 0; + pParse = pWC->pWInfo->pParse; while( pWhere->op==TK_AND ){ - if( !whereUsablePartialIndex(iTab,isLeft,pWC,pWhere->pLeft) ) return 0; + if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0; pWhere = pWhere->pRight; } if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ Expr *pExpr; pExpr = pTerm->pExpr; - if( (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) - && (isLeft==0 || ExprHasProperty(pExpr, EP_FromJoin)) + if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) + && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) && (pTerm->wtFlags & TERM_VNULL)==0 ){ @@ -154262,6 +167402,243 @@ static int whereUsablePartialIndex( return 0; } +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + +/* +** Structure passed to the whereIsCoveringIndex Walker callback. +*/ +typedef struct CoveringIndexCheck CoveringIndexCheck; +struct CoveringIndexCheck { + Index *pIdx; /* The index */ + int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ +}; + +/* +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. +** +** If the Expr node references the table with cursor pCk->iTabCur, then +** make sure that column is covered by the index pCk->pIdx. We know that +** all columns less than 63 (really BMS-1) are covered, so we don't need +** to check them. But we do need to check any column at 63 or greater. +** +** If the index does not cover the column, then set pWalk->eCode to +** non-zero and return WRC_Abort to stop the search. +** +** If this node does not disprove that the index can be a covering index, +** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. +*/ +static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pCk->bUnidx = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; + return WRC_Prune; + } + return WRC_Continue; +} + + +/* +** pIdx is an index that covers all of the low-number columns used by +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. +** +** The return value is one of these: +** +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. +*/ +static SQLITE_NOINLINE u32 whereIsCoveringIndex( + WhereInfo *pWInfo, /* The WHERE clause context */ + Index *pIdx, /* Index that is being tested */ + int iTabCur /* Cursor for the table being indexed */ +){ + int i, rc; + struct CoveringIndexCheck ck; + Walker w; + if( pWInfo->pSelect==0 ){ + /* We don't have access to the full query, so we cannot check to see + ** if pIdx is covering. Assume it is not. */ + return 0; + } + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 0; + } + } + ck.pIdx = pIdx; + ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIsCoveringIndexWalkCallback; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pCovIdxCk = &ck; + sqlite3WalkSelect(&w, pWInfo->pSelect); + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; +} + +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxEpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + IndexedExpr **pp = (IndexedExpr**)pObject; + while( *pp!=0 ){ + IndexedExpr *p = *pp; + *pp = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** This function is called for a partial index - one with a WHERE clause - in +** two scenarios. In both cases, it determines whether or not the WHERE +** clause on the index implies that a column of the table may be safely +** replaced by a constant expression. For example, in the following +** SELECT: +** +** CREATE INDEX i1 ON t1(b, c) WHERE a=; +** SELECT a, b, c FROM t1 WHERE a= AND b=?; +** +** The "a" in the select-list may be replaced by , iff: +** +** (a) is a constant expression, and +** (b) The (a=) comparison uses the BINARY collation sequence, and +** (c) Column "a" has an affinity other than NONE or BLOB. +** +** If argument pItem is NULL, then pMask must not be NULL. In this case this +** function is being called as part of determining whether or not pIdx +** is a covering index. This function clears any bits in (*pMask) +** corresponding to columns that may be replaced by constants as described +** above. +** +** Otherwise, if pItem is not NULL, then this function is being called +** as part of coding a loop that uses index pIdx. In this case, add entries +** to the Parse.pIdxPartExpr list for each column that can be replaced +** by a constant. +*/ +static void wherePartIdxExpr( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Partial index being processed */ + Expr *pPart, /* WHERE clause being processed */ + Bitmask *pMask, /* Mask to clear bits in */ + int iIdxCur, /* Cursor number for index */ + SrcItem *pItem /* The FROM clause entry for the table */ +){ + assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 ); + assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) ); + + if( pPart->op==TK_AND ){ + wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem); + pPart = pPart->pLeft; + } + + if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){ + Expr *pLeft = pPart->pLeft; + Expr *pRight = pPart->pRight; + u8 aff; + + if( pLeft->op!=TK_COLUMN ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; + if( pLeft->iColumn<0 ) return; + aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; + if( aff>=SQLITE_AFF_TEXT ){ + if( pItem ){ + sqlite3 *db = pParse->db; + IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); + if( p ){ + int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; + p->pExpr = sqlite3ExprDup(db, pRight, 0); + p->iDataCur = pItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = pLeft->iColumn; + p->bMaybeNullRow = bNullRow; + p->pIENext = pParse->pIdxPartExpr; + p->aff = aff; + pParse->pIdxPartExpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxPartExpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } + }else if( pLeft->iColumn<(BMS-1) ){ + *pMask &= ~((Bitmask)1 << pLeft->iColumn); + } + } + } +} + + /* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be @@ -154300,7 +167677,7 @@ static int whereUsablePartialIndex( */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mPrereq /* Extra prerequesites for using this table */ + Bitmask mPrereq /* Extra prerequisites for using this table */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ @@ -154344,7 +167721,7 @@ static int whereLoopAddBtree( sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; + sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; @@ -154361,13 +167738,14 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ if( !pBuilder->pOrSet /* Not part of an OR optimization */ - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ + && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */ ){ /* Generate auto-index WhereLoops */ LogEst rLogSize; /* Logarithm of the number of rows in the table */ @@ -154394,7 +167772,8 @@ static int whereLoopAddBtree( if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ pNew->rSetup += 28; }else{ - pNew->rSetup -= 10; + pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes + ** on ephemeral materializations of views */ } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); if( pNew->rSetup<0 ) pNew->rSetup = 0; @@ -154417,9 +167796,8 @@ static int whereLoopAddBtree( for(; rc==SQLITE_OK && pProbe; pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ ){ - int isLeft = (pSrc->fg.jointype & JT_OUTER)!=0; if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pSrc->iCursor, isLeft, pWC, + && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC, pProbe->pPartIdxWhere) ){ testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ @@ -154472,11 +167850,45 @@ static int whereLoopAddBtree( }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + if( pProbe->pPartIdxWhere ){ + wherePartIdxExpr( + pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0 + ); + } + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0x200, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0x200, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0x200, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ + WHERETRACE(0x200, + ("-> %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + } } /* Full scan via index */ @@ -154527,7 +167939,14 @@ static int whereLoopAddBtree( } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); - rc = whereLoopInsert(pBuilder, pNew); + if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){ + /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN + ** because the cursor used to access the index might not be + ** positioned to the correct row during the right-join no-match + ** loop. */ + }else{ + rc = whereLoopInsert(pBuilder, pNew); + } pNew->nOut = rSize; if( rc ) break; } @@ -154540,7 +167959,7 @@ static int whereLoopAddBtree( ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; } #ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); @@ -154553,6 +167972,30 @@ static int whereLoopAddBtree( #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTerm is a virtual table LIMIT or OFFSET term. +*/ +static int isLimitTerm(WhereTerm *pTerm){ + assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 ); + return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT + && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; +} + +/* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; iipNew->iTab. This @@ -154580,9 +168023,11 @@ static int whereLoopAddVirtualOne( u16 mExclude, /* Exclude terms using these operators */ sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ u16 mNoOmit, /* Do not omit these constraints */ - int *pbIn /* OUT: True if plan uses an IN(...) op */ + int *pbIn, /* OUT: True if plan uses an IN(...) op */ + int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */ ){ WhereClause *pWC = pBuilder->pWC; + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; int i; @@ -154605,6 +168050,7 @@ static int whereLoopAddVirtualOne( pIdxCons->usable = 0; if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight && (pTerm->eOperator & mExclude)==0 + && (pbRetryLimit || !isLimitTerm(pTerm)) ){ pIdxCons->usable = 1; } @@ -154620,6 +168066,7 @@ static int whereLoopAddVirtualOne( pIdxInfo->estimatedRows = 25; pIdxInfo->idxFlags = 0; pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + pHidden->mHandleIn = 0; /* Invoke the virtual table xBestIndex() method */ rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); @@ -154629,7 +168076,7 @@ static int whereLoopAddVirtualOne( ** that the particular combination of parameters provided is unusable. ** Make no entries in the loop table. */ - WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); return SQLITE_OK; } return rc; @@ -154637,8 +168084,8 @@ static int whereLoopAddVirtualOne( mxTerm = -1; assert( pNew->nLSlot>=nConstraint ); - for(i=0; iaLTerm[i] = 0; - pNew->u.vtab.omitMask = 0; + memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint ); + memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab)); pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; ieMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){ + pNew->u.vtab.bOmitOffset = 1; + } } - if( (pTerm->eOperator & WO_IN)!=0 ){ + if( SMASKBIT32(i) & pHidden->mHandleIn ){ + pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm); + }else if( (pTerm->eOperator & WO_IN)!=0 ){ /* A virtual table that is constrained by an IN clause may not ** consume the ORDER BY clause because (1) the order of IN terms ** is not necessarily related to the order of output terms and @@ -154683,6 +168135,29 @@ static int whereLoopAddVirtualOne( pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ + assert( pbRetryLimit || !isLimitTerm(pTerm) ); + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ + /* If there is an IN(...) term handled as an == (separate call to + ** xFilter for each value on the RHS of the IN) and a LIMIT or + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->needToFreeIdxStr = 0; + } + *pbRetryLimit = 1; + return SQLITE_OK; + } } } @@ -154719,7 +168194,7 @@ static int whereLoopAddVirtualOne( sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", *pbIn, (sqlite3_uint64)mPrereq, (sqlite3_uint64)(pNew->prereq & ~mPrereq))); @@ -154727,11 +168202,19 @@ static int whereLoopAddVirtualOne( } /* -** If this function is invoked from within an xBestIndex() callback, it -** returns a pointer to a buffer containing the name of the collation -** sequence associated with element iCons of the sqlite3_index_info.aConstraint -** array. Or, if iCons is out of range or there is no active xBestIndex -** call, return NULL. +** Return the collating sequence for a constraint passed into xBestIndex. +** +** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex. +** This routine depends on there being a HiddenIndexInfo structure immediately +** following the sqlite3_index_info structure. +** +** Return a pointer to the collation name: +** +** 1. If there is an explicit COLLATE operator on the constraint, return it. +** +** 2. Else, if the column has an alternative collation, return that. +** +** 3. Otherwise, return "BINARY". */ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; @@ -154748,6 +168231,92 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int return zRet; } +/* +** Return true if constraint iCons is really an IN(...) constraint, or +** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0) +** or clear (if bHandle==0) the flag to handle it using an iterator. +*/ +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + u32 m = SMASKBIT32(iCons); + if( m & pHidden->mIn ){ + if( bHandle==0 ){ + pHidden->mHandleIn &= ~m; + }else if( bHandle>0 ){ + pHidden->mHandleIn |= m; + } + return 1; + } + return 0; +} + +/* +** This interface is callable from within the xBestIndex callback only. +** +** If possible, set (*ppVal) to point to an object containing the value +** on the right-hand-side of constraint iCons. +*/ +SQLITE_API int sqlite3_vtab_rhs_value( + sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */ + int iCons, /* Constraint for which RHS is wanted */ + sqlite3_value **ppVal /* Write value extracted here */ +){ + HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1]; + sqlite3_value *pVal = 0; + int rc = SQLITE_OK; + if( iCons<0 || iCons>=pIdxInfo->nConstraint ){ + rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ + }else{ + if( pH->aRhs[iCons]==0 ){ + WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; + rc = sqlite3ValueFromExpr( + pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), + SQLITE_AFF_BLOB, &pH->aRhs[iCons] + ); + testcase( rc!=SQLITE_OK ); + } + pVal = pH->aRhs[iCons]; + } + *ppVal = pVal; + + if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */ + rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */ + } + + return rc; +} + +/* +** Return true if ORDER BY clause may be handled as DISTINCT. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 ); + return pHidden->eDistinct; +} + +/* +** Cause the prepared statement that is associated with a call to +** xBestIndex to potentially use all schemas. If the statement being +** prepared is read-only, then just start read transactions on all +** schemas. But if this is a write operation, start writes on all +** schemas. +** +** This is used by the (built-in) sqlite_dbpage virtual table. +*/ +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse *pParse){ + int nDb = pParse->db->nDb; + int i; + for(i=0; iwriteMask) ){ + for(i=0; ipNew->iTab. That table is guaranteed to be a virtual table. @@ -154789,6 +168358,7 @@ static int whereLoopAddVirtual( WhereLoop *pNew; Bitmask mBest; /* Tables used by best possible plan */ u16 mNoOmit; + int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */ assert( (mPrereq & mUnusable)==0 ); pWInfo = pBuilder->pWInfo; @@ -154797,8 +168367,7 @@ static int whereLoopAddVirtual( pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; assert( IsVirtual(pSrc->pTab) ); - p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, - &mNoOmit); + p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; @@ -154806,14 +168375,22 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = 0; nConstraint = p->nConstraint; if( whereLoopResize(pParse->db, pNew, nConstraint) ){ - sqlite3DbFree(pParse->db, p); + freeIndexInfo(pParse->db, p); return SQLITE_NOMEM_BKPT; } /* First call xBestIndex() with all constraints usable. */ WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); - WHERETRACE(0x40, (" VirtualOne: all usable\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); + WHERETRACE(0x800, (" VirtualOne: all usable\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry + ); + if( bRetry ){ + assert( rc==SQLITE_OK ); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0 + ); + } /* If the call to xBestIndex() with all terms enabled produced a plan ** that does not require any source tables (IOW: a plan with mBest==0) @@ -154829,9 +168406,9 @@ static int whereLoopAddVirtual( /* If the plan produced by the earlier call uses an IN(...) term, call ** xBestIndex again, this time with IN(...) terms disabled. */ if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn); + pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); assert( bIn==0 ); mBestNoIn = pNew->prereq & ~mPrereq; if( mBestNoIn==0 ){ @@ -154855,10 +168432,10 @@ static int whereLoopAddVirtual( mPrev = mNext; if( mNext==ALLBITS ) break; if( mNext==mBest || mNext==mBestNoIn ) continue; - WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", + WHERETRACE(0x800, (" VirtualOne: mPrev=%04llx mNext=%04llx\n", (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn); + pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0); if( pNew->prereq==mPrereq ){ seenZero = 1; if( bIn==0 ) seenZeroNoIN = 1; @@ -154869,9 +168446,9 @@ static int whereLoopAddVirtual( ** that requires no source tables at all (i.e. one guaranteed to be ** usable), make a call here with all source tables disabled */ if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled\n")); rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn); + pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); if( bIn==0 ) seenZeroNoIN = 1; } @@ -154879,14 +168456,14 @@ static int whereLoopAddVirtual( ** that requires no source tables at all and does not use an IN(...) ** operator, make a final call to obtain one here. */ if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); rc = whereLoopAddVirtualOne( - pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn); + pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); } } if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); - sqlite3DbFreeNN(pParse->db, p); + freeIndexInfo(pParse->db, p); WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); return rc; } @@ -154919,6 +168496,9 @@ static int whereLoopAddOr( pItem = pWInfo->pTabList->a + pNew->iTab; iCur = pItem->iCursor; + /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */ + if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK; + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 @@ -154930,10 +168510,9 @@ static int whereLoopAddOr( int i, j; sSubBuild = *pBuilder; - sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -154942,6 +168521,7 @@ static int whereLoopAddOr( tempWC.pOuter = pWC; tempWC.op = TK_AND; tempWC.nTerm = 1; + tempWC.nBase = 1; tempWC.a = pOrTerm; sSubBuild.pWC = &tempWC; }else{ @@ -154949,9 +168529,9 @@ static int whereLoopAddOr( } sCur.n = 0; #ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ + if( sqlite3WhereTrace & 0x20000 ){ sqlite3WhereClausePrint(sSubBuild.pWC); } #endif @@ -154966,8 +168546,6 @@ static int whereLoopAddOr( if( rc==SQLITE_OK ){ rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } - assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 - || rc==SQLITE_NOMEM ); testcase( rc==SQLITE_NOMEM && sCur.n>0 ); testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ @@ -155013,7 +168591,7 @@ static int whereLoopAddOr( pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); } } return rc; @@ -155032,29 +168610,50 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ SrcItem *pEnd = &pTabList->a[pWInfo->nLevel]; sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; + int bFirstPastRJ = 0; + int hasRightJoin = 0; WhereLoop *pNew; + /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; - whereLoopInit(pNew); + + /* Verify that pNew has already been initialized */ + assert( pNew->nLTerm==0 ); + assert( pNew->wsFlags==0 ); + assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm!=0 ); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItemiTab = iTab; pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); - if( (pItem->fg.jointype & (JT_LEFT|JT_CROSS))!=0 ){ - /* This condition is true when pItem is the FROM clause term on the - ** right-hand-side of a LEFT or CROSS JOIN. */ - mPrereq = mPrior; - }else{ + if( bFirstPastRJ + || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0 + ){ + /* Add prerequisites to prevent reordering of FROM clause terms + ** across CROSS joins and outer joins. The bFirstPastRJ boolean + ** prevents the right operand of a RIGHT JOIN from being swapped with + ** other elements even further to the right. + ** + ** The JT_LTORJ case and the hasRightJoin flag work together to + ** prevent FROM-clause terms from moving from the right side of + ** a LEFT JOIN over to the left side of that join if the LEFT JOIN + ** is itself on the left side of a RIGHT JOIN. + */ + if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; + mPrereq |= mPrior; + bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; + }else if( !hasRightJoin ){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ SrcItem *p; for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){ mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); } } @@ -155179,7 +168778,9 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered && (wctrlFlags & WHERE_DISTINCTBY)==0 ){ + if( pLoop->u.vtab.isOrdered + && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) + ){ obSat = obDone; } break; @@ -155338,8 +168939,8 @@ static i8 wherePathSatisfiesOrderBy( if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; - if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } @@ -155357,16 +168958,18 @@ static i8 wherePathSatisfiesOrderBy( /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ - if( (rev ^ revIdx)!=(pOrderBy->a[i].sortFlags&KEYINFO_ORDER_DESC) ){ + if( (rev ^ revIdx) + != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) + ){ isMatch = 0; } }else{ - rev = revIdx ^ (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC); + rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); if( rev ) *pRevMask |= MASKBIT(iLoop); revSet = 1; } } - if( isMatch && (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ if( j==pLoop->u.btree.nEq ){ pLoop->wsFlags |= WHERE_BIGNULL_SORT; }else{ @@ -155403,7 +169006,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -155446,7 +169049,7 @@ static i8 wherePathSatisfiesOrderBy( ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 */ SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ - assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) ); assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); return pWInfo->sorted; } @@ -155469,37 +169072,56 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ ** order. */ static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted + WhereInfo *pWInfo, /* Query planning context */ + LogEst nRow, /* Estimated number of rows to sort */ + int nOrderBy, /* Number of ORDER BY clause terms */ + int nSorted /* Number of initial ORDER BY terms naturally in order */ ){ - /* TUNING: Estimated cost of a full external sort, where N is + /* Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** - ** cost = (3.0 * N * log(N)). + ** cost = (K * N * log(N)). ** ** Or, if the order-by clause has X terms but only the last Y ** terms are out of order, then block-sorting will reduce the ** sorting cost to: ** - ** cost = (3.0 * N * log(N)) * (Y/X) + ** cost = (K * N * log(N)) * (Y/X) + ** + ** The constant K is at least 2.0 but will be larger if there are a + ** large number of columns to be sorted, as the sorting time is + ** proportional to the amount of content to be sorted. The algorithm + ** does not currently distinguish between fat columns (BLOBs and TEXTs) + ** and skinny columns (INTs). It just uses the number of columns as + ** an approximation for the row width. ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. + ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort + ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; + LogEst rSortCost, nCol; + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->pEList!=0 ); + /* TUNING: sorting cost proportional to the number of output columns: */ + nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); + rSortCost = nRow + nCol; + if( nSorted>0 ){ + /* Scale the result by (Y/X) */ + rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + } /* Multiple by log(M) where M is the number of output rows. ** Use the LIMIT for M if it is smaller. Or if this sort is for ** a DISTINCT operator, M will be the number of distinct output ** rows, so fudge it downwards a bit. */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ + if( nSorted!=0 ){ + rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ + } + if( pWInfo->iLimitiLimit; + } }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT ** reduces the number of output rows by a factor of 2 */ @@ -155525,7 +169147,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ Parse *pParse; /* Parsing context */ - sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ @@ -155544,14 +169165,14 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; - db = pParse->db; nLoop = pWInfo->nLevel; /* TUNING: For simple queries, only the best path is tracked. ** For 2-way joins, the 5 best paths are followed. ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", + nRowEst, pParse->nQueryLoop)); /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this ** case the purpose of this call is to estimate the number of rows returned @@ -155567,7 +169188,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); + pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -155617,9 +169238,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + Bitmask revMask; /* Mask of rev-order loops for (..) */ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; @@ -155638,7 +169259,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; + isOrdered = pFrom->isOrdered; if( isOrdered<0 ){ + revMask = 0; isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); @@ -155651,11 +169274,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - /* TUNING: Add a small extra penalty (5) to sorting as an - ** extra encouragment to the query planner to select a plan + /* TUNING: Add a small extra penalty (3) to sorting as an + ** extra encouragement to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", @@ -155816,7 +169439,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nFrom==0 ){ sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_ERROR; } @@ -155847,12 +169470,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } pWInfo->bOrderedInnerLoop = 0; if( pWInfo->pOrderBy ){ + pWInfo->nOBSat = pFrom->isOrdered; if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ - pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ pWInfo->nOBSat = 0; @@ -155894,14 +169520,90 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ - sqlite3DbFreeNN(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -155931,7 +169633,11 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pItem = pWInfo->pTabList->a; pTab = pItem->pTab; if( IsVirtual(pTab) ) return 0; - if( pItem->fg.isIndexedBy ) return 0; + if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ + testcase( pItem->fg.isIndexedBy ); + testcase( pItem->fg.notIndexed ); + return 0; + } iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; @@ -155992,7 +169698,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->cId = '0'; #endif #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); } #endif @@ -156049,6 +169755,309 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ # define WHERETRACE_ALL_LOOPS(W,C) #endif +/* Attempt to omit tables from a join that do not affect the result. +** For a table to not affect the result, the following must be true: +** +** 1) The query must not be an aggregate. +** 2) The table must be the RHS of a LEFT JOIN. +** 3) Either the query must be DISTINCT, or else the ON or USING clause +** must contain a constraint that limits the scan of the table to +** at most a single row. +** 4) The table must not be referenced by any part of the query apart +** from its own USING or ON clause. +** 5) The table must not have an inner-join ON or USING clause if there is +** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause +** might move from the right side to the left side of the RIGHT JOIN. +** Note: Due to (2), this condition can only arise if the table is +** the right-most table of a subquery that was flattened into the +** main query and that subquery was the right-hand operand of an +** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. +** +** For example, given: +** +** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); +** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); +** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); +** +** then table t2 can be omitted from the following: +** +** SELECT v1, v3 FROM t1 +** LEFT JOIN t2 ON (t1.ipk=t2.ipk) +** LEFT JOIN t3 ON (t1.ipk=t3.ipk) +** +** or from: +** +** SELECT DISTINCT v1, v3 FROM t1 +** LEFT JOIN t2 +** LEFT JOIN t3 ON (t1.ipk=t3.ipk) +*/ +static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( + WhereInfo *pWInfo, + Bitmask notReady +){ + int i; + Bitmask tabUsed; + int hasRightJoin; + + /* Preconditions checked by the caller */ + assert( pWInfo->nLevel>=2 ); + assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) ); + + /* These two preconditions checked by the caller combine to guarantee + ** condition (1) of the header comment */ + assert( pWInfo->pResultSet!=0 ); + assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) ); + + tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet); + if( pWInfo->pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); + } + hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; + for(i=pWInfo->nLevel-1; i>=1; i--){ + WhereTerm *pTerm, *pEnd; + SrcItem *pItem; + WhereLoop *pLoop; + pLoop = pWInfo->a[i].pWLoop; + pItem = &pWInfo->pTabList->a[pLoop->iTab]; + if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + continue; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) continue; + pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm; + for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON) + || pTerm->pExpr->w.iJoin!=pItem->iCursor + ){ + break; + } + } + if( hasRightJoin + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + && pTerm->pExpr->w.iJoin==pItem->iCursor + ){ + break; /* restriction (5) */ + } + } + if( pTerm drop loop %c not used\n", pLoop->cId)); + notReady &= ~pLoop->maskSelf; + for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } + if( i!=pWInfo->nLevel-1 ){ + int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); + memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); + } + pWInfo->nLevel--; + assert( pWInfo->nLevel>0 ); + } + return notReady; +} + +/* +** Check to see if there are any SEARCH loops that might benefit from +** using a Bloom filter. Consider a Bloom filter if: +** +** (1) The SEARCH happens more than N times where N is the number +** of rows in the table that is being considered for the Bloom +** filter. +** (2) Some searches are expected to find zero rows. (This is determined +** by the WHERE_SELFCULL flag on the term.) +** (3) Bloom-filter processing is not disabled. (Checked by the +** caller.) +** (4) The size of the table being searched is known by ANALYZE. +** +** This block of code merely checks to see if a Bloom filter would be +** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the +** WhereLoop. The implementation of the Bloom filter comes further +** down where the code for each WhereLoop is generated. +*/ +static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( + const WhereInfo *pWInfo +){ + int i; + LogEst nSearch = 0; + + assert( pWInfo->nLevel>=2 ); + assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) ); + for(i=0; inLevel; i++){ + WhereLoop *pLoop = pWInfo->a[i].pWLoop; + const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); + SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; + Table *pTab = pItem->pTab; + if( (pTab->tabFlags & TF_HasStat1)==0 ) break; + pTab->tabFlags |= TF_MaybeReanalyze; + if( i>=1 + && (pLoop->wsFlags & reqFlags)==reqFlags + /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ + && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0) + ){ + if( nSearch > pTab->nRowLogEst ){ + testcase( pItem->fg.jointype & JT_LEFT ); + pLoop->wsFlags |= WHERE_BLOOMFILTER; + pLoop->wsFlags &= ~WHERE_IDX_ONLY; + WHERETRACE(0xffffffff, ( + "-> use Bloom-filter on loop %c because there are ~%.1e " + "lookups into %s which has only ~%.1e rows\n", + pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, + (double)sqlite3LogEstToInt(pTab->nRowLogEst))); + } + } + nSearch += pLoop->nOut; + } +} + +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + SrcItem *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + Table *pTab; + assert( pIdx->bHasExpr ); + pTab = pIdx->pTable; + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ + pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); + }else{ + continue; + } + if( sqlite3ExprIsConstant(0,pExpr) ) continue; + if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){ + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index since the index omits the + ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + continue; + } + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxEpr; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr); + } +#endif + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; + if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ + p->aff = pIdx->zColAff[i]; + } +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxEpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxEpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } +} + +/* +** Set the reverse-scan order mask to one for all tables in the query +** with the exception of MATERIALIZED common table expressions that have +** their own internal ORDER BY clauses. +** +** This implements the PRAGMA reverse_unordered_selects=ON setting. +** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER). +*/ +static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ + int ii; + for(ii=0; iipTabList->nSrc; ii++){ + SrcItem *pItem = &pWInfo->pTabList->a[ii]; + if( !pItem->fg.isCte + || pItem->u2.pCteUse->eM10d!=M10d_Yes + || NEVER(pItem->pSelect==0) + || pItem->pSelect->pOrderBy==0 + ){ + pWInfo->revMask |= MASKBIT(ii); + } + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -156107,7 +170116,7 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** ** OUTER JOINS ** -** An outer join of tables t1 and t2 is conceptally coded as follows: +** An outer join of tables t1 and t2 is conceptually coded as follows: ** ** foreach row1 in t1 do ** flag = 0 @@ -156143,6 +170152,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ + Select *pSelect, /* The entire SELECT statement */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ @@ -156176,13 +170186,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); - if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; - sWLB.pOrderBy = pOrderBy; - - /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via - ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + if( pOrderBy && pOrderBy->nExpr>=BMS ){ + pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of @@ -156208,7 +170215,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + nByteWInfo = ROUND8P(sizeof(WhereInfo)); + if( nTabList>1 ){ + nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); + } pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); @@ -156218,7 +170228,9 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; +#if WHERETRACE_ENABLED pWInfo->pWhere = pWhere; +#endif pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; @@ -156226,11 +170238,16 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + pWInfo->pSelect = pSelect; memset(&pWInfo->nOBSat, 0, offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; + pMaskSet->n = 0; + pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be + ** a valid cursor number, to avoid an initial + ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */ sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); @@ -156243,7 +170260,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ - initMaskSet(pMaskSet); sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); @@ -156251,16 +170267,22 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( */ if( nTabList==0 ){ if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; - if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && OptimizationEnabled(db, SQLITE_DistinctOpt) + ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } }else{ /* Assign a bit from the bitmask to every term in the FROM clause. ** ** The N-th term of the FROM clause is assigned a bitmask of 1<sWC); - if( db->mallocFailed ) goto whereBeginError; + if( pSelect && pSelect->pLimit ){ + sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); + } + if( pParse->nErr ) goto whereBeginError; - /* Special case: WHERE terms that do not refer to any tables in the join - ** (constant expressions). Evaluate each such term, and jump over all the - ** generated code if the result is not true. + /* The False-WHERE-Term-Bypass optimization: ** - ** Do not do this if the expression contains non-deterministic functions - ** that are not within a sub-select. This is not strictly required, but - ** preserves SQLite's legacy behaviour in the following two cases: + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. ** - ** FROM ... WHERE random()>0; -- eval random() once per row - ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall - */ - for(ii=0; iinTerm; ii++){ - WhereTerm *pT = &sWLB.pWC->a[ii]; + ** Conditions: + ** + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. + */ + for(ii=0; iinBase; ii++){ + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ if( pT->wtFlags & TERM_VIRTUAL ) continue; - if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ - sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; } } if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT; + }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ @@ -156324,13 +170377,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ + if( sqlite3WhereTrace & 0xffffffff ){ sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); if( wctrlFlags & WHERE_USE_LIMIT ){ sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); - if( sqlite3WhereTrace & 0x100 ){ + if( sqlite3WhereTrace & 0x8000 ){ Select sSelect; memset(&sSelect, 0, sizeof(sSelect)); sSelect.selFlags = SF_WhereBegin; @@ -156340,10 +170393,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sSelect.pEList = pResultSet; sqlite3TreeViewSelect(0, &sSelect, 0); } - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); - sqlite3WhereClausePrint(sWLB.pWC); + if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } } #endif @@ -156359,7 +170412,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** loops will be built using the revised truthProb values. */ if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); - WHERETRACE(0xffff, + WHERETRACE(0xffffffff, ("**** Redo all loop computations due to" " TERM_HIGHTRUTH changes ****\n")); while( pWInfo->pLoops ){ @@ -156376,16 +170429,29 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ + whereInterstageHeuristic(pWInfo); wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } + + /* TUNING: Assume that a DISTINCT clause on a subquery reduces + ** the output size by a factor of 8 (LogEst -30). + */ + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ + WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", + pWInfo->nRowOut, pWInfo->nRowOut-30)); + pWInfo->nRowOut -= 30; + } + } + assert( pWInfo->pTabList!=0 ); if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = ALLBITS; + whereReverseScanOrder(pWInfo); } - if( pParse->nErr || db->mallocFailed ){ + if( pParse->nErr ){ goto whereBeginError; } + assert( db->mallocFailed==0 ); #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); @@ -156413,90 +170479,42 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } #endif - /* Attempt to omit tables from the join that do not affect the result. - ** For a table to not affect the result, the following must be true: + /* Attempt to omit tables from a join that do not affect the result. + ** See the comment on whereOmitNoopJoin() for further information. ** - ** 1) The query must not be an aggregate. - ** 2) The table must be the RHS of a LEFT JOIN. - ** 3) Either the query must be DISTINCT, or else the ON or USING clause - ** must contain a constraint that limits the scan of the table to - ** at most a single row. - ** 4) The table must not be referenced by any part of the query apart - ** from its own USING or ON clause. - ** - ** For example, given: - ** - ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); - ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); - ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); - ** - ** then table t2 can be omitted from the following: - ** - ** SELECT v1, v3 FROM t1 - ** LEFT JOIN t2 ON (t1.ipk=t2.ipk) - ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) - ** - ** or from: - ** - ** SELECT DISTINCT v1, v3 FROM t1 - ** LEFT JOIN t2 - ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) + ** This query optimization is factored out into a separate "no-inline" + ** procedure to keep the sqlite3WhereBegin() procedure from becoming + ** too large. If sqlite3WhereBegin() becomes too large, that prevents + ** some C-compiler optimizers from in-lining the + ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to + ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ + ){ + notReady = whereOmitNoopJoin(pWInfo, notReady); + nTabList = pWInfo->nLevel; + assert( nTabList>0 ); + } + + /* Check to see if there are any SEARCH loops that might benefit from + ** using a Bloom filter. + */ if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + && OptimizationEnabled(db, SQLITE_BloomFilter) ){ - int i; - Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); - if( sWLB.pOrderBy ){ - tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); - } - for(i=pWInfo->nLevel-1; i>=1; i--){ - WhereTerm *pTerm, *pEnd; - SrcItem *pItem; - pLoop = pWInfo->a[i].pWLoop; - pItem = &pWInfo->pTabList->a[pLoop->iTab]; - if( (pItem->fg.jointype & JT_LEFT)==0 ) continue; - if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 - && (pLoop->wsFlags & WHERE_ONEROW)==0 - ){ - continue; - } - if( (tabUsed & pLoop->maskSelf)!=0 ) continue; - pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; - for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 ){ - if( !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - || pTerm->pExpr->iRightJoinTable!=pItem->iCursor - ){ - break; - } - } - } - if( pTerm drop loop %c not used\n", pLoop->cId)); - notReady &= ~pLoop->maskSelf; - for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 ){ - pTerm->wtFlags |= TERM_CODED; - } - } - if( i!=pWInfo->nLevel-1 ){ - int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); - memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); - } - pWInfo->nLevel--; - nTabList--; - } + whereCheckIfBloomFilterIsUseful(pWInfo); } + #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); sqlite3WhereClausePrint(sWLB.pWC); } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); #endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; @@ -156528,6 +170546,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) && !IsVirtual(pTabList->a[0].pTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) + && OptimizationEnabled(db, SQLITE_OnePass) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ @@ -156563,8 +170582,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* noop */ }else #endif - if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ + if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0) + || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0 + ){ int op = OP_OpenRead; if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; @@ -156577,6 +170598,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColtabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 + && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0 ){ /* If we know that only a prefix of the record will be used, ** it is advantageous to reduce the "column count" field in @@ -156588,7 +170610,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif @@ -156630,8 +170652,17 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } + if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){ + wherePartIdxExpr( + pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem + ); + } } pLevel->iIdxCur = iIndexCur; + assert( pIx!=0 ); assert( pIx->pSchema==pTab->pSchema ); assert( iIndexCur>=0 ); if( op ){ @@ -156665,6 +170696,37 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } } if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + if( (pTabItem->fg.jointype & JT_RIGHT)!=0 + && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0 + ){ + WhereRightJoin *pRJ = pLevel->pRJ; + pRJ->iMatch = pParse->nTab++; + pRJ->regBloom = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); + pRJ->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); + assert( pTab==pTabItem->pTab ); + if( HasRowid(pTab) ){ + KeyInfo *pInfo; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); + pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0); + if( pInfo ){ + pInfo->aColl[0] = 0; + pInfo->aSortFlags[0] = 0; + sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO); + } + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + pLoop->wsFlags &= ~WHERE_IDX_ONLY; + /* The nature of RIGHT JOIN processing is such that it messes up + ** the output order. So omit any ORDER BY/GROUP BY elimination + ** optimizations. We need to do an actual sort for RIGHT JOIN. */ + pWInfo->nOBSat = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED; + } } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -156676,16 +170738,31 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( for(ii=0; iinErr ) goto whereBeginError; pLevel = &pWInfo->a[ii]; wsFlags = pLevel->pWLoop->wsFlags; + pSrc = &pTabList->a[pLevel->iFrom]; + if( pSrc->fg.isMaterialized ){ + if( pSrc->fg.isCorrelated ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + }else{ + int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + sqlite3VdbeJumpHere(v, iOnce); + } + } + assert( pTabList == pWInfo->pTabList ); + if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ + if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ #ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ - constructAutomaticIndex(pParse, &pWInfo->sWC, - &pTabList->a[pLevel->iFrom], notReady, pLevel); + constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel); +#endif + }else{ + sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); + } if( db->mallocFailed ) goto whereBeginError; } -#endif addrExplain = sqlite3WhereExplainOneScan( pParse, pTabList, pLevel, wctrlFlags ); @@ -156705,11 +170782,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Jump here if malloc fails */ whereBeginError: if( pWInfo ){ - testcase( pWInfo->pExprMods!=0 ); - whereUndoExprMods(pWInfo); pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } +#ifdef WHERETRACE_ENABLED + /* Prevent harmless compiler warnings about debugging routines + ** being declared but never used */ + sqlite3ShowWhereLoopList(0); +#endif /* WHERETRACE_ENABLED */ return 0; } @@ -156746,6 +170826,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ SrcList *pTabList = pWInfo->pTabList; sqlite3 *db = pParse->db; int iEnd = sqlite3VdbeCurrentAddr(v); + int nRJ = 0; /* Generate loop termination code. */ @@ -156753,6 +170834,17 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; + if( pLevel->pRJ ){ + /* Terminate the subroutine that forms the interior of the loop of + ** the RIGHT JOIN table */ + WhereRightJoin *pRJ = pLevel->pRJ; + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + pLevel->addrCont = 0; + pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); + VdbeCoverage(v); + nRJ++; + } pLoop = pLevel->pWLoop; if( pLevel->op!=OP_Noop ){ #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT @@ -156780,7 +170872,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ /* The common case: Advance to the next row */ - sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); @@ -156795,7 +170887,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); #endif - }else{ + }else if( pLevel->addrCont ){ sqlite3VdbeResolveLabel(v, pLevel->addrCont); } if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ @@ -156845,6 +170937,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->pRJ ){ + sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1); + VdbeCoverage(v); + } if( pLevel->addrSkip ){ sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); @@ -156863,7 +170959,15 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + n = pSrc->regResult; + assert( pSrc->pTab!=0 ); + m = pSrc->pTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) @@ -156888,11 +170992,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); } - /* The "break" point is here, just past the end of the outer loop. - ** Set it. - */ - sqlite3VdbeResolveLabel(v, pWInfo->iBreak); - assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ int k, last; @@ -156903,40 +171002,27 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ assert( pTab!=0 ); pLoop = pLevel->pWLoop; + /* Do RIGHT JOIN processing. Generate code that will output the + ** unmatched rows of the right operand of the RIGHT JOIN with + ** all of the columns of the left operand set to NULL. + */ + if( pLevel->pRJ ){ + sqlite3WhereRightJoinLoop(pWInfo, i, pLevel); + continue; + } + /* For a co-routine, change all OP_Column references to the table of ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); + assert( pTabItem->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, pTabItem->regResult, 0); continue; } -#ifdef SQLITE_ENABLE_EARLY_CURSOR_CLOSE - /* Close all of the cursors that were opened by sqlite3WhereBegin. - ** Except, do not close cursors that will be reused by the OR optimization - ** (WHERE_OR_SUBCLAUSE). And do not close the OP_OpenWrite cursors - ** created for the ONEPASS optimization. - */ - if( (pTab->tabFlags & TF_Ephemeral)==0 - && !IsView(pTab) - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 - ){ - int ws = pLoop->wsFlags; - if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); - } - if( (ws & WHERE_INDEXED)!=0 - && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 - && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] - ){ - sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); - } - } -#endif - /* If this scan uses an index, make VDBE code substitutions to read data ** from the index instead of from the table where possible. In some cases ** this optimization prevents the table from ever being read, which can @@ -156961,10 +171047,28 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxEpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n", + p->iIdxCur, p->iIdxCol); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr); + } +#endif + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ - printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); } /* Proof that the "+1" on the k value above is safe */ pOp = sqlite3VdbeGetOp(v, k - 1); @@ -156985,6 +171089,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ int x = pOp->p2; assert( pIdx->pTable==pTab ); +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + if( pOp->opcode==OP_Offset ){ + /* Do not need to translate the column number */ + }else +#endif if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; @@ -156998,9 +171107,16 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; OpcodeRewriteTrace(db, k, pOp); + }else{ + /* Unable to translate the table reference into an index + ** reference. Verify that this is harmless - that the + ** table being referenced really is open. + */ + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 - || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; @@ -157019,11 +171135,16 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } } + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + /* Final cleanup */ - if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo); pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); + pParse->withinRJSubrtn -= nRJ; return; } @@ -157148,7 +171269,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** ** These are the same built-in window functions supported by Postgres. ** Although the behaviour of aggregate window functions (functions that -** can be used as either aggregates or window funtions) allows them to +** can be used as either aggregates or window functions) allows them to ** be implemented using an API, built-in window functions are much more ** esoteric. Additionally, some window functions (e.g. nth_value()) ** may only be implemented by caching the entire partition in memory. @@ -157678,7 +171799,7 @@ static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ ** is the Window object representing the associated OVER clause. This ** function updates the contents of pWin as follows: ** -** * If the OVER clause refered to a named window (as in "max(x) OVER win"), +** * If the OVER clause referred to a named window (as in "max(x) OVER win"), ** search list pList for a matching WINDOW definition, and update pWin ** accordingly. If no such WINDOW clause can be found, leave an error ** in pParse. @@ -157755,7 +171876,7 @@ SQLITE_PRIVATE void sqlite3WindowUpdate( } } } - pWin->pFunc = pFunc; + pWin->pWFunc = pFunc; } /* @@ -157816,6 +171937,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ } /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: case TK_AGG_FUNCTION: case TK_COLUMN: { int iCol = -1; @@ -157931,7 +172053,6 @@ static ExprList *exprListAppendList( for(i=0; inExpr; i++){ sqlite3 *db = pParse->db; Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); - assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); if( db->mallocFailed ){ sqlite3ExprDelete(db, pDup); break; @@ -157947,7 +172068,7 @@ static ExprList *exprListAppendList( } } pList = sqlite3ExprListAppend(pParse, pList, pDup); - if( pList ) pList->a[nInit+i].sortFlags = pAppend->a[i].sortFlags; + if( pList ) pList->a[nInit+i].fg.sortFlags = pAppend->a[i].fg.sortFlags; } } return pList; @@ -157988,7 +172109,11 @@ static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){ */ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ int rc = SQLITE_OK; - if( p->pWin && p->pPrior==0 && ALWAYS((p->selFlags & SF_WinRewrite)==0) ){ + if( p->pWin + && p->pPrior==0 + && ALWAYS((p->selFlags & SF_WinRewrite)==0) + && ALWAYS(!IN_RENAME_OBJECT) + ){ Vdbe *v = sqlite3GetVdbe(pParse); sqlite3 *db = pParse->db; Select *pSub = 0; /* The subquery */ @@ -158063,8 +172188,9 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ ExprList *pArgs; assert( ExprUseXList(pWin->pOwner) ); + assert( pWin->pWFunc!=0 ); pArgs = pWin->pOwner->x.pList; - if( pWin->pFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ + if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){ selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); pWin->bExprArgs = 1; @@ -158096,7 +172222,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ pSub = sqlite3SelectNew( pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 ); - SELECTTRACE(1,pParse,pSub, + TREETRACE(0x40,pParse,pSub, ("New window-function subquery in FROM clause of (%u/%p)\n", p->selId, p)); p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); @@ -158106,6 +172232,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ if( p->pSrc ){ Table *pTab2; p->pSrc->a[0].pSelect = pSub; + p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); @@ -158137,12 +172264,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ sqlite3ParserAddCleanup(pParse, sqlite3DbFree, pTab); } - if( rc ){ - if( pParse->nErr==0 ){ - assert( pParse->db->mallocFailed ); - sqlite3ErrorToParser(pParse->db, SQLITE_NOMEM); - } - } + assert( rc==SQLITE_OK || pParse->nErr!=0 ); return rc; } @@ -158194,7 +172316,7 @@ SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ ** variable values in the expression tree. */ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ - if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); sqlite3ExprDelete(pParse->db, pExpr); pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); @@ -158298,7 +172420,7 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble( } /* -** Window *pWin has just been created from a WINDOW clause. Tokne pBase +** Window *pWin has just been created from a WINDOW clause. Token pBase ** is the base window. Earlier windows from the same WINDOW clause are ** stored in the linked list starting at pWin->pNextWin. This function ** either updates *pWin according to the base specification, or else @@ -158342,8 +172464,9 @@ SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ if( p ){ assert( p->op==TK_FUNCTION ); assert( pWin ); + assert( ExprIsFullSize(p) ); p->y.pWin = pWin; - ExprSetProperty(p, EP_WinFunc); + ExprSetProperty(p, EP_WinFunc|EP_FullSize); pWin->pOwner = p; if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ sqlite3ErrorMsg(pParse, @@ -158451,7 +172574,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ } for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *p = pWin->pFunc; + FuncDef *p = pWin->pWFunc; if( (p->funcFlags & SQLITE_FUNC_MINMAX) && pWin->eStart!=TK_UNBOUNDED ){ /* The inline versions of min() and max() require a single ephemeral ** table and 3 registers. The registers are used as follows: @@ -158468,7 +172591,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ pWin->csrApp = pParse->nTab++; pWin->regApp = pParse->nMem+1; pParse->nMem += 3; - if( pKeyInfo && pWin->pFunc->zName[1]=='i' ){ + if( pKeyInfo && pWin->pWFunc->zName[1]=='i' ){ assert( pKeyInfo->aSortFlags[0]==0 ); pKeyInfo->aSortFlags[0] = KEYINFO_ORDER_DESC; } @@ -158604,7 +172727,7 @@ struct WindowCsrAndReg { ** ** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) ** -** The windows functions implmentation caches the input rows in a temp +** The windows functions implementation caches the input rows in a temp ** table, sorted by "a, b" (it actually populates the cache lazily, and ** aggressively removes rows once they are no longer required, but that's ** a mere detail). It keeps three cursors open on the temp table. One @@ -158691,7 +172814,7 @@ static void windowAggStep( Vdbe *v = sqlite3GetVdbe(pParse); Window *pWin; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; + FuncDef *pFunc = pWin->pWFunc; int regArg; int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); int i; @@ -158760,7 +172883,7 @@ static void windowAggStep( for(iEnd=sqlite3VdbeCurrentAddr(v); iOpopcode==OP_Column && pOp->p1==pWin->iEphCsr ){ + if( pOp->opcode==OP_Column && pOp->p1==pMWin->iEphCsr ){ pOp->p1 = csr; } } @@ -158805,7 +172928,7 @@ static void windowAggFinal(WindowCodeArg *p, int bFin){ for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ if( pMWin->regStartRowid==0 - && (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->pWFunc->funcFlags & SQLITE_FUNC_MINMAX) && (pWin->eStart!=TK_UNBOUNDED) ){ sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); @@ -158819,12 +172942,12 @@ static void windowAggFinal(WindowCodeArg *p, int bFin){ int nArg = windowArgCount(pWin); if( bFin ){ sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, nArg); - sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAppendP4(v, pWin->pWFunc, P4_FUNCDEF); sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); }else{ sqlite3VdbeAddOp3(v, OP_AggValue,pWin->regAccum,nArg,pWin->regResult); - sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAppendP4(v, pWin->pWFunc, P4_FUNCDEF); } } } @@ -158953,7 +173076,7 @@ static void windowReturnOneRow(WindowCodeArg *p){ Window *pWin; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; + FuncDef *pFunc = pWin->pWFunc; assert( ExprUseXList(pWin->pOwner) ); if( pFunc->zName==nth_valueName || pFunc->zName==first_valueName @@ -159025,7 +173148,7 @@ static int windowInitAccum(Parse *pParse, Window *pMWin){ int nArg = 0; Window *pWin; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; + FuncDef *pFunc = pWin->pWFunc; assert( pWin->regAccum ); sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); nArg = MAX(nArg, windowArgCount(pWin)); @@ -159055,7 +173178,7 @@ static int windowCacheFrame(Window *pMWin){ Window *pWin; if( pMWin->regStartRowid ) return 1; for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; + FuncDef *pFunc = pWin->pWFunc; if( (pFunc->zName==nth_valueName) || (pFunc->zName==first_valueName) || (pFunc->zName==leadName) @@ -159148,7 +173271,7 @@ static void windowCodeRangeTest( assert( op==OP_Ge || op==OP_Gt || op==OP_Le ); assert( pOrderBy && pOrderBy->nExpr==1 ); - if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_DESC ){ + if( pOrderBy->a[0].fg.sortFlags & KEYINFO_ORDER_DESC ){ switch( op ){ case OP_Ge: op = OP_Le; break; case OP_Gt: op = OP_Lt; break; @@ -159181,7 +173304,7 @@ static void windowCodeRangeTest( ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is ** not taken, control jumps over the comparison operator coded below this ** block. */ - if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_BIGNULL ){ + if( pOrderBy->a[0].fg.sortFlags & KEYINFO_ORDER_BIGNULL ){ /* This block runs if reg1 contains a NULL. */ int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v); switch( op ){ @@ -159202,10 +173325,9 @@ static void windowCodeRangeTest( /* This block runs if reg1 is not NULL, but reg2 is. */ sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); - if( op==OP_Gt || op==OP_Ge ){ - sqlite3VdbeChangeP2(v, -1, addrDone); - } + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, + (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); + VdbeCoverage(v); } /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). @@ -159413,7 +173535,7 @@ SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ pNew->zName = sqlite3DbStrDup(db, p->zName); pNew->zBase = sqlite3DbStrDup(db, p->zBase); pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); - pNew->pFunc = p->pFunc; + pNew->pWFunc = p->pWFunc; pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); pNew->eFrmType = p->eFrmType; @@ -159614,7 +173736,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){ ** ** For the most part, the patterns above are adapted to support UNBOUNDED by ** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and -** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING". +** CURRENT ROW by assuming that it is equivalent to "0 PRECEDING/FOLLOWING". ** This is optimized of course - branches that will never be taken and ** conditions that are always true are omitted from the VM code. The only ** exceptional case is: @@ -159893,7 +174015,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( } /* Allocate registers for the array of values from the sub-query, the - ** samve values in record form, and the rowid used to insert said record + ** same values in record form, and the rowid used to insert said record ** into the ephemeral table. */ regNew = pParse->nMem+1; pParse->nMem += nInput; @@ -159977,8 +174099,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound */ VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ windowAggFinal(&s, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); windowReturnOneRow(&s); sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); @@ -159990,13 +174111,10 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( } if( pMWin->eStart!=TK_UNBOUNDED ){ - sqlite3VdbeAddOp2(v, OP_Rewind, s.start.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.start.csr); } - sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); - VdbeCoverageNeverTaken(v); - sqlite3VdbeAddOp2(v, OP_Rewind, s.end.csr, 1); - VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); + sqlite3VdbeAddOp1(v, OP_Rewind, s.end.csr); if( regPeer && pOrderBy ){ sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); @@ -160138,7 +174256,8 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( /************** End of window.c **********************************************/ /************** Begin file parse.c *******************************************/ /* This file is automatically generated by Lemon from input grammar -** source file "parse.y". */ +** source file "parse.y". +*/ /* ** 2001-09-15 ** @@ -160155,7 +174274,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( ** The canonical source code to this file ("parse.y") is a Lemon grammar ** file that specifies the input grammar and actions to take while parsing. ** That input file is processed by Lemon to generate a C-language -** implementation of a parser for the given grammer. You might be reading +** implementation of a parser for the given grammar. You might be reading ** this comment as part of the translated C-code. Edits should be made ** to the original parse.y sources. */ @@ -160267,9 +174386,9 @@ static void updateDeleteLimitError( break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -160289,11 +174408,16 @@ static void updateDeleteLimitError( return pSelect; } - - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } + + + /* Construct a new Expr object from a single token */ static Expr *tokenExpr(Parse *pParse, int op, Token t){ Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); if( p ){ @@ -160302,7 +174426,7 @@ static void updateDeleteLimitError( p->affExpr = 0; p->flags = EP_Leaf; ExprClearVVAProperties(p); - p->iAgg = -1; + /* p->iAgg = -1; // Not required */ p->pLeft = p->pRight = 0; p->pAggInfo = 0; memset(&p->x, 0, sizeof(p->x)); @@ -160313,6 +174437,7 @@ static void updateDeleteLimitError( p->u.zToken = (char*)&p[1]; memcpy(p->u.zToken, t.z, t.n); p->u.zToken[t.n] = 0; + p->w.iOfst = (int)(t.z - pParse->zTail); if( sqlite3Isquote(p->u.zToken[0]) ){ sqlite3DequoteExpr(p); } @@ -160479,78 +174604,80 @@ static void updateDeleteLimitError( #define TK_SLASH 109 #define TK_REM 110 #define TK_CONCAT 111 -#define TK_COLLATE 112 -#define TK_BITNOT 113 -#define TK_ON 114 -#define TK_INDEXED 115 -#define TK_STRING 116 -#define TK_JOIN_KW 117 -#define TK_CONSTRAINT 118 -#define TK_DEFAULT 119 -#define TK_NULL 120 -#define TK_PRIMARY 121 -#define TK_UNIQUE 122 -#define TK_CHECK 123 -#define TK_REFERENCES 124 -#define TK_AUTOINCR 125 -#define TK_INSERT 126 -#define TK_DELETE 127 -#define TK_UPDATE 128 -#define TK_SET 129 -#define TK_DEFERRABLE 130 -#define TK_FOREIGN 131 -#define TK_DROP 132 -#define TK_UNION 133 -#define TK_ALL 134 -#define TK_EXCEPT 135 -#define TK_INTERSECT 136 -#define TK_SELECT 137 -#define TK_VALUES 138 -#define TK_DISTINCT 139 -#define TK_DOT 140 -#define TK_FROM 141 -#define TK_JOIN 142 -#define TK_USING 143 -#define TK_ORDER 144 -#define TK_GROUP 145 -#define TK_HAVING 146 -#define TK_LIMIT 147 -#define TK_WHERE 148 -#define TK_RETURNING 149 -#define TK_INTO 150 -#define TK_NOTHING 151 -#define TK_FLOAT 152 -#define TK_BLOB 153 -#define TK_INTEGER 154 -#define TK_VARIABLE 155 -#define TK_CASE 156 -#define TK_WHEN 157 -#define TK_THEN 158 -#define TK_ELSE 159 -#define TK_INDEX 160 -#define TK_ALTER 161 -#define TK_ADD 162 -#define TK_WINDOW 163 -#define TK_OVER 164 -#define TK_FILTER 165 -#define TK_COLUMN 166 -#define TK_AGG_FUNCTION 167 -#define TK_AGG_COLUMN 168 -#define TK_TRUEFALSE 169 -#define TK_ISNOT 170 -#define TK_FUNCTION 171 -#define TK_UMINUS 172 +#define TK_PTR 112 +#define TK_COLLATE 113 +#define TK_BITNOT 114 +#define TK_ON 115 +#define TK_INDEXED 116 +#define TK_STRING 117 +#define TK_JOIN_KW 118 +#define TK_CONSTRAINT 119 +#define TK_DEFAULT 120 +#define TK_NULL 121 +#define TK_PRIMARY 122 +#define TK_UNIQUE 123 +#define TK_CHECK 124 +#define TK_REFERENCES 125 +#define TK_AUTOINCR 126 +#define TK_INSERT 127 +#define TK_DELETE 128 +#define TK_UPDATE 129 +#define TK_SET 130 +#define TK_DEFERRABLE 131 +#define TK_FOREIGN 132 +#define TK_DROP 133 +#define TK_UNION 134 +#define TK_ALL 135 +#define TK_EXCEPT 136 +#define TK_INTERSECT 137 +#define TK_SELECT 138 +#define TK_VALUES 139 +#define TK_DISTINCT 140 +#define TK_DOT 141 +#define TK_FROM 142 +#define TK_JOIN 143 +#define TK_USING 144 +#define TK_ORDER 145 +#define TK_GROUP 146 +#define TK_HAVING 147 +#define TK_LIMIT 148 +#define TK_WHERE 149 +#define TK_RETURNING 150 +#define TK_INTO 151 +#define TK_NOTHING 152 +#define TK_FLOAT 153 +#define TK_BLOB 154 +#define TK_INTEGER 155 +#define TK_VARIABLE 156 +#define TK_CASE 157 +#define TK_WHEN 158 +#define TK_THEN 159 +#define TK_ELSE 160 +#define TK_INDEX 161 +#define TK_ALTER 162 +#define TK_ADD 163 +#define TK_WINDOW 164 +#define TK_OVER 165 +#define TK_FILTER 166 +#define TK_COLUMN 167 +#define TK_AGG_FUNCTION 168 +#define TK_AGG_COLUMN 169 +#define TK_TRUEFALSE 170 +#define TK_ISNOT 171 +#define TK_FUNCTION 172 #define TK_UPLUS 173 -#define TK_TRUTH 174 -#define TK_REGISTER 175 -#define TK_VECTOR 176 -#define TK_SELECT_COLUMN 177 -#define TK_IF_NULL_ROW 178 -#define TK_ASTERISK 179 -#define TK_SPAN 180 -#define TK_ERROR 181 -#define TK_SPACE 182 -#define TK_ILLEGAL 183 +#define TK_UMINUS 174 +#define TK_TRUTH 175 +#define TK_REGISTER 176 +#define TK_VECTOR 177 +#define TK_SELECT_COLUMN 178 +#define TK_IF_NULL_ROW 179 +#define TK_ASTERISK 180 +#define TK_SPAN 181 +#define TK_ERROR 182 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 #endif /**************** End token definitions ***************************************/ @@ -160591,6 +174718,9 @@ static void updateDeleteLimitError( ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser ** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -160604,36 +174734,39 @@ static void updateDeleteLimitError( ** YY_NO_ACTION The yy_action[] code for no-op ** YY_MIN_REDUCE Minimum value for reduce actions ** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 318 +#define YYNOCODE 322 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 101 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - With* yy43; - u32 yy51; - int yy64; - struct FrameBound yy81; - struct {int value; int mask;} yy83; - TriggerStep* yy95; - Upsert* yy138; - IdList* yy240; - Cte* yy255; - Select* yy303; - Window* yy375; - u8 yy534; - ExprList* yy562; - struct TrigEvent yy570; - const char* yy600; - SrcList* yy607; - Expr* yy626; + ExprList* yy14; + With* yy59; + Cte* yy67; + Upsert* yy122; + IdList* yy132; + int yy144; + const char* yy168; + SrcList* yy203; + Window* yy211; + OnOrUsing yy269; + struct TrigEvent yy286; + struct {int value; int mask;} yy383; + u32 yy391; + TriggerStep* yy427; + Expr* yy454; + u8 yy462; + struct FrameBound yy509; + Select* yy555; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -160643,24 +174776,29 @@ typedef union { #define sqlite3ParserARG_PARAM #define sqlite3ParserARG_FETCH #define sqlite3ParserARG_STORE +#define YYREALLOC parserStackRealloc +#define YYFREE sqlite3_free +#define YYDYNSTACK 1 #define sqlite3ParserCTX_SDECL Parse *pParse; #define sqlite3ParserCTX_PDECL ,Parse *pParse #define sqlite3ParserCTX_PARAM ,pParse #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 572 -#define YYNRULE 401 -#define YYNRULE_WITH_ACTION 339 -#define YYNTOKEN 184 -#define YY_MAX_SHIFT 571 -#define YY_MIN_SHIFTREDUCE 829 -#define YY_MAX_SHIFTREDUCE 1229 -#define YY_ERROR_ACTION 1230 -#define YY_ACCEPT_ACTION 1231 -#define YY_NO_ACTION 1232 -#define YY_MIN_REDUCE 1233 -#define YY_MAX_REDUCE 1633 +#define YYNSTATE 583 +#define YYNRULE 409 +#define YYNRULE_WITH_ACTION 344 +#define YYNTOKEN 186 +#define YY_MAX_SHIFT 582 +#define YY_MIN_SHIFTREDUCE 845 +#define YY_MAX_SHIFTREDUCE 1253 +#define YY_ERROR_ACTION 1254 +#define YY_ACCEPT_ACTION 1255 +#define YY_NO_ACTION 1256 +#define YY_MIN_REDUCE 1257 +#define YY_MAX_REDUCE 1665 +#define YY_MIN_DSTRCTR 205 +#define YY_MAX_DSTRCTR 319 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -160676,6 +174814,22 @@ typedef union { # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -160727,606 +174881,630 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2037) +#define YY_ACTTAB_COUNT (2142) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 564, 115, 112, 220, 169, 199, 115, 112, 220, 564, - /* 10 */ 375, 1266, 564, 376, 564, 270, 1309, 1309, 406, 407, - /* 20 */ 1084, 199, 1513, 41, 41, 515, 489, 521, 558, 558, - /* 30 */ 558, 965, 41, 41, 395, 41, 41, 51, 51, 966, - /* 40 */ 296, 1269, 296, 122, 123, 113, 1207, 1207, 1041, 1044, - /* 50 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 564, 407, - /* 60 */ 275, 275, 275, 275, 1268, 115, 112, 220, 115, 112, - /* 70 */ 220, 1512, 846, 561, 516, 561, 115, 112, 220, 250, - /* 80 */ 217, 71, 71, 122, 123, 113, 1207, 1207, 1041, 1044, - /* 90 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 440, 440, - /* 100 */ 440, 1149, 119, 119, 119, 119, 118, 118, 117, 117, - /* 110 */ 117, 116, 442, 1183, 1149, 116, 442, 1149, 546, 513, - /* 120 */ 1548, 1554, 374, 442, 6, 1183, 1154, 522, 1154, 407, - /* 130 */ 1556, 461, 373, 1554, 535, 99, 463, 332, 121, 121, - /* 140 */ 121, 121, 119, 119, 119, 119, 118, 118, 117, 117, - /* 150 */ 117, 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, - /* 160 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 1257, 1183, - /* 170 */ 1184, 1185, 243, 1064, 564, 502, 499, 498, 567, 124, - /* 180 */ 567, 1183, 1184, 1185, 474, 497, 119, 119, 119, 119, - /* 190 */ 118, 118, 117, 117, 117, 116, 442, 70, 70, 407, - /* 200 */ 121, 121, 121, 121, 114, 117, 117, 117, 116, 442, - /* 210 */ 1409, 1469, 119, 119, 119, 119, 118, 118, 117, 117, - /* 220 */ 117, 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, - /* 230 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 407, 1031, - /* 240 */ 1031, 1042, 1045, 81, 382, 541, 378, 80, 119, 119, - /* 250 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 381, - /* 260 */ 463, 332, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, - /* 270 */ 1034, 120, 120, 121, 121, 121, 121, 262, 215, 512, - /* 280 */ 1424, 422, 119, 119, 119, 119, 118, 118, 117, 117, - /* 290 */ 117, 116, 442, 1231, 1, 1, 571, 2, 1235, 1573, - /* 300 */ 571, 2, 1235, 307, 1149, 141, 1600, 307, 407, 141, - /* 310 */ 1183, 361, 1317, 1035, 866, 531, 1317, 1149, 359, 1567, - /* 320 */ 1149, 119, 119, 119, 119, 118, 118, 117, 117, 117, - /* 330 */ 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, - /* 340 */ 1034, 120, 120, 121, 121, 121, 121, 275, 275, 1001, - /* 350 */ 426, 275, 275, 1128, 1627, 1021, 1627, 137, 542, 1541, - /* 360 */ 561, 272, 950, 950, 561, 1423, 1183, 1184, 1185, 1594, - /* 370 */ 866, 1012, 530, 315, 231, 1011, 468, 1276, 231, 119, - /* 380 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, - /* 390 */ 1570, 119, 119, 119, 119, 118, 118, 117, 117, 117, - /* 400 */ 116, 442, 330, 359, 1567, 564, 446, 1011, 1011, 1013, - /* 410 */ 446, 207, 564, 306, 555, 407, 363, 1021, 363, 346, - /* 420 */ 184, 118, 118, 117, 117, 117, 116, 442, 71, 71, - /* 430 */ 439, 438, 1126, 1012, 472, 71, 71, 1011, 205, 122, - /* 440 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, - /* 450 */ 121, 121, 121, 121, 219, 219, 472, 1183, 407, 570, - /* 460 */ 1183, 1235, 503, 1477, 149, 546, 307, 489, 141, 1011, - /* 470 */ 1011, 1013, 546, 140, 545, 1317, 1214, 191, 1214, 950, - /* 480 */ 950, 514, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, - /* 490 */ 1034, 120, 120, 121, 121, 121, 121, 563, 119, 119, - /* 500 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 283, - /* 510 */ 275, 275, 415, 1183, 1184, 1185, 1183, 1184, 1185, 372, - /* 520 */ 1183, 243, 344, 561, 502, 499, 498, 1539, 407, 1540, - /* 530 */ 1183, 288, 870, 143, 497, 1549, 185, 231, 9, 6, - /* 540 */ 253, 119, 119, 119, 119, 118, 118, 117, 117, 117, - /* 550 */ 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, - /* 560 */ 1034, 120, 120, 121, 121, 121, 121, 407, 137, 446, - /* 570 */ 447, 863, 169, 1183, 397, 1204, 1183, 1184, 1185, 931, - /* 580 */ 526, 1001, 98, 339, 564, 342, 1183, 1184, 1185, 306, - /* 590 */ 555, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, - /* 600 */ 120, 120, 121, 121, 121, 121, 452, 71, 71, 275, - /* 610 */ 275, 119, 119, 119, 119, 118, 118, 117, 117, 117, - /* 620 */ 116, 442, 561, 417, 306, 555, 1183, 1307, 1307, 1183, - /* 630 */ 1184, 1185, 1204, 1149, 330, 458, 318, 407, 363, 470, - /* 640 */ 431, 1167, 32, 541, 527, 350, 1149, 1629, 393, 1149, - /* 650 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, - /* 660 */ 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, - /* 670 */ 120, 120, 121, 121, 121, 121, 407, 199, 472, 1183, - /* 680 */ 1022, 472, 1183, 1184, 1185, 386, 151, 539, 1548, 277, - /* 690 */ 400, 137, 6, 317, 5, 564, 562, 3, 920, 920, - /* 700 */ 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, - /* 710 */ 120, 121, 121, 121, 121, 411, 505, 83, 71, 71, - /* 720 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, - /* 730 */ 442, 1183, 218, 428, 1183, 1183, 1184, 1185, 363, 261, - /* 740 */ 278, 358, 508, 353, 507, 248, 407, 306, 555, 1539, - /* 750 */ 1006, 349, 363, 291, 489, 302, 293, 1542, 281, 119, - /* 760 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, - /* 770 */ 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, - /* 780 */ 120, 121, 121, 121, 121, 407, 148, 1183, 1184, 1185, - /* 790 */ 1183, 1184, 1185, 275, 275, 1304, 1257, 1283, 483, 1476, - /* 800 */ 150, 489, 480, 564, 1187, 1304, 561, 1587, 1255, 122, - /* 810 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, - /* 820 */ 121, 121, 121, 121, 564, 886, 13, 13, 520, 119, - /* 830 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, - /* 840 */ 1183, 420, 417, 564, 269, 269, 1316, 13, 13, 1539, - /* 850 */ 1546, 16, 16, 322, 6, 407, 506, 561, 1089, 1089, - /* 860 */ 486, 1187, 425, 1539, 887, 292, 71, 71, 119, 119, - /* 870 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 122, - /* 880 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, - /* 890 */ 121, 121, 121, 121, 564, 12, 1183, 1184, 1185, 407, - /* 900 */ 275, 275, 451, 303, 834, 835, 836, 417, 489, 276, - /* 910 */ 276, 1547, 284, 561, 319, 6, 321, 71, 71, 429, - /* 920 */ 451, 450, 561, 952, 101, 113, 1207, 1207, 1041, 1044, - /* 930 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 119, 119, - /* 940 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 1105, - /* 950 */ 1183, 489, 564, 1312, 437, 455, 478, 564, 246, 245, - /* 960 */ 244, 1409, 1545, 547, 1106, 405, 6, 1544, 196, 1258, - /* 970 */ 413, 6, 105, 462, 103, 71, 71, 286, 564, 1107, - /* 980 */ 13, 13, 119, 119, 119, 119, 118, 118, 117, 117, - /* 990 */ 117, 116, 442, 451, 104, 427, 337, 320, 275, 275, - /* 1000 */ 906, 13, 13, 564, 1482, 1105, 1183, 1184, 1185, 126, - /* 1010 */ 907, 561, 546, 564, 407, 478, 295, 1321, 253, 200, - /* 1020 */ 1106, 548, 1482, 1484, 280, 1409, 55, 55, 1287, 561, - /* 1030 */ 478, 380, 423, 951, 407, 1107, 71, 71, 122, 123, - /* 1040 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, - /* 1050 */ 121, 121, 121, 1204, 407, 287, 552, 309, 122, 123, - /* 1060 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, - /* 1070 */ 121, 121, 121, 441, 1128, 1628, 146, 1628, 122, 111, - /* 1080 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, - /* 1090 */ 121, 121, 121, 404, 403, 1482, 424, 119, 119, 119, - /* 1100 */ 119, 118, 118, 117, 117, 117, 116, 442, 1183, 564, - /* 1110 */ 1204, 544, 1086, 858, 329, 361, 1086, 119, 119, 119, - /* 1120 */ 119, 118, 118, 117, 117, 117, 116, 442, 564, 294, - /* 1130 */ 144, 523, 56, 56, 224, 564, 510, 119, 119, 119, - /* 1140 */ 119, 118, 118, 117, 117, 117, 116, 442, 484, 1409, - /* 1150 */ 537, 15, 15, 1126, 434, 439, 438, 407, 13, 13, - /* 1160 */ 1523, 12, 926, 1211, 1183, 1184, 1185, 925, 1213, 536, - /* 1170 */ 858, 557, 413, 193, 1525, 494, 1212, 448, 1160, 1222, - /* 1180 */ 1183, 564, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, - /* 1190 */ 120, 120, 121, 121, 121, 121, 1521, 1149, 564, 965, - /* 1200 */ 564, 1214, 247, 1214, 13, 13, 1409, 966, 538, 564, - /* 1210 */ 1149, 108, 556, 1149, 4, 310, 392, 1227, 17, 194, - /* 1220 */ 485, 43, 43, 57, 57, 306, 555, 524, 559, 1160, - /* 1230 */ 464, 564, 44, 44, 392, 1127, 1183, 1184, 1185, 479, - /* 1240 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, - /* 1250 */ 442, 443, 564, 327, 13, 13, 564, 418, 1315, 414, - /* 1260 */ 171, 564, 311, 553, 213, 529, 1253, 564, 517, 543, - /* 1270 */ 412, 108, 556, 137, 4, 58, 58, 435, 314, 59, - /* 1280 */ 59, 274, 217, 549, 60, 60, 349, 476, 559, 1353, - /* 1290 */ 61, 61, 1021, 275, 275, 1228, 213, 564, 106, 106, - /* 1300 */ 8, 275, 275, 275, 275, 107, 561, 443, 566, 565, - /* 1310 */ 564, 443, 1011, 1228, 561, 564, 561, 564, 275, 275, - /* 1320 */ 62, 62, 1352, 553, 247, 456, 564, 98, 110, 306, - /* 1330 */ 555, 561, 564, 45, 45, 405, 1203, 533, 46, 46, - /* 1340 */ 47, 47, 532, 465, 1011, 1011, 1013, 1014, 27, 49, - /* 1350 */ 49, 564, 1021, 405, 469, 50, 50, 564, 106, 106, - /* 1360 */ 305, 564, 84, 204, 405, 107, 564, 443, 566, 565, - /* 1370 */ 405, 564, 1011, 564, 63, 63, 564, 1599, 564, 895, - /* 1380 */ 64, 64, 457, 477, 65, 65, 147, 96, 38, 14, - /* 1390 */ 14, 1528, 412, 564, 66, 66, 128, 128, 926, 67, - /* 1400 */ 67, 52, 52, 925, 1011, 1011, 1013, 1014, 27, 1572, - /* 1410 */ 1171, 445, 208, 1123, 279, 394, 68, 68, 228, 390, - /* 1420 */ 390, 389, 264, 387, 1171, 445, 843, 877, 279, 108, - /* 1430 */ 556, 453, 4, 390, 390, 389, 264, 387, 564, 225, - /* 1440 */ 843, 313, 328, 1003, 98, 252, 559, 544, 471, 312, - /* 1450 */ 252, 564, 208, 225, 564, 313, 473, 30, 252, 279, - /* 1460 */ 466, 69, 69, 312, 390, 390, 389, 264, 387, 443, - /* 1470 */ 333, 843, 98, 564, 53, 53, 323, 157, 157, 227, - /* 1480 */ 495, 553, 249, 289, 225, 564, 313, 162, 31, 1501, - /* 1490 */ 135, 564, 1500, 227, 312, 533, 158, 158, 885, 884, - /* 1500 */ 534, 162, 873, 301, 135, 564, 481, 226, 76, 76, - /* 1510 */ 1021, 347, 1071, 98, 54, 54, 106, 106, 1067, 564, - /* 1520 */ 249, 226, 519, 107, 227, 443, 566, 565, 72, 72, - /* 1530 */ 1011, 334, 162, 564, 230, 135, 108, 556, 959, 4, - /* 1540 */ 252, 408, 129, 129, 564, 1349, 306, 555, 564, 923, - /* 1550 */ 564, 110, 226, 559, 564, 408, 73, 73, 564, 873, - /* 1560 */ 306, 555, 1011, 1011, 1013, 1014, 27, 130, 130, 1071, - /* 1570 */ 449, 131, 131, 127, 127, 357, 443, 156, 156, 892, - /* 1580 */ 893, 155, 155, 338, 449, 356, 408, 564, 553, 968, - /* 1590 */ 969, 306, 555, 1015, 341, 564, 108, 556, 564, 4, - /* 1600 */ 1132, 1286, 533, 564, 856, 343, 145, 532, 345, 1300, - /* 1610 */ 136, 136, 1083, 559, 1083, 449, 564, 1021, 134, 134, - /* 1620 */ 1284, 132, 132, 106, 106, 1285, 133, 133, 564, 352, - /* 1630 */ 107, 564, 443, 566, 565, 1340, 443, 1011, 362, 75, - /* 1640 */ 75, 1082, 564, 1082, 564, 924, 1561, 110, 553, 551, - /* 1650 */ 1015, 77, 77, 1361, 74, 74, 1408, 1336, 1347, 550, - /* 1660 */ 1414, 1265, 1256, 1244, 1243, 42, 42, 48, 48, 1011, - /* 1670 */ 1011, 1013, 1014, 27, 1245, 1580, 490, 1021, 267, 202, - /* 1680 */ 1333, 365, 11, 106, 106, 930, 367, 210, 369, 391, - /* 1690 */ 107, 1395, 443, 566, 565, 223, 1390, 1011, 500, 454, - /* 1700 */ 282, 1400, 285, 108, 556, 214, 4, 325, 1383, 1283, - /* 1710 */ 475, 355, 1473, 1583, 1472, 1399, 371, 1222, 326, 398, - /* 1720 */ 559, 290, 331, 197, 100, 556, 209, 4, 198, 1011, - /* 1730 */ 1011, 1013, 1014, 27, 385, 256, 1520, 1518, 554, 1219, - /* 1740 */ 416, 559, 83, 443, 173, 206, 182, 221, 459, 167, - /* 1750 */ 177, 460, 175, 493, 233, 553, 79, 178, 1396, 179, - /* 1760 */ 35, 180, 96, 1402, 443, 396, 36, 467, 1478, 1401, - /* 1770 */ 482, 237, 1404, 399, 82, 186, 553, 1467, 89, 488, - /* 1780 */ 190, 268, 239, 491, 1021, 340, 240, 401, 1246, 1489, - /* 1790 */ 106, 106, 336, 509, 1294, 241, 1303, 107, 430, 443, - /* 1800 */ 566, 565, 1302, 91, 1011, 1021, 1598, 1301, 1273, 215, - /* 1810 */ 1597, 106, 106, 402, 877, 432, 354, 1272, 107, 1271, - /* 1820 */ 443, 566, 565, 1596, 1566, 1011, 1293, 433, 518, 299, - /* 1830 */ 300, 360, 95, 525, 1344, 364, 1011, 1011, 1013, 1014, - /* 1840 */ 27, 254, 255, 1552, 436, 1551, 125, 544, 10, 379, - /* 1850 */ 1326, 1453, 102, 97, 1345, 528, 304, 1011, 1011, 1013, - /* 1860 */ 1014, 27, 366, 377, 1343, 1342, 368, 370, 1325, 384, - /* 1870 */ 201, 383, 34, 1368, 1367, 568, 1177, 266, 263, 265, - /* 1880 */ 1505, 159, 569, 1241, 1236, 1506, 160, 142, 1504, 1503, - /* 1890 */ 297, 211, 830, 161, 212, 78, 444, 203, 308, 222, - /* 1900 */ 1081, 139, 1079, 316, 174, 163, 1203, 229, 176, 909, - /* 1910 */ 324, 232, 1095, 181, 409, 410, 172, 164, 165, 419, - /* 1920 */ 183, 85, 86, 421, 166, 87, 88, 1098, 1094, 234, - /* 1930 */ 235, 152, 18, 236, 335, 1087, 1216, 252, 187, 487, - /* 1940 */ 238, 188, 37, 845, 492, 356, 242, 496, 351, 501, - /* 1950 */ 189, 90, 19, 504, 348, 20, 875, 92, 298, 168, - /* 1960 */ 888, 153, 93, 511, 94, 1165, 154, 1047, 1134, 39, - /* 1970 */ 216, 1133, 271, 273, 958, 192, 953, 110, 1151, 1155, - /* 1980 */ 251, 7, 21, 1159, 1139, 22, 1153, 33, 23, 24, - /* 1990 */ 25, 540, 1158, 195, 98, 1062, 26, 1048, 1046, 1050, - /* 2000 */ 1104, 1051, 1103, 257, 258, 28, 40, 1173, 1016, 857, - /* 2010 */ 109, 29, 560, 388, 138, 1172, 259, 170, 260, 1232, - /* 2020 */ 1232, 919, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, - /* 2030 */ 1232, 1232, 1589, 1232, 1232, 1232, 1588, + /* 0 */ 576, 128, 125, 232, 1622, 549, 576, 1290, 1281, 576, + /* 10 */ 328, 576, 1300, 212, 576, 128, 125, 232, 578, 412, + /* 20 */ 578, 391, 1542, 51, 51, 523, 405, 1293, 529, 51, + /* 30 */ 51, 983, 51, 51, 81, 81, 1107, 61, 61, 984, + /* 40 */ 1107, 1292, 380, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 50 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 1577, 412, + /* 60 */ 287, 287, 7, 287, 287, 422, 1050, 1050, 1064, 1067, + /* 70 */ 289, 556, 492, 573, 524, 561, 573, 497, 561, 482, + /* 80 */ 530, 262, 229, 135, 136, 90, 1228, 1228, 1063, 1066, + /* 90 */ 1053, 1053, 133, 133, 134, 134, 134, 134, 128, 125, + /* 100 */ 232, 1506, 132, 132, 132, 132, 131, 131, 130, 130, + /* 110 */ 130, 129, 126, 450, 1204, 1255, 1, 1, 582, 2, + /* 120 */ 1259, 1571, 420, 1582, 379, 320, 1174, 153, 1174, 1584, + /* 130 */ 412, 378, 1582, 543, 1341, 330, 111, 570, 570, 570, + /* 140 */ 293, 1054, 132, 132, 132, 132, 131, 131, 130, 130, + /* 150 */ 130, 129, 126, 450, 135, 136, 90, 1228, 1228, 1063, + /* 160 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 287, + /* 170 */ 287, 1204, 1205, 1204, 255, 287, 287, 510, 507, 506, + /* 180 */ 137, 455, 573, 212, 561, 447, 446, 505, 573, 1616, + /* 190 */ 561, 134, 134, 134, 134, 127, 400, 243, 132, 132, + /* 200 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 210 */ 282, 471, 345, 132, 132, 132, 132, 131, 131, 130, + /* 220 */ 130, 130, 129, 126, 450, 574, 155, 936, 936, 454, + /* 230 */ 227, 521, 1236, 412, 1236, 134, 134, 134, 134, 132, + /* 240 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 250 */ 450, 130, 130, 130, 129, 126, 450, 135, 136, 90, + /* 260 */ 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, + /* 270 */ 134, 134, 128, 125, 232, 450, 576, 412, 397, 1249, + /* 280 */ 180, 92, 93, 132, 132, 132, 132, 131, 131, 130, + /* 290 */ 130, 130, 129, 126, 450, 381, 387, 1204, 383, 81, + /* 300 */ 81, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, + /* 310 */ 133, 133, 134, 134, 134, 134, 132, 132, 132, 132, + /* 320 */ 131, 131, 130, 130, 130, 129, 126, 450, 131, 131, + /* 330 */ 130, 130, 130, 129, 126, 450, 556, 1204, 302, 319, + /* 340 */ 567, 121, 568, 480, 4, 555, 1149, 1657, 1628, 1657, + /* 350 */ 45, 128, 125, 232, 1204, 1205, 1204, 1250, 571, 1169, + /* 360 */ 132, 132, 132, 132, 131, 131, 130, 130, 130, 129, + /* 370 */ 126, 450, 1169, 287, 287, 1169, 1019, 576, 422, 1019, + /* 380 */ 412, 451, 1602, 582, 2, 1259, 573, 44, 561, 95, + /* 390 */ 320, 110, 153, 565, 1204, 1205, 1204, 522, 522, 1341, + /* 400 */ 81, 81, 7, 44, 135, 136, 90, 1228, 1228, 1063, + /* 410 */ 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, 295, + /* 420 */ 1149, 1658, 1040, 1658, 1204, 1147, 319, 567, 119, 119, + /* 430 */ 343, 466, 331, 343, 287, 287, 120, 556, 451, 577, + /* 440 */ 451, 1169, 1169, 1028, 319, 567, 438, 573, 210, 561, + /* 450 */ 1339, 1451, 546, 531, 1169, 1169, 1598, 1169, 1169, 416, + /* 460 */ 319, 567, 243, 132, 132, 132, 132, 131, 131, 130, + /* 470 */ 130, 130, 129, 126, 450, 1028, 1028, 1030, 1031, 35, + /* 480 */ 44, 1204, 1205, 1204, 472, 287, 287, 1328, 412, 1307, + /* 490 */ 372, 1595, 359, 225, 454, 1204, 195, 1328, 573, 1147, + /* 500 */ 561, 1333, 1333, 274, 576, 1188, 576, 340, 46, 196, + /* 510 */ 537, 217, 135, 136, 90, 1228, 1228, 1063, 1066, 1053, + /* 520 */ 1053, 133, 133, 134, 134, 134, 134, 19, 19, 19, + /* 530 */ 19, 412, 581, 1204, 1259, 511, 1204, 319, 567, 320, + /* 540 */ 944, 153, 425, 491, 430, 943, 1204, 488, 1341, 1450, + /* 550 */ 532, 1277, 1204, 1205, 1204, 135, 136, 90, 1228, 1228, + /* 560 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 570 */ 575, 132, 132, 132, 132, 131, 131, 130, 130, 130, + /* 580 */ 129, 126, 450, 287, 287, 528, 287, 287, 372, 1595, + /* 590 */ 1204, 1205, 1204, 1204, 1205, 1204, 573, 486, 561, 573, + /* 600 */ 889, 561, 412, 1204, 1205, 1204, 886, 40, 22, 22, + /* 610 */ 220, 243, 525, 1449, 132, 132, 132, 132, 131, 131, + /* 620 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 630 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 640 */ 134, 412, 180, 454, 1204, 879, 255, 287, 287, 510, + /* 650 */ 507, 506, 372, 1595, 1568, 1331, 1331, 576, 889, 505, + /* 660 */ 573, 44, 561, 559, 1207, 135, 136, 90, 1228, 1228, + /* 670 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 680 */ 81, 81, 422, 576, 377, 132, 132, 132, 132, 131, + /* 690 */ 131, 130, 130, 130, 129, 126, 450, 297, 287, 287, + /* 700 */ 460, 1204, 1205, 1204, 1204, 534, 19, 19, 448, 448, + /* 710 */ 448, 573, 412, 561, 230, 436, 1187, 535, 319, 567, + /* 720 */ 363, 432, 1207, 1435, 132, 132, 132, 132, 131, 131, + /* 730 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 740 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 750 */ 134, 412, 211, 949, 1169, 1041, 1110, 1110, 494, 547, + /* 760 */ 547, 1204, 1205, 1204, 7, 539, 1570, 1169, 376, 576, + /* 770 */ 1169, 5, 1204, 486, 3, 135, 136, 90, 1228, 1228, + /* 780 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 790 */ 576, 513, 19, 19, 427, 132, 132, 132, 132, 131, + /* 800 */ 131, 130, 130, 130, 129, 126, 450, 305, 1204, 433, + /* 810 */ 225, 1204, 385, 19, 19, 273, 290, 371, 516, 366, + /* 820 */ 515, 260, 412, 538, 1568, 549, 1024, 362, 437, 1204, + /* 830 */ 1205, 1204, 902, 1552, 132, 132, 132, 132, 131, 131, + /* 840 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 850 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 860 */ 134, 412, 1435, 514, 1281, 1204, 1205, 1204, 1204, 1205, + /* 870 */ 1204, 903, 48, 342, 1568, 1568, 1279, 1627, 1568, 911, + /* 880 */ 576, 129, 126, 450, 110, 135, 136, 90, 1228, 1228, + /* 890 */ 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, 134, + /* 900 */ 265, 576, 459, 19, 19, 132, 132, 132, 132, 131, + /* 910 */ 131, 130, 130, 130, 129, 126, 450, 1345, 204, 576, + /* 920 */ 459, 458, 50, 47, 19, 19, 49, 434, 1105, 573, + /* 930 */ 497, 561, 412, 428, 108, 1224, 1569, 1554, 376, 205, + /* 940 */ 550, 550, 81, 81, 132, 132, 132, 132, 131, 131, + /* 950 */ 130, 130, 130, 129, 126, 450, 135, 136, 90, 1228, + /* 960 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 970 */ 134, 480, 576, 1204, 576, 1541, 412, 1435, 969, 315, + /* 980 */ 1659, 398, 284, 497, 969, 893, 1569, 1569, 376, 376, + /* 990 */ 1569, 461, 376, 1224, 459, 80, 80, 81, 81, 497, + /* 1000 */ 374, 114, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, + /* 1010 */ 133, 134, 134, 134, 134, 132, 132, 132, 132, 131, + /* 1020 */ 131, 130, 130, 130, 129, 126, 450, 1204, 1505, 576, + /* 1030 */ 1204, 1205, 1204, 1366, 316, 486, 281, 281, 497, 431, + /* 1040 */ 557, 288, 288, 402, 1340, 471, 345, 298, 429, 573, + /* 1050 */ 576, 561, 81, 81, 573, 374, 561, 971, 386, 132, + /* 1060 */ 132, 132, 132, 131, 131, 130, 130, 130, 129, 126, + /* 1070 */ 450, 231, 117, 81, 81, 287, 287, 231, 287, 287, + /* 1080 */ 576, 1511, 576, 1336, 1204, 1205, 1204, 139, 573, 556, + /* 1090 */ 561, 573, 412, 561, 441, 456, 969, 213, 558, 1511, + /* 1100 */ 1513, 1550, 969, 143, 143, 145, 145, 1368, 314, 478, + /* 1110 */ 444, 970, 412, 850, 851, 852, 135, 136, 90, 1228, + /* 1120 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1130 */ 134, 357, 412, 397, 1148, 304, 135, 136, 90, 1228, + /* 1140 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1150 */ 134, 1575, 323, 6, 862, 7, 135, 124, 90, 1228, + /* 1160 */ 1228, 1063, 1066, 1053, 1053, 133, 133, 134, 134, 134, + /* 1170 */ 134, 409, 408, 1511, 212, 132, 132, 132, 132, 131, + /* 1180 */ 131, 130, 130, 130, 129, 126, 450, 411, 118, 1204, + /* 1190 */ 116, 10, 352, 265, 355, 132, 132, 132, 132, 131, + /* 1200 */ 131, 130, 130, 130, 129, 126, 450, 576, 324, 306, + /* 1210 */ 576, 306, 1250, 469, 158, 132, 132, 132, 132, 131, + /* 1220 */ 131, 130, 130, 130, 129, 126, 450, 207, 1224, 1126, + /* 1230 */ 65, 65, 470, 66, 66, 412, 447, 446, 882, 531, + /* 1240 */ 335, 258, 257, 256, 1127, 1233, 1204, 1205, 1204, 327, + /* 1250 */ 1235, 874, 159, 576, 16, 480, 1085, 1040, 1234, 1128, + /* 1260 */ 136, 90, 1228, 1228, 1063, 1066, 1053, 1053, 133, 133, + /* 1270 */ 134, 134, 134, 134, 1029, 576, 81, 81, 1028, 1040, + /* 1280 */ 922, 576, 463, 1236, 576, 1236, 1224, 502, 107, 1435, + /* 1290 */ 923, 6, 576, 410, 1498, 882, 1029, 480, 21, 21, + /* 1300 */ 1028, 332, 1380, 334, 53, 53, 497, 81, 81, 874, + /* 1310 */ 1028, 1028, 1030, 445, 259, 19, 19, 533, 132, 132, + /* 1320 */ 132, 132, 131, 131, 130, 130, 130, 129, 126, 450, + /* 1330 */ 551, 301, 1028, 1028, 1030, 107, 532, 545, 121, 568, + /* 1340 */ 1188, 4, 1126, 1576, 449, 576, 462, 7, 1282, 418, + /* 1350 */ 462, 350, 1435, 576, 518, 571, 544, 1127, 121, 568, + /* 1360 */ 442, 4, 1188, 464, 533, 1180, 1223, 9, 67, 67, + /* 1370 */ 487, 576, 1128, 303, 410, 571, 54, 54, 451, 576, + /* 1380 */ 123, 944, 576, 417, 576, 333, 943, 1379, 576, 236, + /* 1390 */ 565, 576, 1574, 564, 68, 68, 7, 576, 451, 362, + /* 1400 */ 419, 182, 69, 69, 541, 70, 70, 71, 71, 540, + /* 1410 */ 565, 72, 72, 484, 55, 55, 473, 1180, 296, 1040, + /* 1420 */ 56, 56, 296, 493, 541, 119, 119, 410, 1573, 542, + /* 1430 */ 569, 418, 7, 120, 1244, 451, 577, 451, 465, 1040, + /* 1440 */ 1028, 576, 1557, 552, 476, 119, 119, 527, 259, 121, + /* 1450 */ 568, 240, 4, 120, 576, 451, 577, 451, 576, 477, + /* 1460 */ 1028, 576, 156, 576, 57, 57, 571, 576, 286, 229, + /* 1470 */ 410, 336, 1028, 1028, 1030, 1031, 35, 59, 59, 219, + /* 1480 */ 983, 60, 60, 220, 73, 73, 74, 74, 984, 451, + /* 1490 */ 75, 75, 1028, 1028, 1030, 1031, 35, 96, 216, 291, + /* 1500 */ 552, 565, 1188, 318, 395, 395, 394, 276, 392, 576, + /* 1510 */ 485, 859, 474, 1311, 410, 541, 576, 417, 1530, 1144, + /* 1520 */ 540, 399, 1188, 292, 237, 1153, 326, 38, 23, 576, + /* 1530 */ 1040, 576, 20, 20, 325, 299, 119, 119, 164, 76, + /* 1540 */ 76, 1529, 121, 568, 120, 4, 451, 577, 451, 203, + /* 1550 */ 576, 1028, 141, 141, 142, 142, 576, 322, 39, 571, + /* 1560 */ 341, 1021, 110, 264, 239, 901, 900, 423, 242, 908, + /* 1570 */ 909, 370, 173, 77, 77, 43, 479, 1310, 264, 62, + /* 1580 */ 62, 369, 451, 1028, 1028, 1030, 1031, 35, 1601, 1192, + /* 1590 */ 453, 1092, 238, 291, 565, 163, 1309, 110, 395, 395, + /* 1600 */ 394, 276, 392, 986, 987, 859, 481, 346, 264, 110, + /* 1610 */ 1032, 489, 576, 1188, 503, 1088, 261, 261, 237, 576, + /* 1620 */ 326, 121, 568, 1040, 4, 347, 1376, 413, 325, 119, + /* 1630 */ 119, 948, 319, 567, 351, 78, 78, 120, 571, 451, + /* 1640 */ 577, 451, 79, 79, 1028, 354, 356, 576, 360, 1092, + /* 1650 */ 110, 576, 974, 942, 264, 123, 457, 358, 239, 576, + /* 1660 */ 519, 451, 939, 1104, 123, 1104, 173, 576, 1032, 43, + /* 1670 */ 63, 63, 1324, 565, 168, 168, 1028, 1028, 1030, 1031, + /* 1680 */ 35, 576, 169, 169, 1308, 872, 238, 157, 1589, 576, + /* 1690 */ 86, 86, 365, 89, 568, 375, 4, 1103, 941, 1103, + /* 1700 */ 123, 576, 1040, 1389, 64, 64, 1188, 1434, 119, 119, + /* 1710 */ 571, 576, 82, 82, 563, 576, 120, 165, 451, 577, + /* 1720 */ 451, 413, 1362, 1028, 144, 144, 319, 567, 576, 1374, + /* 1730 */ 562, 498, 279, 451, 83, 83, 1439, 576, 166, 166, + /* 1740 */ 576, 1289, 554, 576, 1280, 565, 576, 12, 576, 1268, + /* 1750 */ 457, 146, 146, 1267, 576, 1028, 1028, 1030, 1031, 35, + /* 1760 */ 140, 140, 1269, 167, 167, 1609, 160, 160, 1359, 150, + /* 1770 */ 150, 149, 149, 311, 1040, 576, 312, 147, 147, 313, + /* 1780 */ 119, 119, 222, 235, 576, 1188, 396, 576, 120, 576, + /* 1790 */ 451, 577, 451, 1192, 453, 1028, 508, 291, 148, 148, + /* 1800 */ 1421, 1612, 395, 395, 394, 276, 392, 85, 85, 859, + /* 1810 */ 87, 87, 84, 84, 553, 576, 294, 576, 1426, 338, + /* 1820 */ 339, 1425, 237, 300, 326, 1416, 1409, 1028, 1028, 1030, + /* 1830 */ 1031, 35, 325, 344, 403, 483, 226, 1307, 52, 52, + /* 1840 */ 58, 58, 368, 1371, 1502, 566, 1501, 121, 568, 221, + /* 1850 */ 4, 208, 268, 209, 390, 1244, 1549, 1188, 1372, 1370, + /* 1860 */ 1369, 1547, 239, 184, 571, 233, 421, 1241, 95, 218, + /* 1870 */ 173, 1507, 193, 43, 91, 94, 178, 186, 467, 188, + /* 1880 */ 468, 1422, 13, 189, 190, 191, 501, 451, 245, 108, + /* 1890 */ 238, 401, 1428, 1427, 1430, 475, 404, 1496, 197, 565, + /* 1900 */ 14, 490, 249, 101, 1518, 496, 349, 280, 251, 201, + /* 1910 */ 353, 499, 252, 406, 1270, 253, 517, 1327, 1326, 435, + /* 1920 */ 1325, 1318, 103, 893, 1296, 413, 227, 407, 1040, 1626, + /* 1930 */ 319, 567, 1625, 1297, 119, 119, 439, 367, 1317, 1295, + /* 1940 */ 1624, 526, 120, 440, 451, 577, 451, 1594, 309, 1028, + /* 1950 */ 310, 373, 266, 267, 457, 1580, 1579, 443, 138, 1394, + /* 1960 */ 552, 1393, 11, 1483, 384, 115, 317, 1350, 109, 536, + /* 1970 */ 42, 579, 382, 214, 1349, 388, 1198, 389, 275, 277, + /* 1980 */ 278, 1028, 1028, 1030, 1031, 35, 580, 1265, 414, 1260, + /* 1990 */ 170, 415, 183, 1534, 1535, 1533, 171, 154, 307, 1532, + /* 2000 */ 846, 223, 224, 88, 452, 215, 172, 321, 234, 1102, + /* 2010 */ 152, 1188, 1100, 329, 185, 174, 1223, 925, 187, 241, + /* 2020 */ 337, 244, 1116, 192, 175, 176, 424, 426, 97, 194, + /* 2030 */ 98, 99, 100, 177, 1119, 1115, 246, 247, 161, 24, + /* 2040 */ 248, 348, 1238, 264, 1108, 250, 495, 199, 198, 15, + /* 2050 */ 861, 500, 369, 254, 504, 509, 512, 200, 102, 25, + /* 2060 */ 179, 361, 26, 364, 104, 891, 308, 162, 105, 904, + /* 2070 */ 520, 106, 1185, 1069, 1155, 17, 228, 27, 1154, 283, + /* 2080 */ 285, 263, 978, 202, 972, 123, 28, 1175, 29, 30, + /* 2090 */ 1179, 1171, 31, 1173, 1160, 41, 32, 206, 548, 33, + /* 2100 */ 110, 1178, 1083, 8, 112, 1070, 113, 1068, 1072, 34, + /* 2110 */ 1073, 560, 1125, 269, 1124, 270, 36, 18, 1194, 1033, + /* 2120 */ 873, 151, 122, 37, 393, 271, 272, 572, 181, 1193, + /* 2130 */ 1256, 1256, 1256, 935, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2140 */ 1256, 1617, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 192, 273, 274, 275, 192, 192, 273, 274, 275, 192, - /* 10 */ 218, 215, 192, 218, 192, 212, 234, 235, 205, 19, - /* 20 */ 11, 192, 294, 215, 216, 203, 192, 203, 209, 210, - /* 30 */ 211, 31, 215, 216, 205, 215, 216, 215, 216, 39, - /* 40 */ 227, 215, 229, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 192, 19, - /* 60 */ 238, 239, 238, 239, 215, 273, 274, 275, 273, 274, - /* 70 */ 275, 237, 21, 251, 252, 251, 273, 274, 275, 255, - /* 80 */ 256, 215, 216, 43, 44, 45, 46, 47, 48, 49, - /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 209, 210, - /* 100 */ 211, 76, 102, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 112, 59, 89, 111, 112, 92, 252, 307, - /* 120 */ 308, 313, 314, 112, 312, 59, 86, 261, 88, 19, - /* 130 */ 313, 80, 315, 313, 314, 25, 127, 128, 54, 55, - /* 140 */ 56, 57, 102, 103, 104, 105, 106, 107, 108, 109, - /* 150 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, - /* 160 */ 50, 51, 52, 53, 54, 55, 56, 57, 192, 115, - /* 170 */ 116, 117, 118, 122, 192, 121, 122, 123, 202, 69, - /* 180 */ 204, 115, 116, 117, 192, 131, 102, 103, 104, 105, - /* 190 */ 106, 107, 108, 109, 110, 111, 112, 215, 216, 19, - /* 200 */ 54, 55, 56, 57, 58, 108, 109, 110, 111, 112, - /* 210 */ 192, 160, 102, 103, 104, 105, 106, 107, 108, 109, - /* 220 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, - /* 230 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 46, - /* 240 */ 47, 48, 49, 24, 248, 192, 250, 67, 102, 103, - /* 250 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 277, - /* 260 */ 127, 128, 43, 44, 45, 46, 47, 48, 49, 50, - /* 270 */ 51, 52, 53, 54, 55, 56, 57, 26, 164, 165, - /* 280 */ 272, 263, 102, 103, 104, 105, 106, 107, 108, 109, - /* 290 */ 110, 111, 112, 184, 185, 186, 187, 188, 189, 186, - /* 300 */ 187, 188, 189, 194, 76, 196, 229, 194, 19, 196, - /* 310 */ 59, 192, 203, 120, 59, 87, 203, 89, 310, 311, - /* 320 */ 92, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 330 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, - /* 340 */ 51, 52, 53, 54, 55, 56, 57, 238, 239, 73, - /* 350 */ 231, 238, 239, 22, 23, 100, 25, 81, 305, 306, - /* 360 */ 251, 23, 25, 25, 251, 272, 115, 116, 117, 214, - /* 370 */ 115, 116, 144, 192, 265, 120, 114, 222, 265, 102, - /* 380 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 390 */ 192, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 400 */ 111, 112, 126, 310, 311, 192, 297, 152, 153, 154, - /* 410 */ 297, 149, 192, 137, 138, 19, 192, 100, 192, 23, - /* 420 */ 22, 106, 107, 108, 109, 110, 111, 112, 215, 216, - /* 430 */ 106, 107, 101, 116, 192, 215, 216, 120, 149, 43, - /* 440 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 450 */ 54, 55, 56, 57, 117, 117, 192, 59, 19, 187, - /* 460 */ 59, 189, 23, 282, 240, 252, 194, 192, 196, 152, - /* 470 */ 153, 154, 252, 72, 261, 203, 152, 25, 154, 142, - /* 480 */ 142, 261, 43, 44, 45, 46, 47, 48, 49, 50, - /* 490 */ 51, 52, 53, 54, 55, 56, 57, 192, 102, 103, - /* 500 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 267, - /* 510 */ 238, 239, 237, 115, 116, 117, 115, 116, 117, 192, - /* 520 */ 59, 118, 16, 251, 121, 122, 123, 303, 19, 303, - /* 530 */ 59, 267, 23, 72, 131, 308, 22, 265, 22, 312, - /* 540 */ 24, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 550 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, - /* 560 */ 51, 52, 53, 54, 55, 56, 57, 19, 81, 297, - /* 570 */ 295, 23, 192, 59, 203, 59, 115, 116, 117, 108, - /* 580 */ 192, 73, 25, 77, 192, 79, 115, 116, 117, 137, - /* 590 */ 138, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 600 */ 52, 53, 54, 55, 56, 57, 119, 215, 216, 238, - /* 610 */ 239, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 620 */ 111, 112, 251, 192, 137, 138, 59, 234, 235, 115, - /* 630 */ 116, 117, 116, 76, 126, 127, 128, 19, 192, 268, - /* 640 */ 19, 23, 22, 192, 252, 24, 89, 300, 301, 92, - /* 650 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 660 */ 112, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 670 */ 52, 53, 54, 55, 56, 57, 19, 192, 192, 59, - /* 680 */ 23, 192, 115, 116, 117, 200, 240, 307, 308, 22, - /* 690 */ 205, 81, 312, 262, 22, 192, 133, 22, 135, 136, - /* 700 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 710 */ 53, 54, 55, 56, 57, 197, 95, 150, 215, 216, - /* 720 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 730 */ 112, 59, 192, 112, 59, 115, 116, 117, 192, 118, - /* 740 */ 119, 120, 121, 122, 123, 124, 19, 137, 138, 303, - /* 750 */ 23, 130, 192, 267, 192, 252, 267, 306, 203, 102, - /* 760 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 770 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 780 */ 53, 54, 55, 56, 57, 19, 240, 115, 116, 117, - /* 790 */ 115, 116, 117, 238, 239, 222, 192, 224, 280, 237, - /* 800 */ 240, 192, 284, 192, 59, 232, 251, 140, 204, 43, - /* 810 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 820 */ 54, 55, 56, 57, 192, 35, 215, 216, 192, 102, - /* 830 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 840 */ 59, 230, 192, 192, 238, 239, 237, 215, 216, 303, - /* 850 */ 308, 215, 216, 16, 312, 19, 66, 251, 126, 127, - /* 860 */ 128, 116, 230, 303, 74, 203, 215, 216, 102, 103, - /* 870 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 43, - /* 880 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 890 */ 54, 55, 56, 57, 192, 212, 115, 116, 117, 19, - /* 900 */ 238, 239, 192, 252, 7, 8, 9, 192, 192, 238, - /* 910 */ 239, 308, 262, 251, 77, 312, 79, 215, 216, 129, - /* 920 */ 210, 211, 251, 142, 158, 45, 46, 47, 48, 49, - /* 930 */ 50, 51, 52, 53, 54, 55, 56, 57, 102, 103, - /* 940 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 12, - /* 950 */ 59, 192, 192, 237, 252, 243, 192, 192, 126, 127, - /* 960 */ 128, 192, 308, 203, 27, 253, 312, 308, 285, 207, - /* 970 */ 208, 312, 157, 290, 159, 215, 216, 262, 192, 42, - /* 980 */ 215, 216, 102, 103, 104, 105, 106, 107, 108, 109, - /* 990 */ 110, 111, 112, 283, 158, 230, 237, 160, 238, 239, - /* 1000 */ 63, 215, 216, 192, 192, 12, 115, 116, 117, 22, - /* 1010 */ 73, 251, 252, 192, 19, 192, 230, 239, 24, 24, - /* 1020 */ 27, 261, 210, 211, 99, 192, 215, 216, 225, 251, - /* 1030 */ 192, 192, 263, 142, 19, 42, 215, 216, 43, 44, - /* 1040 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 1050 */ 55, 56, 57, 59, 19, 291, 63, 132, 43, 44, - /* 1060 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 1070 */ 55, 56, 57, 252, 22, 23, 22, 25, 43, 44, - /* 1080 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 1090 */ 55, 56, 57, 106, 107, 283, 263, 102, 103, 104, - /* 1100 */ 105, 106, 107, 108, 109, 110, 111, 112, 59, 192, - /* 1110 */ 116, 144, 29, 59, 291, 192, 33, 102, 103, 104, - /* 1120 */ 105, 106, 107, 108, 109, 110, 111, 112, 192, 291, - /* 1130 */ 163, 19, 215, 216, 15, 192, 108, 102, 103, 104, - /* 1140 */ 105, 106, 107, 108, 109, 110, 111, 112, 65, 192, - /* 1150 */ 66, 215, 216, 101, 231, 106, 107, 19, 215, 216, - /* 1160 */ 192, 212, 134, 114, 115, 116, 117, 139, 119, 85, - /* 1170 */ 116, 207, 208, 230, 192, 19, 127, 192, 94, 60, - /* 1180 */ 59, 192, 44, 45, 46, 47, 48, 49, 50, 51, - /* 1190 */ 52, 53, 54, 55, 56, 57, 192, 76, 192, 31, - /* 1200 */ 192, 152, 46, 154, 215, 216, 192, 39, 87, 192, - /* 1210 */ 89, 19, 20, 92, 22, 192, 22, 23, 22, 230, - /* 1220 */ 263, 215, 216, 215, 216, 137, 138, 115, 36, 145, - /* 1230 */ 128, 192, 215, 216, 22, 23, 115, 116, 117, 290, - /* 1240 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 1250 */ 112, 59, 192, 151, 215, 216, 192, 61, 203, 298, - /* 1260 */ 299, 192, 192, 71, 25, 144, 203, 192, 203, 230, - /* 1270 */ 114, 19, 20, 81, 22, 215, 216, 263, 192, 215, - /* 1280 */ 216, 255, 256, 203, 215, 216, 130, 19, 36, 192, - /* 1290 */ 215, 216, 100, 238, 239, 101, 25, 192, 106, 107, - /* 1300 */ 48, 238, 239, 238, 239, 113, 251, 115, 116, 117, - /* 1310 */ 192, 59, 120, 101, 251, 192, 251, 192, 238, 239, - /* 1320 */ 215, 216, 192, 71, 46, 243, 192, 25, 25, 137, - /* 1330 */ 138, 251, 192, 215, 216, 253, 25, 85, 215, 216, - /* 1340 */ 215, 216, 90, 243, 152, 153, 154, 155, 156, 215, - /* 1350 */ 216, 192, 100, 253, 243, 215, 216, 192, 106, 107, - /* 1360 */ 243, 192, 148, 149, 253, 113, 192, 115, 116, 117, - /* 1370 */ 253, 192, 120, 192, 215, 216, 192, 23, 192, 25, - /* 1380 */ 215, 216, 192, 115, 215, 216, 22, 148, 24, 215, - /* 1390 */ 216, 192, 114, 192, 215, 216, 215, 216, 134, 215, - /* 1400 */ 216, 215, 216, 139, 152, 153, 154, 155, 156, 0, - /* 1410 */ 1, 2, 141, 23, 5, 25, 215, 216, 24, 10, - /* 1420 */ 11, 12, 13, 14, 1, 2, 17, 125, 5, 19, - /* 1430 */ 20, 268, 22, 10, 11, 12, 13, 14, 192, 30, - /* 1440 */ 17, 32, 23, 23, 25, 25, 36, 144, 23, 40, - /* 1450 */ 25, 192, 141, 30, 192, 32, 23, 22, 25, 5, - /* 1460 */ 128, 215, 216, 40, 10, 11, 12, 13, 14, 59, - /* 1470 */ 23, 17, 25, 192, 215, 216, 192, 215, 216, 70, - /* 1480 */ 23, 71, 25, 151, 30, 192, 32, 78, 53, 192, - /* 1490 */ 81, 192, 192, 70, 40, 85, 215, 216, 119, 120, - /* 1500 */ 90, 78, 59, 254, 81, 192, 192, 98, 215, 216, - /* 1510 */ 100, 23, 59, 25, 215, 216, 106, 107, 23, 192, - /* 1520 */ 25, 98, 19, 113, 70, 115, 116, 117, 215, 216, - /* 1530 */ 120, 192, 78, 192, 140, 81, 19, 20, 23, 22, - /* 1540 */ 25, 132, 215, 216, 192, 192, 137, 138, 192, 23, - /* 1550 */ 192, 25, 98, 36, 192, 132, 215, 216, 192, 116, - /* 1560 */ 137, 138, 152, 153, 154, 155, 156, 215, 216, 116, - /* 1570 */ 161, 215, 216, 215, 216, 120, 59, 215, 216, 7, - /* 1580 */ 8, 215, 216, 192, 161, 130, 132, 192, 71, 83, - /* 1590 */ 84, 137, 138, 59, 192, 192, 19, 20, 192, 22, - /* 1600 */ 97, 225, 85, 192, 23, 192, 25, 90, 192, 192, - /* 1610 */ 215, 216, 152, 36, 154, 161, 192, 100, 215, 216, - /* 1620 */ 192, 215, 216, 106, 107, 225, 215, 216, 192, 192, - /* 1630 */ 113, 192, 115, 116, 117, 257, 59, 120, 192, 215, - /* 1640 */ 216, 152, 192, 154, 192, 23, 317, 25, 71, 235, - /* 1650 */ 116, 215, 216, 192, 215, 216, 192, 192, 192, 192, - /* 1660 */ 192, 192, 192, 192, 192, 215, 216, 215, 216, 152, - /* 1670 */ 153, 154, 155, 156, 192, 192, 287, 100, 286, 241, - /* 1680 */ 254, 254, 242, 106, 107, 108, 254, 213, 254, 190, - /* 1690 */ 113, 270, 115, 116, 117, 296, 266, 120, 219, 258, - /* 1700 */ 244, 270, 258, 19, 20, 228, 22, 292, 266, 224, - /* 1710 */ 292, 218, 218, 195, 218, 270, 258, 60, 245, 270, - /* 1720 */ 36, 245, 244, 248, 19, 20, 242, 22, 248, 152, - /* 1730 */ 153, 154, 155, 156, 244, 140, 199, 199, 279, 38, - /* 1740 */ 199, 36, 150, 59, 296, 149, 22, 296, 18, 43, - /* 1750 */ 236, 199, 233, 18, 198, 71, 293, 236, 271, 236, - /* 1760 */ 269, 236, 148, 271, 59, 245, 269, 245, 282, 271, - /* 1770 */ 199, 198, 233, 245, 293, 233, 71, 245, 157, 62, - /* 1780 */ 22, 199, 198, 220, 100, 199, 198, 220, 199, 289, - /* 1790 */ 106, 107, 288, 114, 226, 198, 217, 113, 64, 115, - /* 1800 */ 116, 117, 217, 22, 120, 100, 223, 217, 217, 164, - /* 1810 */ 223, 106, 107, 220, 125, 24, 217, 219, 113, 217, - /* 1820 */ 115, 116, 117, 217, 311, 120, 226, 112, 304, 281, - /* 1830 */ 281, 220, 114, 143, 260, 259, 152, 153, 154, 155, - /* 1840 */ 156, 199, 91, 316, 82, 316, 147, 144, 22, 199, - /* 1850 */ 249, 276, 157, 146, 260, 145, 278, 152, 153, 154, - /* 1860 */ 155, 156, 259, 248, 260, 260, 259, 259, 249, 245, - /* 1870 */ 247, 246, 25, 264, 264, 201, 13, 6, 193, 193, - /* 1880 */ 212, 206, 191, 191, 191, 212, 206, 221, 212, 212, - /* 1890 */ 221, 213, 4, 206, 213, 212, 3, 22, 162, 15, - /* 1900 */ 23, 16, 23, 138, 150, 129, 25, 24, 141, 20, - /* 1910 */ 16, 143, 1, 141, 302, 302, 299, 129, 129, 61, - /* 1920 */ 150, 53, 53, 37, 129, 53, 53, 115, 1, 34, - /* 1930 */ 140, 5, 22, 114, 160, 68, 75, 25, 68, 41, - /* 1940 */ 140, 114, 24, 20, 19, 130, 124, 67, 24, 67, - /* 1950 */ 22, 22, 22, 96, 23, 22, 59, 22, 67, 37, - /* 1960 */ 28, 23, 148, 22, 25, 23, 23, 23, 23, 22, - /* 1970 */ 140, 97, 23, 23, 115, 22, 142, 25, 88, 75, - /* 1980 */ 34, 44, 34, 75, 23, 34, 86, 22, 34, 34, - /* 1990 */ 34, 24, 93, 25, 25, 23, 34, 23, 23, 23, - /* 2000 */ 23, 11, 23, 25, 22, 22, 22, 1, 23, 23, - /* 2010 */ 22, 22, 25, 15, 23, 1, 140, 25, 140, 318, - /* 2020 */ 318, 134, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2030 */ 318, 318, 140, 318, 318, 318, 140, 318, 318, 318, - /* 2040 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2050 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2060 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2070 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2080 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2090 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2100 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2110 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2120 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2130 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2140 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2150 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2160 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2170 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2180 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2190 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2200 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2210 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, - /* 2220 */ 318, + /* 0 */ 194, 276, 277, 278, 216, 194, 194, 217, 194, 194, + /* 10 */ 194, 194, 224, 194, 194, 276, 277, 278, 204, 19, + /* 20 */ 206, 202, 297, 217, 218, 205, 207, 217, 205, 217, + /* 30 */ 218, 31, 217, 218, 217, 218, 29, 217, 218, 39, + /* 40 */ 33, 217, 220, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 312, 19, + /* 60 */ 240, 241, 316, 240, 241, 194, 46, 47, 48, 49, + /* 70 */ 22, 254, 65, 253, 254, 255, 253, 194, 255, 194, + /* 80 */ 263, 258, 259, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 276, 277, + /* 100 */ 278, 285, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 59, 186, 187, 188, 189, 190, + /* 120 */ 191, 310, 239, 317, 318, 196, 86, 198, 88, 317, + /* 130 */ 19, 319, 317, 318, 205, 264, 25, 211, 212, 213, + /* 140 */ 205, 121, 102, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 112, 113, 43, 44, 45, 46, 47, 48, + /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 240, + /* 170 */ 241, 116, 117, 118, 119, 240, 241, 122, 123, 124, + /* 180 */ 69, 298, 253, 194, 255, 106, 107, 132, 253, 141, + /* 190 */ 255, 54, 55, 56, 57, 58, 207, 268, 102, 103, + /* 200 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 210 */ 214, 128, 129, 102, 103, 104, 105, 106, 107, 108, + /* 220 */ 109, 110, 111, 112, 113, 134, 25, 136, 137, 300, + /* 230 */ 165, 166, 153, 19, 155, 54, 55, 56, 57, 102, + /* 240 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 250 */ 113, 108, 109, 110, 111, 112, 113, 43, 44, 45, + /* 260 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 270 */ 56, 57, 276, 277, 278, 113, 194, 19, 22, 23, + /* 280 */ 194, 67, 24, 102, 103, 104, 105, 106, 107, 108, + /* 290 */ 109, 110, 111, 112, 113, 220, 250, 59, 252, 217, + /* 300 */ 218, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 310 */ 52, 53, 54, 55, 56, 57, 102, 103, 104, 105, + /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 106, 107, + /* 330 */ 108, 109, 110, 111, 112, 113, 254, 59, 205, 138, + /* 340 */ 139, 19, 20, 194, 22, 263, 22, 23, 231, 25, + /* 350 */ 72, 276, 277, 278, 116, 117, 118, 101, 36, 76, + /* 360 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 370 */ 112, 113, 89, 240, 241, 92, 73, 194, 194, 73, + /* 380 */ 19, 59, 188, 189, 190, 191, 253, 81, 255, 151, + /* 390 */ 196, 25, 198, 71, 116, 117, 118, 311, 312, 205, + /* 400 */ 217, 218, 316, 81, 43, 44, 45, 46, 47, 48, + /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 270, + /* 420 */ 22, 23, 100, 25, 59, 101, 138, 139, 106, 107, + /* 430 */ 127, 128, 129, 127, 240, 241, 114, 254, 116, 117, + /* 440 */ 118, 76, 76, 121, 138, 139, 263, 253, 264, 255, + /* 450 */ 205, 275, 87, 19, 89, 89, 194, 92, 92, 199, + /* 460 */ 138, 139, 268, 102, 103, 104, 105, 106, 107, 108, + /* 470 */ 109, 110, 111, 112, 113, 153, 154, 155, 156, 157, + /* 480 */ 81, 116, 117, 118, 129, 240, 241, 224, 19, 226, + /* 490 */ 314, 315, 23, 25, 300, 59, 22, 234, 253, 101, + /* 500 */ 255, 236, 237, 26, 194, 183, 194, 152, 72, 22, + /* 510 */ 145, 150, 43, 44, 45, 46, 47, 48, 49, 50, + /* 520 */ 51, 52, 53, 54, 55, 56, 57, 217, 218, 217, + /* 530 */ 218, 19, 189, 59, 191, 23, 59, 138, 139, 196, + /* 540 */ 135, 198, 232, 283, 232, 140, 59, 287, 205, 275, + /* 550 */ 116, 205, 116, 117, 118, 43, 44, 45, 46, 47, + /* 560 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 570 */ 194, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 580 */ 111, 112, 113, 240, 241, 194, 240, 241, 314, 315, + /* 590 */ 116, 117, 118, 116, 117, 118, 253, 194, 255, 253, + /* 600 */ 59, 255, 19, 116, 117, 118, 23, 22, 217, 218, + /* 610 */ 142, 268, 205, 275, 102, 103, 104, 105, 106, 107, + /* 620 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, + /* 630 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 640 */ 57, 19, 194, 300, 59, 23, 119, 240, 241, 122, + /* 650 */ 123, 124, 314, 315, 194, 236, 237, 194, 117, 132, + /* 660 */ 253, 81, 255, 205, 59, 43, 44, 45, 46, 47, + /* 670 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 680 */ 217, 218, 194, 194, 194, 102, 103, 104, 105, 106, + /* 690 */ 107, 108, 109, 110, 111, 112, 113, 294, 240, 241, + /* 700 */ 120, 116, 117, 118, 59, 194, 217, 218, 211, 212, + /* 710 */ 213, 253, 19, 255, 194, 19, 23, 254, 138, 139, + /* 720 */ 24, 232, 117, 194, 102, 103, 104, 105, 106, 107, + /* 730 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, + /* 740 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 750 */ 57, 19, 264, 108, 76, 23, 127, 128, 129, 311, + /* 760 */ 312, 116, 117, 118, 316, 87, 306, 89, 308, 194, + /* 770 */ 92, 22, 59, 194, 22, 43, 44, 45, 46, 47, + /* 780 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 790 */ 194, 95, 217, 218, 265, 102, 103, 104, 105, 106, + /* 800 */ 107, 108, 109, 110, 111, 112, 113, 232, 59, 113, + /* 810 */ 25, 59, 194, 217, 218, 119, 120, 121, 122, 123, + /* 820 */ 124, 125, 19, 145, 194, 194, 23, 131, 232, 116, + /* 830 */ 117, 118, 35, 194, 102, 103, 104, 105, 106, 107, + /* 840 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, + /* 850 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 860 */ 57, 19, 194, 66, 194, 116, 117, 118, 116, 117, + /* 870 */ 118, 74, 242, 294, 194, 194, 206, 23, 194, 25, + /* 880 */ 194, 111, 112, 113, 25, 43, 44, 45, 46, 47, + /* 890 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 900 */ 24, 194, 194, 217, 218, 102, 103, 104, 105, 106, + /* 910 */ 107, 108, 109, 110, 111, 112, 113, 241, 232, 194, + /* 920 */ 212, 213, 242, 242, 217, 218, 242, 130, 11, 253, + /* 930 */ 194, 255, 19, 265, 149, 59, 306, 194, 308, 232, + /* 940 */ 309, 310, 217, 218, 102, 103, 104, 105, 106, 107, + /* 950 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, + /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 970 */ 57, 194, 194, 59, 194, 239, 19, 194, 25, 254, + /* 980 */ 303, 304, 23, 194, 25, 126, 306, 306, 308, 308, + /* 990 */ 306, 271, 308, 117, 286, 217, 218, 217, 218, 194, + /* 1000 */ 194, 159, 45, 46, 47, 48, 49, 50, 51, 52, + /* 1010 */ 53, 54, 55, 56, 57, 102, 103, 104, 105, 106, + /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 59, 239, 194, + /* 1030 */ 116, 117, 118, 260, 254, 194, 240, 241, 194, 233, + /* 1040 */ 205, 240, 241, 205, 239, 128, 129, 270, 265, 253, + /* 1050 */ 194, 255, 217, 218, 253, 194, 255, 143, 280, 102, + /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1070 */ 113, 118, 159, 217, 218, 240, 241, 118, 240, 241, + /* 1080 */ 194, 194, 194, 239, 116, 117, 118, 22, 253, 254, + /* 1090 */ 255, 253, 19, 255, 233, 194, 143, 24, 263, 212, + /* 1100 */ 213, 194, 143, 217, 218, 217, 218, 261, 262, 271, + /* 1110 */ 254, 143, 19, 7, 8, 9, 43, 44, 45, 46, + /* 1120 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1130 */ 57, 16, 19, 22, 23, 294, 43, 44, 45, 46, + /* 1140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1150 */ 57, 312, 194, 214, 21, 316, 43, 44, 45, 46, + /* 1160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1170 */ 57, 106, 107, 286, 194, 102, 103, 104, 105, 106, + /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 207, 158, 59, + /* 1190 */ 160, 22, 77, 24, 79, 102, 103, 104, 105, 106, + /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 194, 194, 229, + /* 1210 */ 194, 231, 101, 80, 22, 102, 103, 104, 105, 106, + /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 288, 59, 12, + /* 1230 */ 217, 218, 293, 217, 218, 19, 106, 107, 59, 19, + /* 1240 */ 16, 127, 128, 129, 27, 115, 116, 117, 118, 194, + /* 1250 */ 120, 59, 22, 194, 24, 194, 123, 100, 128, 42, + /* 1260 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 1270 */ 54, 55, 56, 57, 117, 194, 217, 218, 121, 100, + /* 1280 */ 63, 194, 245, 153, 194, 155, 117, 19, 115, 194, + /* 1290 */ 73, 214, 194, 256, 161, 116, 117, 194, 217, 218, + /* 1300 */ 121, 77, 194, 79, 217, 218, 194, 217, 218, 117, + /* 1310 */ 153, 154, 155, 254, 46, 217, 218, 144, 102, 103, + /* 1320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 1330 */ 232, 270, 153, 154, 155, 115, 116, 66, 19, 20, + /* 1340 */ 183, 22, 12, 312, 254, 194, 262, 316, 209, 210, + /* 1350 */ 266, 239, 194, 194, 108, 36, 85, 27, 19, 20, + /* 1360 */ 265, 22, 183, 245, 144, 94, 25, 48, 217, 218, + /* 1370 */ 293, 194, 42, 270, 256, 36, 217, 218, 59, 194, + /* 1380 */ 25, 135, 194, 115, 194, 161, 140, 194, 194, 15, + /* 1390 */ 71, 194, 312, 63, 217, 218, 316, 194, 59, 131, + /* 1400 */ 301, 302, 217, 218, 85, 217, 218, 217, 218, 90, + /* 1410 */ 71, 217, 218, 19, 217, 218, 245, 146, 262, 100, + /* 1420 */ 217, 218, 266, 265, 85, 106, 107, 256, 312, 90, + /* 1430 */ 209, 210, 316, 114, 60, 116, 117, 118, 194, 100, + /* 1440 */ 121, 194, 194, 145, 115, 106, 107, 19, 46, 19, + /* 1450 */ 20, 24, 22, 114, 194, 116, 117, 118, 194, 245, + /* 1460 */ 121, 194, 164, 194, 217, 218, 36, 194, 258, 259, + /* 1470 */ 256, 194, 153, 154, 155, 156, 157, 217, 218, 150, + /* 1480 */ 31, 217, 218, 142, 217, 218, 217, 218, 39, 59, + /* 1490 */ 217, 218, 153, 154, 155, 156, 157, 149, 150, 5, + /* 1500 */ 145, 71, 183, 245, 10, 11, 12, 13, 14, 194, + /* 1510 */ 116, 17, 129, 227, 256, 85, 194, 115, 194, 23, + /* 1520 */ 90, 25, 183, 99, 30, 97, 32, 22, 22, 194, + /* 1530 */ 100, 194, 217, 218, 40, 152, 106, 107, 23, 217, + /* 1540 */ 218, 194, 19, 20, 114, 22, 116, 117, 118, 257, + /* 1550 */ 194, 121, 217, 218, 217, 218, 194, 133, 53, 36, + /* 1560 */ 23, 23, 25, 25, 70, 120, 121, 61, 141, 7, + /* 1570 */ 8, 121, 78, 217, 218, 81, 23, 227, 25, 217, + /* 1580 */ 218, 131, 59, 153, 154, 155, 156, 157, 0, 1, + /* 1590 */ 2, 59, 98, 5, 71, 23, 227, 25, 10, 11, + /* 1600 */ 12, 13, 14, 83, 84, 17, 23, 23, 25, 25, + /* 1610 */ 59, 194, 194, 183, 23, 23, 25, 25, 30, 194, + /* 1620 */ 32, 19, 20, 100, 22, 194, 194, 133, 40, 106, + /* 1630 */ 107, 108, 138, 139, 194, 217, 218, 114, 36, 116, + /* 1640 */ 117, 118, 217, 218, 121, 194, 194, 194, 23, 117, + /* 1650 */ 25, 194, 23, 23, 25, 25, 162, 194, 70, 194, + /* 1660 */ 145, 59, 23, 153, 25, 155, 78, 194, 117, 81, + /* 1670 */ 217, 218, 194, 71, 217, 218, 153, 154, 155, 156, + /* 1680 */ 157, 194, 217, 218, 194, 23, 98, 25, 321, 194, + /* 1690 */ 217, 218, 194, 19, 20, 194, 22, 153, 23, 155, + /* 1700 */ 25, 194, 100, 194, 217, 218, 183, 194, 106, 107, + /* 1710 */ 36, 194, 217, 218, 237, 194, 114, 243, 116, 117, + /* 1720 */ 118, 133, 194, 121, 217, 218, 138, 139, 194, 194, + /* 1730 */ 194, 290, 289, 59, 217, 218, 194, 194, 217, 218, + /* 1740 */ 194, 194, 140, 194, 194, 71, 194, 244, 194, 194, + /* 1750 */ 162, 217, 218, 194, 194, 153, 154, 155, 156, 157, + /* 1760 */ 217, 218, 194, 217, 218, 194, 217, 218, 257, 217, + /* 1770 */ 218, 217, 218, 257, 100, 194, 257, 217, 218, 257, + /* 1780 */ 106, 107, 215, 299, 194, 183, 192, 194, 114, 194, + /* 1790 */ 116, 117, 118, 1, 2, 121, 221, 5, 217, 218, + /* 1800 */ 273, 197, 10, 11, 12, 13, 14, 217, 218, 17, + /* 1810 */ 217, 218, 217, 218, 140, 194, 246, 194, 273, 295, + /* 1820 */ 247, 273, 30, 247, 32, 269, 269, 153, 154, 155, + /* 1830 */ 156, 157, 40, 246, 273, 295, 230, 226, 217, 218, + /* 1840 */ 217, 218, 220, 261, 220, 282, 220, 19, 20, 244, + /* 1850 */ 22, 250, 141, 250, 246, 60, 201, 183, 261, 261, + /* 1860 */ 261, 201, 70, 299, 36, 299, 201, 38, 151, 150, + /* 1870 */ 78, 285, 22, 81, 296, 296, 43, 235, 18, 238, + /* 1880 */ 201, 274, 272, 238, 238, 238, 18, 59, 200, 149, + /* 1890 */ 98, 247, 274, 274, 235, 247, 247, 247, 235, 71, + /* 1900 */ 272, 201, 200, 158, 292, 62, 291, 201, 200, 22, + /* 1910 */ 201, 222, 200, 222, 201, 200, 115, 219, 219, 64, + /* 1920 */ 219, 228, 22, 126, 221, 133, 165, 222, 100, 225, + /* 1930 */ 138, 139, 225, 219, 106, 107, 24, 219, 228, 219, + /* 1940 */ 219, 307, 114, 113, 116, 117, 118, 315, 284, 121, + /* 1950 */ 284, 222, 201, 91, 162, 320, 320, 82, 148, 267, + /* 1960 */ 145, 267, 22, 279, 201, 158, 281, 251, 147, 146, + /* 1970 */ 25, 203, 250, 249, 251, 248, 13, 247, 195, 195, + /* 1980 */ 6, 153, 154, 155, 156, 157, 193, 193, 305, 193, + /* 1990 */ 208, 305, 302, 214, 214, 214, 208, 223, 223, 214, + /* 2000 */ 4, 215, 215, 214, 3, 22, 208, 163, 15, 23, + /* 2010 */ 16, 183, 23, 139, 151, 130, 25, 20, 142, 24, + /* 2020 */ 16, 144, 1, 142, 130, 130, 61, 37, 53, 151, + /* 2030 */ 53, 53, 53, 130, 116, 1, 34, 141, 5, 22, + /* 2040 */ 115, 161, 75, 25, 68, 141, 41, 115, 68, 24, + /* 2050 */ 20, 19, 131, 125, 67, 67, 96, 22, 22, 22, + /* 2060 */ 37, 23, 22, 24, 22, 59, 67, 23, 149, 28, + /* 2070 */ 22, 25, 23, 23, 23, 22, 141, 34, 97, 23, + /* 2080 */ 23, 34, 116, 22, 143, 25, 34, 75, 34, 34, + /* 2090 */ 75, 88, 34, 86, 23, 22, 34, 25, 24, 34, + /* 2100 */ 25, 93, 23, 44, 142, 23, 142, 23, 23, 22, + /* 2110 */ 11, 25, 23, 25, 23, 22, 22, 22, 1, 23, + /* 2120 */ 23, 23, 22, 22, 15, 141, 141, 25, 25, 1, + /* 2130 */ 322, 322, 322, 135, 322, 322, 322, 322, 322, 322, + /* 2140 */ 322, 141, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2150 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2160 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2170 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2180 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2190 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2200 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2210 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2220 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2230 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2240 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2250 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2260 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2270 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2280 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2290 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2300 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2310 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2320 */ 322, 322, 322, 322, 322, 322, 322, 322, }; -#define YY_SHIFT_COUNT (571) +#define YY_SHIFT_COUNT (582) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2014) +#define YY_SHIFT_MAX (2128) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1423, 1409, 1454, 1192, 1192, 610, 1252, 1410, 1517, 1684, - /* 10 */ 1684, 1684, 276, 0, 0, 180, 1015, 1684, 1684, 1684, - /* 20 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, - /* 30 */ 1049, 1049, 1121, 1121, 54, 487, 610, 610, 610, 610, - /* 40 */ 610, 40, 110, 219, 289, 396, 439, 509, 548, 618, - /* 50 */ 657, 727, 766, 836, 995, 1015, 1015, 1015, 1015, 1015, - /* 60 */ 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, - /* 70 */ 1015, 1015, 1015, 1035, 1015, 1138, 880, 880, 1577, 1684, - /* 80 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, - /* 90 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, - /* 100 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, - /* 110 */ 1684, 1684, 1684, 1705, 1684, 1684, 1684, 1684, 1684, 1684, - /* 120 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 146, 84, 84, - /* 130 */ 84, 84, 84, 277, 315, 401, 97, 461, 251, 66, - /* 140 */ 66, 51, 1156, 66, 66, 324, 324, 66, 452, 452, - /* 150 */ 452, 452, 133, 114, 114, 4, 11, 2037, 2037, 621, - /* 160 */ 621, 621, 567, 398, 398, 398, 398, 937, 937, 228, - /* 170 */ 251, 331, 1052, 66, 66, 66, 66, 66, 66, 66, - /* 180 */ 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, - /* 190 */ 66, 66, 66, 557, 557, 66, 9, 25, 25, 745, - /* 200 */ 745, 967, 1088, 2037, 2037, 2037, 2037, 2037, 2037, 2037, - /* 210 */ 255, 317, 317, 514, 403, 620, 471, 672, 781, 891, - /* 220 */ 675, 66, 66, 66, 66, 66, 66, 66, 66, 66, - /* 230 */ 66, 508, 66, 66, 66, 66, 66, 66, 66, 66, - /* 240 */ 66, 66, 66, 66, 790, 790, 790, 66, 66, 66, - /* 250 */ 338, 66, 66, 66, 516, 1084, 66, 66, 993, 66, - /* 260 */ 66, 66, 66, 66, 66, 66, 66, 732, 1083, 563, - /* 270 */ 994, 994, 994, 994, 337, 563, 563, 1028, 987, 897, - /* 280 */ 1119, 262, 1214, 1271, 1112, 1214, 1112, 1268, 1239, 262, - /* 290 */ 262, 1239, 262, 1271, 1268, 1302, 1354, 1278, 1168, 1168, - /* 300 */ 1168, 1112, 1303, 1303, 815, 1311, 1264, 1364, 1657, 1657, - /* 310 */ 1595, 1595, 1701, 1701, 1595, 1592, 1596, 1724, 1706, 1730, - /* 320 */ 1730, 1730, 1730, 1595, 1735, 1614, 1596, 1596, 1614, 1724, - /* 330 */ 1706, 1614, 1706, 1614, 1595, 1735, 1621, 1717, 1595, 1735, - /* 340 */ 1758, 1595, 1735, 1595, 1735, 1758, 1679, 1679, 1679, 1734, - /* 350 */ 1781, 1781, 1758, 1679, 1689, 1679, 1734, 1679, 1679, 1645, - /* 360 */ 1791, 1715, 1715, 1758, 1690, 1718, 1690, 1718, 1690, 1718, - /* 370 */ 1690, 1718, 1595, 1751, 1751, 1762, 1762, 1699, 1703, 1826, - /* 380 */ 1595, 1695, 1699, 1707, 1710, 1614, 1847, 1863, 1863, 1871, - /* 390 */ 1871, 1871, 2037, 2037, 2037, 2037, 2037, 2037, 2037, 2037, - /* 400 */ 2037, 2037, 2037, 2037, 2037, 2037, 2037, 193, 837, 1194, - /* 410 */ 1212, 506, 832, 1054, 1390, 925, 1435, 1394, 1102, 1332, - /* 420 */ 1419, 1196, 1420, 1425, 1433, 1447, 1457, 1488, 1443, 1379, - /* 430 */ 1572, 1455, 1503, 1453, 1495, 1515, 1506, 1526, 1460, 1489, - /* 440 */ 1581, 1622, 1534, 667, 1888, 1893, 1875, 1736, 1884, 1885, - /* 450 */ 1877, 1879, 1765, 1754, 1776, 1881, 1881, 1883, 1767, 1889, - /* 460 */ 1768, 1894, 1911, 1772, 1788, 1881, 1789, 1858, 1886, 1881, - /* 470 */ 1770, 1868, 1869, 1872, 1873, 1795, 1812, 1895, 1790, 1927, - /* 480 */ 1926, 1910, 1819, 1774, 1867, 1912, 1870, 1861, 1898, 1800, - /* 490 */ 1827, 1918, 1923, 1925, 1815, 1822, 1928, 1880, 1929, 1930, - /* 500 */ 1931, 1933, 1882, 1897, 1924, 1857, 1932, 1935, 1891, 1922, - /* 510 */ 1938, 1814, 1941, 1942, 1943, 1944, 1939, 1945, 1947, 1874, - /* 520 */ 1830, 1949, 1950, 1859, 1946, 1953, 1834, 1952, 1948, 1951, - /* 530 */ 1954, 1955, 1890, 1904, 1900, 1937, 1908, 1899, 1956, 1961, - /* 540 */ 1965, 1967, 1968, 1969, 1962, 1972, 1952, 1974, 1975, 1976, - /* 550 */ 1977, 1978, 1979, 1982, 1990, 1983, 1984, 1985, 1986, 1988, - /* 560 */ 1989, 1987, 1887, 1876, 1878, 1892, 1896, 1992, 1991, 1998, - /* 570 */ 2006, 2014, + /* 0 */ 1792, 1588, 1494, 322, 322, 399, 306, 1319, 1339, 1430, + /* 10 */ 1828, 1828, 1828, 580, 399, 399, 399, 399, 399, 0, + /* 20 */ 0, 214, 1093, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 30 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1130, 1130, + /* 40 */ 365, 365, 55, 278, 436, 713, 713, 201, 201, 201, + /* 50 */ 201, 40, 111, 258, 361, 469, 512, 583, 622, 693, + /* 60 */ 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, 1093, + /* 70 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, + /* 80 */ 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1523, 1602, + /* 90 */ 1674, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 100 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 110 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 120 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 130 */ 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, 1828, + /* 140 */ 137, 181, 181, 181, 181, 181, 181, 181, 96, 222, + /* 150 */ 143, 477, 713, 1133, 1268, 713, 713, 79, 79, 713, + /* 160 */ 770, 83, 65, 65, 65, 288, 162, 162, 2142, 2142, + /* 170 */ 696, 696, 696, 238, 474, 474, 474, 474, 1217, 1217, + /* 180 */ 678, 477, 324, 398, 713, 713, 713, 713, 713, 713, + /* 190 */ 713, 713, 713, 713, 713, 713, 713, 713, 713, 713, + /* 200 */ 713, 713, 713, 1220, 366, 366, 713, 917, 283, 283, + /* 210 */ 434, 434, 605, 605, 1298, 2142, 2142, 2142, 2142, 2142, + /* 220 */ 2142, 2142, 1179, 1157, 1157, 487, 527, 585, 645, 749, + /* 230 */ 914, 968, 752, 713, 713, 713, 713, 713, 713, 713, + /* 240 */ 713, 713, 713, 303, 713, 713, 713, 713, 713, 713, + /* 250 */ 713, 713, 713, 713, 713, 713, 797, 797, 797, 713, + /* 260 */ 713, 713, 959, 713, 713, 713, 1169, 1271, 713, 713, + /* 270 */ 1330, 713, 713, 713, 713, 713, 713, 713, 713, 629, + /* 280 */ 7, 91, 876, 876, 876, 876, 953, 91, 91, 1246, + /* 290 */ 1065, 1106, 1374, 1329, 1348, 468, 1348, 1394, 785, 1329, + /* 300 */ 1329, 785, 1329, 468, 1394, 859, 854, 1402, 1449, 1449, + /* 310 */ 1449, 1173, 1173, 1173, 1173, 1355, 1355, 1030, 1341, 405, + /* 320 */ 1230, 1795, 1795, 1711, 1711, 1829, 1829, 1711, 1717, 1719, + /* 330 */ 1850, 1833, 1860, 1860, 1860, 1860, 1711, 1868, 1740, 1719, + /* 340 */ 1719, 1740, 1850, 1833, 1740, 1833, 1740, 1711, 1868, 1745, + /* 350 */ 1843, 1711, 1868, 1887, 1711, 1868, 1711, 1868, 1887, 1801, + /* 360 */ 1801, 1801, 1855, 1900, 1900, 1887, 1801, 1797, 1801, 1855, + /* 370 */ 1801, 1801, 1761, 1912, 1830, 1830, 1887, 1711, 1862, 1862, + /* 380 */ 1875, 1875, 1810, 1815, 1940, 1711, 1807, 1810, 1821, 1823, + /* 390 */ 1740, 1945, 1963, 1963, 1974, 1974, 1974, 2142, 2142, 2142, + /* 400 */ 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, 2142, + /* 410 */ 2142, 2142, 20, 1224, 256, 1111, 1115, 1114, 1192, 1496, + /* 420 */ 1424, 1505, 1427, 355, 1383, 1537, 1506, 1538, 1553, 1583, + /* 430 */ 1584, 1591, 1625, 541, 1445, 1562, 1450, 1572, 1515, 1428, + /* 440 */ 1532, 1592, 1629, 1520, 1630, 1639, 1510, 1544, 1662, 1675, + /* 450 */ 1551, 48, 1996, 2001, 1983, 1844, 1993, 1994, 1986, 1989, + /* 460 */ 1874, 1863, 1885, 1991, 1991, 1995, 1876, 1997, 1877, 2004, + /* 470 */ 2021, 1881, 1894, 1991, 1895, 1965, 1990, 1991, 1878, 1975, + /* 480 */ 1977, 1978, 1979, 1903, 1918, 2002, 1896, 2034, 2033, 2017, + /* 490 */ 1925, 1880, 1976, 2018, 1980, 1967, 2005, 1904, 1932, 2025, + /* 500 */ 2030, 2032, 1921, 1928, 2035, 1987, 2036, 2037, 2038, 2040, + /* 510 */ 1988, 2006, 2039, 1960, 2041, 2042, 1999, 2023, 2044, 2043, + /* 520 */ 1919, 2048, 2049, 2050, 2046, 2051, 2053, 1981, 1935, 2056, + /* 530 */ 2057, 1966, 2047, 2061, 1941, 2060, 2052, 2054, 2055, 2058, + /* 540 */ 2003, 2012, 2007, 2059, 2015, 2008, 2062, 2071, 2073, 2074, + /* 550 */ 2072, 2075, 2065, 1962, 1964, 2079, 2060, 2082, 2084, 2085, + /* 560 */ 2087, 2086, 2089, 2088, 2091, 2093, 2099, 2094, 2095, 2096, + /* 570 */ 2097, 2100, 2101, 2102, 1998, 1984, 1985, 2000, 2103, 2098, + /* 580 */ 2109, 2117, 2128, }; -#define YY_REDUCE_COUNT (406) -#define YY_REDUCE_MIN (-272) -#define YY_REDUCE_MAX (1693) +#define YY_REDUCE_COUNT (411) +#define YY_REDUCE_MIN (-275) +#define YY_REDUCE_MAX (1798) static const short yy_reduce_ofst[] = { - /* 0 */ 109, 113, 272, 760, -178, -176, -192, -183, -180, -134, - /* 10 */ 213, 220, 371, -208, -205, -272, -197, 611, 632, 765, - /* 20 */ 786, 392, 943, 989, 503, 651, 1039, -18, 702, 821, - /* 30 */ 710, 812, -188, 380, -187, 555, 662, 1055, 1063, 1065, - /* 40 */ 1080, -267, -267, -267, -267, -267, -267, -267, -267, -267, - /* 50 */ -267, -267, -267, -267, -267, -267, -267, -267, -267, -267, - /* 60 */ -267, -267, -267, -267, -267, -267, -267, -267, -267, -267, - /* 70 */ -267, -267, -267, -267, -267, -267, -267, -267, 636, 811, - /* 80 */ 917, 936, 1006, 1008, 1017, 1060, 1064, 1069, 1075, 1105, - /* 90 */ 1118, 1123, 1125, 1134, 1140, 1159, 1165, 1169, 1174, 1179, - /* 100 */ 1181, 1184, 1186, 1201, 1246, 1259, 1262, 1281, 1293, 1299, - /* 110 */ 1313, 1327, 1341, 1352, 1356, 1358, 1362, 1366, 1395, 1403, - /* 120 */ 1406, 1411, 1424, 1436, 1439, 1450, 1452, -267, -267, -267, - /* 130 */ -267, -267, -267, -267, -267, 224, -267, 446, -24, 275, - /* 140 */ 546, 518, 573, 560, 53, -181, -111, 485, 606, 671, - /* 150 */ 606, 671, 683, 8, 93, -267, -267, -267, -267, 155, - /* 160 */ 155, 155, 181, 242, 264, 486, 489, -218, 393, 227, - /* 170 */ 604, 347, 347, -171, 431, 650, 715, -166, 562, 609, - /* 180 */ 716, 764, 18, 823, 769, 833, 838, 957, 759, 119, - /* 190 */ 923, 226, 1014, 542, 603, 451, 949, 654, 659, 762, - /* 200 */ 964, -4, 778, 961, 712, 1082, 1100, 1111, 1026, 1117, - /* 210 */ -204, -174, -151, -8, 77, 198, 305, 327, 388, 540, - /* 220 */ 839, 968, 982, 985, 1004, 1023, 1070, 1086, 1097, 1130, - /* 230 */ 1190, 1163, 1199, 1284, 1297, 1300, 1314, 1339, 1353, 1391, - /* 240 */ 1402, 1413, 1416, 1417, 803, 1376, 1400, 1428, 1437, 1446, - /* 250 */ 1378, 1461, 1464, 1465, 1249, 1329, 1466, 1467, 1414, 1468, - /* 260 */ 305, 1469, 1470, 1471, 1472, 1482, 1483, 1389, 1392, 1438, - /* 270 */ 1426, 1427, 1432, 1434, 1378, 1438, 1438, 1440, 1474, 1499, - /* 280 */ 1399, 1421, 1430, 1456, 1441, 1442, 1444, 1415, 1473, 1431, - /* 290 */ 1445, 1476, 1449, 1478, 1418, 1479, 1477, 1485, 1493, 1494, - /* 300 */ 1496, 1458, 1475, 1480, 1459, 1490, 1484, 1518, 1448, 1451, - /* 310 */ 1537, 1538, 1463, 1481, 1541, 1486, 1487, 1491, 1519, 1514, - /* 320 */ 1521, 1523, 1525, 1552, 1556, 1520, 1492, 1498, 1522, 1497, - /* 330 */ 1539, 1528, 1542, 1532, 1571, 1573, 1500, 1504, 1582, 1584, - /* 340 */ 1563, 1586, 1588, 1589, 1597, 1567, 1579, 1585, 1590, 1568, - /* 350 */ 1583, 1587, 1593, 1591, 1598, 1599, 1600, 1602, 1606, 1513, - /* 360 */ 1524, 1548, 1549, 1611, 1574, 1576, 1594, 1603, 1604, 1607, - /* 370 */ 1605, 1608, 1642, 1527, 1529, 1609, 1610, 1601, 1615, 1575, - /* 380 */ 1650, 1578, 1619, 1623, 1625, 1624, 1674, 1685, 1686, 1691, - /* 390 */ 1692, 1693, 1612, 1613, 1617, 1675, 1668, 1673, 1676, 1677, - /* 400 */ 1680, 1666, 1669, 1678, 1681, 1683, 1687, + /* 0 */ -71, 194, 343, 835, -180, -177, 838, -194, -188, -185, + /* 10 */ -183, 82, 183, -65, 133, 245, 346, 407, 458, -178, + /* 20 */ 75, -275, -4, 310, 312, 489, 575, 596, 463, 686, + /* 30 */ 707, 725, 780, 1098, 856, 778, 1059, 1090, 708, 887, + /* 40 */ 86, 448, 980, 630, 680, 681, 684, 796, 801, 796, + /* 50 */ 801, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 60 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 70 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 80 */ -261, -261, -261, -261, -261, -261, -261, -261, 391, 886, + /* 90 */ 888, 1013, 1016, 1081, 1087, 1151, 1159, 1177, 1185, 1188, + /* 100 */ 1190, 1194, 1197, 1203, 1247, 1260, 1264, 1267, 1269, 1273, + /* 110 */ 1315, 1322, 1335, 1337, 1356, 1362, 1418, 1425, 1453, 1457, + /* 120 */ 1465, 1473, 1487, 1495, 1507, 1517, 1521, 1534, 1543, 1546, + /* 130 */ 1549, 1552, 1554, 1560, 1581, 1590, 1593, 1595, 1621, 1623, + /* 140 */ -261, -261, -261, -261, -261, -261, -261, -261, -261, -261, + /* 150 */ -261, -186, -117, 260, 263, 460, 631, -74, 497, -181, + /* 160 */ -261, 939, 176, 274, 338, 676, -261, -261, -261, -261, + /* 170 */ -212, -212, -212, -184, 149, 777, 1061, 1103, 265, 419, + /* 180 */ -254, 670, 677, 677, -11, -129, 184, 488, 736, 789, + /* 190 */ 805, 844, 403, 529, 579, 668, 783, 841, 1158, 1112, + /* 200 */ 806, 861, 1095, 846, 839, 1031, -189, 1077, 1080, 1116, + /* 210 */ 1084, 1156, 1139, 1221, 46, 1099, 1037, 1118, 1171, 1214, + /* 220 */ 1210, 1258, -210, -190, -176, -115, 117, 262, 376, 490, + /* 230 */ 511, 520, 618, 639, 743, 901, 907, 958, 1014, 1055, + /* 240 */ 1108, 1193, 1244, 720, 1248, 1277, 1324, 1347, 1417, 1431, + /* 250 */ 1432, 1440, 1451, 1452, 1463, 1478, 1286, 1350, 1369, 1490, + /* 260 */ 1498, 1501, 773, 1509, 1513, 1528, 1292, 1367, 1535, 1536, + /* 270 */ 1477, 1542, 376, 1547, 1550, 1555, 1559, 1568, 1571, 1441, + /* 280 */ 1443, 1474, 1511, 1516, 1519, 1522, 773, 1474, 1474, 1503, + /* 290 */ 1567, 1594, 1484, 1527, 1556, 1570, 1557, 1524, 1573, 1545, + /* 300 */ 1548, 1576, 1561, 1587, 1540, 1575, 1606, 1611, 1622, 1624, + /* 310 */ 1626, 1582, 1597, 1598, 1599, 1601, 1603, 1563, 1608, 1605, + /* 320 */ 1604, 1564, 1566, 1655, 1660, 1578, 1579, 1665, 1586, 1607, + /* 330 */ 1610, 1642, 1641, 1645, 1646, 1647, 1679, 1688, 1644, 1618, + /* 340 */ 1619, 1648, 1628, 1659, 1649, 1663, 1650, 1700, 1702, 1612, + /* 350 */ 1615, 1706, 1708, 1689, 1709, 1712, 1713, 1715, 1691, 1698, + /* 360 */ 1699, 1701, 1693, 1704, 1707, 1705, 1714, 1703, 1718, 1710, + /* 370 */ 1720, 1721, 1632, 1634, 1664, 1666, 1729, 1751, 1635, 1636, + /* 380 */ 1692, 1694, 1716, 1722, 1684, 1763, 1685, 1723, 1724, 1727, + /* 390 */ 1730, 1768, 1783, 1784, 1793, 1794, 1796, 1683, 1686, 1690, + /* 400 */ 1782, 1779, 1780, 1781, 1785, 1788, 1774, 1775, 1786, 1787, + /* 410 */ 1789, 1798, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1633, 1633, 1633, 1462, 1230, 1341, 1230, 1230, 1230, 1462, - /* 10 */ 1462, 1462, 1230, 1371, 1371, 1515, 1263, 1230, 1230, 1230, - /* 20 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1461, 1230, 1230, - /* 30 */ 1230, 1230, 1550, 1550, 1230, 1230, 1230, 1230, 1230, 1230, - /* 40 */ 1230, 1230, 1380, 1230, 1387, 1230, 1230, 1230, 1230, 1230, - /* 50 */ 1463, 1464, 1230, 1230, 1230, 1514, 1516, 1479, 1394, 1393, - /* 60 */ 1392, 1391, 1497, 1358, 1385, 1378, 1382, 1457, 1458, 1456, - /* 70 */ 1460, 1464, 1463, 1230, 1381, 1428, 1442, 1427, 1230, 1230, - /* 80 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 90 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 100 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 110 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 120 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1436, 1441, 1447, - /* 130 */ 1440, 1437, 1430, 1429, 1431, 1230, 1432, 1230, 1254, 1230, - /* 140 */ 1230, 1251, 1305, 1230, 1230, 1230, 1230, 1230, 1534, 1533, - /* 150 */ 1230, 1230, 1263, 1422, 1421, 1433, 1434, 1444, 1443, 1522, - /* 160 */ 1586, 1585, 1480, 1230, 1230, 1230, 1230, 1230, 1230, 1550, - /* 170 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 180 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 190 */ 1230, 1230, 1230, 1550, 1550, 1230, 1263, 1550, 1550, 1259, - /* 200 */ 1259, 1365, 1230, 1529, 1332, 1332, 1332, 1332, 1341, 1332, - /* 210 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 220 */ 1230, 1230, 1230, 1230, 1230, 1519, 1517, 1230, 1230, 1230, - /* 230 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 240 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 250 */ 1230, 1230, 1230, 1230, 1337, 1230, 1230, 1230, 1230, 1230, - /* 260 */ 1230, 1230, 1230, 1230, 1230, 1230, 1579, 1230, 1492, 1319, - /* 270 */ 1337, 1337, 1337, 1337, 1339, 1320, 1318, 1331, 1264, 1237, - /* 280 */ 1625, 1397, 1386, 1338, 1360, 1386, 1360, 1622, 1384, 1397, - /* 290 */ 1397, 1384, 1397, 1338, 1622, 1280, 1602, 1275, 1371, 1371, - /* 300 */ 1371, 1360, 1365, 1365, 1459, 1338, 1331, 1230, 1625, 1625, - /* 310 */ 1346, 1346, 1624, 1624, 1346, 1480, 1609, 1406, 1308, 1314, - /* 320 */ 1314, 1314, 1314, 1346, 1248, 1384, 1609, 1609, 1384, 1406, - /* 330 */ 1308, 1384, 1308, 1384, 1346, 1248, 1496, 1619, 1346, 1248, - /* 340 */ 1470, 1346, 1248, 1346, 1248, 1470, 1306, 1306, 1306, 1295, - /* 350 */ 1230, 1230, 1470, 1306, 1280, 1306, 1295, 1306, 1306, 1568, - /* 360 */ 1230, 1474, 1474, 1470, 1364, 1359, 1364, 1359, 1364, 1359, - /* 370 */ 1364, 1359, 1346, 1560, 1560, 1374, 1374, 1379, 1365, 1465, - /* 380 */ 1346, 1230, 1379, 1377, 1375, 1384, 1298, 1582, 1582, 1578, - /* 390 */ 1578, 1578, 1630, 1630, 1529, 1595, 1263, 1263, 1263, 1263, - /* 400 */ 1595, 1282, 1282, 1264, 1264, 1263, 1595, 1230, 1230, 1230, - /* 410 */ 1230, 1230, 1230, 1590, 1230, 1524, 1481, 1350, 1230, 1230, - /* 420 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 430 */ 1230, 1230, 1535, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 440 */ 1230, 1230, 1230, 1411, 1230, 1233, 1526, 1230, 1230, 1230, - /* 450 */ 1230, 1230, 1230, 1230, 1230, 1388, 1389, 1351, 1230, 1230, - /* 460 */ 1230, 1230, 1230, 1230, 1230, 1403, 1230, 1230, 1230, 1398, - /* 470 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1621, 1230, - /* 480 */ 1230, 1230, 1230, 1230, 1230, 1495, 1494, 1230, 1230, 1348, - /* 490 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 500 */ 1230, 1230, 1230, 1278, 1230, 1230, 1230, 1230, 1230, 1230, - /* 510 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 520 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1376, 1230, 1230, - /* 530 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 540 */ 1230, 1230, 1565, 1366, 1230, 1230, 1612, 1230, 1230, 1230, - /* 550 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, - /* 560 */ 1230, 1606, 1322, 1413, 1230, 1412, 1416, 1252, 1230, 1242, - /* 570 */ 1230, 1230, + /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, + /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, + /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, + /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, + /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, + /* 60 */ 1492, 1493, 1254, 1254, 1254, 1543, 1545, 1508, 1420, 1419, + /* 70 */ 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, 1486, + /* 80 */ 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, 1254, + /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 140 */ 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, 1456, 1458, + /* 150 */ 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, 1254, 1254, + /* 160 */ 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, 1473, 1472, + /* 170 */ 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, 1254, 1254, + /* 180 */ 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 200 */ 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, 1578, 1578, + /* 210 */ 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, 1358, 1358, + /* 220 */ 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, 1546, 1254, + /* 240 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, 1254, 1254, + /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, 1254, + /* 280 */ 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, 1357, + /* 290 */ 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, 1423, + /* 300 */ 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, 1397, + /* 310 */ 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, 1357, + /* 320 */ 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, 1638, + /* 330 */ 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, 1638, + /* 340 */ 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, 1525, + /* 350 */ 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, 1330, + /* 360 */ 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, 1319, + /* 370 */ 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, 1588, + /* 380 */ 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, 1401, + /* 390 */ 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, 1558, + /* 400 */ 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, 1288, + /* 410 */ 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, 1254, + /* 420 */ 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1564, + /* 440 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 450 */ 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, 1254, + /* 460 */ 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, 1254, + /* 470 */ 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, 1254, + /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, 1254, + /* 490 */ 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, 1254, + /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 510 */ 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 530 */ 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, 1254, + /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 550 */ 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, 1254, + /* 560 */ 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, + /* 580 */ 1266, 1254, 1254, }; /********** End of lemon-generated parsing tables *****************************/ @@ -161458,6 +175636,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* SLASH => nothing */ 0, /* REM => nothing */ 0, /* CONCAT => nothing */ + 0, /* PTR => nothing */ 0, /* COLLATE => nothing */ 0, /* BITNOT => nothing */ 0, /* ON => nothing */ @@ -161518,8 +175697,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* TRUEFALSE => nothing */ 0, /* ISNOT => nothing */ 0, /* FUNCTION => nothing */ - 0, /* UMINUS => nothing */ 0, /* UPLUS => nothing */ + 0, /* UMINUS => nothing */ 0, /* TRUTH => nothing */ 0, /* REGISTER => nothing */ 0, /* VECTOR => nothing */ @@ -161528,6 +175707,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASTERISK => nothing */ 0, /* SPAN => nothing */ 0, /* ERROR => nothing */ + 0, /* QNUMBER => nothing */ 0, /* SPACE => nothing */ 0, /* ILLEGAL => nothing */ }; @@ -161570,14 +175750,9 @@ struct yyParser { #endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; @@ -161730,212 +175905,216 @@ static const char *const yyTokenName[] = { /* 109 */ "SLASH", /* 110 */ "REM", /* 111 */ "CONCAT", - /* 112 */ "COLLATE", - /* 113 */ "BITNOT", - /* 114 */ "ON", - /* 115 */ "INDEXED", - /* 116 */ "STRING", - /* 117 */ "JOIN_KW", - /* 118 */ "CONSTRAINT", - /* 119 */ "DEFAULT", - /* 120 */ "NULL", - /* 121 */ "PRIMARY", - /* 122 */ "UNIQUE", - /* 123 */ "CHECK", - /* 124 */ "REFERENCES", - /* 125 */ "AUTOINCR", - /* 126 */ "INSERT", - /* 127 */ "DELETE", - /* 128 */ "UPDATE", - /* 129 */ "SET", - /* 130 */ "DEFERRABLE", - /* 131 */ "FOREIGN", - /* 132 */ "DROP", - /* 133 */ "UNION", - /* 134 */ "ALL", - /* 135 */ "EXCEPT", - /* 136 */ "INTERSECT", - /* 137 */ "SELECT", - /* 138 */ "VALUES", - /* 139 */ "DISTINCT", - /* 140 */ "DOT", - /* 141 */ "FROM", - /* 142 */ "JOIN", - /* 143 */ "USING", - /* 144 */ "ORDER", - /* 145 */ "GROUP", - /* 146 */ "HAVING", - /* 147 */ "LIMIT", - /* 148 */ "WHERE", - /* 149 */ "RETURNING", - /* 150 */ "INTO", - /* 151 */ "NOTHING", - /* 152 */ "FLOAT", - /* 153 */ "BLOB", - /* 154 */ "INTEGER", - /* 155 */ "VARIABLE", - /* 156 */ "CASE", - /* 157 */ "WHEN", - /* 158 */ "THEN", - /* 159 */ "ELSE", - /* 160 */ "INDEX", - /* 161 */ "ALTER", - /* 162 */ "ADD", - /* 163 */ "WINDOW", - /* 164 */ "OVER", - /* 165 */ "FILTER", - /* 166 */ "COLUMN", - /* 167 */ "AGG_FUNCTION", - /* 168 */ "AGG_COLUMN", - /* 169 */ "TRUEFALSE", - /* 170 */ "ISNOT", - /* 171 */ "FUNCTION", - /* 172 */ "UMINUS", + /* 112 */ "PTR", + /* 113 */ "COLLATE", + /* 114 */ "BITNOT", + /* 115 */ "ON", + /* 116 */ "INDEXED", + /* 117 */ "STRING", + /* 118 */ "JOIN_KW", + /* 119 */ "CONSTRAINT", + /* 120 */ "DEFAULT", + /* 121 */ "NULL", + /* 122 */ "PRIMARY", + /* 123 */ "UNIQUE", + /* 124 */ "CHECK", + /* 125 */ "REFERENCES", + /* 126 */ "AUTOINCR", + /* 127 */ "INSERT", + /* 128 */ "DELETE", + /* 129 */ "UPDATE", + /* 130 */ "SET", + /* 131 */ "DEFERRABLE", + /* 132 */ "FOREIGN", + /* 133 */ "DROP", + /* 134 */ "UNION", + /* 135 */ "ALL", + /* 136 */ "EXCEPT", + /* 137 */ "INTERSECT", + /* 138 */ "SELECT", + /* 139 */ "VALUES", + /* 140 */ "DISTINCT", + /* 141 */ "DOT", + /* 142 */ "FROM", + /* 143 */ "JOIN", + /* 144 */ "USING", + /* 145 */ "ORDER", + /* 146 */ "GROUP", + /* 147 */ "HAVING", + /* 148 */ "LIMIT", + /* 149 */ "WHERE", + /* 150 */ "RETURNING", + /* 151 */ "INTO", + /* 152 */ "NOTHING", + /* 153 */ "FLOAT", + /* 154 */ "BLOB", + /* 155 */ "INTEGER", + /* 156 */ "VARIABLE", + /* 157 */ "CASE", + /* 158 */ "WHEN", + /* 159 */ "THEN", + /* 160 */ "ELSE", + /* 161 */ "INDEX", + /* 162 */ "ALTER", + /* 163 */ "ADD", + /* 164 */ "WINDOW", + /* 165 */ "OVER", + /* 166 */ "FILTER", + /* 167 */ "COLUMN", + /* 168 */ "AGG_FUNCTION", + /* 169 */ "AGG_COLUMN", + /* 170 */ "TRUEFALSE", + /* 171 */ "ISNOT", + /* 172 */ "FUNCTION", /* 173 */ "UPLUS", - /* 174 */ "TRUTH", - /* 175 */ "REGISTER", - /* 176 */ "VECTOR", - /* 177 */ "SELECT_COLUMN", - /* 178 */ "IF_NULL_ROW", - /* 179 */ "ASTERISK", - /* 180 */ "SPAN", - /* 181 */ "ERROR", - /* 182 */ "SPACE", - /* 183 */ "ILLEGAL", - /* 184 */ "input", - /* 185 */ "cmdlist", - /* 186 */ "ecmd", - /* 187 */ "cmdx", - /* 188 */ "explain", - /* 189 */ "cmd", - /* 190 */ "transtype", - /* 191 */ "trans_opt", - /* 192 */ "nm", - /* 193 */ "savepoint_opt", - /* 194 */ "create_table", - /* 195 */ "create_table_args", - /* 196 */ "createkw", - /* 197 */ "temp", - /* 198 */ "ifnotexists", - /* 199 */ "dbnm", - /* 200 */ "columnlist", - /* 201 */ "conslist_opt", - /* 202 */ "table_option_set", - /* 203 */ "select", - /* 204 */ "table_option", - /* 205 */ "columnname", - /* 206 */ "carglist", - /* 207 */ "typetoken", - /* 208 */ "typename", - /* 209 */ "signed", - /* 210 */ "plus_num", - /* 211 */ "minus_num", - /* 212 */ "scanpt", - /* 213 */ "scantok", - /* 214 */ "ccons", - /* 215 */ "term", - /* 216 */ "expr", - /* 217 */ "onconf", - /* 218 */ "sortorder", - /* 219 */ "autoinc", - /* 220 */ "eidlist_opt", - /* 221 */ "refargs", - /* 222 */ "defer_subclause", - /* 223 */ "generated", - /* 224 */ "refarg", - /* 225 */ "refact", - /* 226 */ "init_deferred_pred_opt", - /* 227 */ "conslist", - /* 228 */ "tconscomma", - /* 229 */ "tcons", - /* 230 */ "sortlist", - /* 231 */ "eidlist", - /* 232 */ "defer_subclause_opt", - /* 233 */ "orconf", - /* 234 */ "resolvetype", - /* 235 */ "raisetype", - /* 236 */ "ifexists", - /* 237 */ "fullname", - /* 238 */ "selectnowith", - /* 239 */ "oneselect", - /* 240 */ "wqlist", - /* 241 */ "multiselect_op", - /* 242 */ "distinct", - /* 243 */ "selcollist", - /* 244 */ "from", - /* 245 */ "where_opt", - /* 246 */ "groupby_opt", - /* 247 */ "having_opt", - /* 248 */ "orderby_opt", - /* 249 */ "limit_opt", - /* 250 */ "window_clause", - /* 251 */ "values", - /* 252 */ "nexprlist", - /* 253 */ "sclp", - /* 254 */ "as", - /* 255 */ "seltablist", - /* 256 */ "stl_prefix", - /* 257 */ "joinop", - /* 258 */ "indexed_opt", - /* 259 */ "on_opt", - /* 260 */ "using_opt", - /* 261 */ "exprlist", - /* 262 */ "xfullname", - /* 263 */ "idlist", - /* 264 */ "nulls", - /* 265 */ "with", - /* 266 */ "where_opt_ret", - /* 267 */ "setlist", - /* 268 */ "insert_cmd", - /* 269 */ "idlist_opt", - /* 270 */ "upsert", - /* 271 */ "returning", - /* 272 */ "filter_over", - /* 273 */ "likeop", - /* 274 */ "between_op", - /* 275 */ "in_op", - /* 276 */ "paren_exprlist", - /* 277 */ "case_operand", - /* 278 */ "case_exprlist", - /* 279 */ "case_else", - /* 280 */ "uniqueflag", - /* 281 */ "collate", - /* 282 */ "vinto", - /* 283 */ "nmnum", - /* 284 */ "trigger_decl", - /* 285 */ "trigger_cmd_list", - /* 286 */ "trigger_time", - /* 287 */ "trigger_event", - /* 288 */ "foreach_clause", - /* 289 */ "when_clause", - /* 290 */ "trigger_cmd", - /* 291 */ "trnm", - /* 292 */ "tridxby", - /* 293 */ "database_kw_opt", - /* 294 */ "key_opt", - /* 295 */ "add_column_fullname", - /* 296 */ "kwcolumn_opt", - /* 297 */ "create_vtab", - /* 298 */ "vtabarglist", - /* 299 */ "vtabarg", - /* 300 */ "vtabargtoken", - /* 301 */ "lp", - /* 302 */ "anylist", - /* 303 */ "wqitem", - /* 304 */ "wqas", - /* 305 */ "windowdefn_list", - /* 306 */ "windowdefn", - /* 307 */ "window", - /* 308 */ "frame_opt", - /* 309 */ "part_opt", - /* 310 */ "filter_clause", - /* 311 */ "over_clause", - /* 312 */ "range_or_rows", - /* 313 */ "frame_bound", - /* 314 */ "frame_bound_s", - /* 315 */ "frame_bound_e", - /* 316 */ "frame_exclude_opt", - /* 317 */ "frame_exclude", + /* 174 */ "UMINUS", + /* 175 */ "TRUTH", + /* 176 */ "REGISTER", + /* 177 */ "VECTOR", + /* 178 */ "SELECT_COLUMN", + /* 179 */ "IF_NULL_ROW", + /* 180 */ "ASTERISK", + /* 181 */ "SPAN", + /* 182 */ "ERROR", + /* 183 */ "QNUMBER", + /* 184 */ "SPACE", + /* 185 */ "ILLEGAL", + /* 186 */ "input", + /* 187 */ "cmdlist", + /* 188 */ "ecmd", + /* 189 */ "cmdx", + /* 190 */ "explain", + /* 191 */ "cmd", + /* 192 */ "transtype", + /* 193 */ "trans_opt", + /* 194 */ "nm", + /* 195 */ "savepoint_opt", + /* 196 */ "create_table", + /* 197 */ "create_table_args", + /* 198 */ "createkw", + /* 199 */ "temp", + /* 200 */ "ifnotexists", + /* 201 */ "dbnm", + /* 202 */ "columnlist", + /* 203 */ "conslist_opt", + /* 204 */ "table_option_set", + /* 205 */ "select", + /* 206 */ "table_option", + /* 207 */ "columnname", + /* 208 */ "carglist", + /* 209 */ "typetoken", + /* 210 */ "typename", + /* 211 */ "signed", + /* 212 */ "plus_num", + /* 213 */ "minus_num", + /* 214 */ "scanpt", + /* 215 */ "scantok", + /* 216 */ "ccons", + /* 217 */ "term", + /* 218 */ "expr", + /* 219 */ "onconf", + /* 220 */ "sortorder", + /* 221 */ "autoinc", + /* 222 */ "eidlist_opt", + /* 223 */ "refargs", + /* 224 */ "defer_subclause", + /* 225 */ "generated", + /* 226 */ "refarg", + /* 227 */ "refact", + /* 228 */ "init_deferred_pred_opt", + /* 229 */ "conslist", + /* 230 */ "tconscomma", + /* 231 */ "tcons", + /* 232 */ "sortlist", + /* 233 */ "eidlist", + /* 234 */ "defer_subclause_opt", + /* 235 */ "orconf", + /* 236 */ "resolvetype", + /* 237 */ "raisetype", + /* 238 */ "ifexists", + /* 239 */ "fullname", + /* 240 */ "selectnowith", + /* 241 */ "oneselect", + /* 242 */ "wqlist", + /* 243 */ "multiselect_op", + /* 244 */ "distinct", + /* 245 */ "selcollist", + /* 246 */ "from", + /* 247 */ "where_opt", + /* 248 */ "groupby_opt", + /* 249 */ "having_opt", + /* 250 */ "orderby_opt", + /* 251 */ "limit_opt", + /* 252 */ "window_clause", + /* 253 */ "values", + /* 254 */ "nexprlist", + /* 255 */ "mvalues", + /* 256 */ "sclp", + /* 257 */ "as", + /* 258 */ "seltablist", + /* 259 */ "stl_prefix", + /* 260 */ "joinop", + /* 261 */ "on_using", + /* 262 */ "indexed_by", + /* 263 */ "exprlist", + /* 264 */ "xfullname", + /* 265 */ "idlist", + /* 266 */ "indexed_opt", + /* 267 */ "nulls", + /* 268 */ "with", + /* 269 */ "where_opt_ret", + /* 270 */ "setlist", + /* 271 */ "insert_cmd", + /* 272 */ "idlist_opt", + /* 273 */ "upsert", + /* 274 */ "returning", + /* 275 */ "filter_over", + /* 276 */ "likeop", + /* 277 */ "between_op", + /* 278 */ "in_op", + /* 279 */ "paren_exprlist", + /* 280 */ "case_operand", + /* 281 */ "case_exprlist", + /* 282 */ "case_else", + /* 283 */ "uniqueflag", + /* 284 */ "collate", + /* 285 */ "vinto", + /* 286 */ "nmnum", + /* 287 */ "trigger_decl", + /* 288 */ "trigger_cmd_list", + /* 289 */ "trigger_time", + /* 290 */ "trigger_event", + /* 291 */ "foreach_clause", + /* 292 */ "when_clause", + /* 293 */ "trigger_cmd", + /* 294 */ "trnm", + /* 295 */ "tridxby", + /* 296 */ "database_kw_opt", + /* 297 */ "key_opt", + /* 298 */ "add_column_fullname", + /* 299 */ "kwcolumn_opt", + /* 300 */ "create_vtab", + /* 301 */ "vtabarglist", + /* 302 */ "vtabarg", + /* 303 */ "vtabargtoken", + /* 304 */ "lp", + /* 305 */ "anylist", + /* 306 */ "wqitem", + /* 307 */ "wqas", + /* 308 */ "withnm", + /* 309 */ "windowdefn_list", + /* 310 */ "windowdefn", + /* 311 */ "window", + /* 312 */ "frame_opt", + /* 313 */ "part_opt", + /* 314 */ "filter_clause", + /* 315 */ "over_clause", + /* 316 */ "range_or_rows", + /* 317 */ "frame_bound", + /* 318 */ "frame_bound_s", + /* 319 */ "frame_bound_e", + /* 320 */ "frame_exclude_opt", + /* 321 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -162038,347 +176217,363 @@ static const char *const yyRuleName[] = { /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", /* 94 */ "values ::= VALUES LP nexprlist RP", - /* 95 */ "values ::= values COMMA LP nexprlist RP", - /* 96 */ "distinct ::= DISTINCT", - /* 97 */ "distinct ::= ALL", - /* 98 */ "distinct ::=", - /* 99 */ "sclp ::=", - /* 100 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 101 */ "selcollist ::= sclp scanpt STAR", - /* 102 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 103 */ "as ::= AS nm", - /* 104 */ "as ::=", - /* 105 */ "from ::=", - /* 106 */ "from ::= FROM seltablist", - /* 107 */ "stl_prefix ::= seltablist joinop", - /* 108 */ "stl_prefix ::=", - /* 109 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 110 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 111 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 112 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 113 */ "dbnm ::=", - /* 114 */ "dbnm ::= DOT nm", - /* 115 */ "fullname ::= nm", - /* 116 */ "fullname ::= nm DOT nm", - /* 117 */ "xfullname ::= nm", - /* 118 */ "xfullname ::= nm DOT nm", - /* 119 */ "xfullname ::= nm DOT nm AS nm", - /* 120 */ "xfullname ::= nm AS nm", - /* 121 */ "joinop ::= COMMA|JOIN", - /* 122 */ "joinop ::= JOIN_KW JOIN", - /* 123 */ "joinop ::= JOIN_KW nm JOIN", - /* 124 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 125 */ "on_opt ::= ON expr", - /* 126 */ "on_opt ::=", - /* 127 */ "indexed_opt ::=", - /* 128 */ "indexed_opt ::= INDEXED BY nm", - /* 129 */ "indexed_opt ::= NOT INDEXED", - /* 130 */ "using_opt ::= USING LP idlist RP", - /* 131 */ "using_opt ::=", - /* 132 */ "orderby_opt ::=", - /* 133 */ "orderby_opt ::= ORDER BY sortlist", - /* 134 */ "sortlist ::= sortlist COMMA expr sortorder nulls", - /* 135 */ "sortlist ::= expr sortorder nulls", - /* 136 */ "sortorder ::= ASC", - /* 137 */ "sortorder ::= DESC", - /* 138 */ "sortorder ::=", - /* 139 */ "nulls ::= NULLS FIRST", - /* 140 */ "nulls ::= NULLS LAST", - /* 141 */ "nulls ::=", - /* 142 */ "groupby_opt ::=", - /* 143 */ "groupby_opt ::= GROUP BY nexprlist", - /* 144 */ "having_opt ::=", - /* 145 */ "having_opt ::= HAVING expr", - /* 146 */ "limit_opt ::=", - /* 147 */ "limit_opt ::= LIMIT expr", - /* 148 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 149 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 150 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", - /* 151 */ "where_opt ::=", - /* 152 */ "where_opt ::= WHERE expr", - /* 153 */ "where_opt_ret ::=", - /* 154 */ "where_opt_ret ::= WHERE expr", - /* 155 */ "where_opt_ret ::= RETURNING selcollist", - /* 156 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", - /* 157 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", - /* 158 */ "setlist ::= setlist COMMA nm EQ expr", - /* 159 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 160 */ "setlist ::= nm EQ expr", - /* 161 */ "setlist ::= LP idlist RP EQ expr", - /* 162 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 163 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", - /* 164 */ "upsert ::=", - /* 165 */ "upsert ::= RETURNING selcollist", - /* 166 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", - /* 167 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", - /* 168 */ "upsert ::= ON CONFLICT DO NOTHING returning", - /* 169 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", - /* 170 */ "returning ::= RETURNING selcollist", - /* 171 */ "insert_cmd ::= INSERT orconf", - /* 172 */ "insert_cmd ::= REPLACE", - /* 173 */ "idlist_opt ::=", - /* 174 */ "idlist_opt ::= LP idlist RP", - /* 175 */ "idlist ::= idlist COMMA nm", - /* 176 */ "idlist ::= nm", - /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED", - /* 179 */ "expr ::= JOIN_KW", - /* 180 */ "expr ::= nm DOT nm", - /* 181 */ "expr ::= nm DOT nm DOT nm", - /* 182 */ "term ::= NULL|FLOAT|BLOB", - /* 183 */ "term ::= STRING", - /* 184 */ "term ::= INTEGER", - /* 185 */ "expr ::= VARIABLE", - /* 186 */ "expr ::= expr COLLATE ID|STRING", - /* 187 */ "expr ::= CAST LP expr AS typetoken RP", - /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 189 */ "expr ::= ID|INDEXED LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED LP STAR RP filter_over", - /* 192 */ "term ::= CTIME_KW", - /* 193 */ "expr ::= LP nexprlist COMMA expr RP", - /* 194 */ "expr ::= expr AND expr", - /* 195 */ "expr ::= expr OR expr", - /* 196 */ "expr ::= expr LT|GT|GE|LE expr", - /* 197 */ "expr ::= expr EQ|NE expr", - /* 198 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 199 */ "expr ::= expr PLUS|MINUS expr", - /* 200 */ "expr ::= expr STAR|SLASH|REM expr", - /* 201 */ "expr ::= expr CONCAT expr", - /* 202 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 203 */ "expr ::= expr likeop expr", - /* 204 */ "expr ::= expr likeop expr ESCAPE expr", - /* 205 */ "expr ::= expr ISNULL|NOTNULL", - /* 206 */ "expr ::= expr NOT NULL", - /* 207 */ "expr ::= expr IS expr", - /* 208 */ "expr ::= expr IS NOT expr", - /* 209 */ "expr ::= NOT expr", - /* 210 */ "expr ::= BITNOT expr", - /* 211 */ "expr ::= PLUS|MINUS expr", - /* 212 */ "between_op ::= BETWEEN", - /* 213 */ "between_op ::= NOT BETWEEN", - /* 214 */ "expr ::= expr between_op expr AND expr", - /* 215 */ "in_op ::= IN", - /* 216 */ "in_op ::= NOT IN", - /* 217 */ "expr ::= expr in_op LP exprlist RP", - /* 218 */ "expr ::= LP select RP", - /* 219 */ "expr ::= expr in_op LP select RP", - /* 220 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 221 */ "expr ::= EXISTS LP select RP", - /* 222 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 223 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 224 */ "case_exprlist ::= WHEN expr THEN expr", - /* 225 */ "case_else ::= ELSE expr", - /* 226 */ "case_else ::=", - /* 227 */ "case_operand ::= expr", - /* 228 */ "case_operand ::=", - /* 229 */ "exprlist ::=", - /* 230 */ "nexprlist ::= nexprlist COMMA expr", - /* 231 */ "nexprlist ::= expr", - /* 232 */ "paren_exprlist ::=", - /* 233 */ "paren_exprlist ::= LP exprlist RP", - /* 234 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 235 */ "uniqueflag ::= UNIQUE", - /* 236 */ "uniqueflag ::=", - /* 237 */ "eidlist_opt ::=", - /* 238 */ "eidlist_opt ::= LP eidlist RP", - /* 239 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 240 */ "eidlist ::= nm collate sortorder", - /* 241 */ "collate ::=", - /* 242 */ "collate ::= COLLATE ID|STRING", - /* 243 */ "cmd ::= DROP INDEX ifexists fullname", - /* 244 */ "cmd ::= VACUUM vinto", - /* 245 */ "cmd ::= VACUUM nm vinto", - /* 246 */ "vinto ::= INTO expr", - /* 247 */ "vinto ::=", - /* 248 */ "cmd ::= PRAGMA nm dbnm", - /* 249 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 250 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 251 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 252 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 253 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 254 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 255 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 256 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 257 */ "trigger_time ::= BEFORE|AFTER", - /* 258 */ "trigger_time ::= INSTEAD OF", - /* 259 */ "trigger_time ::=", - /* 260 */ "trigger_event ::= DELETE|INSERT", - /* 261 */ "trigger_event ::= UPDATE", - /* 262 */ "trigger_event ::= UPDATE OF idlist", - /* 263 */ "when_clause ::=", - /* 264 */ "when_clause ::= WHEN expr", - /* 265 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 266 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 267 */ "trnm ::= nm DOT nm", - /* 268 */ "tridxby ::= INDEXED BY nm", - /* 269 */ "tridxby ::= NOT INDEXED", - /* 270 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 271 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 272 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 273 */ "trigger_cmd ::= scanpt select scanpt", - /* 274 */ "expr ::= RAISE LP IGNORE RP", - /* 275 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 276 */ "raisetype ::= ROLLBACK", - /* 277 */ "raisetype ::= ABORT", - /* 278 */ "raisetype ::= FAIL", - /* 279 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 280 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 281 */ "cmd ::= DETACH database_kw_opt expr", - /* 282 */ "key_opt ::=", - /* 283 */ "key_opt ::= KEY expr", - /* 284 */ "cmd ::= REINDEX", - /* 285 */ "cmd ::= REINDEX nm dbnm", - /* 286 */ "cmd ::= ANALYZE", - /* 287 */ "cmd ::= ANALYZE nm dbnm", - /* 288 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 289 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 290 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 291 */ "add_column_fullname ::= fullname", - /* 292 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 293 */ "cmd ::= create_vtab", - /* 294 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 295 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 296 */ "vtabarg ::=", - /* 297 */ "vtabargtoken ::= ANY", - /* 298 */ "vtabargtoken ::= lp anylist RP", - /* 299 */ "lp ::= LP", - /* 300 */ "with ::= WITH wqlist", - /* 301 */ "with ::= WITH RECURSIVE wqlist", - /* 302 */ "wqas ::= AS", - /* 303 */ "wqas ::= AS MATERIALIZED", - /* 304 */ "wqas ::= AS NOT MATERIALIZED", - /* 305 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 306 */ "wqlist ::= wqitem", - /* 307 */ "wqlist ::= wqlist COMMA wqitem", - /* 308 */ "windowdefn_list ::= windowdefn", - /* 309 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 310 */ "windowdefn ::= nm AS LP window RP", - /* 311 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 312 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 313 */ "window ::= ORDER BY sortlist frame_opt", - /* 314 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 315 */ "window ::= frame_opt", - /* 316 */ "window ::= nm frame_opt", - /* 317 */ "frame_opt ::=", - /* 318 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 319 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 320 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 321 */ "frame_bound_s ::= frame_bound", - /* 322 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 323 */ "frame_bound_e ::= frame_bound", - /* 324 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 325 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 326 */ "frame_bound ::= CURRENT ROW", - /* 327 */ "frame_exclude_opt ::=", - /* 328 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 329 */ "frame_exclude ::= NO OTHERS", - /* 330 */ "frame_exclude ::= CURRENT ROW", - /* 331 */ "frame_exclude ::= GROUP|TIES", - /* 332 */ "window_clause ::= WINDOW windowdefn_list", - /* 333 */ "filter_over ::= filter_clause over_clause", - /* 334 */ "filter_over ::= over_clause", - /* 335 */ "filter_over ::= filter_clause", - /* 336 */ "over_clause ::= OVER LP window RP", - /* 337 */ "over_clause ::= OVER nm", - /* 338 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 339 */ "input ::= cmdlist", - /* 340 */ "cmdlist ::= cmdlist ecmd", - /* 341 */ "cmdlist ::= ecmd", - /* 342 */ "ecmd ::= SEMI", - /* 343 */ "ecmd ::= cmdx SEMI", - /* 344 */ "ecmd ::= explain cmdx SEMI", - /* 345 */ "trans_opt ::=", - /* 346 */ "trans_opt ::= TRANSACTION", - /* 347 */ "trans_opt ::= TRANSACTION nm", - /* 348 */ "savepoint_opt ::= SAVEPOINT", - /* 349 */ "savepoint_opt ::=", - /* 350 */ "cmd ::= create_table create_table_args", - /* 351 */ "table_option_set ::= table_option", - /* 352 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 353 */ "columnlist ::= columnname carglist", - /* 354 */ "nm ::= ID|INDEXED", - /* 355 */ "nm ::= STRING", - /* 356 */ "nm ::= JOIN_KW", - /* 357 */ "typetoken ::= typename", - /* 358 */ "typename ::= ID|STRING", - /* 359 */ "signed ::= plus_num", - /* 360 */ "signed ::= minus_num", - /* 361 */ "carglist ::= carglist ccons", - /* 362 */ "carglist ::=", - /* 363 */ "ccons ::= NULL onconf", - /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 365 */ "ccons ::= AS generated", - /* 366 */ "conslist_opt ::= COMMA conslist", - /* 367 */ "conslist ::= conslist tconscomma tcons", - /* 368 */ "conslist ::= tcons", - /* 369 */ "tconscomma ::=", - /* 370 */ "defer_subclause_opt ::= defer_subclause", - /* 371 */ "resolvetype ::= raisetype", - /* 372 */ "selectnowith ::= oneselect", - /* 373 */ "oneselect ::= values", - /* 374 */ "sclp ::= selcollist COMMA", - /* 375 */ "as ::= ID|STRING", - /* 376 */ "returning ::=", - /* 377 */ "expr ::= term", - /* 378 */ "likeop ::= LIKE_KW|MATCH", - /* 379 */ "exprlist ::= nexprlist", - /* 380 */ "nmnum ::= plus_num", - /* 381 */ "nmnum ::= nm", - /* 382 */ "nmnum ::= ON", - /* 383 */ "nmnum ::= DELETE", - /* 384 */ "nmnum ::= DEFAULT", - /* 385 */ "plus_num ::= INTEGER|FLOAT", - /* 386 */ "foreach_clause ::=", - /* 387 */ "foreach_clause ::= FOR EACH ROW", - /* 388 */ "trnm ::= nm", - /* 389 */ "tridxby ::=", - /* 390 */ "database_kw_opt ::= DATABASE", - /* 391 */ "database_kw_opt ::=", - /* 392 */ "kwcolumn_opt ::=", - /* 393 */ "kwcolumn_opt ::= COLUMNKW", - /* 394 */ "vtabarglist ::= vtabarg", - /* 395 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 396 */ "vtabarg ::= vtabarg vtabargtoken", - /* 397 */ "anylist ::=", - /* 398 */ "anylist ::= anylist LP anylist RP", - /* 399 */ "anylist ::= anylist ANY", - /* 400 */ "with ::=", + /* 95 */ "oneselect ::= mvalues", + /* 96 */ "mvalues ::= values COMMA LP nexprlist RP", + /* 97 */ "mvalues ::= mvalues COMMA LP nexprlist RP", + /* 98 */ "distinct ::= DISTINCT", + /* 99 */ "distinct ::= ALL", + /* 100 */ "distinct ::=", + /* 101 */ "sclp ::=", + /* 102 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 103 */ "selcollist ::= sclp scanpt STAR", + /* 104 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 105 */ "as ::= AS nm", + /* 106 */ "as ::=", + /* 107 */ "from ::=", + /* 108 */ "from ::= FROM seltablist", + /* 109 */ "stl_prefix ::= seltablist joinop", + /* 110 */ "stl_prefix ::=", + /* 111 */ "seltablist ::= stl_prefix nm dbnm as on_using", + /* 112 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", + /* 113 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", + /* 114 */ "seltablist ::= stl_prefix LP select RP as on_using", + /* 115 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", + /* 116 */ "dbnm ::=", + /* 117 */ "dbnm ::= DOT nm", + /* 118 */ "fullname ::= nm", + /* 119 */ "fullname ::= nm DOT nm", + /* 120 */ "xfullname ::= nm", + /* 121 */ "xfullname ::= nm DOT nm", + /* 122 */ "xfullname ::= nm DOT nm AS nm", + /* 123 */ "xfullname ::= nm AS nm", + /* 124 */ "joinop ::= COMMA|JOIN", + /* 125 */ "joinop ::= JOIN_KW JOIN", + /* 126 */ "joinop ::= JOIN_KW nm JOIN", + /* 127 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 128 */ "on_using ::= ON expr", + /* 129 */ "on_using ::= USING LP idlist RP", + /* 130 */ "on_using ::=", + /* 131 */ "indexed_opt ::=", + /* 132 */ "indexed_by ::= INDEXED BY nm", + /* 133 */ "indexed_by ::= NOT INDEXED", + /* 134 */ "orderby_opt ::=", + /* 135 */ "orderby_opt ::= ORDER BY sortlist", + /* 136 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 137 */ "sortlist ::= expr sortorder nulls", + /* 138 */ "sortorder ::= ASC", + /* 139 */ "sortorder ::= DESC", + /* 140 */ "sortorder ::=", + /* 141 */ "nulls ::= NULLS FIRST", + /* 142 */ "nulls ::= NULLS LAST", + /* 143 */ "nulls ::=", + /* 144 */ "groupby_opt ::=", + /* 145 */ "groupby_opt ::= GROUP BY nexprlist", + /* 146 */ "having_opt ::=", + /* 147 */ "having_opt ::= HAVING expr", + /* 148 */ "limit_opt ::=", + /* 149 */ "limit_opt ::= LIMIT expr", + /* 150 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 151 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 152 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 153 */ "where_opt ::=", + /* 154 */ "where_opt ::= WHERE expr", + /* 155 */ "where_opt_ret ::=", + /* 156 */ "where_opt_ret ::= WHERE expr", + /* 157 */ "where_opt_ret ::= RETURNING selcollist", + /* 158 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 159 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 160 */ "setlist ::= setlist COMMA nm EQ expr", + /* 161 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 162 */ "setlist ::= nm EQ expr", + /* 163 */ "setlist ::= LP idlist RP EQ expr", + /* 164 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 165 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 166 */ "upsert ::=", + /* 167 */ "upsert ::= RETURNING selcollist", + /* 168 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 169 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 170 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 171 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 172 */ "returning ::= RETURNING selcollist", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "idlist_opt ::=", + /* 176 */ "idlist_opt ::= LP idlist RP", + /* 177 */ "idlist ::= idlist COMMA nm", + /* 178 */ "idlist ::= nm", + /* 179 */ "expr ::= LP expr RP", + /* 180 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 181 */ "expr ::= nm DOT nm", + /* 182 */ "expr ::= nm DOT nm DOT nm", + /* 183 */ "term ::= NULL|FLOAT|BLOB", + /* 184 */ "term ::= STRING", + /* 185 */ "term ::= INTEGER", + /* 186 */ "expr ::= VARIABLE", + /* 187 */ "expr ::= expr COLLATE ID|STRING", + /* 188 */ "expr ::= CAST LP expr AS typetoken RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 193 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 194 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 195 */ "term ::= CTIME_KW", + /* 196 */ "expr ::= LP nexprlist COMMA expr RP", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 206 */ "expr ::= expr likeop expr", + /* 207 */ "expr ::= expr likeop expr ESCAPE expr", + /* 208 */ "expr ::= expr ISNULL|NOTNULL", + /* 209 */ "expr ::= expr NOT NULL", + /* 210 */ "expr ::= expr IS expr", + /* 211 */ "expr ::= expr IS NOT expr", + /* 212 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 213 */ "expr ::= expr IS DISTINCT FROM expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= PLUS|MINUS expr", + /* 217 */ "expr ::= expr PTR expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::=", + /* 234 */ "exprlist ::=", + /* 235 */ "nexprlist ::= nexprlist COMMA expr", + /* 236 */ "nexprlist ::= expr", + /* 237 */ "paren_exprlist ::=", + /* 238 */ "paren_exprlist ::= LP exprlist RP", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "eidlist_opt ::=", + /* 243 */ "eidlist_opt ::= LP eidlist RP", + /* 244 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 245 */ "eidlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM vinto", + /* 250 */ "cmd ::= VACUUM nm vinto", + /* 251 */ "vinto ::= INTO expr", + /* 252 */ "vinto ::=", + /* 253 */ "cmd ::= PRAGMA nm dbnm", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 258 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 259 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 260 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 261 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 262 */ "trigger_time ::= BEFORE|AFTER", + /* 263 */ "trigger_time ::= INSTEAD OF", + /* 264 */ "trigger_time ::=", + /* 265 */ "trigger_event ::= DELETE|INSERT", + /* 266 */ "trigger_event ::= UPDATE", + /* 267 */ "trigger_event ::= UPDATE OF idlist", + /* 268 */ "when_clause ::=", + /* 269 */ "when_clause ::= WHEN expr", + /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 272 */ "trnm ::= nm DOT nm", + /* 273 */ "tridxby ::= INDEXED BY nm", + /* 274 */ "tridxby ::= NOT INDEXED", + /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 278 */ "trigger_cmd ::= scanpt select scanpt", + /* 279 */ "expr ::= RAISE LP IGNORE RP", + /* 280 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 281 */ "raisetype ::= ROLLBACK", + /* 282 */ "raisetype ::= ABORT", + /* 283 */ "raisetype ::= FAIL", + /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 286 */ "cmd ::= DETACH database_kw_opt expr", + /* 287 */ "key_opt ::=", + /* 288 */ "key_opt ::= KEY expr", + /* 289 */ "cmd ::= REINDEX", + /* 290 */ "cmd ::= REINDEX nm dbnm", + /* 291 */ "cmd ::= ANALYZE", + /* 292 */ "cmd ::= ANALYZE nm dbnm", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 296 */ "add_column_fullname ::= fullname", + /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 298 */ "cmd ::= create_vtab", + /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 301 */ "vtabarg ::=", + /* 302 */ "vtabargtoken ::= ANY", + /* 303 */ "vtabargtoken ::= lp anylist RP", + /* 304 */ "lp ::= LP", + /* 305 */ "with ::= WITH wqlist", + /* 306 */ "with ::= WITH RECURSIVE wqlist", + /* 307 */ "wqas ::= AS", + /* 308 */ "wqas ::= AS MATERIALIZED", + /* 309 */ "wqas ::= AS NOT MATERIALIZED", + /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 311 */ "withnm ::= nm", + /* 312 */ "wqlist ::= wqitem", + /* 313 */ "wqlist ::= wqlist COMMA wqitem", + /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 315 */ "windowdefn ::= nm AS LP window RP", + /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 318 */ "window ::= ORDER BY sortlist frame_opt", + /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 320 */ "window ::= nm frame_opt", + /* 321 */ "frame_opt ::=", + /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 325 */ "frame_bound_s ::= frame_bound", + /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 327 */ "frame_bound_e ::= frame_bound", + /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 330 */ "frame_bound ::= CURRENT ROW", + /* 331 */ "frame_exclude_opt ::=", + /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 333 */ "frame_exclude ::= NO OTHERS", + /* 334 */ "frame_exclude ::= CURRENT ROW", + /* 335 */ "frame_exclude ::= GROUP|TIES", + /* 336 */ "window_clause ::= WINDOW windowdefn_list", + /* 337 */ "filter_over ::= filter_clause over_clause", + /* 338 */ "filter_over ::= over_clause", + /* 339 */ "filter_over ::= filter_clause", + /* 340 */ "over_clause ::= OVER LP window RP", + /* 341 */ "over_clause ::= OVER nm", + /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 343 */ "term ::= QNUMBER", + /* 344 */ "input ::= cmdlist", + /* 345 */ "cmdlist ::= cmdlist ecmd", + /* 346 */ "cmdlist ::= ecmd", + /* 347 */ "ecmd ::= SEMI", + /* 348 */ "ecmd ::= cmdx SEMI", + /* 349 */ "ecmd ::= explain cmdx SEMI", + /* 350 */ "trans_opt ::=", + /* 351 */ "trans_opt ::= TRANSACTION", + /* 352 */ "trans_opt ::= TRANSACTION nm", + /* 353 */ "savepoint_opt ::= SAVEPOINT", + /* 354 */ "savepoint_opt ::=", + /* 355 */ "cmd ::= create_table create_table_args", + /* 356 */ "table_option_set ::= table_option", + /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 358 */ "columnlist ::= columnname carglist", + /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 360 */ "nm ::= STRING", + /* 361 */ "typetoken ::= typename", + /* 362 */ "typename ::= ID|STRING", + /* 363 */ "signed ::= plus_num", + /* 364 */ "signed ::= minus_num", + /* 365 */ "carglist ::= carglist ccons", + /* 366 */ "carglist ::=", + /* 367 */ "ccons ::= NULL onconf", + /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 369 */ "ccons ::= AS generated", + /* 370 */ "conslist_opt ::= COMMA conslist", + /* 371 */ "conslist ::= conslist tconscomma tcons", + /* 372 */ "conslist ::= tcons", + /* 373 */ "tconscomma ::=", + /* 374 */ "defer_subclause_opt ::= defer_subclause", + /* 375 */ "resolvetype ::= raisetype", + /* 376 */ "selectnowith ::= oneselect", + /* 377 */ "oneselect ::= values", + /* 378 */ "sclp ::= selcollist COMMA", + /* 379 */ "as ::= ID|STRING", + /* 380 */ "indexed_opt ::= indexed_by", + /* 381 */ "returning ::=", + /* 382 */ "expr ::= term", + /* 383 */ "likeop ::= LIKE_KW|MATCH", + /* 384 */ "case_operand ::= expr", + /* 385 */ "exprlist ::= nexprlist", + /* 386 */ "nmnum ::= plus_num", + /* 387 */ "nmnum ::= nm", + /* 388 */ "nmnum ::= ON", + /* 389 */ "nmnum ::= DELETE", + /* 390 */ "nmnum ::= DEFAULT", + /* 391 */ "plus_num ::= INTEGER|FLOAT", + /* 392 */ "foreach_clause ::=", + /* 393 */ "foreach_clause ::= FOR EACH ROW", + /* 394 */ "trnm ::= nm", + /* 395 */ "tridxby ::=", + /* 396 */ "database_kw_opt ::= DATABASE", + /* 397 */ "database_kw_opt ::=", + /* 398 */ "kwcolumn_opt ::=", + /* 399 */ "kwcolumn_opt ::= COLUMNKW", + /* 400 */ "vtabarglist ::= vtabarg", + /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 402 */ "vtabarg ::= vtabarg vtabargtoken", + /* 403 */ "anylist ::=", + /* 404 */ "anylist ::= anylist LP anylist RP", + /* 405 */ "anylist ::= anylist ANY", + /* 406 */ "with ::=", + /* 407 */ "windowdefn_list ::= windowdefn", + /* 408 */ "window ::= frame_opt", }; #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystack = pNew; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -162398,24 +176593,14 @@ SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL) #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; #endif yypParser->yytos = yypParser->yystack; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Parser_ENGINEALWAYSONSTACK @@ -162469,99 +176654,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 203: /* select */ - case 238: /* selectnowith */ - case 239: /* oneselect */ - case 251: /* values */ + case 205: /* select */ + case 240: /* selectnowith */ + case 241: /* oneselect */ + case 253: /* values */ + case 255: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy303)); -} - break; - case 215: /* term */ - case 216: /* expr */ - case 245: /* where_opt */ - case 247: /* having_opt */ - case 259: /* on_opt */ - case 266: /* where_opt_ret */ - case 277: /* case_operand */ - case 279: /* case_else */ - case 282: /* vinto */ - case 289: /* when_clause */ - case 294: /* key_opt */ - case 310: /* filter_clause */ +sqlite3SelectDelete(pParse->db, (yypminor->yy555)); +} + break; + case 217: /* term */ + case 218: /* expr */ + case 247: /* where_opt */ + case 249: /* having_opt */ + case 269: /* where_opt_ret */ + case 280: /* case_operand */ + case 282: /* case_else */ + case 285: /* vinto */ + case 292: /* when_clause */ + case 297: /* key_opt */ + case 314: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy626)); -} - break; - case 220: /* eidlist_opt */ - case 230: /* sortlist */ - case 231: /* eidlist */ - case 243: /* selcollist */ - case 246: /* groupby_opt */ - case 248: /* orderby_opt */ - case 252: /* nexprlist */ - case 253: /* sclp */ - case 261: /* exprlist */ - case 267: /* setlist */ - case 276: /* paren_exprlist */ - case 278: /* case_exprlist */ - case 309: /* part_opt */ +sqlite3ExprDelete(pParse->db, (yypminor->yy454)); +} + break; + case 222: /* eidlist_opt */ + case 232: /* sortlist */ + case 233: /* eidlist */ + case 245: /* selcollist */ + case 248: /* groupby_opt */ + case 250: /* orderby_opt */ + case 254: /* nexprlist */ + case 256: /* sclp */ + case 263: /* exprlist */ + case 270: /* setlist */ + case 279: /* paren_exprlist */ + case 281: /* case_exprlist */ + case 313: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy562)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 237: /* fullname */ - case 244: /* from */ - case 255: /* seltablist */ - case 256: /* stl_prefix */ - case 262: /* xfullname */ + case 239: /* fullname */ + case 246: /* from */ + case 258: /* seltablist */ + case 259: /* stl_prefix */ + case 264: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy607)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy203)); } break; - case 240: /* wqlist */ + case 242: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy43)); +sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 250: /* window_clause */ - case 305: /* windowdefn_list */ + case 252: /* window_clause */ + case 309: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy375)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy211)); } break; - case 260: /* using_opt */ - case 263: /* idlist */ - case 269: /* idlist_opt */ + case 265: /* idlist */ + case 272: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy240)); +sqlite3IdListDelete(pParse->db, (yypminor->yy132)); } break; - case 272: /* filter_over */ - case 306: /* windowdefn */ - case 307: /* window */ - case 308: /* frame_opt */ - case 311: /* over_clause */ + case 275: /* filter_over */ + case 310: /* windowdefn */ + case 311: /* window */ + case 312: /* frame_opt */ + case 315: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy375)); +sqlite3WindowDelete(pParse->db, (yypminor->yy211)); } break; - case 285: /* trigger_cmd_list */ - case 290: /* trigger_cmd */ + case 288: /* trigger_cmd_list */ + case 293: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy95)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427)); } break; - case 287: /* trigger_event */ + case 290: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy570).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy286).b); } break; - case 313: /* frame_bound */ - case 314: /* frame_bound_s */ - case 315: /* frame_bound_e */ + case 317: /* frame_bound */ + case 318: /* frame_bound_s */ + case 319: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy81).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -162595,9 +176779,26 @@ static void yy_pop_parser_stack(yyParser *pParser){ */ SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } @@ -162780,7 +176981,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); /******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ sqlite3ParserCTX_STORE @@ -162824,25 +177025,19 @@ static void yy_shift( assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ if( yyGrowStack(yypParser) ){ yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif if( yyNewState > YY_MAX_SHIFT ){ yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } - yytos = yypParser->yytos; yytos->stateno = yyNewState; yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; @@ -162852,407 +177047,415 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 188, /* (0) explain ::= EXPLAIN */ - 188, /* (1) explain ::= EXPLAIN QUERY PLAN */ - 187, /* (2) cmdx ::= cmd */ - 189, /* (3) cmd ::= BEGIN transtype trans_opt */ - 190, /* (4) transtype ::= */ - 190, /* (5) transtype ::= DEFERRED */ - 190, /* (6) transtype ::= IMMEDIATE */ - 190, /* (7) transtype ::= EXCLUSIVE */ - 189, /* (8) cmd ::= COMMIT|END trans_opt */ - 189, /* (9) cmd ::= ROLLBACK trans_opt */ - 189, /* (10) cmd ::= SAVEPOINT nm */ - 189, /* (11) cmd ::= RELEASE savepoint_opt nm */ - 189, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - 194, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - 196, /* (14) createkw ::= CREATE */ - 198, /* (15) ifnotexists ::= */ - 198, /* (16) ifnotexists ::= IF NOT EXISTS */ - 197, /* (17) temp ::= TEMP */ - 197, /* (18) temp ::= */ - 195, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ - 195, /* (20) create_table_args ::= AS select */ - 202, /* (21) table_option_set ::= */ - 202, /* (22) table_option_set ::= table_option_set COMMA table_option */ - 204, /* (23) table_option ::= WITHOUT nm */ - 204, /* (24) table_option ::= nm */ - 205, /* (25) columnname ::= nm typetoken */ - 207, /* (26) typetoken ::= */ - 207, /* (27) typetoken ::= typename LP signed RP */ - 207, /* (28) typetoken ::= typename LP signed COMMA signed RP */ - 208, /* (29) typename ::= typename ID|STRING */ - 212, /* (30) scanpt ::= */ - 213, /* (31) scantok ::= */ - 214, /* (32) ccons ::= CONSTRAINT nm */ - 214, /* (33) ccons ::= DEFAULT scantok term */ - 214, /* (34) ccons ::= DEFAULT LP expr RP */ - 214, /* (35) ccons ::= DEFAULT PLUS scantok term */ - 214, /* (36) ccons ::= DEFAULT MINUS scantok term */ - 214, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ - 214, /* (38) ccons ::= NOT NULL onconf */ - 214, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - 214, /* (40) ccons ::= UNIQUE onconf */ - 214, /* (41) ccons ::= CHECK LP expr RP */ - 214, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ - 214, /* (43) ccons ::= defer_subclause */ - 214, /* (44) ccons ::= COLLATE ID|STRING */ - 223, /* (45) generated ::= LP expr RP */ - 223, /* (46) generated ::= LP expr RP ID */ - 219, /* (47) autoinc ::= */ - 219, /* (48) autoinc ::= AUTOINCR */ - 221, /* (49) refargs ::= */ - 221, /* (50) refargs ::= refargs refarg */ - 224, /* (51) refarg ::= MATCH nm */ - 224, /* (52) refarg ::= ON INSERT refact */ - 224, /* (53) refarg ::= ON DELETE refact */ - 224, /* (54) refarg ::= ON UPDATE refact */ - 225, /* (55) refact ::= SET NULL */ - 225, /* (56) refact ::= SET DEFAULT */ - 225, /* (57) refact ::= CASCADE */ - 225, /* (58) refact ::= RESTRICT */ - 225, /* (59) refact ::= NO ACTION */ - 222, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - 222, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - 226, /* (62) init_deferred_pred_opt ::= */ - 226, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - 226, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - 201, /* (65) conslist_opt ::= */ - 228, /* (66) tconscomma ::= COMMA */ - 229, /* (67) tcons ::= CONSTRAINT nm */ - 229, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - 229, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ - 229, /* (70) tcons ::= CHECK LP expr RP onconf */ - 229, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - 232, /* (72) defer_subclause_opt ::= */ - 217, /* (73) onconf ::= */ - 217, /* (74) onconf ::= ON CONFLICT resolvetype */ - 233, /* (75) orconf ::= */ - 233, /* (76) orconf ::= OR resolvetype */ - 234, /* (77) resolvetype ::= IGNORE */ - 234, /* (78) resolvetype ::= REPLACE */ - 189, /* (79) cmd ::= DROP TABLE ifexists fullname */ - 236, /* (80) ifexists ::= IF EXISTS */ - 236, /* (81) ifexists ::= */ - 189, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - 189, /* (83) cmd ::= DROP VIEW ifexists fullname */ - 189, /* (84) cmd ::= select */ - 203, /* (85) select ::= WITH wqlist selectnowith */ - 203, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ - 203, /* (87) select ::= selectnowith */ - 238, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ - 241, /* (89) multiselect_op ::= UNION */ - 241, /* (90) multiselect_op ::= UNION ALL */ - 241, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ - 239, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - 239, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ - 251, /* (94) values ::= VALUES LP nexprlist RP */ - 251, /* (95) values ::= values COMMA LP nexprlist RP */ - 242, /* (96) distinct ::= DISTINCT */ - 242, /* (97) distinct ::= ALL */ - 242, /* (98) distinct ::= */ - 253, /* (99) sclp ::= */ - 243, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - 243, /* (101) selcollist ::= sclp scanpt STAR */ - 243, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - 254, /* (103) as ::= AS nm */ - 254, /* (104) as ::= */ - 244, /* (105) from ::= */ - 244, /* (106) from ::= FROM seltablist */ - 256, /* (107) stl_prefix ::= seltablist joinop */ - 256, /* (108) stl_prefix ::= */ - 255, /* (109) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ - 255, /* (110) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ - 255, /* (111) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ - 255, /* (112) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ - 199, /* (113) dbnm ::= */ - 199, /* (114) dbnm ::= DOT nm */ - 237, /* (115) fullname ::= nm */ - 237, /* (116) fullname ::= nm DOT nm */ - 262, /* (117) xfullname ::= nm */ - 262, /* (118) xfullname ::= nm DOT nm */ - 262, /* (119) xfullname ::= nm DOT nm AS nm */ - 262, /* (120) xfullname ::= nm AS nm */ - 257, /* (121) joinop ::= COMMA|JOIN */ - 257, /* (122) joinop ::= JOIN_KW JOIN */ - 257, /* (123) joinop ::= JOIN_KW nm JOIN */ - 257, /* (124) joinop ::= JOIN_KW nm nm JOIN */ - 259, /* (125) on_opt ::= ON expr */ - 259, /* (126) on_opt ::= */ - 258, /* (127) indexed_opt ::= */ - 258, /* (128) indexed_opt ::= INDEXED BY nm */ - 258, /* (129) indexed_opt ::= NOT INDEXED */ - 260, /* (130) using_opt ::= USING LP idlist RP */ - 260, /* (131) using_opt ::= */ - 248, /* (132) orderby_opt ::= */ - 248, /* (133) orderby_opt ::= ORDER BY sortlist */ - 230, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - 230, /* (135) sortlist ::= expr sortorder nulls */ - 218, /* (136) sortorder ::= ASC */ - 218, /* (137) sortorder ::= DESC */ - 218, /* (138) sortorder ::= */ - 264, /* (139) nulls ::= NULLS FIRST */ - 264, /* (140) nulls ::= NULLS LAST */ - 264, /* (141) nulls ::= */ - 246, /* (142) groupby_opt ::= */ - 246, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 247, /* (144) having_opt ::= */ - 247, /* (145) having_opt ::= HAVING expr */ - 249, /* (146) limit_opt ::= */ - 249, /* (147) limit_opt ::= LIMIT expr */ - 249, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - 249, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - 189, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 245, /* (151) where_opt ::= */ - 245, /* (152) where_opt ::= WHERE expr */ - 266, /* (153) where_opt_ret ::= */ - 266, /* (154) where_opt_ret ::= WHERE expr */ - 266, /* (155) where_opt_ret ::= RETURNING selcollist */ - 266, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - 189, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - 267, /* (158) setlist ::= setlist COMMA nm EQ expr */ - 267, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - 267, /* (160) setlist ::= nm EQ expr */ - 267, /* (161) setlist ::= LP idlist RP EQ expr */ - 189, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - 189, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 270, /* (164) upsert ::= */ - 270, /* (165) upsert ::= RETURNING selcollist */ - 270, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - 270, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - 270, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - 270, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - 271, /* (170) returning ::= RETURNING selcollist */ - 268, /* (171) insert_cmd ::= INSERT orconf */ - 268, /* (172) insert_cmd ::= REPLACE */ - 269, /* (173) idlist_opt ::= */ - 269, /* (174) idlist_opt ::= LP idlist RP */ - 263, /* (175) idlist ::= idlist COMMA nm */ - 263, /* (176) idlist ::= nm */ - 216, /* (177) expr ::= LP expr RP */ - 216, /* (178) expr ::= ID|INDEXED */ - 216, /* (179) expr ::= JOIN_KW */ - 216, /* (180) expr ::= nm DOT nm */ - 216, /* (181) expr ::= nm DOT nm DOT nm */ - 215, /* (182) term ::= NULL|FLOAT|BLOB */ - 215, /* (183) term ::= STRING */ - 215, /* (184) term ::= INTEGER */ - 216, /* (185) expr ::= VARIABLE */ - 216, /* (186) expr ::= expr COLLATE ID|STRING */ - 216, /* (187) expr ::= CAST LP expr AS typetoken RP */ - 216, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - 216, /* (189) expr ::= ID|INDEXED LP STAR RP */ - 216, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - 216, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - 215, /* (192) term ::= CTIME_KW */ - 216, /* (193) expr ::= LP nexprlist COMMA expr RP */ - 216, /* (194) expr ::= expr AND expr */ - 216, /* (195) expr ::= expr OR expr */ - 216, /* (196) expr ::= expr LT|GT|GE|LE expr */ - 216, /* (197) expr ::= expr EQ|NE expr */ - 216, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 216, /* (199) expr ::= expr PLUS|MINUS expr */ - 216, /* (200) expr ::= expr STAR|SLASH|REM expr */ - 216, /* (201) expr ::= expr CONCAT expr */ - 273, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - 216, /* (203) expr ::= expr likeop expr */ - 216, /* (204) expr ::= expr likeop expr ESCAPE expr */ - 216, /* (205) expr ::= expr ISNULL|NOTNULL */ - 216, /* (206) expr ::= expr NOT NULL */ - 216, /* (207) expr ::= expr IS expr */ - 216, /* (208) expr ::= expr IS NOT expr */ - 216, /* (209) expr ::= NOT expr */ - 216, /* (210) expr ::= BITNOT expr */ - 216, /* (211) expr ::= PLUS|MINUS expr */ - 274, /* (212) between_op ::= BETWEEN */ - 274, /* (213) between_op ::= NOT BETWEEN */ - 216, /* (214) expr ::= expr between_op expr AND expr */ - 275, /* (215) in_op ::= IN */ - 275, /* (216) in_op ::= NOT IN */ - 216, /* (217) expr ::= expr in_op LP exprlist RP */ - 216, /* (218) expr ::= LP select RP */ - 216, /* (219) expr ::= expr in_op LP select RP */ - 216, /* (220) expr ::= expr in_op nm dbnm paren_exprlist */ - 216, /* (221) expr ::= EXISTS LP select RP */ - 216, /* (222) expr ::= CASE case_operand case_exprlist case_else END */ - 278, /* (223) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 278, /* (224) case_exprlist ::= WHEN expr THEN expr */ - 279, /* (225) case_else ::= ELSE expr */ - 279, /* (226) case_else ::= */ - 277, /* (227) case_operand ::= expr */ - 277, /* (228) case_operand ::= */ - 261, /* (229) exprlist ::= */ - 252, /* (230) nexprlist ::= nexprlist COMMA expr */ - 252, /* (231) nexprlist ::= expr */ - 276, /* (232) paren_exprlist ::= */ - 276, /* (233) paren_exprlist ::= LP exprlist RP */ - 189, /* (234) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 280, /* (235) uniqueflag ::= UNIQUE */ - 280, /* (236) uniqueflag ::= */ - 220, /* (237) eidlist_opt ::= */ - 220, /* (238) eidlist_opt ::= LP eidlist RP */ - 231, /* (239) eidlist ::= eidlist COMMA nm collate sortorder */ - 231, /* (240) eidlist ::= nm collate sortorder */ - 281, /* (241) collate ::= */ - 281, /* (242) collate ::= COLLATE ID|STRING */ - 189, /* (243) cmd ::= DROP INDEX ifexists fullname */ - 189, /* (244) cmd ::= VACUUM vinto */ - 189, /* (245) cmd ::= VACUUM nm vinto */ - 282, /* (246) vinto ::= INTO expr */ - 282, /* (247) vinto ::= */ - 189, /* (248) cmd ::= PRAGMA nm dbnm */ - 189, /* (249) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 189, /* (250) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 189, /* (251) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 189, /* (252) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 210, /* (253) plus_num ::= PLUS INTEGER|FLOAT */ - 211, /* (254) minus_num ::= MINUS INTEGER|FLOAT */ - 189, /* (255) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 284, /* (256) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 286, /* (257) trigger_time ::= BEFORE|AFTER */ - 286, /* (258) trigger_time ::= INSTEAD OF */ - 286, /* (259) trigger_time ::= */ - 287, /* (260) trigger_event ::= DELETE|INSERT */ - 287, /* (261) trigger_event ::= UPDATE */ - 287, /* (262) trigger_event ::= UPDATE OF idlist */ - 289, /* (263) when_clause ::= */ - 289, /* (264) when_clause ::= WHEN expr */ - 285, /* (265) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 285, /* (266) trigger_cmd_list ::= trigger_cmd SEMI */ - 291, /* (267) trnm ::= nm DOT nm */ - 292, /* (268) tridxby ::= INDEXED BY nm */ - 292, /* (269) tridxby ::= NOT INDEXED */ - 290, /* (270) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 290, /* (271) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 290, /* (272) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 290, /* (273) trigger_cmd ::= scanpt select scanpt */ - 216, /* (274) expr ::= RAISE LP IGNORE RP */ - 216, /* (275) expr ::= RAISE LP raisetype COMMA nm RP */ - 235, /* (276) raisetype ::= ROLLBACK */ - 235, /* (277) raisetype ::= ABORT */ - 235, /* (278) raisetype ::= FAIL */ - 189, /* (279) cmd ::= DROP TRIGGER ifexists fullname */ - 189, /* (280) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 189, /* (281) cmd ::= DETACH database_kw_opt expr */ - 294, /* (282) key_opt ::= */ - 294, /* (283) key_opt ::= KEY expr */ - 189, /* (284) cmd ::= REINDEX */ - 189, /* (285) cmd ::= REINDEX nm dbnm */ - 189, /* (286) cmd ::= ANALYZE */ - 189, /* (287) cmd ::= ANALYZE nm dbnm */ - 189, /* (288) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 189, /* (289) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 189, /* (290) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 295, /* (291) add_column_fullname ::= fullname */ - 189, /* (292) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 189, /* (293) cmd ::= create_vtab */ - 189, /* (294) cmd ::= create_vtab LP vtabarglist RP */ - 297, /* (295) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 299, /* (296) vtabarg ::= */ - 300, /* (297) vtabargtoken ::= ANY */ - 300, /* (298) vtabargtoken ::= lp anylist RP */ - 301, /* (299) lp ::= LP */ - 265, /* (300) with ::= WITH wqlist */ - 265, /* (301) with ::= WITH RECURSIVE wqlist */ - 304, /* (302) wqas ::= AS */ - 304, /* (303) wqas ::= AS MATERIALIZED */ - 304, /* (304) wqas ::= AS NOT MATERIALIZED */ - 303, /* (305) wqitem ::= nm eidlist_opt wqas LP select RP */ - 240, /* (306) wqlist ::= wqitem */ - 240, /* (307) wqlist ::= wqlist COMMA wqitem */ - 305, /* (308) windowdefn_list ::= windowdefn */ - 305, /* (309) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 306, /* (310) windowdefn ::= nm AS LP window RP */ - 307, /* (311) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 307, /* (312) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 307, /* (313) window ::= ORDER BY sortlist frame_opt */ - 307, /* (314) window ::= nm ORDER BY sortlist frame_opt */ - 307, /* (315) window ::= frame_opt */ - 307, /* (316) window ::= nm frame_opt */ - 308, /* (317) frame_opt ::= */ - 308, /* (318) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 308, /* (319) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 312, /* (320) range_or_rows ::= RANGE|ROWS|GROUPS */ - 314, /* (321) frame_bound_s ::= frame_bound */ - 314, /* (322) frame_bound_s ::= UNBOUNDED PRECEDING */ - 315, /* (323) frame_bound_e ::= frame_bound */ - 315, /* (324) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 313, /* (325) frame_bound ::= expr PRECEDING|FOLLOWING */ - 313, /* (326) frame_bound ::= CURRENT ROW */ - 316, /* (327) frame_exclude_opt ::= */ - 316, /* (328) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 317, /* (329) frame_exclude ::= NO OTHERS */ - 317, /* (330) frame_exclude ::= CURRENT ROW */ - 317, /* (331) frame_exclude ::= GROUP|TIES */ - 250, /* (332) window_clause ::= WINDOW windowdefn_list */ - 272, /* (333) filter_over ::= filter_clause over_clause */ - 272, /* (334) filter_over ::= over_clause */ - 272, /* (335) filter_over ::= filter_clause */ - 311, /* (336) over_clause ::= OVER LP window RP */ - 311, /* (337) over_clause ::= OVER nm */ - 310, /* (338) filter_clause ::= FILTER LP WHERE expr RP */ - 184, /* (339) input ::= cmdlist */ - 185, /* (340) cmdlist ::= cmdlist ecmd */ - 185, /* (341) cmdlist ::= ecmd */ - 186, /* (342) ecmd ::= SEMI */ - 186, /* (343) ecmd ::= cmdx SEMI */ - 186, /* (344) ecmd ::= explain cmdx SEMI */ - 191, /* (345) trans_opt ::= */ - 191, /* (346) trans_opt ::= TRANSACTION */ - 191, /* (347) trans_opt ::= TRANSACTION nm */ - 193, /* (348) savepoint_opt ::= SAVEPOINT */ - 193, /* (349) savepoint_opt ::= */ - 189, /* (350) cmd ::= create_table create_table_args */ - 202, /* (351) table_option_set ::= table_option */ - 200, /* (352) columnlist ::= columnlist COMMA columnname carglist */ - 200, /* (353) columnlist ::= columnname carglist */ - 192, /* (354) nm ::= ID|INDEXED */ - 192, /* (355) nm ::= STRING */ - 192, /* (356) nm ::= JOIN_KW */ - 207, /* (357) typetoken ::= typename */ - 208, /* (358) typename ::= ID|STRING */ - 209, /* (359) signed ::= plus_num */ - 209, /* (360) signed ::= minus_num */ - 206, /* (361) carglist ::= carglist ccons */ - 206, /* (362) carglist ::= */ - 214, /* (363) ccons ::= NULL onconf */ - 214, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - 214, /* (365) ccons ::= AS generated */ - 201, /* (366) conslist_opt ::= COMMA conslist */ - 227, /* (367) conslist ::= conslist tconscomma tcons */ - 227, /* (368) conslist ::= tcons */ - 228, /* (369) tconscomma ::= */ - 232, /* (370) defer_subclause_opt ::= defer_subclause */ - 234, /* (371) resolvetype ::= raisetype */ - 238, /* (372) selectnowith ::= oneselect */ - 239, /* (373) oneselect ::= values */ - 253, /* (374) sclp ::= selcollist COMMA */ - 254, /* (375) as ::= ID|STRING */ - 271, /* (376) returning ::= */ - 216, /* (377) expr ::= term */ - 273, /* (378) likeop ::= LIKE_KW|MATCH */ - 261, /* (379) exprlist ::= nexprlist */ - 283, /* (380) nmnum ::= plus_num */ - 283, /* (381) nmnum ::= nm */ - 283, /* (382) nmnum ::= ON */ - 283, /* (383) nmnum ::= DELETE */ - 283, /* (384) nmnum ::= DEFAULT */ - 210, /* (385) plus_num ::= INTEGER|FLOAT */ - 288, /* (386) foreach_clause ::= */ - 288, /* (387) foreach_clause ::= FOR EACH ROW */ - 291, /* (388) trnm ::= nm */ - 292, /* (389) tridxby ::= */ - 293, /* (390) database_kw_opt ::= DATABASE */ - 293, /* (391) database_kw_opt ::= */ - 296, /* (392) kwcolumn_opt ::= */ - 296, /* (393) kwcolumn_opt ::= COLUMNKW */ - 298, /* (394) vtabarglist ::= vtabarg */ - 298, /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ - 299, /* (396) vtabarg ::= vtabarg vtabargtoken */ - 302, /* (397) anylist ::= */ - 302, /* (398) anylist ::= anylist LP anylist RP */ - 302, /* (399) anylist ::= anylist ANY */ - 265, /* (400) with ::= */ + 190, /* (0) explain ::= EXPLAIN */ + 190, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 189, /* (2) cmdx ::= cmd */ + 191, /* (3) cmd ::= BEGIN transtype trans_opt */ + 192, /* (4) transtype ::= */ + 192, /* (5) transtype ::= DEFERRED */ + 192, /* (6) transtype ::= IMMEDIATE */ + 192, /* (7) transtype ::= EXCLUSIVE */ + 191, /* (8) cmd ::= COMMIT|END trans_opt */ + 191, /* (9) cmd ::= ROLLBACK trans_opt */ + 191, /* (10) cmd ::= SAVEPOINT nm */ + 191, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 191, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 196, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 198, /* (14) createkw ::= CREATE */ + 200, /* (15) ifnotexists ::= */ + 200, /* (16) ifnotexists ::= IF NOT EXISTS */ + 199, /* (17) temp ::= TEMP */ + 199, /* (18) temp ::= */ + 197, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 197, /* (20) create_table_args ::= AS select */ + 204, /* (21) table_option_set ::= */ + 204, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 206, /* (23) table_option ::= WITHOUT nm */ + 206, /* (24) table_option ::= nm */ + 207, /* (25) columnname ::= nm typetoken */ + 209, /* (26) typetoken ::= */ + 209, /* (27) typetoken ::= typename LP signed RP */ + 209, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 210, /* (29) typename ::= typename ID|STRING */ + 214, /* (30) scanpt ::= */ + 215, /* (31) scantok ::= */ + 216, /* (32) ccons ::= CONSTRAINT nm */ + 216, /* (33) ccons ::= DEFAULT scantok term */ + 216, /* (34) ccons ::= DEFAULT LP expr RP */ + 216, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 216, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 216, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 216, /* (38) ccons ::= NOT NULL onconf */ + 216, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 216, /* (40) ccons ::= UNIQUE onconf */ + 216, /* (41) ccons ::= CHECK LP expr RP */ + 216, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 216, /* (43) ccons ::= defer_subclause */ + 216, /* (44) ccons ::= COLLATE ID|STRING */ + 225, /* (45) generated ::= LP expr RP */ + 225, /* (46) generated ::= LP expr RP ID */ + 221, /* (47) autoinc ::= */ + 221, /* (48) autoinc ::= AUTOINCR */ + 223, /* (49) refargs ::= */ + 223, /* (50) refargs ::= refargs refarg */ + 226, /* (51) refarg ::= MATCH nm */ + 226, /* (52) refarg ::= ON INSERT refact */ + 226, /* (53) refarg ::= ON DELETE refact */ + 226, /* (54) refarg ::= ON UPDATE refact */ + 227, /* (55) refact ::= SET NULL */ + 227, /* (56) refact ::= SET DEFAULT */ + 227, /* (57) refact ::= CASCADE */ + 227, /* (58) refact ::= RESTRICT */ + 227, /* (59) refact ::= NO ACTION */ + 224, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 224, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 228, /* (62) init_deferred_pred_opt ::= */ + 228, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 228, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 203, /* (65) conslist_opt ::= */ + 230, /* (66) tconscomma ::= COMMA */ + 231, /* (67) tcons ::= CONSTRAINT nm */ + 231, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 231, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 231, /* (70) tcons ::= CHECK LP expr RP onconf */ + 231, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 234, /* (72) defer_subclause_opt ::= */ + 219, /* (73) onconf ::= */ + 219, /* (74) onconf ::= ON CONFLICT resolvetype */ + 235, /* (75) orconf ::= */ + 235, /* (76) orconf ::= OR resolvetype */ + 236, /* (77) resolvetype ::= IGNORE */ + 236, /* (78) resolvetype ::= REPLACE */ + 191, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 238, /* (80) ifexists ::= IF EXISTS */ + 238, /* (81) ifexists ::= */ + 191, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 191, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 191, /* (84) cmd ::= select */ + 205, /* (85) select ::= WITH wqlist selectnowith */ + 205, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 205, /* (87) select ::= selectnowith */ + 240, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 243, /* (89) multiselect_op ::= UNION */ + 243, /* (90) multiselect_op ::= UNION ALL */ + 243, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 241, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 241, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 253, /* (94) values ::= VALUES LP nexprlist RP */ + 241, /* (95) oneselect ::= mvalues */ + 255, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + 255, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + 244, /* (98) distinct ::= DISTINCT */ + 244, /* (99) distinct ::= ALL */ + 244, /* (100) distinct ::= */ + 256, /* (101) sclp ::= */ + 245, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + 245, /* (103) selcollist ::= sclp scanpt STAR */ + 245, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + 257, /* (105) as ::= AS nm */ + 257, /* (106) as ::= */ + 246, /* (107) from ::= */ + 246, /* (108) from ::= FROM seltablist */ + 259, /* (109) stl_prefix ::= seltablist joinop */ + 259, /* (110) stl_prefix ::= */ + 258, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + 258, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + 258, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + 258, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + 258, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 201, /* (116) dbnm ::= */ + 201, /* (117) dbnm ::= DOT nm */ + 239, /* (118) fullname ::= nm */ + 239, /* (119) fullname ::= nm DOT nm */ + 264, /* (120) xfullname ::= nm */ + 264, /* (121) xfullname ::= nm DOT nm */ + 264, /* (122) xfullname ::= nm DOT nm AS nm */ + 264, /* (123) xfullname ::= nm AS nm */ + 260, /* (124) joinop ::= COMMA|JOIN */ + 260, /* (125) joinop ::= JOIN_KW JOIN */ + 260, /* (126) joinop ::= JOIN_KW nm JOIN */ + 260, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + 261, /* (128) on_using ::= ON expr */ + 261, /* (129) on_using ::= USING LP idlist RP */ + 261, /* (130) on_using ::= */ + 266, /* (131) indexed_opt ::= */ + 262, /* (132) indexed_by ::= INDEXED BY nm */ + 262, /* (133) indexed_by ::= NOT INDEXED */ + 250, /* (134) orderby_opt ::= */ + 250, /* (135) orderby_opt ::= ORDER BY sortlist */ + 232, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + 232, /* (137) sortlist ::= expr sortorder nulls */ + 220, /* (138) sortorder ::= ASC */ + 220, /* (139) sortorder ::= DESC */ + 220, /* (140) sortorder ::= */ + 267, /* (141) nulls ::= NULLS FIRST */ + 267, /* (142) nulls ::= NULLS LAST */ + 267, /* (143) nulls ::= */ + 248, /* (144) groupby_opt ::= */ + 248, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 249, /* (146) having_opt ::= */ + 249, /* (147) having_opt ::= HAVING expr */ + 251, /* (148) limit_opt ::= */ + 251, /* (149) limit_opt ::= LIMIT expr */ + 251, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + 251, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + 191, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 247, /* (153) where_opt ::= */ + 247, /* (154) where_opt ::= WHERE expr */ + 269, /* (155) where_opt_ret ::= */ + 269, /* (156) where_opt_ret ::= WHERE expr */ + 269, /* (157) where_opt_ret ::= RETURNING selcollist */ + 269, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 191, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 270, /* (160) setlist ::= setlist COMMA nm EQ expr */ + 270, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 270, /* (162) setlist ::= nm EQ expr */ + 270, /* (163) setlist ::= LP idlist RP EQ expr */ + 191, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 191, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 273, /* (166) upsert ::= */ + 273, /* (167) upsert ::= RETURNING selcollist */ + 273, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 273, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 273, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + 273, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 274, /* (172) returning ::= RETURNING selcollist */ + 271, /* (173) insert_cmd ::= INSERT orconf */ + 271, /* (174) insert_cmd ::= REPLACE */ + 272, /* (175) idlist_opt ::= */ + 272, /* (176) idlist_opt ::= LP idlist RP */ + 265, /* (177) idlist ::= idlist COMMA nm */ + 265, /* (178) idlist ::= nm */ + 218, /* (179) expr ::= LP expr RP */ + 218, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + 218, /* (181) expr ::= nm DOT nm */ + 218, /* (182) expr ::= nm DOT nm DOT nm */ + 217, /* (183) term ::= NULL|FLOAT|BLOB */ + 217, /* (184) term ::= STRING */ + 217, /* (185) term ::= INTEGER */ + 218, /* (186) expr ::= VARIABLE */ + 218, /* (187) expr ::= expr COLLATE ID|STRING */ + 218, /* (188) expr ::= CAST LP expr AS typetoken RP */ + 218, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 218, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 218, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 218, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 218, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 218, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 217, /* (195) term ::= CTIME_KW */ + 218, /* (196) expr ::= LP nexprlist COMMA expr RP */ + 218, /* (197) expr ::= expr AND expr */ + 218, /* (198) expr ::= expr OR expr */ + 218, /* (199) expr ::= expr LT|GT|GE|LE expr */ + 218, /* (200) expr ::= expr EQ|NE expr */ + 218, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 218, /* (202) expr ::= expr PLUS|MINUS expr */ + 218, /* (203) expr ::= expr STAR|SLASH|REM expr */ + 218, /* (204) expr ::= expr CONCAT expr */ + 276, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + 218, /* (206) expr ::= expr likeop expr */ + 218, /* (207) expr ::= expr likeop expr ESCAPE expr */ + 218, /* (208) expr ::= expr ISNULL|NOTNULL */ + 218, /* (209) expr ::= expr NOT NULL */ + 218, /* (210) expr ::= expr IS expr */ + 218, /* (211) expr ::= expr IS NOT expr */ + 218, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + 218, /* (213) expr ::= expr IS DISTINCT FROM expr */ + 218, /* (214) expr ::= NOT expr */ + 218, /* (215) expr ::= BITNOT expr */ + 218, /* (216) expr ::= PLUS|MINUS expr */ + 218, /* (217) expr ::= expr PTR expr */ + 277, /* (218) between_op ::= BETWEEN */ + 277, /* (219) between_op ::= NOT BETWEEN */ + 218, /* (220) expr ::= expr between_op expr AND expr */ + 278, /* (221) in_op ::= IN */ + 278, /* (222) in_op ::= NOT IN */ + 218, /* (223) expr ::= expr in_op LP exprlist RP */ + 218, /* (224) expr ::= LP select RP */ + 218, /* (225) expr ::= expr in_op LP select RP */ + 218, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + 218, /* (227) expr ::= EXISTS LP select RP */ + 218, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + 281, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 281, /* (230) case_exprlist ::= WHEN expr THEN expr */ + 282, /* (231) case_else ::= ELSE expr */ + 282, /* (232) case_else ::= */ + 280, /* (233) case_operand ::= */ + 263, /* (234) exprlist ::= */ + 254, /* (235) nexprlist ::= nexprlist COMMA expr */ + 254, /* (236) nexprlist ::= expr */ + 279, /* (237) paren_exprlist ::= */ + 279, /* (238) paren_exprlist ::= LP exprlist RP */ + 191, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 283, /* (240) uniqueflag ::= UNIQUE */ + 283, /* (241) uniqueflag ::= */ + 222, /* (242) eidlist_opt ::= */ + 222, /* (243) eidlist_opt ::= LP eidlist RP */ + 233, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + 233, /* (245) eidlist ::= nm collate sortorder */ + 284, /* (246) collate ::= */ + 284, /* (247) collate ::= COLLATE ID|STRING */ + 191, /* (248) cmd ::= DROP INDEX ifexists fullname */ + 191, /* (249) cmd ::= VACUUM vinto */ + 191, /* (250) cmd ::= VACUUM nm vinto */ + 285, /* (251) vinto ::= INTO expr */ + 285, /* (252) vinto ::= */ + 191, /* (253) cmd ::= PRAGMA nm dbnm */ + 191, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 191, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 191, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 191, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 212, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + 213, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + 191, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 287, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 289, /* (262) trigger_time ::= BEFORE|AFTER */ + 289, /* (263) trigger_time ::= INSTEAD OF */ + 289, /* (264) trigger_time ::= */ + 290, /* (265) trigger_event ::= DELETE|INSERT */ + 290, /* (266) trigger_event ::= UPDATE */ + 290, /* (267) trigger_event ::= UPDATE OF idlist */ + 292, /* (268) when_clause ::= */ + 292, /* (269) when_clause ::= WHEN expr */ + 288, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 288, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + 294, /* (272) trnm ::= nm DOT nm */ + 295, /* (273) tridxby ::= INDEXED BY nm */ + 295, /* (274) tridxby ::= NOT INDEXED */ + 293, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 293, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 293, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 293, /* (278) trigger_cmd ::= scanpt select scanpt */ + 218, /* (279) expr ::= RAISE LP IGNORE RP */ + 218, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + 237, /* (281) raisetype ::= ROLLBACK */ + 237, /* (282) raisetype ::= ABORT */ + 237, /* (283) raisetype ::= FAIL */ + 191, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + 191, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 191, /* (286) cmd ::= DETACH database_kw_opt expr */ + 297, /* (287) key_opt ::= */ + 297, /* (288) key_opt ::= KEY expr */ + 191, /* (289) cmd ::= REINDEX */ + 191, /* (290) cmd ::= REINDEX nm dbnm */ + 191, /* (291) cmd ::= ANALYZE */ + 191, /* (292) cmd ::= ANALYZE nm dbnm */ + 191, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 191, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 191, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 298, /* (296) add_column_fullname ::= fullname */ + 191, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 191, /* (298) cmd ::= create_vtab */ + 191, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + 300, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 302, /* (301) vtabarg ::= */ + 303, /* (302) vtabargtoken ::= ANY */ + 303, /* (303) vtabargtoken ::= lp anylist RP */ + 304, /* (304) lp ::= LP */ + 268, /* (305) with ::= WITH wqlist */ + 268, /* (306) with ::= WITH RECURSIVE wqlist */ + 307, /* (307) wqas ::= AS */ + 307, /* (308) wqas ::= AS MATERIALIZED */ + 307, /* (309) wqas ::= AS NOT MATERIALIZED */ + 306, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 308, /* (311) withnm ::= nm */ + 242, /* (312) wqlist ::= wqitem */ + 242, /* (313) wqlist ::= wqlist COMMA wqitem */ + 309, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 310, /* (315) windowdefn ::= nm AS LP window RP */ + 311, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (318) window ::= ORDER BY sortlist frame_opt */ + 311, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + 311, /* (320) window ::= nm frame_opt */ + 312, /* (321) frame_opt ::= */ + 312, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 312, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 316, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + 318, /* (325) frame_bound_s ::= frame_bound */ + 318, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + 319, /* (327) frame_bound_e ::= frame_bound */ + 319, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 317, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + 317, /* (330) frame_bound ::= CURRENT ROW */ + 320, /* (331) frame_exclude_opt ::= */ + 320, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 321, /* (333) frame_exclude ::= NO OTHERS */ + 321, /* (334) frame_exclude ::= CURRENT ROW */ + 321, /* (335) frame_exclude ::= GROUP|TIES */ + 252, /* (336) window_clause ::= WINDOW windowdefn_list */ + 275, /* (337) filter_over ::= filter_clause over_clause */ + 275, /* (338) filter_over ::= over_clause */ + 275, /* (339) filter_over ::= filter_clause */ + 315, /* (340) over_clause ::= OVER LP window RP */ + 315, /* (341) over_clause ::= OVER nm */ + 314, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + 217, /* (343) term ::= QNUMBER */ + 186, /* (344) input ::= cmdlist */ + 187, /* (345) cmdlist ::= cmdlist ecmd */ + 187, /* (346) cmdlist ::= ecmd */ + 188, /* (347) ecmd ::= SEMI */ + 188, /* (348) ecmd ::= cmdx SEMI */ + 188, /* (349) ecmd ::= explain cmdx SEMI */ + 193, /* (350) trans_opt ::= */ + 193, /* (351) trans_opt ::= TRANSACTION */ + 193, /* (352) trans_opt ::= TRANSACTION nm */ + 195, /* (353) savepoint_opt ::= SAVEPOINT */ + 195, /* (354) savepoint_opt ::= */ + 191, /* (355) cmd ::= create_table create_table_args */ + 204, /* (356) table_option_set ::= table_option */ + 202, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + 202, /* (358) columnlist ::= columnname carglist */ + 194, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + 194, /* (360) nm ::= STRING */ + 209, /* (361) typetoken ::= typename */ + 210, /* (362) typename ::= ID|STRING */ + 211, /* (363) signed ::= plus_num */ + 211, /* (364) signed ::= minus_num */ + 208, /* (365) carglist ::= carglist ccons */ + 208, /* (366) carglist ::= */ + 216, /* (367) ccons ::= NULL onconf */ + 216, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + 216, /* (369) ccons ::= AS generated */ + 203, /* (370) conslist_opt ::= COMMA conslist */ + 229, /* (371) conslist ::= conslist tconscomma tcons */ + 229, /* (372) conslist ::= tcons */ + 230, /* (373) tconscomma ::= */ + 234, /* (374) defer_subclause_opt ::= defer_subclause */ + 236, /* (375) resolvetype ::= raisetype */ + 240, /* (376) selectnowith ::= oneselect */ + 241, /* (377) oneselect ::= values */ + 256, /* (378) sclp ::= selcollist COMMA */ + 257, /* (379) as ::= ID|STRING */ + 266, /* (380) indexed_opt ::= indexed_by */ + 274, /* (381) returning ::= */ + 218, /* (382) expr ::= term */ + 276, /* (383) likeop ::= LIKE_KW|MATCH */ + 280, /* (384) case_operand ::= expr */ + 263, /* (385) exprlist ::= nexprlist */ + 286, /* (386) nmnum ::= plus_num */ + 286, /* (387) nmnum ::= nm */ + 286, /* (388) nmnum ::= ON */ + 286, /* (389) nmnum ::= DELETE */ + 286, /* (390) nmnum ::= DEFAULT */ + 212, /* (391) plus_num ::= INTEGER|FLOAT */ + 291, /* (392) foreach_clause ::= */ + 291, /* (393) foreach_clause ::= FOR EACH ROW */ + 294, /* (394) trnm ::= nm */ + 295, /* (395) tridxby ::= */ + 296, /* (396) database_kw_opt ::= DATABASE */ + 296, /* (397) database_kw_opt ::= */ + 299, /* (398) kwcolumn_opt ::= */ + 299, /* (399) kwcolumn_opt ::= COLUMNKW */ + 301, /* (400) vtabarglist ::= vtabarg */ + 301, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + 302, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 305, /* (403) anylist ::= */ + 305, /* (404) anylist ::= anylist LP anylist RP */ + 305, /* (405) anylist ::= anylist ANY */ + 268, /* (406) with ::= */ + 309, /* (407) windowdefn_list ::= windowdefn */ + 311, /* (408) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -163353,312 +177556,320 @@ static const signed char yyRuleInfoNRhs[] = { -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ -4, /* (94) values ::= VALUES LP nexprlist RP */ - -5, /* (95) values ::= values COMMA LP nexprlist RP */ - -1, /* (96) distinct ::= DISTINCT */ - -1, /* (97) distinct ::= ALL */ - 0, /* (98) distinct ::= */ - 0, /* (99) sclp ::= */ - -5, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - -3, /* (101) selcollist ::= sclp scanpt STAR */ - -5, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - -2, /* (103) as ::= AS nm */ - 0, /* (104) as ::= */ - 0, /* (105) from ::= */ - -2, /* (106) from ::= FROM seltablist */ - -2, /* (107) stl_prefix ::= seltablist joinop */ - 0, /* (108) stl_prefix ::= */ - -7, /* (109) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ - -9, /* (110) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ - -7, /* (111) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ - -7, /* (112) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ - 0, /* (113) dbnm ::= */ - -2, /* (114) dbnm ::= DOT nm */ - -1, /* (115) fullname ::= nm */ - -3, /* (116) fullname ::= nm DOT nm */ - -1, /* (117) xfullname ::= nm */ - -3, /* (118) xfullname ::= nm DOT nm */ - -5, /* (119) xfullname ::= nm DOT nm AS nm */ - -3, /* (120) xfullname ::= nm AS nm */ - -1, /* (121) joinop ::= COMMA|JOIN */ - -2, /* (122) joinop ::= JOIN_KW JOIN */ - -3, /* (123) joinop ::= JOIN_KW nm JOIN */ - -4, /* (124) joinop ::= JOIN_KW nm nm JOIN */ - -2, /* (125) on_opt ::= ON expr */ - 0, /* (126) on_opt ::= */ - 0, /* (127) indexed_opt ::= */ - -3, /* (128) indexed_opt ::= INDEXED BY nm */ - -2, /* (129) indexed_opt ::= NOT INDEXED */ - -4, /* (130) using_opt ::= USING LP idlist RP */ - 0, /* (131) using_opt ::= */ - 0, /* (132) orderby_opt ::= */ - -3, /* (133) orderby_opt ::= ORDER BY sortlist */ - -5, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - -3, /* (135) sortlist ::= expr sortorder nulls */ - -1, /* (136) sortorder ::= ASC */ - -1, /* (137) sortorder ::= DESC */ - 0, /* (138) sortorder ::= */ - -2, /* (139) nulls ::= NULLS FIRST */ - -2, /* (140) nulls ::= NULLS LAST */ - 0, /* (141) nulls ::= */ - 0, /* (142) groupby_opt ::= */ - -3, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 0, /* (144) having_opt ::= */ - -2, /* (145) having_opt ::= HAVING expr */ - 0, /* (146) limit_opt ::= */ - -2, /* (147) limit_opt ::= LIMIT expr */ - -4, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - -4, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - -6, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 0, /* (151) where_opt ::= */ - -2, /* (152) where_opt ::= WHERE expr */ - 0, /* (153) where_opt_ret ::= */ - -2, /* (154) where_opt_ret ::= WHERE expr */ - -2, /* (155) where_opt_ret ::= RETURNING selcollist */ - -4, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - -9, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - -5, /* (158) setlist ::= setlist COMMA nm EQ expr */ - -7, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - -3, /* (160) setlist ::= nm EQ expr */ - -5, /* (161) setlist ::= LP idlist RP EQ expr */ - -7, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - -8, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 0, /* (164) upsert ::= */ - -2, /* (165) upsert ::= RETURNING selcollist */ - -12, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - -9, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - -5, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - -8, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - -2, /* (170) returning ::= RETURNING selcollist */ - -2, /* (171) insert_cmd ::= INSERT orconf */ - -1, /* (172) insert_cmd ::= REPLACE */ - 0, /* (173) idlist_opt ::= */ - -3, /* (174) idlist_opt ::= LP idlist RP */ - -3, /* (175) idlist ::= idlist COMMA nm */ - -1, /* (176) idlist ::= nm */ - -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED */ - -1, /* (179) expr ::= JOIN_KW */ - -3, /* (180) expr ::= nm DOT nm */ - -5, /* (181) expr ::= nm DOT nm DOT nm */ - -1, /* (182) term ::= NULL|FLOAT|BLOB */ - -1, /* (183) term ::= STRING */ - -1, /* (184) term ::= INTEGER */ - -1, /* (185) expr ::= VARIABLE */ - -3, /* (186) expr ::= expr COLLATE ID|STRING */ - -6, /* (187) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - -4, /* (189) expr ::= ID|INDEXED LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - -5, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - -1, /* (192) term ::= CTIME_KW */ - -5, /* (193) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (194) expr ::= expr AND expr */ - -3, /* (195) expr ::= expr OR expr */ - -3, /* (196) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (197) expr ::= expr EQ|NE expr */ - -3, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (199) expr ::= expr PLUS|MINUS expr */ - -3, /* (200) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (201) expr ::= expr CONCAT expr */ - -2, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (203) expr ::= expr likeop expr */ - -5, /* (204) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (205) expr ::= expr ISNULL|NOTNULL */ - -3, /* (206) expr ::= expr NOT NULL */ - -3, /* (207) expr ::= expr IS expr */ - -4, /* (208) expr ::= expr IS NOT expr */ - -2, /* (209) expr ::= NOT expr */ - -2, /* (210) expr ::= BITNOT expr */ - -2, /* (211) expr ::= PLUS|MINUS expr */ - -1, /* (212) between_op ::= BETWEEN */ - -2, /* (213) between_op ::= NOT BETWEEN */ - -5, /* (214) expr ::= expr between_op expr AND expr */ - -1, /* (215) in_op ::= IN */ - -2, /* (216) in_op ::= NOT IN */ - -5, /* (217) expr ::= expr in_op LP exprlist RP */ - -3, /* (218) expr ::= LP select RP */ - -5, /* (219) expr ::= expr in_op LP select RP */ - -5, /* (220) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (221) expr ::= EXISTS LP select RP */ - -5, /* (222) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (223) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (224) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (225) case_else ::= ELSE expr */ - 0, /* (226) case_else ::= */ - -1, /* (227) case_operand ::= expr */ - 0, /* (228) case_operand ::= */ - 0, /* (229) exprlist ::= */ - -3, /* (230) nexprlist ::= nexprlist COMMA expr */ - -1, /* (231) nexprlist ::= expr */ - 0, /* (232) paren_exprlist ::= */ - -3, /* (233) paren_exprlist ::= LP exprlist RP */ - -12, /* (234) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (235) uniqueflag ::= UNIQUE */ - 0, /* (236) uniqueflag ::= */ - 0, /* (237) eidlist_opt ::= */ - -3, /* (238) eidlist_opt ::= LP eidlist RP */ - -5, /* (239) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (240) eidlist ::= nm collate sortorder */ - 0, /* (241) collate ::= */ - -2, /* (242) collate ::= COLLATE ID|STRING */ - -4, /* (243) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (244) cmd ::= VACUUM vinto */ - -3, /* (245) cmd ::= VACUUM nm vinto */ - -2, /* (246) vinto ::= INTO expr */ - 0, /* (247) vinto ::= */ - -3, /* (248) cmd ::= PRAGMA nm dbnm */ - -5, /* (249) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (250) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (251) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (252) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (253) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (254) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (255) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (256) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (257) trigger_time ::= BEFORE|AFTER */ - -2, /* (258) trigger_time ::= INSTEAD OF */ - 0, /* (259) trigger_time ::= */ - -1, /* (260) trigger_event ::= DELETE|INSERT */ - -1, /* (261) trigger_event ::= UPDATE */ - -3, /* (262) trigger_event ::= UPDATE OF idlist */ - 0, /* (263) when_clause ::= */ - -2, /* (264) when_clause ::= WHEN expr */ - -3, /* (265) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (266) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (267) trnm ::= nm DOT nm */ - -3, /* (268) tridxby ::= INDEXED BY nm */ - -2, /* (269) tridxby ::= NOT INDEXED */ - -9, /* (270) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (271) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (272) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (273) trigger_cmd ::= scanpt select scanpt */ - -4, /* (274) expr ::= RAISE LP IGNORE RP */ - -6, /* (275) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (276) raisetype ::= ROLLBACK */ - -1, /* (277) raisetype ::= ABORT */ - -1, /* (278) raisetype ::= FAIL */ - -4, /* (279) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (280) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (281) cmd ::= DETACH database_kw_opt expr */ - 0, /* (282) key_opt ::= */ - -2, /* (283) key_opt ::= KEY expr */ - -1, /* (284) cmd ::= REINDEX */ - -3, /* (285) cmd ::= REINDEX nm dbnm */ - -1, /* (286) cmd ::= ANALYZE */ - -3, /* (287) cmd ::= ANALYZE nm dbnm */ - -6, /* (288) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (289) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (290) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (291) add_column_fullname ::= fullname */ - -8, /* (292) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (293) cmd ::= create_vtab */ - -4, /* (294) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (295) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (296) vtabarg ::= */ - -1, /* (297) vtabargtoken ::= ANY */ - -3, /* (298) vtabargtoken ::= lp anylist RP */ - -1, /* (299) lp ::= LP */ - -2, /* (300) with ::= WITH wqlist */ - -3, /* (301) with ::= WITH RECURSIVE wqlist */ - -1, /* (302) wqas ::= AS */ - -2, /* (303) wqas ::= AS MATERIALIZED */ - -3, /* (304) wqas ::= AS NOT MATERIALIZED */ - -6, /* (305) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (306) wqlist ::= wqitem */ - -3, /* (307) wqlist ::= wqlist COMMA wqitem */ - -1, /* (308) windowdefn_list ::= windowdefn */ - -3, /* (309) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (310) windowdefn ::= nm AS LP window RP */ - -5, /* (311) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (312) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (313) window ::= ORDER BY sortlist frame_opt */ - -5, /* (314) window ::= nm ORDER BY sortlist frame_opt */ - -1, /* (315) window ::= frame_opt */ - -2, /* (316) window ::= nm frame_opt */ - 0, /* (317) frame_opt ::= */ - -3, /* (318) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (319) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (320) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (321) frame_bound_s ::= frame_bound */ - -2, /* (322) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (323) frame_bound_e ::= frame_bound */ - -2, /* (324) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (325) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (326) frame_bound ::= CURRENT ROW */ - 0, /* (327) frame_exclude_opt ::= */ - -2, /* (328) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (329) frame_exclude ::= NO OTHERS */ - -2, /* (330) frame_exclude ::= CURRENT ROW */ - -1, /* (331) frame_exclude ::= GROUP|TIES */ - -2, /* (332) window_clause ::= WINDOW windowdefn_list */ - -2, /* (333) filter_over ::= filter_clause over_clause */ - -1, /* (334) filter_over ::= over_clause */ - -1, /* (335) filter_over ::= filter_clause */ - -4, /* (336) over_clause ::= OVER LP window RP */ - -2, /* (337) over_clause ::= OVER nm */ - -5, /* (338) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (339) input ::= cmdlist */ - -2, /* (340) cmdlist ::= cmdlist ecmd */ - -1, /* (341) cmdlist ::= ecmd */ - -1, /* (342) ecmd ::= SEMI */ - -2, /* (343) ecmd ::= cmdx SEMI */ - -3, /* (344) ecmd ::= explain cmdx SEMI */ - 0, /* (345) trans_opt ::= */ - -1, /* (346) trans_opt ::= TRANSACTION */ - -2, /* (347) trans_opt ::= TRANSACTION nm */ - -1, /* (348) savepoint_opt ::= SAVEPOINT */ - 0, /* (349) savepoint_opt ::= */ - -2, /* (350) cmd ::= create_table create_table_args */ - -1, /* (351) table_option_set ::= table_option */ - -4, /* (352) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (353) columnlist ::= columnname carglist */ - -1, /* (354) nm ::= ID|INDEXED */ - -1, /* (355) nm ::= STRING */ - -1, /* (356) nm ::= JOIN_KW */ - -1, /* (357) typetoken ::= typename */ - -1, /* (358) typename ::= ID|STRING */ - -1, /* (359) signed ::= plus_num */ - -1, /* (360) signed ::= minus_num */ - -2, /* (361) carglist ::= carglist ccons */ - 0, /* (362) carglist ::= */ - -2, /* (363) ccons ::= NULL onconf */ - -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (365) ccons ::= AS generated */ - -2, /* (366) conslist_opt ::= COMMA conslist */ - -3, /* (367) conslist ::= conslist tconscomma tcons */ - -1, /* (368) conslist ::= tcons */ - 0, /* (369) tconscomma ::= */ - -1, /* (370) defer_subclause_opt ::= defer_subclause */ - -1, /* (371) resolvetype ::= raisetype */ - -1, /* (372) selectnowith ::= oneselect */ - -1, /* (373) oneselect ::= values */ - -2, /* (374) sclp ::= selcollist COMMA */ - -1, /* (375) as ::= ID|STRING */ - 0, /* (376) returning ::= */ - -1, /* (377) expr ::= term */ - -1, /* (378) likeop ::= LIKE_KW|MATCH */ - -1, /* (379) exprlist ::= nexprlist */ - -1, /* (380) nmnum ::= plus_num */ - -1, /* (381) nmnum ::= nm */ - -1, /* (382) nmnum ::= ON */ - -1, /* (383) nmnum ::= DELETE */ - -1, /* (384) nmnum ::= DEFAULT */ - -1, /* (385) plus_num ::= INTEGER|FLOAT */ - 0, /* (386) foreach_clause ::= */ - -3, /* (387) foreach_clause ::= FOR EACH ROW */ - -1, /* (388) trnm ::= nm */ - 0, /* (389) tridxby ::= */ - -1, /* (390) database_kw_opt ::= DATABASE */ - 0, /* (391) database_kw_opt ::= */ - 0, /* (392) kwcolumn_opt ::= */ - -1, /* (393) kwcolumn_opt ::= COLUMNKW */ - -1, /* (394) vtabarglist ::= vtabarg */ - -3, /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (396) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (397) anylist ::= */ - -4, /* (398) anylist ::= anylist LP anylist RP */ - -2, /* (399) anylist ::= anylist ANY */ - 0, /* (400) with ::= */ + -1, /* (95) oneselect ::= mvalues */ + -5, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + -5, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + -1, /* (98) distinct ::= DISTINCT */ + -1, /* (99) distinct ::= ALL */ + 0, /* (100) distinct ::= */ + 0, /* (101) sclp ::= */ + -5, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (103) selcollist ::= sclp scanpt STAR */ + -5, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (105) as ::= AS nm */ + 0, /* (106) as ::= */ + 0, /* (107) from ::= */ + -2, /* (108) from ::= FROM seltablist */ + -2, /* (109) stl_prefix ::= seltablist joinop */ + 0, /* (110) stl_prefix ::= */ + -5, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + -6, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + -8, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + -6, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + -6, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 0, /* (116) dbnm ::= */ + -2, /* (117) dbnm ::= DOT nm */ + -1, /* (118) fullname ::= nm */ + -3, /* (119) fullname ::= nm DOT nm */ + -1, /* (120) xfullname ::= nm */ + -3, /* (121) xfullname ::= nm DOT nm */ + -5, /* (122) xfullname ::= nm DOT nm AS nm */ + -3, /* (123) xfullname ::= nm AS nm */ + -1, /* (124) joinop ::= COMMA|JOIN */ + -2, /* (125) joinop ::= JOIN_KW JOIN */ + -3, /* (126) joinop ::= JOIN_KW nm JOIN */ + -4, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (128) on_using ::= ON expr */ + -4, /* (129) on_using ::= USING LP idlist RP */ + 0, /* (130) on_using ::= */ + 0, /* (131) indexed_opt ::= */ + -3, /* (132) indexed_by ::= INDEXED BY nm */ + -2, /* (133) indexed_by ::= NOT INDEXED */ + 0, /* (134) orderby_opt ::= */ + -3, /* (135) orderby_opt ::= ORDER BY sortlist */ + -5, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (137) sortlist ::= expr sortorder nulls */ + -1, /* (138) sortorder ::= ASC */ + -1, /* (139) sortorder ::= DESC */ + 0, /* (140) sortorder ::= */ + -2, /* (141) nulls ::= NULLS FIRST */ + -2, /* (142) nulls ::= NULLS LAST */ + 0, /* (143) nulls ::= */ + 0, /* (144) groupby_opt ::= */ + -3, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (146) having_opt ::= */ + -2, /* (147) having_opt ::= HAVING expr */ + 0, /* (148) limit_opt ::= */ + -2, /* (149) limit_opt ::= LIMIT expr */ + -4, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (153) where_opt ::= */ + -2, /* (154) where_opt ::= WHERE expr */ + 0, /* (155) where_opt_ret ::= */ + -2, /* (156) where_opt_ret ::= WHERE expr */ + -2, /* (157) where_opt_ret ::= RETURNING selcollist */ + -4, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (160) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (162) setlist ::= nm EQ expr */ + -5, /* (163) setlist ::= LP idlist RP EQ expr */ + -7, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (166) upsert ::= */ + -2, /* (167) upsert ::= RETURNING selcollist */ + -12, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (172) returning ::= RETURNING selcollist */ + -2, /* (173) insert_cmd ::= INSERT orconf */ + -1, /* (174) insert_cmd ::= REPLACE */ + 0, /* (175) idlist_opt ::= */ + -3, /* (176) idlist_opt ::= LP idlist RP */ + -3, /* (177) idlist ::= idlist COMMA nm */ + -1, /* (178) idlist ::= nm */ + -3, /* (179) expr ::= LP expr RP */ + -1, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (181) expr ::= nm DOT nm */ + -5, /* (182) expr ::= nm DOT nm DOT nm */ + -1, /* (183) term ::= NULL|FLOAT|BLOB */ + -1, /* (184) term ::= STRING */ + -1, /* (185) term ::= INTEGER */ + -1, /* (186) expr ::= VARIABLE */ + -3, /* (187) expr ::= expr COLLATE ID|STRING */ + -6, /* (188) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -8, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (195) term ::= CTIME_KW */ + -5, /* (196) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (197) expr ::= expr AND expr */ + -3, /* (198) expr ::= expr OR expr */ + -3, /* (199) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (200) expr ::= expr EQ|NE expr */ + -3, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (202) expr ::= expr PLUS|MINUS expr */ + -3, /* (203) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (204) expr ::= expr CONCAT expr */ + -2, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (206) expr ::= expr likeop expr */ + -5, /* (207) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (208) expr ::= expr ISNULL|NOTNULL */ + -3, /* (209) expr ::= expr NOT NULL */ + -3, /* (210) expr ::= expr IS expr */ + -4, /* (211) expr ::= expr IS NOT expr */ + -6, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (213) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (214) expr ::= NOT expr */ + -2, /* (215) expr ::= BITNOT expr */ + -2, /* (216) expr ::= PLUS|MINUS expr */ + -3, /* (217) expr ::= expr PTR expr */ + -1, /* (218) between_op ::= BETWEEN */ + -2, /* (219) between_op ::= NOT BETWEEN */ + -5, /* (220) expr ::= expr between_op expr AND expr */ + -1, /* (221) in_op ::= IN */ + -2, /* (222) in_op ::= NOT IN */ + -5, /* (223) expr ::= expr in_op LP exprlist RP */ + -3, /* (224) expr ::= LP select RP */ + -5, /* (225) expr ::= expr in_op LP select RP */ + -5, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (227) expr ::= EXISTS LP select RP */ + -5, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (230) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (231) case_else ::= ELSE expr */ + 0, /* (232) case_else ::= */ + 0, /* (233) case_operand ::= */ + 0, /* (234) exprlist ::= */ + -3, /* (235) nexprlist ::= nexprlist COMMA expr */ + -1, /* (236) nexprlist ::= expr */ + 0, /* (237) paren_exprlist ::= */ + -3, /* (238) paren_exprlist ::= LP exprlist RP */ + -12, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (240) uniqueflag ::= UNIQUE */ + 0, /* (241) uniqueflag ::= */ + 0, /* (242) eidlist_opt ::= */ + -3, /* (243) eidlist_opt ::= LP eidlist RP */ + -5, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (245) eidlist ::= nm collate sortorder */ + 0, /* (246) collate ::= */ + -2, /* (247) collate ::= COLLATE ID|STRING */ + -4, /* (248) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (249) cmd ::= VACUUM vinto */ + -3, /* (250) cmd ::= VACUUM nm vinto */ + -2, /* (251) vinto ::= INTO expr */ + 0, /* (252) vinto ::= */ + -3, /* (253) cmd ::= PRAGMA nm dbnm */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (262) trigger_time ::= BEFORE|AFTER */ + -2, /* (263) trigger_time ::= INSTEAD OF */ + 0, /* (264) trigger_time ::= */ + -1, /* (265) trigger_event ::= DELETE|INSERT */ + -1, /* (266) trigger_event ::= UPDATE */ + -3, /* (267) trigger_event ::= UPDATE OF idlist */ + 0, /* (268) when_clause ::= */ + -2, /* (269) when_clause ::= WHEN expr */ + -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (272) trnm ::= nm DOT nm */ + -3, /* (273) tridxby ::= INDEXED BY nm */ + -2, /* (274) tridxby ::= NOT INDEXED */ + -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (278) trigger_cmd ::= scanpt select scanpt */ + -4, /* (279) expr ::= RAISE LP IGNORE RP */ + -6, /* (280) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (281) raisetype ::= ROLLBACK */ + -1, /* (282) raisetype ::= ABORT */ + -1, /* (283) raisetype ::= FAIL */ + -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (286) cmd ::= DETACH database_kw_opt expr */ + 0, /* (287) key_opt ::= */ + -2, /* (288) key_opt ::= KEY expr */ + -1, /* (289) cmd ::= REINDEX */ + -3, /* (290) cmd ::= REINDEX nm dbnm */ + -1, /* (291) cmd ::= ANALYZE */ + -3, /* (292) cmd ::= ANALYZE nm dbnm */ + -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (296) add_column_fullname ::= fullname */ + -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (298) cmd ::= create_vtab */ + -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (301) vtabarg ::= */ + -1, /* (302) vtabargtoken ::= ANY */ + -3, /* (303) vtabargtoken ::= lp anylist RP */ + -1, /* (304) lp ::= LP */ + -2, /* (305) with ::= WITH wqlist */ + -3, /* (306) with ::= WITH RECURSIVE wqlist */ + -1, /* (307) wqas ::= AS */ + -2, /* (308) wqas ::= AS MATERIALIZED */ + -3, /* (309) wqas ::= AS NOT MATERIALIZED */ + -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (311) withnm ::= nm */ + -1, /* (312) wqlist ::= wqitem */ + -3, /* (313) wqlist ::= wqlist COMMA wqitem */ + -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (315) windowdefn ::= nm AS LP window RP */ + -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (318) window ::= ORDER BY sortlist frame_opt */ + -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (320) window ::= nm frame_opt */ + 0, /* (321) frame_opt ::= */ + -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (325) frame_bound_s ::= frame_bound */ + -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (327) frame_bound_e ::= frame_bound */ + -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (330) frame_bound ::= CURRENT ROW */ + 0, /* (331) frame_exclude_opt ::= */ + -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (333) frame_exclude ::= NO OTHERS */ + -2, /* (334) frame_exclude ::= CURRENT ROW */ + -1, /* (335) frame_exclude ::= GROUP|TIES */ + -2, /* (336) window_clause ::= WINDOW windowdefn_list */ + -2, /* (337) filter_over ::= filter_clause over_clause */ + -1, /* (338) filter_over ::= over_clause */ + -1, /* (339) filter_over ::= filter_clause */ + -4, /* (340) over_clause ::= OVER LP window RP */ + -2, /* (341) over_clause ::= OVER nm */ + -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (343) term ::= QNUMBER */ + -1, /* (344) input ::= cmdlist */ + -2, /* (345) cmdlist ::= cmdlist ecmd */ + -1, /* (346) cmdlist ::= ecmd */ + -1, /* (347) ecmd ::= SEMI */ + -2, /* (348) ecmd ::= cmdx SEMI */ + -3, /* (349) ecmd ::= explain cmdx SEMI */ + 0, /* (350) trans_opt ::= */ + -1, /* (351) trans_opt ::= TRANSACTION */ + -2, /* (352) trans_opt ::= TRANSACTION nm */ + -1, /* (353) savepoint_opt ::= SAVEPOINT */ + 0, /* (354) savepoint_opt ::= */ + -2, /* (355) cmd ::= create_table create_table_args */ + -1, /* (356) table_option_set ::= table_option */ + -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (358) columnlist ::= columnname carglist */ + -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (360) nm ::= STRING */ + -1, /* (361) typetoken ::= typename */ + -1, /* (362) typename ::= ID|STRING */ + -1, /* (363) signed ::= plus_num */ + -1, /* (364) signed ::= minus_num */ + -2, /* (365) carglist ::= carglist ccons */ + 0, /* (366) carglist ::= */ + -2, /* (367) ccons ::= NULL onconf */ + -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (369) ccons ::= AS generated */ + -2, /* (370) conslist_opt ::= COMMA conslist */ + -3, /* (371) conslist ::= conslist tconscomma tcons */ + -1, /* (372) conslist ::= tcons */ + 0, /* (373) tconscomma ::= */ + -1, /* (374) defer_subclause_opt ::= defer_subclause */ + -1, /* (375) resolvetype ::= raisetype */ + -1, /* (376) selectnowith ::= oneselect */ + -1, /* (377) oneselect ::= values */ + -2, /* (378) sclp ::= selcollist COMMA */ + -1, /* (379) as ::= ID|STRING */ + -1, /* (380) indexed_opt ::= indexed_by */ + 0, /* (381) returning ::= */ + -1, /* (382) expr ::= term */ + -1, /* (383) likeop ::= LIKE_KW|MATCH */ + -1, /* (384) case_operand ::= expr */ + -1, /* (385) exprlist ::= nexprlist */ + -1, /* (386) nmnum ::= plus_num */ + -1, /* (387) nmnum ::= nm */ + -1, /* (388) nmnum ::= ON */ + -1, /* (389) nmnum ::= DELETE */ + -1, /* (390) nmnum ::= DEFAULT */ + -1, /* (391) plus_num ::= INTEGER|FLOAT */ + 0, /* (392) foreach_clause ::= */ + -3, /* (393) foreach_clause ::= FOR EACH ROW */ + -1, /* (394) trnm ::= nm */ + 0, /* (395) tridxby ::= */ + -1, /* (396) database_kw_opt ::= DATABASE */ + 0, /* (397) database_kw_opt ::= */ + 0, /* (398) kwcolumn_opt ::= */ + -1, /* (399) kwcolumn_opt ::= COLUMNKW */ + -1, /* (400) vtabarglist ::= vtabarg */ + -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (403) anylist ::= */ + -4, /* (404) anylist ::= anylist LP anylist RP */ + -2, /* (405) anylist ::= anylist ANY */ + 0, /* (406) with ::= */ + -1, /* (407) windowdefn_list ::= windowdefn */ + -1, /* (408) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -163701,25 +177912,25 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* explain ::= EXPLAIN */ -{ pParse->explain = 1; } +{ if( pParse->pReprepare==0 ) pParse->explain = 1; } break; case 1: /* explain ::= EXPLAIN QUERY PLAN */ -{ pParse->explain = 2; } +{ if( pParse->pReprepare==0 ) pParse->explain = 2; } break; case 2: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy64);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy64 = TK_DEFERRED;} +{yymsp[1].minor.yy144 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 320: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==320); -{yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-X*/} + case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -163742,7 +177953,7 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy64,0,0,yymsp[-2].minor.yy64); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144); } break; case 14: /* createkw ::= CREATE */ @@ -163754,40 +177965,40 @@ static YYACTIONTYPE yy_reduce( case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); - case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 241: /* collate ::= */ yytestcase(yyruleno==241); -{yymsp[1].minor.yy64 = 0;} + case 100: /* distinct ::= */ yytestcase(yyruleno==100); + case 246: /* collate ::= */ yytestcase(yyruleno==246); +{yymsp[1].minor.yy144 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy64 = 1;} +{yymsp[-2].minor.yy144 = 1;} break; case 17: /* temp ::= TEMP */ -{yymsp[0].minor.yy64 = pParse->db->init.busy==0;} +{yymsp[0].minor.yy144 = pParse->db->init.busy==0;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy51,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy303); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy303); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 21: /* table_option_set ::= */ -{yymsp[1].minor.yy51 = 0;} +{yymsp[1].minor.yy391 = 0;} break; case 22: /* table_option_set ::= table_option_set COMMA table_option */ -{yylhsminor.yy51 = yymsp[-2].minor.yy51|yymsp[0].minor.yy51;} - yymsp[-2].minor.yy51 = yylhsminor.yy51; +{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;} + yymsp[-2].minor.yy391 = yylhsminor.yy391; break; case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy51 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy51 = 0; + yymsp[-1].minor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -163795,20 +178006,20 @@ static YYACTIONTYPE yy_reduce( case 24: /* table_option ::= nm */ { if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ - yylhsminor.yy51 = TF_Strict; + yylhsminor.yy391 = TF_Strict; }else{ - yylhsminor.yy51 = 0; + yylhsminor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } - yymsp[0].minor.yy51 = yylhsminor.yy51; + yymsp[0].minor.yy391 = yylhsminor.yy391; break; case 25: /* columnname ::= nm typetoken */ {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} break; case 26: /* typetoken ::= */ case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); - case 104: /* as ::= */ yytestcase(yyruleno==104); + case 106: /* as ::= */ yytestcase(yyruleno==106); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 27: /* typetoken ::= typename LP signed RP */ @@ -163827,7 +178038,7 @@ static YYACTIONTYPE yy_reduce( case 30: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy600 = yyLookaheadToken.z; + yymsp[1].minor.yy168 = yyLookaheadToken.z; } break; case 31: /* scantok ::= */ @@ -163841,17 +178052,17 @@ static YYACTIONTYPE yy_reduce( {pParse->constraintName = yymsp[0].minor.yy0;} break; case 33: /* ccons ::= DEFAULT scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy626,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 34: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy626,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 35: /* ccons ::= DEFAULT PLUS scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy626,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy626, 0); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0); sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; @@ -163866,307 +178077,314 @@ static YYACTIONTYPE yy_reduce( } break; case 38: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy64);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);} break; case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy64,yymsp[0].minor.yy64,yymsp[-2].minor.yy64);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);} break; case 40: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy64,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 41: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy626,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy562,yymsp[0].minor.yy64);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);} break; case 43: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy64);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);} break; case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 45: /* generated ::= LP expr RP */ -{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy626,0);} +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);} break; case 46: /* generated ::= LP expr RP ID */ -{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy626,&yymsp[0].minor.yy0);} +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);} break; case 48: /* autoinc ::= AUTOINCR */ -{yymsp[0].minor.yy64 = 1;} +{yymsp[0].minor.yy144 = 1;} break; case 49: /* refargs ::= */ -{ yymsp[1].minor.yy64 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 50: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy64 = (yymsp[-1].minor.yy64 & ~yymsp[0].minor.yy83.mask) | yymsp[0].minor.yy83.value; } +{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; } break; case 51: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy83.value = 0; yymsp[-1].minor.yy83.mask = 0x000000; } +{ yymsp[-1].minor.yy383.value = 0; yymsp[-1].minor.yy383.mask = 0x000000; } break; case 52: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy83.value = 0; yymsp[-2].minor.yy83.mask = 0x000000; } +{ yymsp[-2].minor.yy383.value = 0; yymsp[-2].minor.yy383.mask = 0x000000; } break; case 53: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy83.value = yymsp[0].minor.yy64; yymsp[-2].minor.yy83.mask = 0x0000ff; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144; yymsp[-2].minor.yy383.mask = 0x0000ff; } break; case 54: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy83.value = yymsp[0].minor.yy64<<8; yymsp[-2].minor.yy83.mask = 0x00ff00; } +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8; yymsp[-2].minor.yy383.mask = 0x00ff00; } break; case 55: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy64 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetNull; /* EV: R-33326-45252 */} break; case 56: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy64 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 57: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy64 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Cascade; /* EV: R-33326-45252 */} break; case 58: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy64 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */} break; case 59: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy64 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy144 = OE_None; /* EV: R-33326-45252 */} break; case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy64 = 0;} +{yymsp[-2].minor.yy144 = 0;} break; case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); - case 171: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==171); -{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} + case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); +{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;} break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 213: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==213); - case 216: /* in_op ::= NOT IN */ yytestcase(yyruleno==216); - case 242: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==242); -{yymsp[-1].minor.yy64 = 1;} + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); +{yymsp[-1].minor.yy144 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy64 = 0;} +{yymsp[-1].minor.yy144 = 0;} break; case 66: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy562,yymsp[0].minor.yy64,yymsp[-2].minor.yy64,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);} break; case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy562,yymsp[0].minor.yy64,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 70: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy626,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy562, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy562, yymsp[-1].minor.yy64); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy64); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144); } break; case 73: /* onconf ::= */ case 75: /* orconf ::= */ yytestcase(yyruleno==75); -{yymsp[1].minor.yy64 = OE_Default;} +{yymsp[1].minor.yy144 = OE_Default;} break; case 74: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy64 = yymsp[0].minor.yy64;} +{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;} break; case 77: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy64 = OE_Ignore;} +{yymsp[0].minor.yy144 = OE_Ignore;} break; case 78: /* resolvetype ::= REPLACE */ - case 172: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==172); -{yymsp[0].minor.yy64 = OE_Replace;} + case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy144 = OE_Replace;} break; case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy607, 0, yymsp[-1].minor.yy64); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144); } break; case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy562, yymsp[0].minor.yy303, yymsp[-7].minor.yy64, yymsp[-5].minor.yy64); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144); } break; case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy607, 1, yymsp[-1].minor.yy64); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144); } break; case 84: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy303, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy303); + sqlite3Select(pParse, yymsp[0].minor.yy555, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; case 85: /* select ::= WITH wqlist selectnowith */ -{yymsp[-2].minor.yy303 = attachWithToSelect(pParse,yymsp[0].minor.yy303,yymsp[-1].minor.yy43);} +{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{yymsp[-3].minor.yy303 = attachWithToSelect(pParse,yymsp[0].minor.yy303,yymsp[-1].minor.yy43);} +{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} break; case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy303; + Select *p = yymsp[0].minor.yy555; if( p ){ parserDoubleLinkSelect(pParse, p); } - yymsp[0].minor.yy303 = p; /*A-overwrites-X*/ } break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy303; - Select *pLhs = yymsp[-2].minor.yy303; + Select *pRhs = yymsp[0].minor.yy555; + Select *pLhs = yymsp[-2].minor.yy555; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; x.n = 0; parserDoubleLinkSelect(pParse, pRhs); - pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0); pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy64; + pRhs->op = (u8)yymsp[-1].minor.yy144; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy64!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy303 = pRhs; + yymsp[-2].minor.yy555 = pRhs; } break; case 89: /* multiselect_op ::= UNION */ case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); -{yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/} break; case 90: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy64 = TK_ALL;} +{yymsp[-1].minor.yy144 = TK_ALL;} break; case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy562,yymsp[-5].minor.yy607,yymsp[-4].minor.yy626,yymsp[-3].minor.yy562,yymsp[-2].minor.yy626,yymsp[-1].minor.yy562,yymsp[-7].minor.yy64,yymsp[0].minor.yy626); + yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454); } break; case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy562,yymsp[-6].minor.yy607,yymsp[-5].minor.yy626,yymsp[-4].minor.yy562,yymsp[-3].minor.yy626,yymsp[-1].minor.yy562,yymsp[-8].minor.yy64,yymsp[0].minor.yy626); - if( yymsp[-9].minor.yy303 ){ - yymsp[-9].minor.yy303->pWinDefn = yymsp[-2].minor.yy375; + yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454); + if( yymsp[-9].minor.yy555 ){ + yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy375); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211); } } break; case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy562,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0); } break; - case 95: /* values ::= values COMMA LP nexprlist RP */ + case 95: /* oneselect ::= mvalues */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy303; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy562,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy303 = pRight; - }else{ - yymsp[-4].minor.yy303 = pLeft; - } + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555); } break; - case 96: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy64 = SF_Distinct;} + case 96: /* mvalues ::= values COMMA LP nexprlist RP */ + case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); +{ + yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14); +} + break; + case 98: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy144 = SF_Distinct;} break; - case 97: /* distinct ::= ALL */ -{yymsp[0].minor.yy64 = SF_All;} + case 99: /* distinct ::= ALL */ +{yymsp[0].minor.yy144 = SF_All;} break; - case 99: /* sclp ::= */ - case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); - case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 229: /* exprlist ::= */ yytestcase(yyruleno==229); - case 232: /* paren_exprlist ::= */ yytestcase(yyruleno==232); - case 237: /* eidlist_opt ::= */ yytestcase(yyruleno==237); -{yymsp[1].minor.yy562 = 0;} + case 101: /* sclp ::= */ + case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); + case 144: /* groupby_opt ::= */ yytestcase(yyruleno==144); + case 234: /* exprlist ::= */ yytestcase(yyruleno==234); + case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); + case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); +{yymsp[1].minor.yy14 = 0;} break; - case 100: /* selcollist ::= sclp scanpt expr scanpt as */ + case 102: /* selcollist ::= sclp scanpt expr scanpt as */ { - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy562, yymsp[-2].minor.yy626); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy562, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy562,yymsp[-3].minor.yy600,yymsp[-1].minor.yy600); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168); } break; - case 101: /* selcollist ::= sclp scanpt STAR */ + case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy562, p); + sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p); } break; - case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); - Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, pDot); -} - break; - case 103: /* as ::= AS nm */ - case 114: /* dbnm ::= DOT nm */ yytestcase(yyruleno==114); - case 253: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==253); - case 254: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==254); + Expr *pRight, *pLeft, *pDot; + pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); + pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); + pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot); +} + break; + case 105: /* as ::= AS nm */ + case 117: /* dbnm ::= DOT nm */ yytestcase(yyruleno==117); + case 258: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==258); + case 259: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==259); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 105: /* from ::= */ - case 108: /* stl_prefix ::= */ yytestcase(yyruleno==108); -{yymsp[1].minor.yy607 = 0;} + case 107: /* from ::= */ + case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); +{yymsp[1].minor.yy203 = 0;} break; - case 106: /* from ::= FROM seltablist */ + case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy607 = yymsp[0].minor.yy607; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy607); + yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203); } break; - case 107: /* stl_prefix ::= seltablist joinop */ + case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy607 && yymsp[-1].minor.yy607->nSrc>0) ) yymsp[-1].minor.yy607->a[yymsp[-1].minor.yy607->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy64; + if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144; } break; - case 109: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy607, &yymsp[-2].minor.yy0); + yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); } break; - case 110: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-8].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy607,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy607, yymsp[-4].minor.yy562); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0); } break; - case 111: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy303,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14); +} + break; + case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ +{ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269); } break; - case 112: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-6].minor.yy607==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy626==0 && yymsp[0].minor.yy240==0 ){ - yymsp[-6].minor.yy607 = yymsp[-4].minor.yy607; - }else if( yymsp[-4].minor.yy607->nSrc==1 ){ - yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); - if( yymsp[-6].minor.yy607 ){ - SrcItem *pNew = &yymsp[-6].minor.yy607->a[yymsp[-6].minor.yy607->nSrc-1]; - SrcItem *pOld = yymsp[-4].minor.yy607->a; + if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){ + yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203; + }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + if( yymsp[-5].minor.yy203 ){ + SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy203->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; + if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->fg.isNestedFrom = 1; + } if( pOld->fg.isTabFunc ){ pNew->u1.pFuncArg = pOld->u1.pFuncArg; pOld->u1.pFuncArg = 0; @@ -164176,267 +178394,276 @@ static YYACTIONTYPE yy_reduce( pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy607); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy607); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy607,0,0,0,0,SF_NestedFrom,0); - yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269); } } break; - case 113: /* dbnm ::= */ - case 127: /* indexed_opt ::= */ yytestcase(yyruleno==127); + case 116: /* dbnm ::= */ + case 131: /* indexed_opt ::= */ yytestcase(yyruleno==131); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 115: /* fullname ::= nm */ + case 118: /* fullname ::= nm */ { - yylhsminor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy607 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy607->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy607 = yylhsminor.yy607; + yymsp[0].minor.yy203 = yylhsminor.yy203; break; - case 116: /* fullname ::= nm DOT nm */ + case 119: /* fullname ::= nm DOT nm */ { - yylhsminor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy607 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy607->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy607 = yylhsminor.yy607; + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 117: /* xfullname ::= nm */ -{yymsp[0].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + case 120: /* xfullname ::= nm */ +{yymsp[0].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 118: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 121: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 119: /* xfullname ::= nm DOT nm AS nm */ + case 122: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-4].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy607 ) yymsp[-4].minor.yy607->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-4].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy203 ) yymsp[-4].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 120: /* xfullname ::= nm AS nm */ + case 123: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy607 ) yymsp[-2].minor.yy607->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy203 ) yymsp[-2].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 121: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy64 = JT_INNER; } + case 124: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy144 = JT_INNER; } break; - case 122: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 125: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 123: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 126: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 124: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 127: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 125: /* on_opt ::= ON expr */ - case 145: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==145); - case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); - case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 225: /* case_else ::= ELSE expr */ yytestcase(yyruleno==225); - case 246: /* vinto ::= INTO expr */ yytestcase(yyruleno==246); -{yymsp[-1].minor.yy626 = yymsp[0].minor.yy626;} + case 128: /* on_using ::= ON expr */ +{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;} break; - case 126: /* on_opt ::= */ - case 144: /* having_opt ::= */ yytestcase(yyruleno==144); - case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); - case 151: /* where_opt ::= */ yytestcase(yyruleno==151); - case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 226: /* case_else ::= */ yytestcase(yyruleno==226); - case 228: /* case_operand ::= */ yytestcase(yyruleno==228); - case 247: /* vinto ::= */ yytestcase(yyruleno==247); -{yymsp[1].minor.yy626 = 0;} + case 129: /* on_using ::= USING LP idlist RP */ +{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;} break; - case 128: /* indexed_opt ::= INDEXED BY nm */ + case 130: /* on_using ::= */ +{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;} + break; + case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 129: /* indexed_opt ::= NOT INDEXED */ + case 133: /* indexed_by ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 130: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy240 = yymsp[-1].minor.yy240;} - break; - case 131: /* using_opt ::= */ - case 173: /* idlist_opt ::= */ yytestcase(yyruleno==173); -{yymsp[1].minor.yy240 = 0;} - break; - case 133: /* orderby_opt ::= ORDER BY sortlist */ - case 143: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==143); -{yymsp[-2].minor.yy562 = yymsp[0].minor.yy562;} + case 135: /* orderby_opt ::= ORDER BY sortlist */ + case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); +{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;} break; - case 134: /* sortlist ::= sortlist COMMA expr sortorder nulls */ + case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562,yymsp[-2].minor.yy626); - sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy562,yymsp[-1].minor.yy64,yymsp[0].minor.yy64); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 135: /* sortlist ::= expr sortorder nulls */ + case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy626); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy562,yymsp[-1].minor.yy64,yymsp[0].minor.yy64); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 136: /* sortorder ::= ASC */ -{yymsp[0].minor.yy64 = SQLITE_SO_ASC;} + case 138: /* sortorder ::= ASC */ +{yymsp[0].minor.yy144 = SQLITE_SO_ASC;} + break; + case 139: /* sortorder ::= DESC */ +{yymsp[0].minor.yy144 = SQLITE_SO_DESC;} + break; + case 140: /* sortorder ::= */ + case 143: /* nulls ::= */ yytestcase(yyruleno==143); +{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;} break; - case 137: /* sortorder ::= DESC */ -{yymsp[0].minor.yy64 = SQLITE_SO_DESC;} + case 141: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;} break; - case 138: /* sortorder ::= */ - case 141: /* nulls ::= */ yytestcase(yyruleno==141); -{yymsp[1].minor.yy64 = SQLITE_SO_UNDEFINED;} + case 142: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;} break; - case 139: /* nulls ::= NULLS FIRST */ -{yymsp[-1].minor.yy64 = SQLITE_SO_ASC;} + case 146: /* having_opt ::= */ + case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); + case 153: /* where_opt ::= */ yytestcase(yyruleno==153); + case 155: /* where_opt_ret ::= */ yytestcase(yyruleno==155); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 233: /* case_operand ::= */ yytestcase(yyruleno==233); + case 252: /* vinto ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy454 = 0;} break; - case 140: /* nulls ::= NULLS LAST */ -{yymsp[-1].minor.yy64 = SQLITE_SO_DESC;} + case 147: /* having_opt ::= HAVING expr */ + case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); + case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); +{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;} break; - case 147: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy626,0);} + case 149: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);} break; - case 148: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 149: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy626,yymsp[-2].minor.yy626);} + case 151: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);} break; - case 150: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy607, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy607,yymsp[0].minor.yy626,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy203, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy203,yymsp[0].minor.yy454,0,0); } break; - case 155: /* where_opt_ret ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy562); yymsp[-1].minor.yy626 = 0;} + case 157: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;} break; - case 156: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy562); yymsp[-3].minor.yy626 = yymsp[-2].minor.yy626;} + case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;} break; - case 157: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy607, &yymsp[-4].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy562,"set list"); - yymsp[-5].minor.yy607 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy607, yymsp[-1].minor.yy607); - sqlite3Update(pParse,yymsp[-5].minor.yy607,yymsp[-2].minor.yy562,yymsp[0].minor.yy626,yymsp[-6].minor.yy64,0,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy14,"set list"); + if( yymsp[-1].minor.yy203 ){ + SrcList *pFromClause = yymsp[-1].minor.yy203; + if( pFromClause->nSrc>1 ){ + Select *pSubquery; + Token as; + pSubquery = sqlite3SelectNew(pParse,0,pFromClause,0,0,0,0,SF_NestedFrom,0); + as.n = 0; + as.z = 0; + pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); + } + yymsp[-5].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy203, pFromClause); + } + sqlite3Update(pParse,yymsp[-5].minor.yy203,yymsp[-2].minor.yy14,yymsp[0].minor.yy454,yymsp[-6].minor.yy144,0,0,0); } break; - case 158: /* setlist ::= setlist COMMA nm EQ expr */ + case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy562, yymsp[0].minor.yy626); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy562, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 159: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy562 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy562, yymsp[-3].minor.yy240, yymsp[0].minor.yy626); + yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 160: /* setlist ::= nm EQ expr */ + case 162: /* setlist ::= nm EQ expr */ { - yylhsminor.yy562 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy626); - sqlite3ExprListSetName(pParse, yylhsminor.yy562, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy562 = yylhsminor.yy562; + yymsp[-2].minor.yy14 = yylhsminor.yy14; break; - case 161: /* setlist ::= LP idlist RP EQ expr */ + case 163: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy562 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy240, yymsp[0].minor.yy626); + yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 162: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy607, yymsp[-1].minor.yy303, yymsp[-2].minor.yy240, yymsp[-5].minor.yy64, yymsp[0].minor.yy138); + sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122); } break; - case 163: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - sqlite3Insert(pParse, yymsp[-4].minor.yy607, 0, yymsp[-3].minor.yy240, yymsp[-6].minor.yy64, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0); } break; - case 164: /* upsert ::= */ -{ yymsp[1].minor.yy138 = 0; } + case 166: /* upsert ::= */ +{ yymsp[1].minor.yy122 = 0; } + break; + case 167: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); } break; - case 165: /* upsert ::= RETURNING selcollist */ -{ yymsp[-1].minor.yy138 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy562); } + case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);} break; - case 166: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ -{ yymsp[-11].minor.yy138 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy562,yymsp[-6].minor.yy626,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626,yymsp[0].minor.yy138);} + case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); } break; - case 167: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ -{ yymsp[-8].minor.yy138 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy562,yymsp[-3].minor.yy626,0,0,yymsp[0].minor.yy138); } + case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } break; - case 168: /* upsert ::= ON CONFLICT DO NOTHING returning */ -{ yymsp[-4].minor.yy138 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);} break; - case 169: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ -{ yymsp[-7].minor.yy138 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626,0);} + case 172: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);} break; - case 170: /* returning ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy562);} + case 175: /* idlist_opt ::= */ +{yymsp[1].minor.yy132 = 0;} break; - case 174: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy240 = yymsp[-1].minor.yy240;} + case 176: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;} break; - case 175: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy240 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy240,&yymsp[0].minor.yy0);} + case 177: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);} break; - case 176: /* idlist ::= nm */ -{yymsp[0].minor.yy240 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 178: /* idlist ::= nm */ +{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 177: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy626 = yymsp[-1].minor.yy626;} + case 179: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;} break; - case 178: /* expr ::= ID|INDEXED */ - case 179: /* expr ::= JOIN_KW */ yytestcase(yyruleno==179); -{yymsp[0].minor.yy626=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 180: /* expr ::= ID|INDEXED|JOIN_KW */ +{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 180: /* expr ::= nm DOT nm */ + case 181: /* expr ::= nm DOT nm */ { - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); - sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); - } - yylhsminor.yy626 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); + Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy626 = yylhsminor.yy626; + yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 181: /* expr ::= nm DOT nm DOT nm */ + case 182: /* expr ::= nm DOT nm DOT nm */ { - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); + Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); + Expr *temp3 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); - sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); + sqlite3RenameTokenRemap(pParse, 0, temp1); } - yylhsminor.yy626 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy626 = yylhsminor.yy626; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 182: /* term ::= NULL|FLOAT|BLOB */ - case 183: /* term ::= STRING */ yytestcase(yyruleno==183); -{yymsp[0].minor.yy626=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 183: /* term ::= NULL|FLOAT|BLOB */ + case 184: /* term ::= STRING */ yytestcase(yyruleno==184); +{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 184: /* term ::= INTEGER */ + case 185: /* term ::= INTEGER */ { - yylhsminor.yy626 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } - yymsp[0].minor.yy626 = yylhsminor.yy626; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 185: /* expr ::= VARIABLE */ + case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy626 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy626, n); + yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -164445,159 +178672,203 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy626 = 0; + yymsp[0].minor.yy454 = 0; }else{ - yymsp[0].minor.yy626 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy626 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy626->iTable); + yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable); } } } break; - case 186: /* expr ::= expr COLLATE ID|STRING */ + case 187: /* expr ::= expr COLLATE ID|STRING */ +{ + yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1); +} + break; + case 188: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0); +} + break; + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - yymsp[-2].minor.yy626 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy626, &yymsp[0].minor.yy0, 1); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144); } + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 187: /* expr ::= CAST LP expr AS typetoken RP */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yymsp[-5].minor.yy626 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy626, yymsp[-3].minor.yy626, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14); } + yymsp[-7].minor.yy454 = yylhsminor.yy454; break; - case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy626 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy562, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy64); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-4].minor.yy626 = yylhsminor.yy626; + yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 189: /* expr ::= ID|INDEXED LP STAR RP */ + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-3].minor.yy626 = yylhsminor.yy626; + yymsp[-5].minor.yy454 = yylhsminor.yy454; break; - case 190: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - yylhsminor.yy626 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy562, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy64); - sqlite3WindowAttach(pParse, yylhsminor.yy626, yymsp[0].minor.yy375); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14); } - yymsp[-5].minor.yy626 = yylhsminor.yy626; + yymsp[-8].minor.yy454 = yylhsminor.yy454; break; - case 191: /* expr ::= ID|INDEXED LP STAR RP filter_over */ + case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy626, yymsp[0].minor.yy375); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } - yymsp[-4].minor.yy626 = yylhsminor.yy626; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 192: /* term ::= CTIME_KW */ + case 195: /* term ::= CTIME_KW */ { - yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy626 = yylhsminor.yy626; + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 193: /* expr ::= LP nexprlist COMMA expr RP */ + case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy562, yymsp[-1].minor.yy626); - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy626 ){ - yymsp[-4].minor.yy626->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; if( ALWAYS(pList->nExpr) ){ - yymsp[-4].minor.yy626->flags |= pList->a[0].pExpr->flags & EP_Propagate; + yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate; } }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 194: /* expr ::= expr AND expr */ -{yymsp[-2].minor.yy626=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + case 197: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 195: /* expr ::= expr OR expr */ - case 196: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==196); - case 197: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==201); -{yymsp[-2].minor.yy626=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); +{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} break; - case 202: /* likeop ::= NOT LIKE_KW|MATCH */ + case 205: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 203: /* expr ::= expr likeop expr */ + case 206: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy626); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy626); - yymsp[-2].minor.yy626 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy626, 0); - if( yymsp[-2].minor.yy626 ) yymsp[-2].minor.yy626->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454); + yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0); + if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc; } break; - case 204: /* expr ::= expr likeop expr ESCAPE expr */ + case 207: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy626); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy626); - yymsp[-4].minor.yy626 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); - if( yymsp[-4].minor.yy626 ) yymsp[-4].minor.yy626->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy626,0);} + case 208: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy454,0);} break; - case 206: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy626 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy626,0);} + case 209: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy454,0);} break; - case 207: /* expr ::= expr IS expr */ + case 210: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy626 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy626,yymsp[0].minor.yy626); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy626, yymsp[-2].minor.yy626, TK_ISNULL); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-2].minor.yy454, TK_ISNULL); } break; - case 208: /* expr ::= expr IS NOT expr */ + case 211: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy626,yymsp[0].minor.yy626); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy626, yymsp[-3].minor.yy626, TK_NOTNULL); + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-3].minor.yy454, TK_NOTNULL); } break; - case 209: /* expr ::= NOT expr */ - case 210: /* expr ::= BITNOT expr */ yytestcase(yyruleno==210); -{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy626, 0);/*A-overwrites-B*/} + case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ +{ + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-5].minor.yy454, TK_ISNULL); +} break; - case 211: /* expr ::= PLUS|MINUS expr */ + case 213: /* expr ::= expr IS DISTINCT FROM expr */ { - yymsp[-1].minor.yy626 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy626, 0); - /*A-overwrites-B*/ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-4].minor.yy454, TK_NOTNULL); } break; - case 212: /* between_op ::= BETWEEN */ - case 215: /* in_op ::= IN */ yytestcase(yyruleno==215); -{yymsp[0].minor.yy64 = 0;} + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/} break; - case 214: /* expr ::= expr between_op expr AND expr */ + case 216: /* expr ::= PLUS|MINUS expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy626); - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy626, 0); - if( yymsp[-4].minor.yy626 ){ - yymsp[-4].minor.yy626->x.pList = pList; + Expr *p = yymsp[0].minor.yy454; + u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + yymsp[-1].minor.yy454 = p; + }else{ + yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } +} + break; + case 217: /* expr ::= expr PTR expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); +} + yymsp[-2].minor.yy454 = yylhsminor.yy454; + break; + case 218: /* between_op ::= BETWEEN */ + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yymsp[0].minor.yy144 = 0;} + break; + case 220: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 217: /* expr ::= expr in_op LP exprlist RP */ + case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy562==0 ){ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -164606,205 +178877,208 @@ static YYACTIONTYPE yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy626); - yymsp[-4].minor.yy626 = sqlite3Expr(pParse->db, TK_INTEGER, yymsp[-3].minor.yy64 ? "1" : "0"); - }else{ - Expr *pRHS = yymsp[-1].minor.yy562->a[0].pExpr; - if( yymsp[-1].minor.yy562->nExpr==1 && sqlite3ExprIsConstant(pRHS) && yymsp[-4].minor.yy626->op!=TK_VECTOR ){ - yymsp[-1].minor.yy562->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy562); + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false"); + if( yymsp[-4].minor.yy454 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy454); + }else{ + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){ + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy626, pRHS); - }else{ - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); - if( yymsp[-4].minor.yy626==0 ){ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy562); - }else if( yymsp[-4].minor.yy626->pLeft->op==TK_VECTOR ){ - int nExpr = yymsp[-4].minor.yy626->pLeft->x.pList->nExpr; - Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy562); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS); + }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect); + pRHS->x.pSelect = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else{ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14); if( pSelectRHS ){ parserDoubleLinkSelect(pParse, pSelectRHS); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS); } }else{ - yymsp[-4].minor.yy626->x.pList = yymsp[-1].minor.yy562; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy626); + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); } } - if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } } break; - case 218: /* expr ::= LP select RP */ + case 224: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy626 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy626, yymsp[-1].minor.yy303); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555); } break; - case 219: /* expr ::= expr in_op LP select RP */ + case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, yymsp[-1].minor.yy303); - if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 220: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy562 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy562); - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, pSelect); - if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + if( yymsp[0].minor.yy14 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 221: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy626 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy303); + p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555); } break; - case 222: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy626, 0); - if( yymsp[-4].minor.yy626 ){ - yymsp[-4].minor.yy626->x.pList = yymsp[-1].minor.yy626 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626) : yymsp[-2].minor.yy562; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy626); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy562); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy626); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } } break; - case 223: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, yymsp[-2].minor.yy626); - yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, yymsp[0].minor.yy626); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454); } break; - case 224: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy562 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); - yymsp[-3].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy562, yymsp[0].minor.yy626); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454); } break; - case 227: /* case_operand ::= expr */ -{yymsp[0].minor.yy626 = yymsp[0].minor.yy626; /*A-overwrites-X*/} + case 235: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);} break; - case 230: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy562,yymsp[0].minor.yy626);} + case 236: /* nexprlist ::= expr */ +{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/} break; - case 231: /* nexprlist ::= expr */ -{yymsp[0].minor.yy562 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy626); /*A-overwrites-Y*/} + case 238: /* paren_exprlist ::= LP exprlist RP */ + case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); +{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;} break; - case 233: /* paren_exprlist ::= LP exprlist RP */ - case 238: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==238); -{yymsp[-2].minor.yy562 = yymsp[-1].minor.yy562;} - break; - case 234: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy562, yymsp[-10].minor.yy64, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy626, SQLITE_SO_ASC, yymsp[-8].minor.yy64, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; - case 235: /* uniqueflag ::= UNIQUE */ - case 277: /* raisetype ::= ABORT */ yytestcase(yyruleno==277); -{yymsp[0].minor.yy64 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); +{yymsp[0].minor.yy144 = OE_Abort;} break; - case 236: /* uniqueflag ::= */ -{yymsp[1].minor.yy64 = OE_None;} + case 241: /* uniqueflag ::= */ +{yymsp[1].minor.yy144 = OE_None;} break; - case 239: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy562 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy562, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy64, yymsp[0].minor.yy64); + yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); } break; - case 240: /* eidlist ::= nm collate sortorder */ + case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy562 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy64, yymsp[0].minor.yy64); /*A-overwrites-Y*/ + yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/ } break; - case 243: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy607, yymsp[-1].minor.yy64);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);} break; - case 244: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy626);} + case 249: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);} break; - case 245: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy626);} + case 250: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);} break; - case 248: /* cmd ::= PRAGMA nm dbnm */ + case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 249: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 250: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 251: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 252: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 257: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 255: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 260: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy95, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all); } break; - case 256: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy64, yymsp[-4].minor.yy570.a, yymsp[-4].minor.yy570.b, yymsp[-2].minor.yy607, yymsp[0].minor.yy626, yymsp[-10].minor.yy64, yymsp[-8].minor.yy64); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 257: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-X*/ } + case 262: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 258: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy64 = TK_INSTEAD;} + case 263: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy144 = TK_INSTEAD;} break; - case 259: /* trigger_time ::= */ -{ yymsp[1].minor.yy64 = TK_BEFORE; } + case 264: /* trigger_time ::= */ +{ yymsp[1].minor.yy144 = TK_BEFORE; } break; - case 260: /* trigger_event ::= DELETE|INSERT */ - case 261: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==261); -{yymsp[0].minor.yy570.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy570.b = 0;} + case 265: /* trigger_event ::= DELETE|INSERT */ + case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;} break; - case 262: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy570.a = TK_UPDATE; yymsp[-2].minor.yy570.b = yymsp[0].minor.yy240;} + case 267: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;} break; - case 263: /* when_clause ::= */ - case 282: /* key_opt ::= */ yytestcase(yyruleno==282); -{ yymsp[1].minor.yy626 = 0; } + case 268: /* when_clause ::= */ + case 287: /* key_opt ::= */ yytestcase(yyruleno==287); +{ yymsp[1].minor.yy454 = 0; } break; - case 264: /* when_clause ::= WHEN expr */ - case 283: /* key_opt ::= KEY expr */ yytestcase(yyruleno==283); -{ yymsp[-1].minor.yy626 = yymsp[0].minor.yy626; } + case 269: /* when_clause ::= WHEN expr */ + case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); +{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; } break; - case 265: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy95!=0 ); - yymsp[-2].minor.yy95->pLast->pNext = yymsp[-1].minor.yy95; - yymsp[-2].minor.yy95->pLast = yymsp[-1].minor.yy95; + assert( yymsp[-2].minor.yy427!=0 ); + yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427; + yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 266: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy95!=0 ); - yymsp[-1].minor.yy95->pLast = yymsp[-1].minor.yy95; + assert( yymsp[-1].minor.yy427!=0 ); + yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 267: /* trnm ::= nm DOT nm */ + case 272: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -164812,369 +179086,377 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 268: /* tridxby ::= INDEXED BY nm */ + case 273: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 269: /* tridxby ::= NOT INDEXED */ + case 274: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 270: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ -{yylhsminor.yy95 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy607, yymsp[-3].minor.yy562, yymsp[-1].minor.yy626, yymsp[-7].minor.yy64, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy600);} - yymsp[-8].minor.yy95 = yylhsminor.yy95; + case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-8].minor.yy427 = yylhsminor.yy427; break; - case 271: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy95 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy240,yymsp[-2].minor.yy303,yymsp[-6].minor.yy64,yymsp[-1].minor.yy138,yymsp[-7].minor.yy600,yymsp[0].minor.yy600);/*yylhsminor.yy95-overwrites-yymsp[-6].minor.yy64*/ + yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/ } - yymsp[-7].minor.yy95 = yylhsminor.yy95; + yymsp[-7].minor.yy427 = yylhsminor.yy427; break; - case 272: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy95 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy626, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy600);} - yymsp[-5].minor.yy95 = yylhsminor.yy95; + case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-5].minor.yy427 = yylhsminor.yy427; break; - case 273: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy95 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy303, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); /*yylhsminor.yy95-overwrites-yymsp[-1].minor.yy303*/} - yymsp[-2].minor.yy95 = yylhsminor.yy95; + case 278: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/} + yymsp[-2].minor.yy427 = yylhsminor.yy427; break; - case 274: /* expr ::= RAISE LP IGNORE RP */ + case 279: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy626 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy626 ){ - yymsp[-3].minor.yy626->affExpr = OE_Ignore; + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy454 ){ + yymsp[-3].minor.yy454->affExpr = OE_Ignore; } } break; - case 275: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 280: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yymsp[-5].minor.yy626 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy626 ) { - yymsp[-5].minor.yy626->affExpr = (char)yymsp[-3].minor.yy64; + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy454 ) { + yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144; } } break; - case 276: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy64 = OE_Rollback;} + case 281: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy144 = OE_Rollback;} break; - case 278: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy64 = OE_Fail;} + case 283: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy144 = OE_Fail;} break; - case 279: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy607,yymsp[-1].minor.yy64); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144); } break; - case 280: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy626, yymsp[-1].minor.yy626, yymsp[0].minor.yy626); + sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454); } break; - case 281: /* cmd ::= DETACH database_kw_opt expr */ + case 286: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy626); + sqlite3Detach(pParse, yymsp[0].minor.yy454); } break; - case 284: /* cmd ::= REINDEX */ + case 289: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 285: /* cmd ::= REINDEX nm dbnm */ + case 290: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 286: /* cmd ::= ANALYZE */ + case 291: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 287: /* cmd ::= ANALYZE nm dbnm */ + case 292: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 288: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy607,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0); } break; - case 289: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 290: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { - sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy607, &yymsp[0].minor.yy0); + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0); } break; - case 291: /* add_column_fullname ::= fullname */ + case 296: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy607); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy203); } break; - case 292: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy607, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 293: /* cmd ::= create_vtab */ + case 298: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 294: /* cmd ::= create_vtab LP vtabarglist RP */ + case 299: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 295: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy64); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144); } break; - case 296: /* vtabarg ::= */ + case 301: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 297: /* vtabargtoken ::= ANY */ - case 298: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==298); - case 299: /* lp ::= LP */ yytestcase(yyruleno==299); + case 302: /* vtabargtoken ::= ANY */ + case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); + case 304: /* lp ::= LP */ yytestcase(yyruleno==304); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 300: /* with ::= WITH wqlist */ - case 301: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==301); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy43, 1); } + case 305: /* with ::= WITH wqlist */ + case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); } break; - case 302: /* wqas ::= AS */ -{yymsp[0].minor.yy534 = M10d_Any;} + case 307: /* wqas ::= AS */ +{yymsp[0].minor.yy462 = M10d_Any;} break; - case 303: /* wqas ::= AS MATERIALIZED */ -{yymsp[-1].minor.yy534 = M10d_Yes;} + case 308: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy462 = M10d_Yes;} break; - case 304: /* wqas ::= AS NOT MATERIALIZED */ -{yymsp[-2].minor.yy534 = M10d_No;} + case 309: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy462 = M10d_No;} break; - case 305: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy255 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy562, yymsp[-1].minor.yy303, yymsp[-3].minor.yy534); /*A-overwrites-X*/ + yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/ } break; - case 306: /* wqlist ::= wqitem */ -{ - yymsp[0].minor.yy43 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy255); /*A-overwrites-X*/ -} + case 311: /* withnm ::= nm */ +{pParse->bHasWith = 1;} break; - case 307: /* wqlist ::= wqlist COMMA wqitem */ + case 312: /* wqlist ::= wqitem */ { - yymsp[-2].minor.yy43 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy43, yymsp[0].minor.yy255); + yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/ } break; - case 308: /* windowdefn_list ::= windowdefn */ -{ yylhsminor.yy375 = yymsp[0].minor.yy375; } - yymsp[0].minor.yy375 = yylhsminor.yy375; - break; - case 309: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 313: /* wqlist ::= wqlist COMMA wqitem */ { - assert( yymsp[0].minor.yy375!=0 ); - sqlite3WindowChain(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy375); - yymsp[0].minor.yy375->pNextWin = yymsp[-2].minor.yy375; - yylhsminor.yy375 = yymsp[0].minor.yy375; + yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67); } - yymsp[-2].minor.yy375 = yylhsminor.yy375; break; - case 310: /* windowdefn ::= nm AS LP window RP */ + case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - if( ALWAYS(yymsp[-1].minor.yy375) ){ - yymsp[-1].minor.yy375->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); - } - yylhsminor.yy375 = yymsp[-1].minor.yy375; + assert( yymsp[0].minor.yy211!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211); + yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211; + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-4].minor.yy375 = yylhsminor.yy375; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 311: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 315: /* windowdefn ::= nm AS LP window RP */ { - yymsp[-4].minor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy562, yymsp[-1].minor.yy562, 0); + if( ALWAYS(yymsp[-1].minor.yy211) ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + } + yylhsminor.yy211 = yymsp[-1].minor.yy211; } + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 312: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy562, yymsp[-1].minor.yy562, &yymsp[-5].minor.yy0); + yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0); } - yymsp[-5].minor.yy375 = yylhsminor.yy375; break; - case 313: /* window ::= ORDER BY sortlist frame_opt */ + case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-3].minor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, yymsp[-1].minor.yy562, 0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0); } + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 314: /* window ::= nm ORDER BY sortlist frame_opt */ + case 318: /* window ::= ORDER BY sortlist frame_opt */ { - yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, yymsp[-1].minor.yy562, &yymsp[-4].minor.yy0); + yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0); } - yymsp[-4].minor.yy375 = yylhsminor.yy375; break; - case 315: /* window ::= frame_opt */ - case 334: /* filter_over ::= over_clause */ yytestcase(yyruleno==334); + case 319: /* window ::= nm ORDER BY sortlist frame_opt */ { - yylhsminor.yy375 = yymsp[0].minor.yy375; + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); } - yymsp[0].minor.yy375 = yylhsminor.yy375; + yymsp[-4].minor.yy211 = yylhsminor.yy211; break; - case 316: /* window ::= nm frame_opt */ + case 320: /* window ::= nm frame_opt */ { - yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, 0, &yymsp[-1].minor.yy0); + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy375 = yylhsminor.yy375; + yymsp[-1].minor.yy211 = yylhsminor.yy211; break; - case 317: /* frame_opt ::= */ + case 321: /* frame_opt ::= */ { - yymsp[1].minor.yy375 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); + yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 318: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy375 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy64, yymsp[-1].minor.yy81.eType, yymsp[-1].minor.yy81.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy534); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462); } - yymsp[-2].minor.yy375 = yylhsminor.yy375; + yymsp[-2].minor.yy211 = yylhsminor.yy211; break; - case 319: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy375 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy64, yymsp[-3].minor.yy81.eType, yymsp[-3].minor.yy81.pExpr, yymsp[-1].minor.yy81.eType, yymsp[-1].minor.yy81.pExpr, yymsp[0].minor.yy534); + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462); } - yymsp[-5].minor.yy375 = yylhsminor.yy375; + yymsp[-5].minor.yy211 = yylhsminor.yy211; break; - case 321: /* frame_bound_s ::= frame_bound */ - case 323: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==323); -{yylhsminor.yy81 = yymsp[0].minor.yy81;} - yymsp[0].minor.yy81 = yylhsminor.yy81; + case 325: /* frame_bound_s ::= frame_bound */ + case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); +{yylhsminor.yy509 = yymsp[0].minor.yy509;} + yymsp[0].minor.yy509 = yylhsminor.yy509; break; - case 322: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 324: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==324); - case 326: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==326); -{yylhsminor.yy81.eType = yymsp[-1].major; yylhsminor.yy81.pExpr = 0;} - yymsp[-1].minor.yy81 = yylhsminor.yy81; + case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); + case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 325: /* frame_bound ::= expr PRECEDING|FOLLOWING */ -{yylhsminor.yy81.eType = yymsp[0].major; yylhsminor.yy81.pExpr = yymsp[-1].minor.yy626;} - yymsp[-1].minor.yy81 = yylhsminor.yy81; + case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; break; - case 327: /* frame_exclude_opt ::= */ -{yymsp[1].minor.yy534 = 0;} + case 331: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy462 = 0;} break; - case 328: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ -{yymsp[-1].minor.yy534 = yymsp[0].minor.yy534;} + case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;} break; - case 329: /* frame_exclude ::= NO OTHERS */ - case 330: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==330); -{yymsp[-1].minor.yy534 = yymsp[-1].major; /*A-overwrites-X*/} + case 333: /* frame_exclude ::= NO OTHERS */ + case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 331: /* frame_exclude ::= GROUP|TIES */ -{yymsp[0].minor.yy534 = yymsp[0].major; /*A-overwrites-X*/} + case 335: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/} break; - case 332: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy375 = yymsp[0].minor.yy375; } + case 336: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; } break; - case 333: /* filter_over ::= filter_clause over_clause */ + case 337: /* filter_over ::= filter_clause over_clause */ { - if( yymsp[0].minor.yy375 ){ - yymsp[0].minor.yy375->pFilter = yymsp[-1].minor.yy626; + if( yymsp[0].minor.yy211 ){ + yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy626); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } - yylhsminor.yy375 = yymsp[0].minor.yy375; + yylhsminor.yy211 = yymsp[0].minor.yy211; +} + yymsp[-1].minor.yy211 = yylhsminor.yy211; + break; + case 338: /* filter_over ::= over_clause */ +{ + yylhsminor.yy211 = yymsp[0].minor.yy211; } - yymsp[-1].minor.yy375 = yylhsminor.yy375; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 335: /* filter_over ::= filter_clause */ + case 339: /* filter_over ::= filter_clause */ { - yylhsminor.yy375 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy375 ){ - yylhsminor.yy375->eFrmType = TK_FILTER; - yylhsminor.yy375->pFilter = yymsp[0].minor.yy626; + yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy211 ){ + yylhsminor.yy211->eFrmType = TK_FILTER; + yylhsminor.yy211->pFilter = yymsp[0].minor.yy454; }else{ - sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy626); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454); } } - yymsp[0].minor.yy375 = yylhsminor.yy375; + yymsp[0].minor.yy211 = yylhsminor.yy211; break; - case 336: /* over_clause ::= OVER LP window RP */ + case 340: /* over_clause ::= OVER LP window RP */ { - yymsp[-3].minor.yy375 = yymsp[-1].minor.yy375; - assert( yymsp[-3].minor.yy375!=0 ); + yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211; + assert( yymsp[-3].minor.yy211!=0 ); } break; - case 337: /* over_clause ::= OVER nm */ + case 341: /* over_clause ::= OVER nm */ { - yymsp[-1].minor.yy375 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yymsp[-1].minor.yy375 ){ - yymsp[-1].minor.yy375->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy211 ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); } } break; - case 338: /* filter_clause ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy626 = yymsp[-1].minor.yy626; } + case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; } + break; + case 343: /* term ::= QNUMBER */ +{ + yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy454); +} + yymsp[0].minor.yy454 = yylhsminor.yy454; break; default: - /* (339) input ::= cmdlist */ yytestcase(yyruleno==339); - /* (340) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==340); - /* (341) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=341); - /* (342) ecmd ::= SEMI */ yytestcase(yyruleno==342); - /* (343) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==343); - /* (344) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=344); - /* (345) trans_opt ::= */ yytestcase(yyruleno==345); - /* (346) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==346); - /* (347) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==347); - /* (348) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==348); - /* (349) savepoint_opt ::= */ yytestcase(yyruleno==349); - /* (350) cmd ::= create_table create_table_args */ yytestcase(yyruleno==350); - /* (351) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=351); - /* (352) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==352); - /* (353) columnlist ::= columnname carglist */ yytestcase(yyruleno==353); - /* (354) nm ::= ID|INDEXED */ yytestcase(yyruleno==354); - /* (355) nm ::= STRING */ yytestcase(yyruleno==355); - /* (356) nm ::= JOIN_KW */ yytestcase(yyruleno==356); - /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); - /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); - /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); - /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); - /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); - /* (362) carglist ::= */ yytestcase(yyruleno==362); - /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); - /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); - /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); - /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); - /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); - /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); - /* (369) tconscomma ::= */ yytestcase(yyruleno==369); - /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); - /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); - /* (373) oneselect ::= values */ yytestcase(yyruleno==373); - /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); - /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); - /* (376) returning ::= */ yytestcase(yyruleno==376); - /* (377) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=377); - /* (378) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==378); - /* (379) exprlist ::= nexprlist */ yytestcase(yyruleno==379); - /* (380) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=380); - /* (381) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=381); - /* (382) nmnum ::= ON */ yytestcase(yyruleno==382); - /* (383) nmnum ::= DELETE */ yytestcase(yyruleno==383); - /* (384) nmnum ::= DEFAULT */ yytestcase(yyruleno==384); - /* (385) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==385); - /* (386) foreach_clause ::= */ yytestcase(yyruleno==386); - /* (387) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==387); - /* (388) trnm ::= nm */ yytestcase(yyruleno==388); - /* (389) tridxby ::= */ yytestcase(yyruleno==389); - /* (390) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==390); - /* (391) database_kw_opt ::= */ yytestcase(yyruleno==391); - /* (392) kwcolumn_opt ::= */ yytestcase(yyruleno==392); - /* (393) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==393); - /* (394) vtabarglist ::= vtabarg */ yytestcase(yyruleno==394); - /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==395); - /* (396) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==396); - /* (397) anylist ::= */ yytestcase(yyruleno==397); - /* (398) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==398); - /* (399) anylist ::= anylist ANY */ yytestcase(yyruleno==399); - /* (400) with ::= */ yytestcase(yyruleno==400); + /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); + /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); + /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); + /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); + /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); + /* (350) trans_opt ::= */ yytestcase(yyruleno==350); + /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); + /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); + /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); + /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); + /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); + /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); + /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); + /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); + /* (360) nm ::= STRING */ yytestcase(yyruleno==360); + /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); + /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); + /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); + /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); + /* (366) carglist ::= */ yytestcase(yyruleno==366); + /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); + /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); + /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); + /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); + /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); + /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) tconscomma ::= */ yytestcase(yyruleno==373); + /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); + /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) oneselect ::= values */ yytestcase(yyruleno==377); + /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); + /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); + /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) returning ::= */ yytestcase(yyruleno==381); + /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); + /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); + /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); + /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); + /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); + /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); + /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); + /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); + /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); + /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); + /* (394) trnm ::= nm */ yytestcase(yyruleno==394); + /* (395) tridxby ::= */ yytestcase(yyruleno==395); + /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); + /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); + /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); + /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); + /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); + /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); + /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); + /* (403) anylist ::= */ yytestcase(yyruleno==403); + /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); + /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); + /* (406) with ::= */ yytestcase(yyruleno==406); + /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); + /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); break; /********** End reduce actions ************************************************/ }; @@ -165361,19 +179643,12 @@ SQLITE_PRIVATE void sqlite3Parser( (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); break; } } -#endif } yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ @@ -165750,7 +180025,7 @@ static const unsigned char aKWHash[127] = { /* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 ** then the i-th keyword has no more hash collisions. Otherwise, ** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[147] = { +static const unsigned char aKWNext[148] = {0, 0, 0, 0, 0, 4, 0, 43, 0, 0, 106, 114, 0, 0, 0, 2, 0, 0, 143, 0, 0, 0, 13, 0, 0, 0, 0, 141, 0, 0, 119, 52, 0, 0, 137, 12, 0, 0, 62, 0, @@ -165765,7 +180040,7 @@ static const unsigned char aKWNext[147] = { 102, 0, 0, 87, }; /* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[147] = { +static const unsigned char aKWLen[148] = {0, 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, @@ -165781,7 +180056,7 @@ static const unsigned char aKWLen[147] = { }; /* aKWOffset[i] is the index into zKWText[] of the start of ** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[147] = { +static const unsigned short int aKWOffset[148] = {0, 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, @@ -165796,7 +180071,7 @@ static const unsigned short int aKWOffset[147] = { 648, 650, 655, 659, }; /* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[147] = { +static const unsigned char aKWCode[148] = {0, TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, @@ -165963,185 +180238,185 @@ static const unsigned char aKWCode[147] = { static int keywordCode(const char *z, int n, int *pType){ int i, j; const char *zKW; - if( n>=2 ){ - i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; - for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ - if( aKWLen[i]!=n ) continue; - zKW = &zKWText[aKWOffset[i]]; + assert( n>=2 ); + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; + for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; #ifdef SQLITE_ASCII - if( (z[0]&~0x20)!=zKW[0] ) continue; - if( (z[1]&~0x20)!=zKW[1] ) continue; - j = 2; - while( j=2 ) keywordCode((char*)z, n, &id); return id; } #define SQLITE_N_KEYWORD 147 SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){ if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + i++; *pzName = zKWText + aKWOffset[i]; *pnName = aKWLen[i]; return SQLITE_OK; @@ -166297,6 +180572,9 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; + }else if( z[1]=='>' ){ + *tokenType = TK_PTR; + return 2 + (z[2]=='>'); } *tokenType = TK_MINUS; return 1; @@ -166437,31 +180715,62 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); + testcase( z[0]=='9' ); testcase( z[0]=='.' ); *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -166509,7 +180818,8 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } case CC_KYWD0: { - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( aiClass[z[1]]>CC_KYWD ){ i = 1; break; } + for(i=2; aiClass[z[i]]<=CC_KYWD; i++){} if( IdChar(z[i]) ){ /* This token started out using characters that can appear in keywords, ** but z[i] is a character not allowed within keywords, so this must @@ -166566,13 +180876,9 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** Run the parser on the given SQL string. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ int nErr = 0; /* Number of errors encountered */ void *pEngine; /* The LEMON-generated LALR(1) parser */ int n = 0; /* Length of the next token token */ @@ -166593,7 +180899,6 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr } pParse->rc = SQLITE_OK; pParse->zTail = zSql; - assert( pzErrMsg!=0 ); #ifdef SQLITE_DEBUG if( db->flags & SQLITE_ParserTrace ){ printf("parser: [[[%s]]]\n", zSql); @@ -166623,19 +180928,24 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr mxSqlLen -= n; if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; break; } #ifndef SQLITE_OMIT_WINDOWFUNC if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; + pParse->nErr++; break; } if( tokenType==TK_SPACE ){ @@ -166664,8 +180974,11 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); + }else if( tokenType!=TK_QNUMBER ){ + Token x; + x.z = zSql; + x.n = n; + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x); break; } } @@ -166693,44 +181006,29 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM_BKPT; } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); - } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s in \"%s\"", - *pzErrMsg, pParse->zTail); - pParse->zErrMsg = 0; + if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){ + if( pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); nErr++; } pParse->zTail = zSql; - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; - } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; - } -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3_free(pParse->apVtabLock); #endif - if( !IN_SPECIAL_PARSE ){ + if( pParse->pNewTable && !IN_SPECIAL_PARSE ){ /* If the pParse->declareVtab flag is set, do not delete any table ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ sqlite3DeleteTable(db, pParse->pNewTable); } - if( !IN_RENAME_OBJECT ){ + if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){ sqlite3DeleteTrigger(db, pParse->pNewTrigger); } - sqlite3DbFree(db, pParse->pVList); + if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList); db->pParse = pParentParse; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; @@ -167254,6 +181552,7 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); /************** End of rtree.h ***********************************************/ /************** Continuing where we left off in main.c ***********************/ #endif +#include "sqlite3tokenizer.h" #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) /************** Include sqliteicu.h in the middle of main.c ******************/ /************** Begin file sqliteicu.h ***************************************/ @@ -167278,13 +181577,54 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); extern "C" { #endif /* __cplusplus */ -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); +SQLITE_PRIVATE int sqlite3IcuInitInner(sqlite3 *db); #if 0 } /* extern "C" */ #endif /* __cplusplus */ /************** End of sqliteicu.h *******************************************/ +#ifndef _WIN32 +#include +#endif + +typedef void (*sqlite3Fts3IcuTokenizerModule_ptr)(sqlite3_tokenizer_module const** ppModule); +typedef int (*sqlite3IcuInit_ptr)(sqlite3 *db); +static sqlite3Fts3IcuTokenizerModule_ptr tokenModulePtr = NULL; +static sqlite3IcuInit_ptr icuInitPtr = NULL; +static u32 icuEnable = 0u; +static u32 icuInit = 0u; +static void *g_library = NULL; + +int sqlite3IcuModuleInit(){ + int rc = SQLITE_OK; + if( icuInit ){ + return rc; + } +#ifndef _WIN32 + g_library = dlopen("libsqliteicu.z.so", RTLD_LAZY); + if( g_library==NULL ){ + sqlite3_log(SQLITE_ERROR, "load icu so failed"); + return SQLITE_ERROR; + } + tokenModulePtr = (sqlite3Fts3IcuTokenizerModule_ptr)dlsym(g_library, "sqlite3Fts3IcuTokenizerModule"); + icuInitPtr = (sqlite3IcuInit_ptr)dlsym(g_library, "sqlite3IcuInit"); + if( tokenModulePtr==NULL || icuInitPtr==NULL ){ + sqlite3_log(SQLITE_ERROR, "load icu init function failed"); + return SQLITE_ERROR; + } + icuInit = 1u; +#endif + return rc; +} + +SQLITE_PRIVATE int sqlite3IcuInitInner(sqlite3 *db) +{ + if( !icuEnable ){ + return SQLITE_OK; + } + return icuInitPtr(db); +} /************** Continuing where we left off in main.c ***********************/ #endif @@ -167303,33 +181643,20 @@ static int sqlite3TestExtInit(sqlite3 *db){ ** Forward declarations of external module initializer functions ** for modules that need them. */ -#ifdef SQLITE_ENABLE_FTS1 -SQLITE_PRIVATE int sqlite3Fts1Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_FTS2 -SQLITE_PRIVATE int sqlite3Fts2Init(sqlite3*); -#endif #ifdef SQLITE_ENABLE_FTS5 SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); #endif -#ifdef SQLITE_ENABLE_JSON1 -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); -#endif #ifdef SQLITE_ENABLE_STMTVTAB SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); #endif - +#ifdef SQLITE_EXTRA_AUTOEXT +int SQLITE_EXTRA_AUTOEXT(sqlite3*); +#endif /* ** An array of pointers to extension initializer functions for ** built-in extensions. */ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { -#ifdef SQLITE_ENABLE_FTS1 - sqlite3Fts1Init, -#endif -#ifdef SQLITE_ENABLE_FTS2 - sqlite3Fts2Init, -#endif #ifdef SQLITE_ENABLE_FTS3 sqlite3Fts3Init, #endif @@ -167337,7 +181664,7 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { sqlite3Fts5Init, #endif #if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) - sqlite3IcuInit, + sqlite3IcuInitInner, #endif #ifdef SQLITE_ENABLE_RTREE sqlite3RtreeInit, @@ -167349,8 +181676,8 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { sqlite3DbstatRegister, #endif sqlite3TestExtInit, -#ifdef SQLITE_ENABLE_JSON1 - sqlite3Json1Init, +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) + sqlite3JsonTableFunctions, #endif #ifdef SQLITE_ENABLE_STMTVTAB sqlite3StmtVtabInit, @@ -167358,6 +181685,9 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { #ifdef SQLITE_ENABLE_BYTECODE_VTAB sqlite3VdbeBytecodeVtabInit, #endif +#ifdef SQLITE_EXTRA_AUTOEXT + SQLITE_EXTRA_AUTOEXT, +#endif }; #ifndef SQLITE_AMALGAMATION @@ -167431,6 +181761,32 @@ SQLITE_API char *sqlite3_temp_directory = 0; */ SQLITE_API char *sqlite3_data_directory = 0; +/* +** Determine whether or not high-precision (long double) floating point +** math works correctly on CPU currently running. +*/ +static SQLITE_NOINLINE int hasHighPrecisionDouble(int rc){ + if( sizeof(LONGDOUBLE_TYPE)<=8 ){ + /* If the size of "long double" is not more than 8, then + ** high-precision math is not possible. */ + return 0; + }else{ + /* Just because sizeof(long double)>8 does not mean that the underlying + ** hardware actually supports high-precision floating point. For example, + ** clearing the 0x100 bit in the floating-point control word on Intel + ** processors will make long double work like double, even though long + ** double takes up more space. The only way to determine if long double + ** actually works is to run an experiment. */ + LONGDOUBLE_TYPE a, b, c; + rc++; + a = 1.0+rc*0.1; + b = 1.0e+18+rc*25.0; + c = a+b; + return b!=c; + } +} + + /* ** Initialize SQLite. ** @@ -167626,6 +181982,12 @@ SQLITE_API int sqlite3_initialize(void){ } #endif + /* Experimentally determine if high-precision floating point is + ** available. */ +#ifndef SQLITE_OMIT_WSD + sqlite3Config.bUseLongDouble = hasHighPrecisionDouble(rc); +#endif + return rc; } @@ -167694,12 +182056,36 @@ SQLITE_API int sqlite3_shutdown(void){ SQLITE_API int sqlite3_config(int op, ...){ va_list ap; int rc = SQLITE_OK; + va_start(ap, op); - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + if( op==SQLITE_CONFIG_ENABLE_ICU ){ + int iVal = va_arg(ap, int); + if( iVal==0 ){ + icuEnable = 0u; + }else{ + icuEnable = 1u; + } + return rc; + } +#endif /* SQLITE_ENABLE_ICU */ + + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes + ** are allowed. + */ + if( sqlite3GlobalConfig.isInit ){ + static const u64 mAnytimeConfigOption = 0 + | MASKBIT64( SQLITE_CONFIG_LOG ) + | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) + ; + if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ + return SQLITE_MISUSE_BKPT; + } + testcase( op==SQLITE_CONFIG_LOG ); + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } - va_start(ap, op); switch( op ){ /* Mutex configuration options are only available in a threadsafe @@ -167766,6 +182152,7 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_MEMSTATUS: { + assert( !sqlite3GlobalConfig.isInit ); /* Cannot change at runtime */ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ @@ -167889,8 +182276,10 @@ SQLITE_API int sqlite3_config(int op, ...){ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); */ typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t); + void *pLogArg = va_arg(ap, void*); + AtomicStore(&sqlite3GlobalConfig.xLog, xLog); + AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg); break; } @@ -167904,7 +182293,8 @@ SQLITE_API int sqlite3_config(int op, ...){ ** argument of type int. If non-zero, then URI handling is globally ** enabled. If the parameter is zero, then URI handling is globally ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + int bOpenUri = va_arg(ap, int); + AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri); break; } @@ -167989,6 +182379,24 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + case SQLITE_CONFIG_CORRUPTION: { + typedef void(*CORRUPTION_FUNC_t)(void*, const void*); + sqlite3GlobalConfig.xCorruption = va_arg(ap, CORRUPTION_FUNC_t); + sqlite3GlobalConfig.pCorruptionArg = va_arg(ap, void*); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -168089,18 +182497,19 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ db->lookaside.bMalloced = pBuf==0 ?1:0; db->lookaside.nSlot = nBig+nSm; }else{ - db->lookaside.pStart = db; + db->lookaside.pStart = 0; #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE db->lookaside.pSmallInit = 0; db->lookaside.pSmallFree = 0; - db->lookaside.pMiddle = db; + db->lookaside.pMiddle = 0; #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - db->lookaside.pEnd = db; + db->lookaside.pEnd = 0; db->lookaside.bDisable = 1; db->lookaside.sz = 0; db->lookaside.bMalloced = 0; db->lookaside.nSlot = 0; } + db->lookaside.pTrueEnd = db->lookaside.pEnd; assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; @@ -168179,6 +182588,11 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_MAINDBNAME: { @@ -168217,7 +182631,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ rc = SQLITE_OK; break; } -#endif +#endif /* SQLITE_SHARED_BLOCK_OPTIMIZATION */ default: { static const struct { int op; /* The opcode */ @@ -168240,6 +182654,8 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, + { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -168267,6 +182683,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ } } va_end(ap); + sqlite3_mutex_leave(db->mutex); return rc; } @@ -168527,6 +182944,14 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ } #endif + while( db->pDbData ){ + DbClientData *p = db->pDbData; + db->pDbData = p->pNext; + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + sqlite3_free(p); + } + /* Convert the connection into a zombie and then close it. */ db->eOpenState = SQLITE_STATE_ZOMBIE; @@ -168851,6 +183276,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; case SQLITE_NOTICE_RECOVER_ROLLBACK: zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_NOTICE_RBU: zName = "SQLITE_NOTICE_RBU"; break; case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; case SQLITE_DONE: zName = "SQLITE_DONE"; break; @@ -168919,6 +183345,10 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ zErr = "no more rows available"; break; } + case SQLITE_META_RECOVERED: { + zErr = "warning meta recover message"; + break; + } default: { rc &= 0xff; if( ALWAYS(rc>=0) && rceOpenState!=SQLITE_STATE_ZOMBIE) ){ + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ (void)SQLITE_MISUSE_BKPT; return; } @@ -169088,6 +183520,21 @@ SQLITE_API void sqlite3_interrupt(sqlite3 *db){ AtomicStore(&db->u1.isInterrupted, 1); } +/* +** Return true or false depending on whether or not an interrupt is +** pending on connection db. +*/ +SQLITE_API int sqlite3_is_interrupted(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return AtomicLoad(&db->u1.isInterrupted)!=0; +} /* ** This function is exactly the same as sqlite3_create_function(), except @@ -169126,13 +183573,13 @@ SQLITE_PRIVATE int sqlite3CreateFunc( assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| - SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + SQLITE_SUBTYPE|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE); enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But ** the meaning is inverted. So flip the bit. */ assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); - extraFlags ^= SQLITE_FUNC_UNSAFE; + extraFlags ^= SQLITE_FUNC_UNSAFE; /* tag-20230109-1 */ #ifndef SQLITE_OMIT_UTF16 @@ -169150,11 +183597,11 @@ SQLITE_PRIVATE int sqlite3CreateFunc( case SQLITE_ANY: { int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, - (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, + (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, /* tag-20230109-1 */ pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, - (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, + (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, /* tag-20230109-1*/ pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); } if( rc!=SQLITE_OK ){ @@ -169403,7 +183850,7 @@ SQLITE_API int sqlite3_overload_function( rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; sqlite3_mutex_leave(db->mutex); if( rc ) return SQLITE_OK; - zCopy = sqlite3_mprintf(zName); + zCopy = sqlite3_mprintf("%s", zName); if( zCopy==0 ) return SQLITE_NOMEM; return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); @@ -169583,6 +184030,12 @@ SQLITE_API void *sqlite3_preupdate_hook( void *pArg /* First callback argument */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 ){ + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pPreUpdateArg; db->xPreUpdateCallback = xCallback; @@ -169729,7 +184182,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -169882,6 +184335,19 @@ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ return z; } +/* +** Return the byte offset of the most recent error +*/ +SQLITE_API int sqlite3_error_offset(sqlite3 *db){ + int iOffset = -1; + if( db && sqlite3SafetyCheckSickOrOk(db) && db->errCode ){ + sqlite3_mutex_enter(db->mutex); + iOffset = db->errByteOffset; + sqlite3_mutex_leave(db->mutex); + } + return iOffset; +} + #ifndef SQLITE_OMIT_UTF16 /* ** Return UTF-16 encoded English language explanation of the most recent @@ -170193,9 +184659,9 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -170424,7 +184890,7 @@ static const char *uriParameter(const char *zFilename, const char *zParam){ return 0; } -#if defined(SQLITE_HAS_CODEC) +#ifdef SQLITE_HAS_CODEC /* ** Process URI filename query parameters relevant to the SQLite Encryption ** Extension. Return true if any of the relevant query parameters are @@ -170458,8 +184924,7 @@ SQLITE_PRIVATE int sqlite3CodecQueryParameters( return 0; } } -#endif - +#endif /* SQLITE_HAS_CODEC */ /* ** This routine does the work of opening a database on behalf of @@ -170588,7 +185053,7 @@ static int openDatabase( ** 0 off off ** ** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) -** and so that is the default. But developers are encouranged to use +** and so that is the default. But developers are encouraged to use ** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. */ #if !defined(SQLITE_DQS) @@ -170636,6 +185101,9 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) | SQLITE_LegacyAlter +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) + | SQLITE_StmtScanStatus #endif ; sqlite3HashInit(&db->aCollSeq); @@ -170659,6 +185127,19 @@ static int openDatabase( goto opendb_out; } +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) + /* Process magic filenames ":localStorage:" and ":sessionStorage:" */ + if( zFilename && zFilename[0]==':' ){ + if( strcmp(zFilename, ":localStorage:")==0 ){ + zFilename = "file:local?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + }else if( strcmp(zFilename, ":sessionStorage:")==0 ){ + zFilename = "file:session?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + } + } +#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */ + /* Parse the filename/URI argument ** ** Only allow sensible combinations of bits in the flags argument. @@ -170689,6 +185170,12 @@ static int openDatabase( sqlite3_free(zErrMsg); goto opendb_out; } + assert( db->pVfs!=0 ); +#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL) + if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){ + db->temp_store = 2; + } +#endif /* Open the backend database driver */ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, @@ -170729,6 +185216,12 @@ static int openDatabase( sqlite3RegisterPerConnectionBuiltinFunctions(db); rc = sqlite3_errcode(db); +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + if( icuEnable ){ + rc = sqlite3IcuModuleInit(); + if( rc!=SQLITE_OK ) return rc; + } +#endif /* Load compiled-in extensions */ for(i=0; rc==SQLITE_OK && imDropTableName = NULL; db->mDropSchemaName = NULL; db->xDropTableHandle = NULL; -#endif +#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */ *ppDb = db; #ifdef SQLITE_ENABLE_SQLLOG if( sqlite3GlobalConfig.xSqllog ){ @@ -170799,9 +185292,14 @@ opendb_out: sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } #endif -#if defined(SQLITE_HAS_CODEC) - if( rc==SQLITE_OK ) sqlite3CodecQueryParameters(db, 0, zOpen); -#endif +#ifdef SQLITE_HAS_CODEC + if( rc==SQLITE_OK ) { +#ifdef SQLITE_CODEC_ATTACH_CHANGED + sqlite3CodecResetParameters(&db->codecParm); +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ + sqlite3CodecQueryParameters(db, 0, zOpen); + } +#endif /* SQLITE_HAS_CODEC */ sqlite3_free_filename(zOpen); return rc; } @@ -170975,6 +185473,69 @@ SQLITE_API int sqlite3_collation_needed16( } #endif /* SQLITE_OMIT_UTF16 */ +/* +** Find existing client data. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ + DbClientData *p; + sqlite3_mutex_enter(db->mutex); + for(p=db->pDbData; p; p=p->pNext){ + if( strcmp(p->zName, zName)==0 ){ + void *pResult = p->pData; + sqlite3_mutex_leave(db->mutex); + return pResult; + } + } + sqlite3_mutex_leave(db->mutex); + return 0; +} + +/* +** Add new client data to a database connection. +*/ +SQLITE_API int sqlite3_set_clientdata( + sqlite3 *db, /* Attach client data to this connection */ + const char *zName, /* Name of the client data */ + void *pData, /* The client data itself */ + void (*xDestructor)(void*) /* Destructor */ +){ + DbClientData *p, **pp; + sqlite3_mutex_enter(db->mutex); + pp = &db->pDbData; + for(p=db->pDbData; p && strcmp(p->zName,zName); p=p->pNext){ + pp = &p->pNext; + } + if( p ){ + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + if( pData==0 ){ + *pp = p->pNext; + sqlite3_free(p); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + } + }else if( pData==0 ){ + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + }else{ + size_t n = strlen(zName); + p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + if( p==0 ){ + if( xDestructor ) xDestructor(pData); + sqlite3_mutex_leave(db->mutex); + return SQLITE_NOMEM; + } + memcpy(p->zName, zName, n+1); + p->pNext = db->pDbData; + db->pDbData = p; + } + p->pData = pData; + p->xDestructor = xDestructor; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + + #ifndef SQLITE_OMIT_DEPRECATED /* ** This function is now an anachronism. It used to be used to recover from a @@ -171017,9 +185578,21 @@ SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ zType, lineno, 20+sqlite3_sourceid()); return iErr; } -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ +SQLITE_PRIVATE int sqlite3CorruptError(int lineno, sqlite3CorruptContext *context){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); + if (context!=NULL && sqlite3GlobalConfig.xCorruption != 0) { + char zMsg[SQLITE_PRINT_BUF_SIZE] = {0}; /* Complete corruption log message */ + sqlite3_snprintf(sizeof(zMsg), zMsg, "pgno:%u,type:%d,range:{%d,%d},line:%d", + context->pgno, (int)context->type, (int)context->zoneRange.offset, (int)context->zoneRange.size, lineno); + sqlite3GlobalConfig.xCorruption(sqlite3GlobalConfig.pCorruptionArg, zMsg); + } + char zCorruptMsg[SQLITE_PRINT_BUF_SIZE * 10] = {0}; + if (context!=NULL && context->zMsg != NULL){ + sqlite3_snprintf(sizeof(zCorruptMsg), zCorruptMsg, "database corruption, %s", context->zMsg); + } else { + sqlite3_snprintf(sizeof(zCorruptMsg), zCorruptMsg, "database corruption"); + } + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zCorruptMsg); } SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); @@ -171110,7 +185683,7 @@ SQLITE_API int sqlite3_table_column_metadata( /* Find the column for which info is requested */ if( zColumnName==0 ){ - /* Query for existance of table only */ + /* Query for existence of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; @@ -171191,7 +185764,7 @@ SQLITE_API int sqlite3_sleep(int ms){ /* This function works in milliseconds, but the underlying OsSleep() ** API uses microseconds. Hence the 1000's. */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); return rc; } @@ -171241,12 +185814,19 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ +#ifndef SQLITE_CKSUMVFS_STATIC + rc = SQLITE_OK; +#else int iNew = *(int*)pArg; *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); if( iNew>=0 && iNew<=255 ){ sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); } rc = SQLITE_OK; +#endif + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; }else{ int nSave = db->busyHandler.nBusy; rc = sqlite3OsFileControl(fd, op, pArg); @@ -171321,6 +185901,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ } #endif + /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); + ** + ** If b is true, then activate the SQLITE_FkNoAction setting. If b is + ** false then clearn that setting. If the SQLITE_FkNoAction setting is + ** abled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** they were NO ACTION, regardless of how they are defined. + ** + ** NB: One must usually run "PRAGMA writable_schema=RESET" after + ** using this test-control, before it will take full effect. failing + ** to reset the schema can result in some unexpected behavior. + */ + case SQLITE_TESTCTRL_FK_NO_ACTION: { + sqlite3 *db = va_arg(ap, sqlite3*); + int b = va_arg(ap, int); + if( b ){ + db->flags |= SQLITE_FkNoAction; + }else{ + db->flags &= ~SQLITE_FkNoAction; + } + break; + } + /* ** sqlite3_test_control(BITVEC_TEST, size, program) ** @@ -171348,12 +185950,16 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** sqlite3_test_control(). */ case SQLITE_TESTCTRL_FAULT_INSTALL: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 + /* A bug in MSVC prevents it from understanding pointers to functions + ** types in the second argument to va_arg(). Work around the problem + ** using a typedef. + ** http://support.microsoft.com/kb/47961 <-- dead hyperlink + ** Search at http://web.archive.org/ to find the 2015-03-16 archive + ** of the link above to see the original text. ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); */ - typedef int(*TESTCALLBACKFUNC_t)(int); - sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + typedef int(*sqlite3FaultFuncType)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, sqlite3FaultFuncType); rc = sqlite3FaultSim(0); break; } @@ -171412,6 +186018,30 @@ SQLITE_API int sqlite3_test_control(int op, ...){ volatile int x = 0; assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); rc = x; +#if defined(SQLITE_DEBUG) + /* Invoke these debugging routines so that the compiler does not + ** issue "defined but not used" warnings. */ + if( x==9999 ){ + sqlite3ShowExpr(0); + sqlite3ShowExpr(0); + sqlite3ShowExprList(0); + sqlite3ShowIdList(0); + sqlite3ShowSrcList(0); + sqlite3ShowWith(0); + sqlite3ShowUpsert(0); +#ifndef SQLITE_OMIT_TRIGGER + sqlite3ShowTriggerStep(0); + sqlite3ShowTriggerStepList(0); + sqlite3ShowTrigger(0); + sqlite3ShowTriggerList(0); +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC + sqlite3ShowWindow(0); + sqlite3ShowWinFunc(0); +#endif + sqlite3ShowSelect(0); + } +#endif break; } @@ -171480,13 +186110,27 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); + ** + ** If parameter onoff is 1, subsequent calls to localtime() fail. + ** If 2, then invoke xAlt() instead of localtime(). If 0, normal + ** processing. + ** + ** xAlt arguments are void pointers, but they really want to be: ** - ** If parameter onoff is non-zero, subsequent calls to localtime() - ** and its variants fail. If onoff is zero, undo this setting. + ** int xAlt(const time_t*, struct tm*); + ** + ** xAlt should write results in to struct tm object of its 2nd argument + ** and return zero on success, or return non-zero on failure. */ case SQLITE_TESTCTRL_LOCALTIME_FAULT: { sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + if( sqlite3GlobalConfig.bLocaltimeFault==2 ){ + typedef int(*sqlite3LocaltimeType)(const void*,void*); + sqlite3GlobalConfig.xAltLocaltime = va_arg(ap, sqlite3LocaltimeType); + }else{ + sqlite3GlobalConfig.xAltLocaltime = 0; + } break; } @@ -171507,7 +186151,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** formed and never corrupt. This flag is clear by default, indicating that ** database files might have arbitrary corruption. Setting the flag during ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. + ** that demonstrate invariants on well-formed database files. */ case SQLITE_TESTCTRL_NEVER_CORRUPT: { sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); @@ -171659,23 +186303,62 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** ** "ptr" is a pointer to a u32. ** - ** op==0 Store the current sqlite3SelectTrace in *ptr - ** op==1 Set sqlite3SelectTrace to the value *ptr - ** op==3 Store the current sqlite3WhereTrace in *ptr + ** op==0 Store the current sqlite3TreeTrace in *ptr + ** op==1 Set sqlite3TreeTrace to the value *ptr + ** op==2 Store the current sqlite3WhereTrace in *ptr ** op==3 Set sqlite3WhereTrace to the value *ptr */ case SQLITE_TESTCTRL_TRACEFLAGS: { int opTrace = va_arg(ap, int); u32 *ptr = va_arg(ap, u32*); switch( opTrace ){ - case 0: *ptr = sqlite3SelectTrace; break; - case 1: sqlite3SelectTrace = *ptr; break; - case 2: *ptr = sqlite3WhereTrace; break; - case 3: sqlite3WhereTrace = *ptr; break; + case 0: *ptr = sqlite3TreeTrace; break; + case 1: sqlite3TreeTrace = *ptr; break; + case 2: *ptr = sqlite3WhereTrace; break; + case 3: sqlite3WhereTrace = *ptr; break; } break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOGEST, + ** double fIn, // Input value + ** int *pLogEst, // sqlite3LogEstFromDouble(fIn) + ** u64 *pInt, // sqlite3LogEstToInt(*pLogEst) + ** int *pLogEst2 // sqlite3LogEst(*pInt) + ** ); + ** + ** Test access for the LogEst conversion routines. + */ + case SQLITE_TESTCTRL_LOGEST: { + double rIn = va_arg(ap, double); + LogEst rLogEst = sqlite3LogEstFromDouble(rIn); + int *pI1 = va_arg(ap,int*); + u64 *pU64 = va_arg(ap,u64*); + int *pI2 = va_arg(ap,int*); + *pI1 = rLogEst; + *pU64 = sqlite3LogEstToInt(rLogEst); + *pI2 = sqlite3LogEst(*pU64); + break; + } + +#if !defined(SQLITE_OMIT_WSD) + /* sqlite3_test_control(SQLITE_TESTCTRL_USELONGDOUBLE, int X); + ** + ** X<0 Make no changes to the bUseLongDouble. Just report value. + ** X==0 Disable bUseLongDouble + ** X==1 Enable bUseLongDouble + ** X>=2 Set bUseLongDouble to its default value for this platform + */ + case SQLITE_TESTCTRL_USELONGDOUBLE: { + int b = va_arg(ap, int); + if( b>=2 ) b = hasHighPrecisionDouble(b); + if( b>=0 ) sqlite3Config.bUseLongDouble = b>0; + rc = sqlite3Config.bUseLongDouble!=0; + break; + } +#endif + + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -171705,6 +186388,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } #endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff); + ** + ** Activate or deactivate validation of JSONB that is generated from + ** text. Off by default, as the validation is slow. Validation is + ** only available if compiled using SQLITE_DEBUG. + ** + ** If onOff is initially 1, then turn it on. If onOff is initially + ** off, turn it off. If onOff is initially -1, then change onOff + ** to be the current setting. + */ + case SQLITE_TESTCTRL_JSON_SELFCHECK: { +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) + int *pOnOff = va_arg(ap, int*); + if( *pOnOff<0 ){ + *pOnOff = sqlite3Config.bJsonSelfcheck; + }else{ + sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff); + } +#endif + break; + } } va_end(ap); #endif /* SQLITE_UNTESTABLE */ @@ -171745,7 +186450,7 @@ static char *appendText(char *p, const char *z){ ** Memory layout must be compatible with that generated by the pager ** and expected by sqlite3_uri_parameter() and databaseName(). */ -SQLITE_API char *sqlite3_create_filename( +SQLITE_API const char *sqlite3_create_filename( const char *zDatabase, const char *zJournal, const char *zWal, @@ -171781,10 +186486,10 @@ SQLITE_API char *sqlite3_create_filename( ** error to call this routine with any parameter other than a pointer ** previously obtained from sqlite3_create_filename() or a NULL pointer. */ -SQLITE_API void sqlite3_free_filename(char *p){ +SQLITE_API void sqlite3_free_filename(const char *p){ if( p==0 ) return; - p = (char*)databaseName(p); - sqlite3_free(p - 4); + p = databaseName(p); + sqlite3_free((char*)p - 4); } @@ -171886,6 +186591,24 @@ SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ return iDb<0 ? 0 : db->aDb[iDb].pBt; } +/* +** Return the name of the N-th database schema. Return NULL if N is out +** of range. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + if( N<0 || N>=db->nDb ){ + return 0; + }else{ + return db->aDb[N].zDbSName; + } +} + /* ** Return the filename of the database associated with a database ** connection. @@ -171957,7 +186680,7 @@ SQLITE_API int sqlite3_snapshot_get( } /* -** Open a read-transaction on the snapshot idendified by pSnapshot. +** Open a read-transaction on the snapshot identified by pSnapshot. */ SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, @@ -172017,8 +186740,8 @@ SQLITE_API int sqlite3_snapshot_open( */ SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ int rc = SQLITE_ERROR; - int iDb; #ifndef SQLITE_OMIT_WAL + int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -172064,7 +186787,7 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ int nOpt; const char **azCompileOpt; -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; @@ -172259,6 +186982,9 @@ SQLITE_API int sqlite3_unlock_notify( ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); enterMutex(); @@ -172825,114 +187551,6 @@ SQLITE_EXTENSION_INIT3 ** the tokenization rules supplied by a specific sqlite3_tokenizer ** object. */ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; - -struct sqlite3_tokenizer_module { - - /* - ** Structure version. Should always be set to 0 or 1. - */ - int iVersion; - - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) - ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". - ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialized by - ** this callback. The caller will do so. - */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ - ); - - /* - ** Destroy an existing tokenizer. The fts3 module calls this method - ** exactly once for each successful call to xCreate(). - */ - int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); - - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); - - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. - ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. - ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. - */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); - - /*********************************************************************** - ** Methods below this point are only available if iVersion>=1. - */ - - /* - ** Configure the language id of a tokenizer cursor. - */ - int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); -}; - -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ -}; - -struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ - /* Tokenizer implementations will typically add additional fields */ -}; int fts3_global_term_cnt(int iTerm, int iCol); int fts3_term_cnt(int iTerm, int iCol); @@ -173280,6 +187898,7 @@ struct Fts3Table { int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + int iSavepoint; /* ** The following array of hash tables is used to buffer pending index @@ -173573,7 +188192,7 @@ struct Fts3MultiSegReader { int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; @@ -173656,7 +188275,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *) SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); /* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -173665,6 +188284,10 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); #endif +SQLITE_PRIVATE int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); + +SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); + #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ @@ -173689,6 +188312,12 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); SQLITE_EXTENSION_INIT1 #endif +typedef struct Fts3HashWrapper Fts3HashWrapper; +struct Fts3HashWrapper { + Fts3Hash hash; /* Hash table */ + int nRef; /* Number of pointers to this object */ +}; + static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( @@ -174015,6 +188644,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS); /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); @@ -174553,7 +189183,7 @@ static int fts3InitVtab( sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ - Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Hash *pHash = &((Fts3HashWrapper*)pAux)->hash; Fts3Table *p = 0; /* Pointer to allocated vtab */ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ @@ -176263,7 +190893,7 @@ static int fts3TermSelectMerge( ** ** Similar padding is added in the fts3DoclistOrMerge() function. */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -177264,6 +191894,8 @@ static int fts3RenameMethod( rc = sqlite3Fts3PendingTermsFlush(p); } + p->bIgnoreSavepoint = 1; + if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", @@ -177291,6 +191923,8 @@ static int fts3RenameMethod( "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); + + p->bIgnoreSavepoint = 0; return rc; } @@ -177301,12 +191935,28 @@ static int fts3RenameMethod( */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint<=iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + + if( pTab->bIgnoreSavepoint==0 ){ + if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){ + char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->zDb, pTab->zName, pTab->zName + ); + if( zSql ){ + pTab->bIgnoreSavepoint = 1; + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + pTab->bIgnoreSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } } return rc; } @@ -177317,12 +191967,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint >= iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint-1 ); + pTab->iSavepoint = iSavepoint; return SQLITE_OK; } @@ -177332,11 +191981,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** Discard the contents of the pending terms table. */ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; + Fts3Table *pTab = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); + assert( pTab->inTransaction ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + sqlite3Fts3PendingTermsClear(pTab); + } return SQLITE_OK; } @@ -177355,8 +192006,42 @@ static int fts3ShadowName(const char *zName){ return 0; } +/* +** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual +** table. +*/ +static int fts3IntegrityMethod( + sqlite3_vtab *pVtab, /* The virtual table to be checked */ + const char *zSchema, /* Name of schema in which pVtab lives */ + const char *zTabname, /* Name of the pVTab table */ + int isQuick, /* True if this is a quick_check */ + char **pzErr /* Write error message here */ +){ + Fts3Table *p = (Fts3Table*)pVtab; + int rc = SQLITE_OK; + int bOk = 0; + + UNUSED_PARAMETER(isQuick); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS%d table %s.%s: %s", + p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + + + static const sqlite3_module fts3Module = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -177380,6 +192065,7 @@ static const sqlite3_module fts3Module = { /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, /* xShadowName */ fts3ShadowName, + /* xIntegrity */ fts3IntegrityMethod, }; /* @@ -177388,9 +192074,12 @@ static const sqlite3_module fts3Module = { ** allocated for the tokenizer hash table. */ static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); + Fts3HashWrapper *pHash = (Fts3HashWrapper *)p; + pHash->nRef--; + if( pHash->nRef<=0 ){ + sqlite3Fts3HashClear(&pHash->hash); + sqlite3_free(pHash); + } } /* @@ -177408,9 +192097,6 @@ SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module co #ifndef SQLITE_DISABLE_FTS3_UNICODE SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif -#ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#endif /* ** Initialize the fts3 extension. If this extension is built as part @@ -177420,7 +192106,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const */ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; - Fts3Hash *pHash = 0; + Fts3HashWrapper *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -177429,7 +192115,14 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); + if( icuEnable ){ + if( tokenModulePtr!=NULL ){ + tokenModulePtr(&pIcu); + }else{ + sqlite3_log(SQLITE_ERROR, "icu module ptr is null"); + return SQLITE_ERROR; + } + } #endif #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -177448,23 +192141,24 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); + pHash = sqlite3_malloc(sizeof(Fts3HashWrapper)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1); + pHash->nRef = 0; } /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter) #ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) + || sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) + || (icuEnable && pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) #endif ){ rc = SQLITE_NOMEM; @@ -177473,7 +192167,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ #ifdef SQLITE_TEST if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db, pHash); + rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash); } #endif @@ -177482,23 +192176,26 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ ** module with sqlite. */ if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ + pHash->nRef++; rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); if( rc==SQLITE_OK ){ + pHash->nRef++; rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 + db, "fts4", &fts3Module, (void *)pHash, hashDestroy ); } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); + pHash->nRef++; + rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy); } return rc; } @@ -177507,7 +192204,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ - sqlite3Fts3HashClear(pHash); + sqlite3Fts3HashClear(&pHash->hash); sqlite3_free(pHash); } return rc; @@ -177676,8 +192373,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); + char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0); for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; @@ -177691,6 +192387,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ if( pList==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -177711,6 +192408,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -177743,13 +192441,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; } pPhrase->doclist.pList = aOut; + assert( p1 && p2 ); if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); @@ -177762,6 +192461,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } } + if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree); return SQLITE_OK; } #endif /* SQLITE_DISABLE_FTS4_DEFERRED */ @@ -178110,7 +192810,7 @@ static int fts3EvalIncrPhraseNext( if( bEof==0 ){ int nList = 0; int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); if( !aDoclist ) return SQLITE_NOMEM; memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); @@ -178652,9 +193352,8 @@ static void fts3EvalNextRow( Fts3Expr *pExpr, /* Expr. to advance to next matching row */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ + if( *pRc==SQLITE_OK && pExpr->bEof==0 ){ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -178936,11 +193635,10 @@ static int fts3EvalTestExpr( default: { #ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ + if( pCsr->pDeferred && (pExpr->bDeferred || ( + pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList + ))){ Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ fts3EvalInvalidatePoslist(pPhrase); } @@ -179131,6 +193829,22 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ } } +/* +** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array +** has not yet been allocated, allocate and zero it. Otherwise, just zero +** it. +*/ +static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){ + Fts3Table *pTab = (Fts3Table*)pCtx; + UNUSED_PARAMETER(iPhrase); + if( pExpr->aMI==0 ){ + pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( pExpr->aMI==0 ) return SQLITE_NOMEM; + } + memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + return SQLITE_OK; +} + /* ** Expression pExpr must be of type FTSQUERY_PHRASE. ** @@ -179152,7 +193866,6 @@ static int fts3EvalGatherStats( if( pExpr->aMI==0 ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ sqlite3_int64 iPrevId = pCsr->iPrevId; sqlite3_int64 iDocid; @@ -179160,7 +193873,9 @@ static int fts3EvalGatherStats( /* Find the root of the NEAR expression */ pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + while( pRoot->pParent + && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) + ){ pRoot = pRoot->pParent; } iDocid = pRoot->iDocid; @@ -179168,14 +193883,8 @@ static int fts3EvalGatherStats( assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); - } - + rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab); + if( rc!=SQLITE_OK ) return rc; fts3EvalRestart(pCsr, pRoot, &rc); while( pCsr->isEof==0 && rc==SQLITE_OK ){ @@ -179331,6 +194040,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */ int bMatch; /* Check if this phrase descends from an OR expression node. If not, @@ -179345,25 +194055,30 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( if( p->bEof ) bTreeEof = 1; } if( bOr==0 ) return SQLITE_OK; + pRun = pNear; + while( pRun->bDeferred ){ + assert( pRun->pParent ); + pRun = pRun->pParent; + } /* This is the descendent of an OR node. In this case we cannot use ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; + int bEofSave = pRun->bEof; + fts3EvalRestart(pCsr, pRun, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); + if( bEofSave==0 && pRun->iDocid==iDocid ) break; } assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){ + if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){ rc = FTS_CORRUPT_VTAB; } } if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); } } if( rc!=SQLITE_OK ) return rc; @@ -180030,7 +194745,8 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ @@ -182347,7 +197063,7 @@ static int porterNext( if( n>c->nAllocated ){ char *pNew; c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); if( !pNew ) return SQLITE_NOMEM; c->zToken = pNew; } @@ -183099,7 +197815,7 @@ static int simpleNext( if( n>c->nTokenAllocated ){ char *pNew; c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); if( !pNew ) return SQLITE_NOMEM; c->pToken = pNew; } @@ -183571,7 +198287,7 @@ static int fts3tokRowidMethod( ** Register the fts3tok module with database connection db. Return SQLITE_OK ** if successful or an error code if sqlite3_create_module() fails. */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){ static const sqlite3_module fts3tok_module = { 0, /* iVersion */ fts3tokConnectMethod, /* xCreate */ @@ -183596,11 +198312,14 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + rc = sqlite3_create_module_v2( + db, "fts3tokenize", &fts3tok_module, (void*)pHash, xDestroy + ); return rc; } @@ -184259,7 +198978,7 @@ static int fts3PendingListAppendVarint( /* Allocate or grow the PendingList as required. */ if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); + p = sqlite3_malloc64(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } @@ -184268,14 +198987,14 @@ static int fts3PendingListAppendVarint( p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } - p->nSpace = nNew; + p->nSpace = (int)nNew; p->aData = (char *)&p[1]; } @@ -184832,7 +199551,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int nByte = sqlite3_blob_bytes(p->pSegments); *pnBlob = nByte; if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ @@ -184949,7 +199668,7 @@ static int fts3SegReaderNext( int nTerm = fts3HashKeysize(pElem); if( (nTerm+1)>pReader->nTermAlloc ){ sqlite3_free(pReader->zTerm); - pReader->zTerm = (char*)sqlite3_malloc((nTerm+1)*2); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); if( !pReader->zTerm ) return SQLITE_NOMEM; pReader->nTermAlloc = (nTerm+1)*2; } @@ -184957,7 +199676,7 @@ static int fts3SegReaderNext( pReader->zTerm[nTerm] = '\0'; pReader->nTerm = nTerm; - aCopy = (char*)sqlite3_malloc(nCopy); + aCopy = (char*)sqlite3_malloc64(nCopy); if( !aCopy ) return SQLITE_NOMEM; memcpy(aCopy, pList->aData, nCopy); pReader->nNode = pReader->nDoclist = nCopy; @@ -185244,7 +199963,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( nExtra = nRoot + FTS3_NODE_PADDING; } - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } @@ -185336,7 +200055,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( if( nElem==nAlloc ){ Fts3HashElem **aElem2; nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem2 = (Fts3HashElem **)sqlite3_realloc64( aElem, nAlloc*sizeof(Fts3HashElem *) ); if( !aElem2 ){ @@ -185670,7 +200389,7 @@ static int fts3NodeAddTerm( ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); + pTree->aData = (char *)sqlite3_malloc64(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } @@ -185688,7 +200407,7 @@ static int fts3NodeAddTerm( if( isCopyTerm ){ if( pTree->nMalloczMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -185714,7 +200433,7 @@ static int fts3NodeAddTerm( ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); if( !pNew ){ return SQLITE_NOMEM; } @@ -185852,7 +200571,7 @@ static int fts3SegWriterAdd( ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ + i64 nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; @@ -185861,13 +200580,13 @@ static int fts3SegWriterAdd( sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = p->nNodeSize; @@ -185942,7 +200661,7 @@ static int fts3SegWriterAdd( ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; @@ -185967,7 +200686,7 @@ static int fts3SegWriterAdd( */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -186275,18 +200994,20 @@ static void fts3ColumnFilter( static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, - int nList + i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } @@ -186336,7 +201057,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); if( rc!=SQLITE_OK ) return rc; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); pList = pMsr->aBuffer; @@ -186473,11 +201194,11 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ return SQLITE_OK; } -static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ if( nReq>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = nReq*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } @@ -186568,7 +201289,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; @@ -186621,7 +201343,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; if( isFirst ){ @@ -186647,7 +201370,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ - rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); pCsr->aDoclist = pCsr->aBuffer; @@ -186931,7 +201654,6 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written @@ -186953,6 +201675,10 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = sqlite3_reset(pStmt); } } + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } return rc; } @@ -187360,7 +202086,7 @@ struct NodeReader { static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); if( a ){ pBlob->nAlloc = nAlloc; pBlob->a = a; @@ -187509,6 +202235,8 @@ static int fts3IncrmergePush( pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); } pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + assert( nPrefix+nSuffix<=nTerm ); + assert( nPrefix>=0 ); memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); pBlk->n += nSuffix; @@ -187582,6 +202310,8 @@ static int fts3AppendToNode( blobGrowBuffer(pPrev, nTerm, &rc); if( rc!=SQLITE_OK ) return rc; + assert( pPrev!=0 ); + assert( pPrev->a!=0 ); nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); nSuffix = nTerm - nPrefix; @@ -187631,15 +202361,20 @@ static int fts3IncrmergeAppend( pLeaf = &pWriter->aNodeWriter[0]; nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; nSpace = sqlite3Fts3VarintLen(nPrefix); nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + ** to the current block would make it larger than Fts3Table.nNodeSize bytes, + ** and if there is still room for another leaf page, write this block out to + ** the database. */ + if( pLeaf->block.n>0 + && (pLeaf->block.n + nSpace)>p->nNodeSize + && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst) + ){ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); pWriter->nWork++; @@ -187950,6 +202685,7 @@ static int fts3IncrmergeLoad( for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ NodeReader reader; + memset(&reader, 0, sizeof(reader)); pNode = &pWriter->aNodeWriter[i]; if( pNode->block.a){ @@ -187970,7 +202706,7 @@ static int fts3IncrmergeLoad( rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc - ); + ); if( rc==SQLITE_OK ){ memcpy(pNode->block.a, aBlock, nBlock); pNode->block.n = nBlock; @@ -188154,7 +202890,7 @@ static int fts3RepackSegdirLevel( if( nIdx>=nAlloc ){ int *aNew; nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); if( !aNew ){ rc = SQLITE_NOMEM; break; @@ -188528,7 +203264,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; @@ -188820,7 +203556,7 @@ static u64 fts3ChecksumIndex( int rc; u64 cksum = 0; - assert( *pRc==SQLITE_OK ); + if( *pRc ) return 0; memset(&filter, 0, sizeof(filter)); memset(&csr, 0, sizeof(csr)); @@ -188887,7 +203623,7 @@ static u64 fts3ChecksumIndex( ** If an error occurs (e.g. an OOM or IO error), return an SQLite error ** code. The final value of *pbOk is undefined in this case. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ +SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ @@ -188965,7 +203701,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -189005,7 +203746,7 @@ static int fts3DoIntegrityCheck( ){ int rc; int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } @@ -189035,8 +203776,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoIncrmerge(p, &zVal[6]); }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ rc = fts3DoAutoincrmerge(p, &zVal[10]); + }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - }else{ + else{ int v; if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ v = atoi(&zVal[9]); @@ -189054,8 +203798,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; rc = SQLITE_OK; } -#endif } +#endif return rc; } @@ -189164,7 +203908,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( return SQLITE_OK; } - pRet = (char *)sqlite3_malloc(p->pList->nData); + pRet = (char *)sqlite3_malloc64(p->pList->nData); if( !pRet ) return SQLITE_NOMEM; nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); @@ -189184,7 +203928,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } @@ -189463,7 +204207,7 @@ typedef sqlite3_int64 i64; /* -** Used as an fts3ExprIterate() context when loading phrase doclists to +** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; @@ -189507,7 +204251,7 @@ struct SnippetFragment { }; /* -** This type is used as an fts3ExprIterate() context object while +** This type is used as an sqlite3Fts3ExprIterate() context object while ** accumulating the data returned by the matchinfo() function. */ typedef struct MatchInfo MatchInfo; @@ -189666,7 +204410,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){ } /* -** Helper function for fts3ExprIterate() (see below). +** Helper function for sqlite3Fts3ExprIterate() (see below). */ static int fts3ExprIterate2( Fts3Expr *pExpr, /* Expression to iterate phrases of */ @@ -189700,7 +204444,7 @@ static int fts3ExprIterate2( ** Otherwise, SQLITE_OK is returned after a callback has been made for ** all eligible phrase nodes. */ -static int fts3ExprIterate( +SQLITE_PRIVATE int sqlite3Fts3ExprIterate( Fts3Expr *pExpr, /* Expression to iterate phrases of */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ @@ -189709,10 +204453,9 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } - /* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** This is an sqlite3Fts3ExprIterate() callback used while loading the +** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also ** fts3ExprLoadDoclists(). */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ @@ -189744,9 +204487,9 @@ static int fts3ExprLoadDoclists( int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ - LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */ sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx); if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; @@ -189759,7 +204502,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); return nPhrase; } @@ -189869,7 +204612,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -189887,8 +204630,9 @@ static void fts3SnippetDetails( } /* -** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). -** Each invocation populates an element of the SnippetIter.aPhrase[] array. +** This function is an sqlite3Fts3ExprIterate() callback used by +** fts3BestSnippet(). Each invocation populates an element of the +** SnippetIter.aPhrase[] array. */ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ SnippetIter *p = (SnippetIter *)ctx; @@ -189978,7 +204722,9 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter + ); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ @@ -190339,10 +205085,10 @@ static int fts3ExprLHitGather( } /* -** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. +** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo +** stats for a single query. ** -** fts3ExprIterate() callback to load the 'global' elements of a +** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements ** of the matchinfo array that are constant for all rows returned by the ** current query. @@ -190377,7 +205123,7 @@ static int fts3ExprGlobalHitsCb( } /* -** fts3ExprIterate() callback used to collect the "local" part of the +** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the ** array that are different for each row returned by the query. */ @@ -190573,7 +205319,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ **/ aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase); if( !aIter ) return SQLITE_NOMEM; - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; @@ -190750,11 +205496,11 @@ static int fts3MatchinfoValues( rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); if( rc!=SQLITE_OK ) break; } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); sqlite3Fts3EvalTestDeferred(pCsr, &rc); if( rc!=SQLITE_OK ) break; } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); break; } } @@ -190977,7 +205723,7 @@ struct TermOffsetCtx { }; /* -** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). +** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets(). */ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ TermOffsetCtx *p = (TermOffsetCtx *)ctx; @@ -191005,6 +205751,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ return rc; } +/* +** Expression node pExpr is an MSR phrase. This function restarts pExpr +** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +*/ +int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc = SQLITE_OK; + if( pExpr->bEof==0 ){ + i64 iDocid = pExpr->iDocid; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB; + } + } + return rc; +} + +/* +** If expression pExpr is a phrase expression that uses an MSR query, +** restart it as a regular, non-incremental query. Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx*)ctx; + int rc = SQLITE_OK; + if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){ + rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr); + pExpr->pPhrase->bIncr = 0; + } + return rc; +} + /* ** Implementation of offsets() function. */ @@ -191041,6 +205820,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( sCtx.iDocid = pCsr->iPrevId; sCtx.pCsr = pCsr; + /* If a query restart will be required, do it here, rather than later of + ** after pointers to poslist buffers that may be invalidated by a restart + ** have been saved. */ + rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx); + if( rc!=SQLITE_OK ) goto offsets_out; + /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ @@ -191059,7 +205844,9 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( */ sCtx.iCol = iCol; sCtx.iTerm = 0; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx + ); if( rc!=SQLITE_OK ) goto offsets_out; /* Retreive the text stored in column iCol. If an SQL NULL is stored @@ -191959,7 +206746,7 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ /************** End of fts3_unicode2.c ***************************************/ -/************** Begin file json1.c *******************************************/ +/************** Begin file json.c ********************************************/ /* ** 2015-08-12 ** @@ -191972,129 +206759,242 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ ** ****************************************************************************** ** -** This SQLite extension implements JSON functions. The interface is -** modeled after MySQL JSON functions: -** -** https://dev.mysql.com/doc/refman/5.7/en/json.html +** SQLite JSON functions. +** +** This file began as an extension in ext/misc/json1.c in 2015. That +** extension proved so useful that it has now been moved into the core. +** +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.45.0 (circa 2024-01-01), these routines also +** accept BLOB values that have JSON encoded using a binary representation +** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk +** format SQLite JSONB is completely different and incompatible with +** PostgreSQL JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. ** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) +** Additional information can be found in the doc/jsonb.md file of the +** canonical SQLite source tree. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(SQLITEINT_H) -/* #include "sqlite3ext.h" */ -#endif -SQLITE_EXTENSION_INIT1 -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* Mark a function parameter as unused, to suppress nuisance compiler -** warnings. */ -#ifndef UNUSED_PARAM -# define UNUSED_PARAM(X) (void)(X) -#endif - -#ifndef LARGEST_INT64 -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - -#ifndef deliberate_fall_through -# define deliberate_fall_through -#endif +#ifndef SQLITE_OMIT_JSON +/* #include "sqliteInt.h" */ -/* -** Versions of isspace(), isalnum() and isdigit() to which it is safe -** to pass signed char values. -*/ -#ifdef sqlite3Isdigit - /* Use the SQLite core versions if this routine is part of the - ** SQLite amalgamation */ -# define safe_isdigit(x) sqlite3Isdigit(x) -# define safe_isalnum(x) sqlite3Isalnum(x) -# define safe_isxdigit(x) sqlite3Isxdigit(x) -#else - /* Use the standard library for separate compilation */ -#include /* amalgamator: keep */ -# define safe_isdigit(x) isdigit((unsigned char)(x)) -# define safe_isalnum(x) isalnum((unsigned char)(x)) -# define safe_isxdigit(x) isxdigit((unsigned char)(x)) -#endif +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + +/* Human-readable names for the JSONB values. The index for each +** string must correspond to the JSONB_* integer above. +*/ +static const char * const jsonbType[] = { + "null", "true", "false", "integer", "integer", + "real", "real", "text", "text", "text", + "text", "array", "object", "", "", "", "" +}; /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) + +/* +** The set of all space characters recognized by jsonIsspace(). +** Useful as the second argument to strspn(). +*/ +static const char jsonSpaces[] = "\011\012\015\040"; + +/* +** Characters that are special to JSON. Control characters, +** '"' and '\\' and '\''. Actually, '\'' is not special to +** canonical JSON, but it is special in JSON-5, so we include +** it in the set of special characters. +*/ +static const char jsonIsOk[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; -#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) -#ifndef SQLITE_AMALGAMATION - /* Unsigned integer types. These are already defined in the sqliteInt.h, - ** but the definitions need to be repeated for separate compilation. */ - typedef sqlite3_uint64 u64; - typedef unsigned int u32; - typedef unsigned short int u16; - typedef unsigned char u8; -# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) -# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 -# endif -# if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -# elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -# else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -# endif -# define testcase(X) -#endif -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define VVA(X) -#else -# define VVA(X) X -#endif +/* Objects */ +typedef struct JsonCache JsonCache; +typedef struct JsonString JsonString; +typedef struct JsonParse JsonParse; /* -** Some of the testcase() macros in this file are problematic for gcov -** in that they generate false-miss errors randomly. This is a gcov problem, -** not a problem in this case. But to work around it, we disable the -** problematic test cases for production builds. +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() */ -#define json_testcase(X) +#define JSON_CACHE_ID (-429938) /* Cache entry */ +#define JSON_CACHE_SIZE 4 /* Max number of cache entries */ -/* Objects */ -typedef struct JsonString JsonString; -typedef struct JsonNode JsonNode; -typedef struct JsonParse JsonParse; +/* +** jsonUnescapeOneChar() returns this invalid code point if it encounters +** a syntax error. +*/ +#define JSON_INVALID_CHAR 0x99999 + +/* A cache mapping JSON text into JSONB blobs. +** +** Each cache entry is a JsonParse object with the following restrictions: +** +** * The bReadOnly flag must be set +** +** * The aBlob[] array must be owned by the JsonParse object. In other +** words, nBlobAlloc must be non-zero. +** +** * eEdit and delta must be zero. +** +** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. +*/ +struct JsonCache { + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */ +}; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. +** +** If the generated string is longer than will fit into the zSpace[] buffer, +** then it will be an RCStr string. This aids with caching of large +** JSON strings. */ struct JsonString { sqlite3_context *pCtx; /* Function context - put error messages here */ @@ -192102,89 +207002,227 @@ struct JsonString { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ + u8 eErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; -/* JSON type values -*/ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 +/* Allowed values for JsonString.eErr */ +#define JSTRING_OOM 0x01 /* Out of memory */ +#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ +#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ -/* The "subtype" set for JSON values */ +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ #define JSON_SUBTYPE 74 /* Ascii for "J" */ /* -** Names of the various JSON types: -*/ -static const char * const jsonType[] = { - "null", "true", "false", "integer", "real", "text", "array", "object" -}; - -/* Bit values for the JsonNode.jnFlag field +** Bit values for the flags passed into various SQL function implementations +** via the sqlite3_user_data() value. */ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ -/* A single node of parsed JSON -*/ -struct JsonNode { - u8 eType; /* One of the JSON_ type values */ - u8 jnFlags; /* JNODE flags */ - u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content, or number of sub-nodes */ - union { - const char *zJContent; /* 1: Content for INT, REAL, and STRING */ - u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ - u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ - } u; -}; - -/* A completely parsed JSON string +/* A parsed JSON value. Lifecycle: +** +** 1. JSON comes in and is parsed into a JSONB value in aBlob. The +** original text is stored in zJson. This step is skipped if the +** input is JSONB instead of text JSON. +** +** 2. The aBlob[] array is searched using the JSON path notation, if needed. +** +** 3. Zero or more changes are made to aBlob[] (via json_remove() or +** json_replace() or json_patch() or similar). +** +** 4. New JSON text is generated from the aBlob[] for output. This step +** is skipped if the function is one of the jsonb_* functions that +** returns JSONB instead of text JSON. */ struct JsonParse { - u32 nNode; /* Number of slots of aNode[] used */ - u32 nAlloc; /* Number of slots of aNode[] allocated */ - JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ - u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ - u16 iDepth; /* Nesting depth */ + u8 *aBlob; /* JSONB representation of JSON value */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ + char *zJson; /* Json text used for parsing */ + sqlite3 *db; /* The database connection to which this object belongs */ int nJson; /* Length of the zJson string in bytes */ - u32 iHold; /* Replace cache line with the lowest iHold value */ + u32 nJPRef; /* Number of references to this object */ + u32 iErr; /* Error location in zJson[] */ + u16 iDepth; /* Nesting depth */ + u8 nErr; /* Number of errors seen */ + u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ + u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 bReadOnly; /* Do not modify. */ + /* Search and edit information. See jsonLookupStep() */ + u8 eEdit; /* Edit operation to apply */ + int delta; /* Size change due to the edit */ + u32 nIns; /* Number of bytes to insert */ + u32 iLabel; /* Location of label if search landed on an object value */ + u8 *aIns; /* Content to be inserted */ }; +/* Allowed values for JsonParse.eEdit */ +#define JEDIT_DEL 1 /* Delete if exists */ +#define JEDIT_REPL 2 /* Overwrite if exists */ +#define JEDIT_INS 3 /* Insert if not exists */ +#define JEDIT_SET 4 /* Insert or overwrite */ + /* ** Maximum nesting depth of JSON for this implementation. ** ** This limit is needed to avoid a stack overflow in the recursive -** descent parser. A depth of 2000 is far deeper than any sane JSON -** should go. +** descent parser. A depth of 1000 is far deeper than any sane JSON +** should go. Historical note: This limit was 2000 prior to version 3.42.0 +*/ +#ifndef SQLITE_JSON_MAX_DEPTH +# define JSON_MAX_DEPTH 1000 +#else +# define JSON_MAX_DEPTH SQLITE_JSON_MAX_DEPTH +#endif + +/* +** Allowed values for the flgs argument to jsonParseFuncArg(); +*/ +#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ +#define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */ + +/************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static void jsonReturnParse(sqlite3_context*,JsonParse*); +static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +static void jsonParseFree(JsonParse*); +static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); +static u32 jsonUnescapeOneChar(const char*, u32, u32*); + +/************************************************************************** +** Utility routines for dealing with JsonCache objects +**************************************************************************/ + +/* +** Free a JsonCache object. +*/ +static void jsonCacheDelete(JsonCache *p){ + int i; + for(i=0; inUsed; i++){ + jsonParseFree(p->a[i]); + } + sqlite3DbFree(p->db, p); +} +static void jsonCacheDeleteGeneric(void *p){ + jsonCacheDelete((JsonCache*)p); +} + +/* +** Insert a new entry into the cache. If the cache is full, expel +** the least recently used entry. Return SQLITE_OK on success or a +** result code otherwise. +** +** Cache entries are stored in age order, oldest first. +*/ +static int jsonCacheInsert( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + JsonParse *pParse /* The parse object to be added to the cache */ +){ + JsonCache *p; + + assert( pParse->zJson!=0 ); + assert( pParse->bJsonIsRCStr ); + assert( pParse->delta==0 ); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->db = db; + sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ) return SQLITE_NOMEM; + } + if( p->nUsed >= JSON_CACHE_SIZE ){ + jsonParseFree(p->a[0]); + memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); + p->nUsed = JSON_CACHE_SIZE-1; + } + assert( pParse->nBlobAlloc>0 ); + pParse->eEdit = 0; + pParse->nJPRef++; + pParse->bReadOnly = 1; + p->a[p->nUsed] = pParse; + p->nUsed++; + return SQLITE_OK; +} + +/* +** Search for a cached translation the json text supplied by pArg. Return +** the JsonParse object if found. Return NULL if not found. +** +** When a match if found, the matching entry is moved to become the +** most-recently used entry if it isn't so already. +** +** The JsonParse object returned still belongs to the Cache and might +** be deleted at any moment. If the caller whants the JsonParse to +** linger, it needs to increment the nPJRef reference counter. */ -#define JSON_MAX_DEPTH 2000 +static JsonParse *jsonCacheSearch( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + sqlite3_value *pArg /* Function argument containing SQL text */ +){ + JsonCache *p; + int i; + const char *zJson; + int nJson; + + if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){ + return 0; + } + zJson = (const char*)sqlite3_value_text(pArg); + if( zJson==0 ) return 0; + nJson = sqlite3_value_bytes(pArg); + + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + return 0; + } + for(i=0; inUsed; i++){ + if( p->a[i]->zJson==zJson ) break; + } + if( i>=p->nUsed ){ + for(i=0; inUsed; i++){ + if( p->a[i]->nJson!=nJson ) continue; + if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break; + } + } + if( inUsed ){ + if( inUsed-1 ){ + /* Make the matching entry the most recently used entry */ + JsonParse *tmp = p->a[i]; + memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); + p->a[p->nUsed-1] = tmp; + i = p->nUsed - 1; + } + assert( p->a[i]->delta==0 ); + return p->a[i]; + }else{ + return 0; + } +} /************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ -/* Set the JsonString object to an empty string +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. */ -static void jsonZero(JsonString *p){ +static void jsonStringZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; @@ -192193,53 +207231,51 @@ static void jsonZero(JsonString *p){ /* Initialize the JsonString object */ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->bErr = 0; - jsonZero(p); + p->eErr = 0; + jsonStringZero(p); } - /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); - jsonZero(p); +static void jsonStringReset(JsonString *p){ + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); + jsonStringZero(p); } - /* Report an out-of-memory (OOM) condition */ -static void jsonOom(JsonString *p){ - p->bErr = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); +static void jsonStringOom(JsonString *p){ + p->eErr |= JSTRING_OOM; + if( p->pCtx ) sqlite3_result_error_nomem(p->pCtx); + jsonStringReset(p); } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(JsonString *p, u32 N){ +static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); + if( p->eErr ) return 1; + zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ - jsonOom(p); + jsonStringOom(p); return SQLITE_NOMEM; } memcpy(zNew, p->zBuf, (size_t)p->nUsed); p->zBuf = zNew; p->bStatic = 0; }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->eErr |= JSTRING_OOM; + jsonStringZero(p); return SQLITE_NOMEM; } - p->zBuf = zNew; } p->nAlloc = nTotal; return SQLITE_OK; @@ -192247,18 +207283,40 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( N==0 ) return; - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; +static SQLITE_NOINLINE void jsonStringExpandAndAppend( + JsonString *p, + const char *zIn, + u32 N +){ + assert( N>0 ); + if( jsonStringGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ + assert( N>0 ); + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; va_start(ap, zFormat); sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); va_end(ap); @@ -192267,10 +207325,38 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ + if( jsonStringGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc ){ + jsonAppendCharExpand(p,c); + }else{ + p->zBuf[p->nUsed++] = c; + } +} + +/* Remove a single character from the end of the string +*/ +static void jsonStringTrimOneChar(JsonString *p){ + if( p->eErr==0 ){ + assert( p->nUsed>0 ); + p->nUsed--; + } +} + + +/* Make sure there is a zero terminator on p->zBuf[] +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonStringTerminate(JsonString *p){ + jsonAppendChar(p, 0); + jsonStringTrimOneChar(p); + return p->eErr==0; +} /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. @@ -192279,68 +207365,137 @@ static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); + if( c=='[' || c=='{' ) return; + jsonAppendChar(p, ','); +} + +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } } /* Append the N-byte string in zIn to the end of the JsonString string -** under construction. Enclose the string in "..." and escape -** any double-quotes or backslash characters contained within the +** under construction. Enclose the string in double-quotes ("...") and +** escape any double-quotes or backslash characters contained within the ** string. +** +** This routine is a high-runner. There is a measurable performance +** increase associated with unwinding the jsonIsOk[] loop. */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; + u32 k; + u8 c; + const u8 *z = (const u8*)zIn; + if( z==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; - for(i=0; i=N ){ + while( k=N ){ + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + } + break; + } + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + z += k; + N -= k; + } + c = z[0]; if( c=='"' || c=='\\' ){ - json_simple_escape: - if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - }else if( c<=0x1f ){ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - if( aSpecial[c] ){ - c = aSpecial[c]; - goto json_simple_escape; - } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0' + (c>>4); - c = "0123456789abcdef"[c&0xf]; + p->zBuf[p->nUsed++] = c; + }else if( c=='\'' ){ + p->zBuf[p->nUsed++] = c; + }else{ + if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; + jsonAppendControlChar(p, c); } - p->zBuf[p->nUsed++] = c; + z++; + N--; } p->zBuf[p->nUsed++] = '"'; assert( p->nUsednAlloc ); } /* -** Append a function parameter value to the JSON string under -** construction. +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. */ -static void jsonAppendValue( +static void jsonAppendSqlValue( JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); + jsonAppendRawNZ(p, "null", 4); break; } - case SQLITE_INTEGER: case SQLITE_FLOAT: { + jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); + break; + } + case SQLITE_INTEGER: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); jsonAppendRaw(p, z, n); @@ -192357,184 +207512,127 @@ static void jsonAppendValue( break; } default: { - if( p->bErr==0 ){ + if( jsonFuncArgMightBeBinary(pValue) ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(pValue); + px.nBlob = sqlite3_value_bytes(pValue); + jsonTranslateBlobToText(&px, 0, p); + }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; - jsonReset(p); + p->eErr = JSTRING_ERR; + jsonStringReset(p); } break; } } } - -/* Make the JSON in p the result of the SQL function. +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. +** +** The JsonString is reset. +** +** If pParse and ctx are both non-NULL, then the SQL string in p is +** loaded into the zJson field of the pParse object as a RCStr and the +** pParse is added to the cache. */ -static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); +static void jsonReturnString( + JsonString *p, /* String to return */ + JsonParse *pParse, /* JSONB source or NULL */ + sqlite3_context *ctx /* Where to cache */ +){ + assert( (pParse!=0)==(ctx!=0) ); + assert( ctx==0 || ctx==p->pCtx ); + if( p->eErr==0 ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonStringTerminate(p) ){ + if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ + int rc; + pParse->zJson = sqlite3RCStrRef(p->zBuf); + pParse->nJson = p->nUsed; + pParse->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, pParse); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(ctx); + jsonStringReset(p); + return; + } + } + sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, + sqlite3RCStrUnref, + SQLITE_UTF8); + }else{ + sqlite3_result_error_nomem(p->pCtx); + } + }else if( p->eErr & JSTRING_OOM ){ + sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_MALFORMED ){ + sqlite3_result_error(p->pCtx, "malformed JSON", -1); } - assert( p->bStatic ); + jsonStringReset(p); } /************************************************************************** -** Utility routines for dealing with JsonNode and JsonParse objects +** Utility routines for dealing with JsonParse objects **************************************************************************/ -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonNodeSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} - /* ** Reclaim all memory allocated by a JsonParse object. But do not ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; + assert( pParse->nJPRef<=1 ); + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->nJson = 0; + pParse->bJsonIsRCStr = 0; + } + if( pParse->nBlobAlloc ){ + sqlite3DbFree(pParse->db, pParse->aBlob); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } } /* -** Free a JsonParse object that was obtained from sqlite3_malloc(). +** Decrement the reference count on the JsonParse object. When the +** count reaches zero, free the object. */ static void jsonParseFree(JsonParse *pParse){ - jsonParseReset(pParse); - sqlite3_free(pParse); -} - -/* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. -*/ -static void jsonRenderNode( - JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ -){ - assert( pNode!=0 ); - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ - assert( pNode->eU==4 ); - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; - } - assert( pNode->eU==5 ); - pNode = pNode->u.pPatch; - } - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); - break; - } - case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); - break; - } - case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; - } - /* no break */ deliberate_fall_through - } - case JSON_REAL: - case JSON_INT: { - assert( pNode->eU==1 ); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; + if( pParse ){ + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3DbFree(pParse->db, pParse); } } } -/* -** Return a JsonNode and all its descendents as a JSON string. -*/ -static void jsonReturnJson( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); -} +/************************************************************************** +** Utility routines for the JSON text parser +**************************************************************************/ /* ** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F +** This routine only gives a correct answer if h really is a valid hexadecimal +** character: 0..9a..fA..F. But unlike sqlite3HexToInt(), it does not +** assert() if the digit is not hex. */ static u8 jsonHexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif #ifdef SQLITE_EBCDIC h += 9*(1&~(h>>4)); -#else - h += 9*(1&(h>>6)); #endif return (u8)(h & 0xf); } @@ -192544,10 +207642,6 @@ static u8 jsonHexToInt(int h){ */ static u32 jsonHexToInt4(const char *z){ u32 v; - assert( safe_isxdigit(z[0]) ); - assert( safe_isxdigit(z[1]) ); - assert( safe_isxdigit(z[2]) ); - assert( safe_isxdigit(z[3]) ); v = (jsonHexToInt(z[0])<<12) + (jsonHexToInt(z[1])<<8) + (jsonHexToInt(z[2])<<4) @@ -192556,420 +207650,1108 @@ static u32 jsonHexToInt4(const char *z){ } /* -** Make the JsonNode the return value of the function. +** Return true if z[] begins with 2 (or more) hexadecimal digits */ -static void jsonReturn( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - const char *z; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; +static int jsonIs2Hex(const char *z){ + return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]); +} + +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + return jsonIs2Hex(z) && jsonIs2Hex(&z[2]); +} + +/* +** Return the number of bytes of JSON5 whitespace at the beginning of +** the input string z[]. +** +** JSON5 whitespace consists of any of the following characters: +** +** Unicode UTF-8 Name +** U+0009 09 horizontal tab +** U+000a 0a line feed +** U+000b 0b vertical tab +** U+000c 0c form feed +** U+000d 0d carriage return +** U+0020 20 space +** U+00a0 c2 a0 non-breaking space +** U+1680 e1 9a 80 ogham space mark +** U+2000 e2 80 80 en quad +** U+2001 e2 80 81 em quad +** U+2002 e2 80 82 en space +** U+2003 e2 80 83 em space +** U+2004 e2 80 84 three-per-em space +** U+2005 e2 80 85 four-per-em space +** U+2006 e2 80 86 six-per-em space +** U+2007 e2 80 87 figure space +** U+2008 e2 80 88 punctuation space +** U+2009 e2 80 89 thin space +** U+200a e2 80 8a hair space +** U+2028 e2 80 a8 line separator +** U+2029 e2 80 a9 paragraph separator +** U+202f e2 80 af narrow no-break space (NNBSP) +** U+205f e2 81 9f medium mathematical space (MMSP) +** U+3000 e3 80 80 ideographical space +** U+FEFF ef bb bf byte order mark +** +** In addition, comments between '/', '*' and '*', '/' and +** from '/', '/' to end-of-line are also considered to be whitespace. +*/ +static int json5Whitespace(const char *zIn){ + int n = 0; + const u8 *z = (u8*)zIn; + while( 1 /*exit by "goto whitespace_done"*/ ){ + switch( z[n] ){ + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x20: { + n++; + break; + } + case '/': { + if( z[n+1]=='*' && z[n+2]!=0 ){ + int j; + for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ + if( z[j]==0 ) goto whitespace_done; + } + n = j+1; + break; + }else if( z[n+1]=='/' ){ + int j; + char c; + for(j=n+2; (c = z[j])!=0; j++){ + if( c=='\n' || c=='\r' ) break; + if( 0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]) + ){ + j += 2; + break; } } + n = j; + if( z[n] ) n++; + break; } - i = i*10 + v; + goto whitespace_done; } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: - break; - int_as_real: ; /* no break */ deliberate_fall_through - } - case JSON_REAL: { - double r; -#ifdef SQLITE_AMALGAMATION - const char *z; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - assert( pNode->eU==1 ); - r = strtod(pNode->u.zJContent, 0); -#endif - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ - if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z; - char *zOut; - u32 j; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - zOut = sqlite3_malloc( n+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); + case 0xc2: { + if( z[n+1]==0xa0 ){ + n += 2; break; } - for(i=1, j=0; i>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - u32 vlo; - if( (v&0xfc00)==0xd800 - && i>18); - zOut[j++] = 0x80 | ((v>>12)&0x3f); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = 0xe0 | (v>>12); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - } - }else{ - if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - } - zOut[j++] = c; - } + goto whitespace_done; + } + case 0xe1: { + if( z[n+1]==0x9a && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe2: { + if( z[n+1]==0x80 ){ + u8 c = z[n+2]; + if( c<0x80 ) goto whitespace_done; + if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){ + n += 3; + break; } + }else if( z[n+1]==0x81 && z[n+2]==0x9f ){ + n += 3; + break; } - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + goto whitespace_done; + } + case 0xe3: { + if( z[n+1]==0x80 && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xef: { + if( z[n+1]==0xbb && z[n+2]==0xbf ){ + n += 3; + break; + } + goto whitespace_done; + } + default: { + goto whitespace_done; } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); - break; } } + whitespace_done: + return n; } -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - /* -** A macro to hint to the compiler that a function should not be -** inlined. +** Extra floating-point literals to allow in JSON. */ -#if defined(__GNUC__) -# define JSON_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define JSON_NOINLINE __declspec(noinline) -#else -# define JSON_NOINLINE -#endif +static const struct NanInfName { + char c1; + char c2; + char n; + char eType; + char nRepl; + char *zMatch; + char *zRepl; +} aNanInfName[] = { + { 'i', 'I', 3, JSONB_FLOAT, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSONB_FLOAT, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSONB_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSONB_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" }, +}; -static JSON_NOINLINE int jsonParseAddNodeExpand( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName ){ - u32 nNew; - JsonNode *pNew; - assert( pParse->nNode>=pParse->nAlloc ); - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u32 t; + assert( N>pParse->nBlobAlloc ); + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; } - pParse->nAlloc = nNew; - pParse->aNode = pNew; - assert( pParse->nNodenAlloc ); - return jsonParseAddNode(pParse, eType, n, zContent); + if( tdb, pParse->aBlob, t); + if( aNew==0 ){ pParse->oom = 1; return 1; } + pParse->aBlob = aNew; + pParse->nBlobAlloc = t; + return 0; } /* -** Create a new JsonNode instance based on the arguments and append that -** instance to the JsonParse. Return the index in pParse->aNode[] of the -** new node, or -1 if a memory allocation fails. +** If pParse->aBlob is not previously editable (because it is taken +** from sqlite3_value_blob(), as indicated by the fact that +** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable +** by making a copy into space obtained from malloc. +** +** Return true on success. Return false on OOM. */ -static int jsonParseAddNode( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ +static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ + u8 *aOld; + u32 nSize; + assert( !pParse->bReadOnly ); + if( pParse->oom ) return 0; + if( pParse->nBlobAlloc>0 ) return 1; + aOld = pParse->aBlob; + nSize = pParse->nBlob + nExtra; + pParse->aBlob = 0; + if( jsonBlobExpand(pParse, nSize) ){ + return 0; + } + assert( pParse->nBlobAlloc >= pParse->nBlob + nExtra ); + memcpy(pParse->aBlob, aOld, pParse->nBlob); + return 1; +} + +/* Expand pParse->aBlob and append one bytes. +*/ +static SQLITE_NOINLINE void jsonBlobExpandAndAppendOneByte( + JsonParse *pParse, + u8 c ){ - JsonNode *p; - if( pParse->aNode==0 || pParse->nNode>=pParse->nAlloc ){ - return jsonParseAddNodeExpand(pParse, eType, n, zContent); + jsonBlobExpand(pParse, pParse->nBlob+1); + if( pParse->oom==0 ){ + assert( pParse->nBlob+1<=pParse->nBlobAlloc ); + pParse->aBlob[pParse->nBlob++] = c; } - p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; - VVA( p->eU = zContent ? 1 : 0 ); - p->n = n; - p->u.zJContent = zContent; - return pParse->nNode++; } -/* -** Return true if z[] begins with 4 (or more) hexadecimal digits +/* Append a single character. */ -static int jsonIs4Hex(const char *z){ - int i; - for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; +static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendOneByte(pParse, c); + }else{ + pParse->aBlob[pParse->nBlob++] = c; + } +} + +/* Slow version of jsonBlobAppendNode() that first resizes the +** pParse->aBlob structure. +*/ +static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( + JsonParse *pParse, + u8 eType, + u32 szPayload, + const void *aPayload +){ + if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; + jsonBlobAppendNode(pParse, eType, szPayload, aPayload); +} + + +/* Append an node type byte together with the payload size and +** possibly also the payload. +** +** If aPayload is not NULL, then it is a pointer to the payload which +** is also appended. If aPayload is NULL, the pParse->aBlob[] array +** is resized (if necessary) so that it is big enough to hold the +** payload, but the payload is not appended and pParse->nBlob is left +** pointing to where the first byte of payload will eventually be. +*/ +static void jsonBlobAppendNode( + JsonParse *pParse, /* The JsonParse object under construction */ + u8 eType, /* Node type. One of JSONB_* */ + u32 szPayload, /* Number of bytes of payload */ + const void *aPayload /* The payload. Might be NULL */ +){ + u8 *a; + if( pParse->nBlob+szPayload+9 > pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendNode(pParse,eType,szPayload,aPayload); + return; + } + assert( pParse->aBlob!=0 ); + a = &pParse->aBlob[pParse->nBlob]; + if( szPayload<=11 ){ + a[0] = eType | (szPayload<<4); + pParse->nBlob += 1; + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + pParse->nBlob += 2; + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + pParse->nBlob += 3; + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + pParse->nBlob += 5; + } + if( aPayload ){ + pParse->nBlob += szPayload; + memcpy(&pParse->aBlob[pParse->nBlob-szPayload], aPayload, szPayload); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static int jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a; + u8 szType; + u8 nExtra; + u8 nNeeded; + int delta; + if( pParse->oom ) return 0; + a = &pParse->aBlob[i]; + szType = a[0]>>4; + if( szType<=11 ){ + nExtra = 0; + }else if( szType==12 ){ + nExtra = 1; + }else if( szType==13 ){ + nExtra = 2; + }else{ + nExtra = 4; + } + if( szPayload<=11 ){ + nNeeded = 0; + }else if( szPayload<=0xff ){ + nNeeded = 1; + }else if( szPayload<=0xffff ){ + nNeeded = 2; + }else{ + nNeeded = 4; + } + delta = nNeeded - nExtra; + if( delta ){ + u32 newSize = pParse->nBlob + delta; + if( delta>0 ){ + if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ + return 0; /* OOM error. Error state recorded in pParse->oom. */ + } + a = &pParse->aBlob[i]; + memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); + }else{ + memmove(&a[1], &a[1-delta], pParse->nBlob - (i+1-delta)); + } + pParse->nBlob = newSize; + } + if( nNeeded==0 ){ + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( nNeeded==1 ){ + a[0] = (a[0] & 0x0f) | 0xc0; + a[1] = szPayload & 0xff; + }else if( nNeeded==2 ){ + a[0] = (a[0] & 0x0f) | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + a[0] = (a[0] & 0x0f) | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } + return delta; +} + +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !jsonIs4Hex(&z[1]) ) return 0; + *pOp = JSONB_TEXTJ; return 1; } /* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck( + const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */ + u32 i, /* Start of element as pParse->aBlob[i] */ + u32 iEnd, /* One more than the last byte of the element */ + u32 iDepth /* Current nesting depth */ +){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( NEVER(n==0) ) return i+1; /* Checked by caller */ + if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */ + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return i+1; + } + k = i+n+sz; + while( jk ) return j+1; + if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1; + j++; + } + for(; j0 ) return j+1; + if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ + return j+1; + } + seen = 1; + continue; + } + if( z[j]=='e' || z[j]=='E' ){ + if( seen==2 ) return j+1; + if( j==k-1 ) return j+1; + if( z[j+1]=='+' || z[j+1]=='-' ){ + j++; + if( j==k-1 ) return j+1; + } + seen = 2; + continue; + } + return j+1; + } + if( seen==0 ) return i+1; + return 0; + } + case JSONB_TEXT: { + j = i+n; + k = j+sz; + while( j=k ){ + return j+1; + }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ + j++; + }else if( z[j+1]=='u' ){ + if( j+5>=k ) return j+1; + if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1; + j++; + }else if( x!=JSONB_TEXT5 ){ + return j+1; + }else{ + u32 c = 0; + u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); + if( c==JSON_INVALID_CHAR ) return j+1; + j += szC - 1; + } + } + j++; + } + return 0; + } + case JSONB_TEXTRAW: { + return 0; + } + case JSONB_ARRAY: { + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + default: { + return i+1; + } + } +} + +/* +** Translate a single element of JSON text at pParse->zJson[i] into +** its equivalent binary JSONB representation. Append the translation into +** pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. ** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. +** Return the index of the first character past the end of the element parsed, +** or one of the following special result codes: +** +** 0 End of input +** -1 Syntax error or OOM +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / */ -static int jsonParseValue(JsonParse *pParse, u32 i){ +static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ char c; u32 j; - int iThis; + u32 iThis, iStart; int x; - JsonNode *pNode; + u8 t; const char *z = pParse->zJson; - while( safe_isspace(z[i]) ){ i++; } - if( (c = z[i])=='{' ){ +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { /* Parse object */ - iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; + iThis = pParse->nBlob; + jsonBlobAppendNode(pParse, JSONB_OBJECT, pParse->nJson-i, 0); + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + iStart = pParse->nBlob; for(j=i+1;;j++){ - while( safe_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; - x = jsonParseValue(pParse, j); - if( x<0 ){ - pParse->iDepth--; - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; + u32 iBlob = pParse->nBlob; + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + int op; + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) + ){ + k++; + } + assert( iBlob==pParse->nBlob ); + jsonBlobAppendNode(pParse, op, k-j, &z[j]); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } } if( pParse->oom ) return -1; - pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; - pNode->jnFlags |= JNODE_LABEL; + t = pParse->aBlob[iBlob] & 0x0f; + if( tJSONB_TEXTRAW ){ + pParse->iErr = j; + return -1; + } j = x; - while( safe_isspace(z[j]) ){ j++; } - if( z[j]!=':' ) return -1; - j++; - x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ) return -1; + if( z[j]==':' ){ + j++; + }else{ + if( jsonIsspace(z[j]) ){ + /* strspn() is not helpful here */ + do{ j++; }while( jsonIsspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } j = x; - while( safe_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; return j+1; - }else if( c=='[' ){ + } + case '[': { /* Parse array */ - iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; - memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); + iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); + jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); + iStart = pParse->nBlob; + if( pParse->oom ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } for(j=i+1;;j++){ - while( safe_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; - x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; return -1; } j = x; - while( safe_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; return j+1; - }else if( c=='"' ){ + } + case '\'': { + u8 opcode; + char cDelim; + pParse->hasNonstd = 1; + opcode = JSONB_TEXT; + goto parse_string; + case '"': /* Parse string */ - u8 jnFlags = 0; + opcode = JSONB_TEXT; + parse_string: + cDelim = z[i]; j = i+1; - for(;;){ - c = z[j]; - if( (c & ~0x1f)==0 ){ - /* Control characters are not allowed in strings */ - return -1; + while( 1 /*exit-by-break*/ ){ + if( jsonIsOk[(u8)z[j]] ){ + if( !jsonIsOk[(u8)z[j+1]] ){ + j += 1; + }else if( !jsonIsOk[(u8)z[j+2]] ){ + j += 2; + }else{ + j += 3; + continue; + } } - if( c=='\\' ){ + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(z+j+1)) ){ - jnFlags = JNODE_ESCAPE; + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else{ + pParse->iErr = j; + return -1; + } + }else if( c<=0x1f ){ + if( c==0 ){ + pParse->iErr = j; return -1; } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else if( c=='"' ){ - break; + opcode = JSONB_TEXT5; } j++; } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + jsonBlobAppendNode(pParse, opcode, j-1-i, &z[i+1]); return j+1; - }else if( c=='n' - && strncmp(z+i,"null",4)==0 - && !safe_isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(z+i,"true",4)==0 - && !safe_isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(z+i,"false",5)==0 - && !safe_isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_TRUE); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonBlobAppendOneByte(pParse, JSONB_FALSE); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenE; + pParse->hasNonstd = 1; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + seenE = 0; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + parse_number: + seenE = 0; assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + if( c<='0' ){ - j = c=='-' ? i+1 : i; - if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( t==0x00 ); + pParse->hasNonstd = 1; + t = 0x01; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + t |= 0x01; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } } - j = i+1; - for(;; j++){ + + parse_number_2: + for(j=i+1;; j++){ c = z[j]; - if( c>='0' && c<='9' ) continue; + if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( z[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; + if( (t & 0x02)!=0 ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; continue; } if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ) return -1; - if( seenE ) return -1; - seenDP = seenE = 1; + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ j++; c = z[j+1]; } - if( c<'0' || c>'9' ) return -1; + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } continue; } break; } - if( z[j-1]<'0' ) return -1; - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &z[i]); + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; + jsonBlobAppendNode(pParse, JSONB_INT+t, j-i, &z[i]); return j; - }else if( c=='}' ){ + } + case '}': { + pParse->iErr = i; return -2; /* End of {...} */ - }else if( c==']' ){ + } + case ']': { + pParse->iErr = i; return -3; /* End of [...] */ - }else if( c==0 ){ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { return 0; /* End of file */ - }else{ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + i += 1 + (u32)strspn(&z[i+1], jsonSpaces); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; khasNonstd = 1; + return i + nn; + } + pParse->iErr = i; return -1; /* Syntax error */ } + } /* End switch(z[i]) */ } + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. ** -** pParse is uninitialized when this routine is called. +** pParse must be initialized to an empty parse object prior to calling +** this routine. */ -static int jsonParse( +static int jsonConvertTextToBlob( JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + sqlite3_context *pCtx /* Report errors here */ ){ int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; - i = jsonParseValue(pParse, 0); + const char *zJson = pParse->zJson; + i = jsonTranslateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ +#ifdef SQLITE_DEBUG assert( pParse->iDepth==0 ); - while( safe_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; + if( sqlite3Config.bJsonSelfcheck ){ + assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); + } +#endif + while( jsonIsspace(zJson[i]) ) i++; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + if( pCtx ) sqlite3_result_error(pCtx, "malformed JSON", -1); + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } } if( i<=0 ){ if( pCtx!=0 ){ @@ -192985,161 +208767,832 @@ static int jsonParse( return 0; } -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. */ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + px.db = sqlite3_context_db_handle(pStr->pCtx); + (void)jsonTranslateTextToBlob(&px, 0); + if( px.oom ){ + sqlite3DbFree(px.db, px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + assert( px.nBlobAlloc>0 ); + assert( !px.bReadOnly ); + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, SQLITE_DYNAMIC); + } +} + +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. Return 0 on error. +*/ +static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + if( NEVER(i>pParse->nBlob) ){ + *pSz = 0; + return 0; + } + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else if( x==13 ){ + if( i+2>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else if( x==14 ){ + if( i+4>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + }else{ + if( i+8>=pParse->nBlob + || pParse->aBlob[i+1]!=0 + || pParse->aBlob[i+2]!=0 + || pParse->aBlob[i+3]!=0 + || pParse->aBlob[i+4]!=0 + ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; + n = 9; + } + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta + ){ + sz = 0; + n = 0; + } + *pSz = sz; + return n; +} + + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToText( + const JsonParse *pParse, /* the complete parse of the JSON */ + u32 i, /* Start rendering at this index */ + JsonString *pOut /* Write JSON here */ +){ + u32 sz, n, j, iEnd; + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + jsonAppendRawNZ(pOut, "null", 4); + return i+1; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + return i+1; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + return i+1; + } + case JSONB_INT: + case JSONB_FLOAT: { + if( sz==0 ) goto malformed_jsonb; + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + u32 k = 2; + sqlite3_uint64 u = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + int bOverflow = 0; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + }else if( zIn[0]=='+' ){ + k++; } + for(; keErr |= JSTRING_MALFORMED; + break; + }else if( (u>>60)!=0 ){ + bOverflow = 1; + }else{ + u = u*16 + sqlite3HexToInt(zIn[k]); + } + } + jsonPrintf(100,pOut,bOverflow?"9.0e999":"%llu", u); break; } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); + case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ + u32 k = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + } + if( zIn[k]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(; kaBlob[i+n], sz); + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + u32 sz2 = sz; + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz2>0 ){ + for(k=0; k0 ){ + jsonAppendRawNZ(pOut, zIn, k); + if( k>=sz2 ){ + break; + } + zIn += k; + sz2 -= k; + } + if( zIn[0]=='"' ){ + jsonAppendRawNZ(pOut, "\\\"", 2); + zIn++; + sz2--; + continue; + } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } + assert( zIn[0]=='\\' ); + assert( sz2>=1 ); + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u0009", 6); + break; + case 'x': + if( sz2<4 ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz2 -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( sz2>2 && zIn[2]=='\n' ){ + zIn++; + sz2--; + } + break; + case '\n': + break; + case 0xe2: + /* '\' followed by either U+2028 or U+2029 is ignored as + ** whitespace. Not that in UTF8, U+2028 is 0xe2 0x80 0x29. + ** U+2029 is the same except for the last byte */ + if( sz2<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + zIn += 2; + sz2 -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + assert( sz2>=2 ); + zIn += 2; + sz2 -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXTRAW: { + jsonAppendString(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, ','); + } + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, '}'); + break; + } + default: { + malformed_jsonb: + pOut->eErr |= JSTRING_MALFORMED; break; } } + return i+n+sz; +} + +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } } /* -** Compute the parentage of all nodes in a completed parse. +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; } - jsonParseFillInParentage(pParse, 0, 0); - return SQLITE_OK; + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; +} + + +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + aBlob = sqlite3_value_blob(pJson); + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + if( sz+n!=(u32)nBlob ) return 0; + if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; + return sz+n==(u32)nBlob; } /* -** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. */ -#define JSON_CACHE_ID (-429938) /* First cache entry */ -#define JSON_CACHE_SZ 4 /* Max number of cache entries */ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; n>0 && idelta. */ -static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value **argv, - sqlite3_context *pErrCtx +static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ + u32 sz = 0; + u32 nBlob; + assert( pParse->delta!=0 ); + assert( pParse->nBlobAlloc >= pParse->nBlob ); + nBlob = pParse->nBlob; + pParse->nBlob = pParse->nBlobAlloc; + (void)jsonbPayloadSize(pParse, iRoot, &sz); + pParse->nBlob = nBlob; + sz += pParse->delta; + pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); +} + +/* +** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of +** content beginning at iDel, and replacing them with nIns bytes of +** content given by aIns. +** +** nDel may be zero, in which case no bytes are removed. But iDel is +** still important as new bytes will be insert beginning at iDel. +** +** aIns may be zero, in which case space is created to hold nIns bytes +** beginning at iDel, but that space is uninitialized. +** +** Set pParse->oom if an OOM occurs. +*/ +static void jsonBlobEdit( + JsonParse *pParse, /* The JSONB to be modified is in pParse->aBlob */ + u32 iDel, /* First byte to be removed */ + u32 nDel, /* Number of bytes to remove */ + const u8 *aIns, /* Content to insert */ + u32 nIns /* Bytes of content to insert */ ){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); - JsonParse *p; - JsonParse *pMatch = 0; - int iKey; - int iMinKey = 0; - u32 iMinHold = 0xffffffff; - u32 iMaxHold = 0; - if( zJson==0 ) return 0; - for(iKey=0; iKeynBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return; } - if( pMatch==0 - && p->nJson==nJson - && memcmp(p->zJson,zJson,nJson)==0 - ){ - p->nErr = 0; - pMatch = p; - }else if( p->iHoldiHold; - iMinKey = iKey; + memmove(&pParse->aBlob[iDel+nIns], + &pParse->aBlob[iDel+nDel], + pParse->nBlob - (iDel+nDel)); + pParse->nBlob += d; + pParse->delta += d; + } + if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); +} + +/* +** Return the number of escaped newlines to be ignored. +** An escaped newline is a one of the following byte sequences: +** +** 0x5c 0x0a +** 0x5c 0x0d +** 0x5c 0x0d 0x0a +** 0x5c 0xe2 0x80 0xa8 +** 0x5c 0xe2 0x80 0xa9 +*/ +static u32 jsonBytesToBypass(const char *z, u32 n){ + u32 i = 0; + while( i+1iHold>iMaxHold ){ - iMaxHold = p->iHold; + if( z[i+1]=='\r' ){ + if( i+2nErr = 0; - pMatch->iHold = iMaxHold+1; - return pMatch; + return i; +} + +/* +** Input z[0..n] defines JSON escape sequence including the leading '\\'. +** Decode that escape sequence into a single character. Write that +** character into *piOut. Return the number of bytes in the escape sequence. +** +** If there is a syntax error of some kind (for example too few characters +** after the '\\' to complete the encoding) then *piOut is set to +** JSON_INVALID_CHAR. +*/ +static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ + assert( n>0 ); + assert( z[0]=='\\' ); + if( n<2 ){ + *piOut = JSON_INVALID_CHAR; + return n; } - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); - if( p==0 ){ - sqlite3_result_error_nomem(pCtx); - return 0; + switch( (u8)z[1] ){ + case 'u': { + u32 v, vlo; + if( n<6 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + v = jsonHexToInt4(&z[2]); + if( (v & 0xfc00)==0xd800 + && n>=12 + && z[6]=='\\' + && z[7]=='u' + && ((vlo = jsonHexToInt4(&z[8]))&0xfc00)==0xdc00 + ){ + *piOut = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + return 12; + }else{ + *piOut = v; + return 6; + } + } + case 'b': { *piOut = '\b'; return 2; } + case 'f': { *piOut = '\f'; return 2; } + case 'n': { *piOut = '\n'; return 2; } + case 'r': { *piOut = '\r'; return 2; } + case 't': { *piOut = '\t'; return 2; } + case 'v': { *piOut = '\v'; return 2; } + case '0': { *piOut = 0; return 2; } + case '\'': + case '"': + case '/': + case '\\':{ *piOut = z[1]; return 2; } + case 'x': { + if( n<4 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); + return 4; + } + case 0xe2: + case '\r': + case '\n': { + u32 nSkip = jsonBytesToBypass(z, n); + if( nSkip==0 ){ + *piOut = JSON_INVALID_CHAR; + return n; + }else if( nSkip==n ){ + *piOut = 0; + return n; + }else if( z[nSkip]=='\\' ){ + return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); + }else{ + int sz = sqlite3Utf8ReadLimited((u8*)&z[nSkip], n-nSkip, piOut); + return nSkip + sz; + } + } + default: { + *piOut = JSON_INVALID_CHAR; + return 2; + } } - memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ - sqlite3_free(p); - return 0; +} + + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. +** +** In this version, we know that one or the other or both of the +** two comparands contains an escape sequence. +*/ +static SQLITE_NOINLINE int jsonLabelCompareEscaped( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + u32 cLeft, cRight; + assert( rawLeft==0 || rawRight==0 ); + while( 1 /*exit-by-return*/ ){ + if( nLeft==0 ){ + cLeft = 0; + }else if( rawLeft || zLeft[0]!='\\' ){ + cLeft = ((u8*)zLeft)[0]; + if( cLeft>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zLeft, nLeft, &cLeft); + zLeft += sz; + nLeft -= sz; + }else{ + zLeft++; + nLeft--; + } + }else{ + u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); + zLeft += n; + assert( n<=nLeft ); + nLeft -= n; + } + if( nRight==0 ){ + cRight = 0; + }else if( rawRight || zRight[0]!='\\' ){ + cRight = ((u8*)zRight)[0]; + if( cRight>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zRight, nRight, &cRight); + zRight += sz; + nRight -= sz; + }else{ + zRight++; + nRight--; + } + }else{ + u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); + zRight += n; + assert( n<=nRight ); + nRight -= n; + } + if( cLeft!=cRight ) return 0; + if( cLeft==0 ) return 1; } - p->nJson = nJson; - p->iHold = iMaxHold+1; - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, - (void(*)(void*))jsonParseFree); - return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); } /* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. Return -1 if an OOM occurs. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; +static int jsonLabelCompare( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + if( rawLeft && rawRight ){ + /* Simpliest case: Neither label contains escapes. A simple + ** memcmp() is sufficient. */ + if( nLeft!=nRight ) return 0; + return memcmp(zLeft, zRight, nLeft)==0; }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + return jsonLabelCompareEscaped(zLeft, nLeft, rawLeft, + zRight, nRight, rawRight); } } -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); +/* +** Error returns from jsonLookupStep() +*/ +#define JSON_LOOKUP_ERROR 0xffffffff +#define JSON_LOOKUP_NOTFOUND 0xfffffffe +#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) + +/* Forward declaration */ +static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); + + +/* This helper routine for jsonLookupStep() populates pIns with +** binary data that is to be inserted into pParse. +** +** In the common case, pIns just points to pParse->aIns and pParse->nIns. +** But if the zPath of the original edit operation includes path elements +** that go deeper, additional substructure must be created. +** +** For example: +** +** json_insert('{}', '$.a.b.c', 123); +** +** The search stops at '$.a' But additional substructure must be +** created for the ".b.c" part of the patch so that the final result +** is: {"a":{"b":{"c"::123}}}. This routine populates pIns with +** the binary equivalent of {"b":{"c":123}} so that it can be inserted. +** +** The caller is responsible for resetting pIns when it has finished +** using the substructure. +*/ +static u32 jsonCreateEditSubstructure( + JsonParse *pParse, /* The original JSONB that is being edited */ + JsonParse *pIns, /* Populate this with the blob data to insert */ + const char *zTail /* Tail of the path that determins substructure */ +){ + static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; + int rc; + memset(pIns, 0, sizeof(*pIns)); + pIns->db = pParse->db; + if( zTail[0]==0 ){ + /* No substructure. Just insert what is given in pParse. */ + pIns->aBlob = pParse->aIns; + pIns->nBlob = pParse->nIns; + rc = 0; + }else{ + /* Construct the binary substructure */ + pIns->nBlob = 1; + pIns->aBlob = (u8*)&emptyObject[zTail[0]=='.']; + pIns->eEdit = pParse->eEdit; + pIns->nIns = pParse->nIns; + pIns->aIns = pParse->aIns; + rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->oom |= pIns->oom; + } + return rc; /* Error code only */ +} /* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. ** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. +** If the value found by this routine is the value half of label/value pair +** within an object, then set pPath->iLabel to the start of the corresponding +** label, before returning. +** +** Return one of the JSON_LOOKUP error codes if problems are seen. +** +** This routine will also modify the blob. If pParse->eEdit is one of +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be +** made to the selected value. If an edit is performed, then the return +** value does not necessarily point to the select element. If an edit +** is performed, the return value is only useful for detecting error +** conditions. */ -static JsonNode *jsonLookupStep( +static u32 jsonLookupStep( JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ + u32 iLabel /* Label if iRoot is a value of in an object */ ){ - u32 i, j, nKey; + u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; - if( zPath[0]==0 ) return pRoot; - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + u8 x; + + if( zPath[0]==0 ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ + n = jsonbPayloadSize(pParse, iRoot, &sz); + sz += n; + if( pParse->eEdit==JEDIT_DEL ){ + if( iLabel>0 ){ + sz += iRoot - iLabel; + iRoot = iLabel; + } + jsonBlobEdit(pParse, iRoot, sz, 0, 0); + }else if( pParse->eEdit==JEDIT_INS ){ + /* Already exists, so json_insert() is a no-op */ + }else{ + /* json_set() or json_replace() */ + jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); + } + } + pParse->iLabel = iLabel; + return iRoot; + } if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; + int rawKey = 1; + x = pParse->aBlob[iRoot]; zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; @@ -193148,302 +209601,851 @@ static JsonNode *jsonLookupStep( if( zPath[i] ){ i++; }else{ - *pzErr = zPath; - return 0; + return JSON_LOOKUP_PATHERROR; } + testcase( nKey==0 ); + rawKey = memchr(zKey, '\\', nKey)==0; }else{ zKey = zPath; for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; - } - if( nKey==0 ){ - *pzErr = zPath; - return 0; - } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); - } - j++; - j += jsonNodeSize(&pRoot[j]); + if( nKey==0 ){ + return JSON_LOOKUP_PATHERROR; + } + } + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; /* j is the index of a label */ + iEnd = j+sz; + while( jaBlob[j] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + k = j+n; /* k is the index of the label text */ + if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; + zLabel = (const char*)&pParse->aBlob[k]; + rawLabel = x==JSONB_TEXT || x==JSONB_TEXTRAW; + if( jsonLabelCompare(zKey, nKey, rawKey, zLabel, sz, rawLabel) ){ + u32 v = k+sz; /* v is the index of the value */ + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, v, &sz); + if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; + assert( j>0 ); + rc = jsonLookupStep(pParse, v, &zPath[i], j); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( pParse->eEdit>=JEDIT_INS ){ + u32 nIns; /* Total bytes to insert (label+value) */ + JsonParse v; /* BLOB encoding of the value to be inserted */ + JsonParse ix; /* Header of the label to be inserted */ + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + memset(&ix, 0, sizeof(ix)); + ix.db = pParse->db; + jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); + pParse->oom |= ix.oom; + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) + ){ + assert( !pParse->oom ); + nIns = ix.nBlob + nKey + v.nBlob; + jsonBlobEdit(pParse, j, 0, 0, nIns); + if( !pParse->oom ){ + assert( pParse->aBlob!=0 ); /* Because pParse->oom!=0 */ + assert( ix.aBlob!=0 ); /* Because pPasre->oom!=0 */ + memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); + k = j + ix.nBlob; + memcpy(&pParse->aBlob[k], zKey, nKey); + k += nKey; + memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); + if( ALWAYS(pParse->delta) ) jsonAfterEditSizeAdjust(pParse, iRoot); + } + } + jsonParseReset(&v); + jsonParseReset(&ix); + return rc; } }else if( zPath[0]=='[' ){ - i = 0; - j = 1; - while( safe_isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; } - if( j<2 || zPath[j]!=']' ){ + if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - assert( pBase->eU==2 ); - iBase += pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; - if( zPath[2]=='-' && safe_isdigit(zPath[3]) ){ - unsigned int x = 0; - j = 3; + k = jsonbArrayCount(pParse, iRoot); + i = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int nn = 0; + i = 3; do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( safe_isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; + nn = nn*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( nn>k ) return JSON_LOOKUP_NOTFOUND; + k -= nn; } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; + if( zPath[i]!=']' ){ + return JSON_LOOKUP_PATHERROR; } }else{ - *pzErr = zPath; - return 0; + return JSON_LOOKUP_PATHERROR; } } - if( pRoot->eType!=JSON_ARRAY ) return 0; - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + j = iRoot+n; + iEnd = j+sz; + while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + k--; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( pParse->eEdit>=JEDIT_INS ){ + JsonParse v; + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, v.nBlob) + ){ + assert( !pParse->oom ); + jsonBlobEdit(pParse, j, 0, v.aBlob, v.nBlob); } - return pNode; + jsonParseReset(&v); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } }else{ - *pzErr = zPath; + return JSON_LOOKUP_PATHERROR; } - return 0; + return JSON_LOOKUP_NOTFOUND; } /* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. */ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob ){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; - } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; - } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); + JsonParse x; + JsonString s; + + if( NEVER(aBlob==0) ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonStringInit(&s, ctx); + jsonTranslateBlobToText(&x, 0, &s); + jsonReturnString(&s, 0, 0); } + /* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. */ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx, /* Return value for this function */ + int textOnly /* return text JSON. Disregard user-data */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x; + if( sz==0 ) goto returnfromblob_malformed; + x = (char)pParse->aBlob[i+n]; + if( x=='-' ){ + if( sz<2 ) goto returnfromblob_malformed; + n++; + sz--; + bNeg = 1; + } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc==0 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else if( rc==1 ){ + goto returnfromblob_malformed; + }else{ + if( bNeg ){ n--; sz++; } + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + if( sz==0 ) goto returnfromblob_malformed; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + if( rc<=0 ) goto returnfromblob_malformed; + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3DbMallocRaw(db, nOut+1); + if( zOut==0 ) goto returnfromblob_oom; + for(iIn=iOut=0; iIn=2 ); + zOut[iOut++] = (char)(0xc0 | (v>>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v<0x10000 ){ + assert( szEscape>=3 ); + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v==JSON_INVALID_CHAR ){ + /* Silently ignore illegal unicode */ + }else{ + assert( szEscape>=4 ); + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + iIn += szEscape - 1; + }else{ + zOut[iOut++] = c; + } + } /* end for() */ + assert( iOut<=nOut ); + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, SQLITE_DYNAMIC); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + int flags = textOnly ? 0 : SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } + break; + } + default: { + goto returnfromblob_malformed; + } + } + return; + +returnfromblob_oom: + sqlite3_result_error_nomem(pCtx); + return; + +returnfromblob_malformed: + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; } /* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. +** pArg is a function argument that might be an SQL value or a JSON +** value. Figure out what it is and encode it as a JSONB blob. +** Return the results in pParse. ** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. +** pParse is uninitialized upon entry. This routine will handle the +** initialization of pParse. The result will be contained in +** pParse->aBlob and pParse->nBlob. pParse->aBlob might be dynamically +** allocated (if pParse->nBlobAlloc is greater than zero) in which case +** the caller is responsible for freeing the space allocated to pParse->aBlob +** when it has finished with it. Or pParse->aBlob might be a static string +** or a value obtained from sqlite3_value_blob(pArg). ** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. +** If the argument is a BLOB that is clearly not a JSONB, then this +** function might set an error message in ctx and return non-zero. +** It might also set an error message and return non-zero on an OOM error. */ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ -){ - const char *zErr = 0; - JsonNode *pNode = 0; - char *zMsg; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; +static int jsonFunctionArgToBlob( + sqlite3_context *ctx, + sqlite3_value *pArg, + JsonParse *pParse +){ + int eType = sqlite3_value_type(pArg); + static u8 aNull[] = { 0x00 }; + memset(pParse, 0, sizeof(pParse[0])); + pParse->db = sqlite3_context_db_handle(ctx); + switch( eType ){ + default: { + pParse->aBlob = aNull; + pParse->nBlob = 1; + return 0; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(pArg) ){ + pParse->aBlob = (u8*)sqlite3_value_blob(pArg); + pParse->nBlob = sqlite3_value_bytes(pArg); + }else{ + sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); + return 1; + } + break; + } + case SQLITE_TEXT: { + const char *zJson = (const char*)sqlite3_value_text(pArg); + int nJson = sqlite3_value_bytes(pArg); + if( zJson==0 ) return 1; + if( sqlite3_value_subtype(pArg)==JSON_SUBTYPE ){ + pParse->zJson = (char*)zJson; + pParse->nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3DbFree(pParse->db, pParse->aBlob); + memset(pParse, 0, sizeof(pParse[0])); + return 1; + } + }else{ + jsonBlobAppendNode(pParse, JSONB_TEXTRAW, nJson, zJson); + } + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + if( NEVER(sqlite3IsNaN(r)) ){ + jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); + }else{ + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + if( z[0]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else if( z[0]=='-' && z[1]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, n, z); + } + } + break; + } + case SQLITE_INTEGER: { + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + jsonBlobAppendNode(pParse, JSONB_INT, n, z); + break; + } + } + if( pParse->oom ){ + sqlite3_result_error_nomem(ctx); + return 1; + }else{ + return 0; } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; +} -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); +/* +** Generate a bad path error. +** +** If ctx is not NULL then push the error message into ctx and return NULL. +** If ctx is NULL, then return the text of the error message. +*/ +static char *jsonBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ +){ + char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + if( ctx==0 ) return zMsg; if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_result_error(ctx, zMsg, -1); sqlite3_free(zMsg); }else{ - sqlite3_result_error_nomem(pCtx); + sqlite3_result_error_nomem(ctx); } return 0; } +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments come in parse where each pair contains a JSON path and +** content to insert or set at that patch. Do the updates +** and return the result. +** +** The specific operation is determined by eEdit, which can be one +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +*/ +static void jsonInsertIntoBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv, + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ +){ + int i; + u32 rc = 0; + const char *zPath = 0; + int flgs; + JsonParse *p; + JsonParse ax; + + assert( (argc&1)==1 ); + flgs = argc==1 ? 0 : JSON_EDITABLE; + p = jsonParseFuncArg(ctx, argv[0], flgs); + if( p==0 ) return; + for(i=1; inBlob, ax.aBlob, ax.nBlob); + } + rc = 0; + }else{ + p->eEdit = eEdit; + p->nIns = ax.nBlob; + p->aIns = ax.aBlob; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + } + jsonParseReset(&ax); + if( rc==JSON_LOOKUP_NOTFOUND ) continue; + if( JSON_LOOKUP_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; + } + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +jsonInsertIntoBlob_patherror: + jsonParseFree(p); + if( rc==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + jsonBadPathError(ctx, zPath); + } + return; +} /* -** Report the wrong number of arguments for json_insert(), json_replace() -** or json_set(). +** If pArg is a blob that seems like a JSONB blob, then initialize +** p to point to that JSONB and return TRUE. If pArg does not seem like +** a JSONB blob, then return FALSE; +** +** This routine is only called if it is already known that pArg is a +** blob. The only open question is whether or not the blob appears +** to be a JSONB blob. */ -static void jsonWrongNumArgs( - sqlite3_context *pCtx, - const char *zFuncName -){ - char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", - zFuncName); - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); +static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ + u32 n, sz = 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob==0 ){ + p->aBlob = 0; + return 0; + } + if( NEVER(p->aBlob==0) ){ + return 0; + } + if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + && (n = jsonbPayloadSize(p, 0, &sz))>0 + && sz+n==p->nBlob + && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + ){ + return 1; + } + p->aBlob = 0; + p->nBlob = 0; + return 0; } /* -** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, +** from the SQL function argument pArg. Return a pointer to the new +** JsonParse object. +** +** Ownership of the new JsonParse object is passed to the caller. The +** caller should invoke jsonParseFree() on the return value when it +** has finished using it. +** +** If any errors are detected, an appropriate error messages is set +** using sqlite3_result_error() or the equivalent and this routine +** returns NULL. This routine also returns NULL if the pArg argument +** is an SQL NULL value, but no error message is set in that case. This +** is so that SQL functions that are given NULL arguments will return +** a NULL value. */ -static void jsonRemoveAllNulls(JsonNode *pNode){ - int i, n; - assert( pNode->eType==JSON_OBJECT ); - n = pNode->n; - for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ - switch( pNode[i].eType ){ - case JSON_NULL: - pNode[i].jnFlags |= JNODE_REMOVE; - break; - case JSON_OBJECT: - jsonRemoveAllNulls(&pNode[i]); - break; +static JsonParse *jsonParseFuncArg( + sqlite3_context *ctx, + sqlite3_value *pArg, + u32 flgs +){ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + JsonParse *pFromCache = 0; /* Value taken from cache */ + sqlite3 *db; /* The database connection */ + + assert( ctx!=0 ); + eType = sqlite3_value_type(pArg); + if( eType==SQLITE_NULL ){ + return 0; + } + pFromCache = jsonCacheSearch(ctx, pArg); + if( pFromCache ){ + pFromCache->nJPRef++; + if( (flgs & JSON_EDITABLE)==0 ){ + return pFromCache; } } -} + db = sqlite3_context_db_handle(ctx); +rebuild_from_cache: + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) goto json_pfa_oom; + memset(p, 0, sizeof(*p)); + p->db = db; + p->nJPRef = 1; + if( pFromCache!=0 ){ + u32 nBlob = pFromCache->nBlob; + p->aBlob = sqlite3DbMallocRaw(db, nBlob); + if( p->aBlob==0 ) goto json_pfa_oom; + memcpy(p->aBlob, pFromCache->aBlob, nBlob); + p->nBlobAlloc = p->nBlob = nBlob; + p->hasNonstd = pFromCache->hasNonstd; + jsonParseFree(pFromCache); + return p; + } + if( eType==SQLITE_BLOB ){ + if( jsonArgIsJsonb(pArg,p) ){ + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; + } + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, espeically when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ + } + p->zJson = (char*)sqlite3_value_text(pArg); + p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; + if( p->nJson==0 ) goto json_pfa_malformed; + assert( p->zJson!=0 ); + if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + return 0; + } + }else{ + int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); + int rc; + if( !isRCStr ){ + char *zNew = sqlite3RCStrNew( p->nJson ); + if( zNew==0 ) goto json_pfa_oom; + memcpy(zNew, p->zJson, p->nJson); + p->zJson = zNew; + p->zJson[p->nJson] = 0; + }else{ + sqlite3RCStrRef(p->zJson); + } + p->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, p); + if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + if( flgs & JSON_EDITABLE ){ + pFromCache = p; + p = 0; + goto rebuild_from_cache; + } + } + return p; +json_pfa_malformed: + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + sqlite3_result_error(ctx, "malformed JSON", -1); + return 0; + } -/**************************************************************************** -** SQL functions used for testing and debugging -****************************************************************************/ +json_pfa_oom: + jsonParseFree(pFromCache); + jsonParseFree(p); + sqlite3_result_error_nomem(ctx); + return 0; +} -#ifdef SQLITE_DEBUG /* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. +** Make the return value of a JSON function either the raw JSONB blob +** or make it JSON text, depending on whether the JSON_BLOB flag is +** set on the function. */ -static void jsonParseFunc( +static void jsonReturnParse( sqlite3_context *ctx, - int argc, - sqlite3_value **argv + JsonParse *p ){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; - - assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; ioom ){ + sqlite3_result_error_nomem(ctx); + return; + } + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + if( p->nBlobAlloc>0 && !p->bReadOnly ){ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_DYNAMIC); + p->nBlobAlloc = 0; }else{ - zType = jsonType[x.aNode[i].eType]; + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_TRANSIENT); } - jsonPrintf(100, &s,"node %3u: %7s n=%-4d up=%-4d", - i, zType, x.aNode[i].n, x.aUp[i]); - assert( x.aNode[i].eU==0 || x.aNode[i].eU==1 ); - if( x.aNode[i].u.zJContent!=0 ){ - assert( x.aNode[i].eU==1 ); - jsonAppendRaw(&s, " ", 1); - jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); - }else{ - assert( x.aNode[i].eU==0 ); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + p->delta = 0; + jsonTranslateBlobToText(p, 0, &s); + jsonReturnString(&s, p, ctx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } +} + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#if SQLITE_DEBUG +/* +** Decode JSONB bytes in aBlob[] starting at iStart through but not +** including iEnd. Indent the +** content by nIndent spaces. +*/ +static void jsonDebugPrintBlob( + JsonParse *pParse, /* JSON content */ + u32 iStart, /* Start rendering here */ + u32 iEnd, /* Do not render this byte or any byte after this one */ + int nIndent, /* Indent by this many spaces */ + sqlite3_str *pOut /* Generate output into this sqlite3_str object */ +){ + while( iStartaBlob[iStart] & 0x0f; + u32 savedNBlob = pParse->nBlob; + sqlite3_str_appendf(pOut, "%5d:%*s", iStart, nIndent, ""); + if( pParse->nBlobAlloc>pParse->nBlob ){ + pParse->nBlob = pParse->nBlobAlloc; + } + nn = n = jsonbPayloadSize(pParse, iStart, &sz); + if( nn==0 ) nn = 1; + if( sz>0 && xaBlob[iStart+i]); + } + if( n==0 ){ + sqlite3_str_appendf(pOut, " ERROR invalid node size\n"); + iStart = n==0 ? iStart+1 : iEnd; + continue; + } + pParse->nBlob = savedNBlob; + if( iStart+n+sz>iEnd ){ + iEnd = iStart+n+sz; + if( iEnd>pParse->nBlob ){ + if( pParse->nBlobAlloc>0 && iEnd>pParse->nBlobAlloc ){ + iEnd = pParse->nBlobAlloc; + }else{ + iEnd = pParse->nBlob; + } + } } - jsonAppendRaw(&s, "\n", 1); + sqlite3_str_appendall(pOut," <-- "); + switch( x ){ + case JSONB_NULL: sqlite3_str_appendall(pOut,"null"); break; + case JSONB_TRUE: sqlite3_str_appendall(pOut,"true"); break; + case JSONB_FALSE: sqlite3_str_appendall(pOut,"false"); break; + case JSONB_INT: sqlite3_str_appendall(pOut,"int"); break; + case JSONB_INT5: sqlite3_str_appendall(pOut,"int5"); break; + case JSONB_FLOAT: sqlite3_str_appendall(pOut,"float"); break; + case JSONB_FLOAT5: sqlite3_str_appendall(pOut,"float5"); break; + case JSONB_TEXT: sqlite3_str_appendall(pOut,"text"); break; + case JSONB_TEXTJ: sqlite3_str_appendall(pOut,"textj"); break; + case JSONB_TEXT5: sqlite3_str_appendall(pOut,"text5"); break; + case JSONB_TEXTRAW: sqlite3_str_appendall(pOut,"textraw"); break; + case JSONB_ARRAY: { + sqlite3_str_appendf(pOut,"array, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + case JSONB_OBJECT: { + sqlite3_str_appendf(pOut, "object, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + default: { + sqlite3_str_appendall(pOut, "ERROR: unknown node type\n"); + showContent = 0; + break; + } + } + if( showContent ){ + if( sz==0 && x<=JSONB_FALSE ){ + sqlite3_str_append(pOut, "\n", 1); + }else{ + u32 j; + sqlite3_str_appendall(pOut, ": \""); + for(j=iStart+n; jaBlob[j]; + if( c<0x20 || c>=0x7f ) c = '.'; + sqlite3_str_append(pOut, (char*)&c, 1); + } + sqlite3_str_append(pOut, "\"\n", 2); + } + } + iStart += n + sz; } - jsonParseReset(&x); - jsonResult(&s); } +static void jsonShowParse(JsonParse *pParse){ + sqlite3_str out; + char zBuf[1000]; + if( pParse==0 ){ + printf("NULL pointer\n"); + return; + }else{ + printf("nBlobAlloc = %u\n", pParse->nBlobAlloc); + printf("nBlob = %u\n", pParse->nBlob); + printf("delta = %d\n", pParse->delta); + if( pParse->nBlob==0 ) return; + printf("content (bytes 0..%u):\n", pParse->nBlob-1); + } + sqlite3StrAccumInit(&out, 0, zBuf, sizeof(zBuf), 1000000); + jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0, &out); + printf("%s", sqlite3_str_value(&out)); + sqlite3_str_reset(&out); +} +#endif /* SQLITE_DEBUG */ +#ifdef SQLITE_DEBUG /* -** The json_test1(JSON) function return true (1) if the input is JSON -** text generated by another json function. It returns (0) if the input -** is not known to be JSON. +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseFuncArg(). Return text that is a +** human-readable dump of the binary JSONB for the input parameter. */ -static void jsonTest1Func( +static void jsonParseFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - UNUSED_PARAM(argc); - sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); + JsonParse *p; /* The parse */ + sqlite3_str out; + + assert( argc>=1 ); + sqlite3StrAccumInit(&out, 0, 0, 0, 1000000); + p = jsonParseFuncArg(ctx, argv[0], 0); + if( p==0 ) return; + if( argc==1 ){ + jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); + }else{ + jsonShowParse(p); + } + jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -193452,7 +210454,7 @@ static void jsonTest1Func( ****************************************************************************/ /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. @@ -193463,11 +210465,11 @@ static void jsonQuoteFunc( sqlite3_value **argv ){ JsonString jx; - UNUSED_PARAM(argc); + UNUSED_PARAMETER(argc); - jsonInit(&jx, ctx); - jsonAppendValue(&jx, argv[0]); - jsonResult(&jx); + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx, 0, 0); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -193484,18 +210486,17 @@ static void jsonArrayFunc( int i; JsonString jx; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=0; inNode ); if( argc==2 ){ const char *zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); + if( zPath==0 ){ + jsonParseFree(p); + return; + } + i = jsonLookupStep(p, 0, zPath[0]=='$' ? zPath+1 : "@", 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + eErr = 1; + i = 0; + } }else{ - pNode = p->aNode; - } - if( pNode==0 ){ - return; + i = 0; } - if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); - } + if( (p->aBlob[i] & 0x0f)==JSONB_ARRAY ){ + cnt = jsonbArrayCount(p, i); } - sqlite3_result_int64(ctx, n); + if( !eErr ) sqlite3_result_int64(ctx, cnt); + jsonParseFree(p); +} + +/* True if the string is all alphanumerics and underscores */ +static int jsonAllAlphanum(const char *z, int n){ + int i; + for(i=0; i"(JSON,PATH) +** "->>"(JSON,PATH) +** +** Return the element described by PATH. Return NULL if that PATH element +** is not found. +** +** If JSON_JSON is set or if more that one PATH argument is supplied then +** always return a JSON representation of the result. If JSON_SQL is set, +** then always return an SQL representation of the result. If neither flag +** is present and argc==2, then return JSON for objects and arrays and SQL +** for all other values. ** -** Return the element described by PATH. Return NULL if there is no -** PATH element. If there are multiple PATHs, then return a JSON array -** with the result from each path. Throw an error if the JSON or any PATH -** is malformed. +** When multiple PATH arguments are supplied, the result is a JSON array +** containing the result of each PATH. +** +** Abbreviated JSON path expressions are allows if JSON_ABPATH, for +** compatibility with PG. */ static void jsonExtractFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonParse *p; /* The parse */ - JsonNode *pNode; - const char *zPath; - JsonString jx; - int i; + JsonParse *p = 0; /* The parse */ + int flags; /* Flags associated with the function */ + int i; /* Loop counter */ + JsonString jx; /* String for array result */ if( argc<2 ) return; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '['); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + jsonStringInit(&jx, ctx); + if( argc>2 ){ + jsonAppendChar(&jx, '['); + } for(i=1; inErr ) break; - if( argc>2 ){ - jsonAppendSeparator(&jx); - if( pNode ){ - jsonRenderNode(pNode, &jx, 0); + /* With a single PATH argument */ + const char *zPath = (const char*)sqlite3_value_text(argv[i]); + int nPath; + u32 j; + if( zPath==0 ) goto json_extract_error; + nPath = sqlite3Strlen30(zPath); + if( zPath[0]=='$' ){ + j = jsonLookupStep(p, 0, zPath+1, 0); + }else if( (flags & JSON_ABPATH) ){ + /* The -> and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + */ + jsonStringInit(&jx, ctx); + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ + jsonAppendRawNZ(&jx, "[", 1); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "]", 2); + }else if( jsonAllAlphanum(zPath, nPath) ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, nPath); + }else if( zPath[0]=='[' && nPath>=3 && zPath[nPath-1]==']' ){ + jsonAppendRaw(&jx, zPath, nPath); + }else{ + jsonAppendRawNZ(&jx, ".\"", 2); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "\"", 1); + } + jsonStringTerminate(&jx); + j = jsonLookupStep(p, 0, jx.zBuf, 0); + jsonStringReset(&jx); + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; + } + if( jnBlob ){ + if( argc==2 ){ + if( flags & JSON_JSON ){ + jsonStringInit(&jx, ctx); + jsonTranslateBlobToText(p, j, &jx); + jsonReturnString(&jx, 0, 0); + jsonStringReset(&jx); + assert( (flags & JSON_BLOB)==0 ); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + }else{ + jsonReturnFromBlob(p, j, ctx, 0); + if( (flags & (JSON_SQL|JSON_BLOB))==0 + && (p->aBlob[j]&0x0f)>=JSONB_ARRAY + ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + } + }else{ + jsonAppendSeparator(&jx); + jsonTranslateBlobToText(p, j, &jx); + } + }else if( j==JSON_LOOKUP_NOTFOUND ){ + if( argc==2 ){ + goto json_extract_error; /* Return NULL if not found */ }else{ - jsonAppendRaw(&jx, "null", 4); + jsonAppendSeparator(&jx); + jsonAppendRawNZ(&jx, "null", 4); } - }else if( pNode ){ - jsonReturn(pNode, ctx, 0); + }else if( j==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + goto json_extract_error; + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; } } - if( argc>2 && i==argc ){ + if( argc>2 ){ jsonAppendChar(&jx, ']'); - jsonResult(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); + jsonReturnString(&jx, 0, 0); + if( (flags & JSON_BLOB)==0 ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } } - jsonReset(&jx); +json_extract_error: + jsonStringReset(&jx); + jsonParseFree(p); + return; } -/* This is the RFC 7396 MergePatch algorithm. -*/ -static JsonNode *jsonMergePatch( - JsonParse *pParse, /* The JSON parser that contains the TARGET */ - u32 iTarget, /* Node of the TARGET in pParse */ - JsonNode *pPatch /* The PATCH */ -){ - u32 i, j; - u32 iRoot; - JsonNode *pTarget; - if( pPatch->eType!=JSON_OBJECT ){ - return pPatch; - } - assert( iTarget>=0 && iTargetnNode ); - pTarget = &pParse->aNode[iTarget]; - assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); - if( pTarget->eType!=JSON_OBJECT ){ - jsonRemoveAllNulls(pPatch); - return pPatch; - } - iRoot = iTarget; - for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ - u32 nKey; - const char *zKey; - assert( pPatch[i].eType==JSON_STRING ); - assert( pPatch[i].jnFlags & JNODE_LABEL ); - assert( pPatch[i].eU==1 ); - nKey = pPatch[i].n; - zKey = pPatch[i].u.zJContent; - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ - assert( pTarget[j].eType==JSON_STRING ); - assert( pTarget[j].jnFlags & JNODE_LABEL ); - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; - if( pPatch[i+1].eType==JSON_NULL ){ - pTarget[j+1].jnFlags |= JNODE_REMOVE; - }else{ - JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); - if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - assert( pTarget[j+1].eU==0 - || pTarget[j+1].eU==1 - || pTarget[j+1].eU==2 ); - testcase( pTarget[j+1].eU==1 ); - testcase( pTarget[j+1].eU==2 ); - VVA( pTarget[j+1].eU = 5 ); - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; - } - } - break; +/* +** Return codes for jsonMergePatch() +*/ +#define JSON_MERGE_OK 0 /* Success */ +#define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ +#define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ +#define JSON_MERGE_OOM 3 /* Out-of-memory condition */ + +/* +** RFC-7396 MergePatch for two JSONB blobs. +** +** pTarget is the target. pPatch is the patch. The target is updated +** in place. The patch is read-only. +** +** The original RFC-7396 algorithm is this: +** +** define MergePatch(Target, Patch): +** if Patch is an Object: +** if Target is not an Object: +** Target = {} # Ignore the contents and set it to an empty Object +** for each Name/Value pair in Patch: +** if Value is null: +** if Name exists in Target: +** remove the Name/Value pair from Target +** else: +** Target[Name] = MergePatch(Target[Name], Value) +** return Target +** else: +** return Patch +** +** Here is an equivalent algorithm restructured to show the actual +** implementation: +** +** 01 define MergePatch(Target, Patch): +** 02 if Patch is not an Object: +** 03 return Patch +** 04 else: // if Patch is an Object +** 05 if Target is not an Object: +** 06 Target = {} +** 07 for each Name/Value pair in Patch: +** 08 if Name exists in Target: +** 09 if Value is null: +** 10 remove the Name/Value pair from Target +** 11 else +** 12 Target[name] = MergePatch(Target[Name], Value) +** 13 else if Value is not NULL: +** 14 if Value is not an Object: +** 15 Target[name] = Value +** 16 else: +** 17 Target[name] = MergePatch('{}',value) +** 18 return Target +** | +** ^---- Line numbers referenced in comments in the implementation +*/ +static int jsonMergePatch( + JsonParse *pTarget, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ + const JsonParse *pPatch, /* The PATCH */ + u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ +){ + u8 x; /* Type of a single node */ + u32 n, sz=0; /* Return values from jsonbPayloadSize() */ + u32 iTCursor; /* Cursor position while scanning the target object */ + u32 iTStart; /* First label in the target object */ + u32 iTEndBE; /* Original first byte past end of target, before edit */ + u32 iTEnd; /* Current first byte past end of target */ + u8 eTLabel; /* Node type of the target label */ + u32 iTLabel = 0; /* Index of the label */ + u32 nTLabel = 0; /* Header size in bytes for the target label */ + u32 szTLabel = 0; /* Size of the target label payload */ + u32 iTValue = 0; /* Index of the target value */ + u32 nTValue = 0; /* Header size of the target value */ + u32 szTValue = 0; /* Payload size for the target value */ + + u32 iPCursor; /* Cursor position while scanning the patch */ + u32 iPEnd; /* First byte past the end of the patch */ + u8 ePLabel; /* Node type of the patch label */ + u32 iPLabel; /* Start of patch label */ + u32 nPLabel; /* Size of header on the patch label */ + u32 szPLabel; /* Payload size of the patch label */ + u32 iPValue; /* Start of patch value */ + u32 nPValue; /* Header size for the patch value */ + u32 szPValue; /* Payload size of the patch value */ + + assert( iTarget>=0 && iTargetnBlob ); + assert( iPatch>=0 && iPatchnBlob ); + x = pPatch->aBlob[iPatch] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 02 */ + u32 szPatch; /* Total size of the patch, header+payload */ + u32 szTarget; /* Total size of the target, header+payload */ + n = jsonbPayloadSize(pPatch, iPatch, &sz); + szPatch = n+sz; + sz = 0; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + szTarget = n+sz; + jsonBlobEdit(pTarget, iTarget, szTarget, pPatch->aBlob+iPatch, szPatch); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; /* Line 03 */ + } + x = pTarget->aBlob[iTarget] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ + n = jsonbPayloadSize(pTarget, iTarget, &sz); + jsonBlobEdit(pTarget, iTarget+n, sz, 0, 0); + x = pTarget->aBlob[iTarget]; + pTarget->aBlob[iTarget] = (x & 0xf0) | JSONB_OBJECT; + } + n = jsonbPayloadSize(pPatch, iPatch, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADPATCH; + iPCursor = iPatch+n; + iPEnd = iPCursor+sz; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADTARGET; + iTStart = iTarget+n; + iTEndBE = iTStart+sz; + + while( iPCursoraBlob[iPCursor] & 0x0f; + if( ePLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADPATCH; + } + nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); + if( nPLabel==0 ) return JSON_MERGE_BADPATCH; + iPValue = iPCursor + nPLabel + szPLabel; + if( iPValue>=iPEnd ) return JSON_MERGE_BADPATCH; + nPValue = jsonbPayloadSize(pPatch, iPValue, &szPValue); + if( nPValue==0 ) return JSON_MERGE_BADPATCH; + iPCursor = iPValue + nPValue + szPValue; + if( iPCursor>iPEnd ) return JSON_MERGE_BADPATCH; + + iTCursor = iTStart; + iTEnd = iTEndBE + pTarget->delta; + while( iTCursoraBlob[iTCursor] & 0x0f; + if( eTLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADTARGET; + } + nTLabel = jsonbPayloadSize(pTarget, iTCursor, &szTLabel); + if( nTLabel==0 ) return JSON_MERGE_BADTARGET; + iTValue = iTLabel + nTLabel + szTLabel; + if( iTValue>=iTEnd ) return JSON_MERGE_BADTARGET; + nTValue = jsonbPayloadSize(pTarget, iTValue, &szTValue); + if( nTValue==0 ) return JSON_MERGE_BADTARGET; + if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; + isEqual = jsonLabelCompare( + (const char*)&pPatch->aBlob[iPLabel+nPLabel], + szPLabel, + (ePLabel==JSONB_TEXT || ePLabel==JSONB_TEXTRAW), + (const char*)&pTarget->aBlob[iTLabel+nTLabel], + szTLabel, + (eTLabel==JSONB_TEXT || eTLabel==JSONB_TEXTRAW)); + if( isEqual ) break; + iTCursor = iTValue + nTValue + szTValue; + } + x = pPatch->aBlob[iPValue] & 0x0f; + if( iTCursoroom) ) return JSON_MERGE_OOM; + }else{ + /* Algorithm line 12 */ + int rc, savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; + } + }else if( x>0 ){ /* Algorithm line 13 */ + /* No match and patch value is not NULL */ + u32 szNew = szPLabel+nPLabel; + if( (pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ /* Line 14 */ + jsonBlobEdit(pTarget, iTEnd, 0, 0, szPValue+nPValue+szNew); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + memcpy(&pTarget->aBlob[iTEnd+szNew], + &pPatch->aBlob[iPValue], szPValue+nPValue); + }else{ + int rc, savedDelta; + jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + pTarget->aBlob[iTEnd+szNew] = 0x00; + savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; } } - if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); - testcase( pParse->aNode[iRoot].eU==2 ); - pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; - VVA( pParse->aNode[iRoot].eU = 2 ); - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; - iRoot = iStart; - assert( pParse->aNode[iPatch].eU==0 ); - VVA( pParse->aNode[iPatch].eU = 5 ); - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; - } } - return pTarget; + if( pTarget->delta ) jsonAfterEditSizeAdjust(pTarget, iTarget); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; } + /* ** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON ** object that is the result of running the RFC 7396 MergePatch() algorithm @@ -193670,25 +210887,27 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ - JsonNode *pResult; /* The result of the merge */ + JsonParse *pTarget; /* The TARGET */ + JsonParse *pPatch; /* The PATCH */ + int rc; /* Result code */ - UNUSED_PARAM(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult ){ - jsonReturnJson(pResult, ctx, 0); - }else{ - sqlite3_result_error_nomem(ctx); + UNUSED_PARAMETER(argc); + assert( argc==2 ); + pTarget = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + if( pTarget==0 ) return; + pPatch = jsonParseFuncArg(ctx, argv[1], 0); + if( pPatch ){ + rc = jsonMergePatch(pTarget, 0, pPatch, 0); + if( rc==JSON_MERGE_OK ){ + jsonReturnParse(ctx, pTarget); + }else if( rc==JSON_MERGE_OOM ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + jsonParseFree(pPatch); } - jsonParseReset(&x); - jsonParseReset(&y); + jsonParseFree(pTarget); } @@ -193712,23 +210931,23 @@ static void jsonObjectFunc( "of arguments", -1); return; } - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '{'); for(i=0; i1 ? JSON_EDITABLE : 0); + if( p==0 ) return; + for(i=1; ijnFlags |= JNODE_REMOVE; - } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); + if( zPath==0 ){ + goto json_remove_done; + } + if( zPath[0]!='$' ){ + goto json_remove_patherror; + } + if( zPath[1]==0 ){ + /* json_remove(j,'$') returns NULL */ + goto json_remove_done; + } + p->eEdit = JEDIT_DEL; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(rc) ){ + if( rc==JSON_LOOKUP_NOTFOUND ){ + continue; /* No-op */ + }else if( rc==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_remove_done; + } } -remove_done: - jsonParseReset(&x); + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +json_remove_patherror: + jsonBadPathError(ctx, zPath); + +json_remove_done: + jsonParseFree(p); + return; } /* @@ -193777,40 +211020,15 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - if( argc<1 ) return; if( (argc&1)==0 ) { jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; - if( pNode ){ - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); - json_testcase( pNode->eU!=0 && pNode->eU!=1 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - VVA( pNode->eU = 4 ); - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -replace_err: - jsonParseReset(&x); + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); } + /* ** json_set(JSON, PATH, VALUE, ...) ** @@ -193828,53 +211046,24 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - int bApnd; - int bIsSet = *(int*)sqlite3_user_data(ctx); + + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; if( (argc&1)==0 ) { jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ - sqlite3_result_error_nomem(ctx); - goto jsonSetDone; - }else if( x.nErr ){ - goto jsonSetDone; - }else if( pNode && (bApnd || bIsSet) ){ - json_testcase( pNode->eU!=0 && pNode->eU!=1 && pNode->eU!=4 ); - assert( pNode->eU!=3 || pNode->eU!=5 ); - VVA( pNode->eU = 4 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -jsonSetDone: - jsonParseReset(&x); + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); } /* ** json_type(JSON) ** json_type(JSON, PATH) ** -** Return the top-level "type" of a JSON string. Throw an error if -** either the JSON or PATH inputs are not well-formed. +** Return the top-level "type" of a JSON string. json_type() raises an +** error if either the JSON or PATH inputs are not well-formed. */ static void jsonTypeFunc( sqlite3_context *ctx, @@ -193882,27 +211071,127 @@ static void jsonTypeFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - const char *zPath; - JsonNode *pNode; + const char *zPath = 0; + u32 i; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); + if( zPath==0 ) goto json_type_done; + if( zPath[0]!='$' ){ + jsonBadPathError(ctx, zPath); + goto json_type_done; + } + i = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_type_done; + } }else{ - pNode = p->aNode; + i = 0; } - if( pNode ){ - sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + sqlite3_result_text(ctx, jsonbType[p->aBlob[i]&0x0f], -1, SQLITE_STATIC); +json_type_done: + jsonParseFree(p); +} + +/* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); } /* ** json_valid(JSON) -** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. +** json_valid(JSON, FLAGS) +** +** Check the JSON argument to see if it is well-formed. The FLAGS argument +** encodes the various constraints on what is meant by "well-formed": +** +** 0x01 Canonical RFC-8259 JSON text +** 0x02 JSON text with optional JSON-5 extensions +** 0x04 Superficially appears to be JSONB +** 0x08 Strictly well-formed JSONB +** +** If the FLAGS argument is omitted, it defaults to 1. Useful values for +** FLAGS include: +** +** 1 Strict canonical JSON text +** 2 JSON text perhaps with JSON-5 extensions +** 4 Superficially appears to be JSONB +** 5 Canonical JSON text or superficial JSONB +** 6 JSON-5 text or superficial JSONB +** 8 Strict JSONB +** 9 Canonical JSON text or strict JSONB +** 10 JSON-5 text or strict JSONB +** +** Other flag combinations are redundant. For example, every canonical +** JSON text is also well-formed JSON-5 text, so FLAG values 2 and 3 +** are the same. Similarly, any input that passes a strict JSONB validation +** will also pass the superficial validation so 12 through 15 are the same +** as 8 through 11 respectively. +** +** This routine runs in linear time to validate text and when doing strict +** JSONB validation. Superficial JSONB validation is constant time, +** assuming the BLOB is already in memory. The performance advantage +** of superficial JSONB validation is why that option is provided. +** Application developers can choose to do fast superficial validation or +** slower strict validation, according to their specific needs. +** +** Only the lower four bits of the FLAGS argument are currently used. +** Higher bits are reserved for future expansion. To facilitate +** compatibility, the current implementation raises an error if any bit +** in FLAGS is set other than the lower four bits. +** +** The original circa 2015 implementation of the JSON routines in +** SQLite only supported canonical RFC-8259 JSON text and the json_valid() +** function only accepted one argument. That is why the default value +** for the FLAGS argument is 1, since FLAGS=1 causes this routine to only +** recognize canonical RFC-8259 JSON text as valid. The extra FLAGS +** argument was added when the JSON routines were extended to support +** JSON5-like extensions and binary JSONB stored in BLOBs. +** +** Return Values: +** +** * Raise an error if FLAGS is outside the range of 1 to 15. +** * Return NULL if the input is NULL +** * Return 1 if the input is well-formed. +** * Return 0 if the input is not well-formed. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -193910,11 +211199,127 @@ static void jsonValidFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - UNUSED_PARAM(argc); - p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0); + u8 flags = 1; + u8 res = 0; + if( argc==2 ){ + i64 f = sqlite3_value_int64(argv[1]); + if( f<1 || f>15 ){ + sqlite3_result_error(ctx, "FLAGS parameter to json_valid() must be" + " between 1 and 15", -1); + return; + } + flags = f & 0x0f; + } + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_NULL: { +#ifdef SQLITE_LEGACY_JSON_VALID + /* Incorrect legacy behavior was to return FALSE for a NULL input */ + sqlite3_result_int(ctx, 0); +#endif + return; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(argv[0]) ){ + if( flags & 0x04 ){ + /* Superficial checking only - accomplished by the + ** jsonFuncArgMightBeBinary() call above. */ + res = 1; + }else if( flags & 0x08 ){ + /* Strict checking. Check by translating BLOB->TEXT->BLOB. If + ** no errors occur, call that a "strict check". */ + JsonParse px; + u32 iErr; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + px.nBlob = sqlite3_value_bytes(argv[0]); + iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); + res = iErr==0; + } + break; + } + /* Fall through into interpreting the input as text. See note + ** above at tag-20240123-a. */ + /* no break */ deliberate_fall_through + } + default: { + JsonParse px; + if( (flags & 0x3)==0 ) break; + memset(&px, 0, sizeof(px)); + + p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); + if( p ){ + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( p->nErr ){ + /* no-op */ + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 ){ + res = 1; + } + jsonParseFree(p); + }else{ + sqlite3_result_error_nomem(ctx); + } + break; + } + } + sqlite3_result_int(ctx, res); } +/* +** json_error_position(JSON) +** +** If the argument is NULL, return NULL +** +** If the argument is BLOB, do a full validity check and return non-zero +** if the check fails. The return value is the approximate 1-based offset +** to the byte of the element that contains the first error. +** +** Otherwise interpret the argument is TEXT (even if it is numeric) and +** return the 1-based character position for where the parser first recognized +** that the input was not valid JSON, or return 0 if the input text looks +** ok. JSON-5 extensions are accepted. +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + i64 iErrPos = 0; /* Error position to be returned */ + JsonParse s; + + assert( argc==1 ); + UNUSED_PARAMETER(argc); + memset(&s, 0, sizeof(s)); + s.db = sqlite3_context_db_handle(ctx); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + s.aBlob = (u8*)sqlite3_value_blob(argv[0]); + s.nBlob = sqlite3_value_bytes(argv[0]); + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); + }else{ + s.zJson = (char*)sqlite3_value_text(argv[0]); + if( s.zJson==0 ) return; /* NULL input or OOM */ + s.nJson = sqlite3_value_bytes(argv[0]); + if( jsonConvertTextToBlob(&s,0) ){ + if( s.oom ){ + iErrPos = -1; + }else{ + /* Convert byte-offset s.iErr into a character offset */ + u32 k; + assert( s.zJson!=0 ); /* Because s.oom is false */ + for(k=0; kzBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '['); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - jsonAppendValue(pStr, argv[0]); + jsonAppendSqlValue(pStr, argv[0]); } } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); @@ -193990,12 +211406,12 @@ static void jsonGroupInverse( char *z; char c; JsonString *pStr; - UNUSED_PARAM(argc); - UNUSED_PARAM(argv); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); #ifdef NEVER /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will - ** always have been called to initalize it */ + ** always have been called to initialize it */ if( NEVER(!pStr) ) return; #endif z = pStr->zBuf; @@ -194035,38 +211451,50 @@ static void jsonObjectStep( JsonString *pStr; const char *z; u32 n; - UNUSED_PARAM(argc); + UNUSED_PARAMETER(argc); pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; z = (const char*)sqlite3_value_text(argv[0]); - n = (u32)sqlite3_value_bytes(argv[0]); + n = sqlite3Strlen30(z); jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); + jsonAppendSqlValue(pStr, argv[1]); } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); @@ -194086,19 +211514,37 @@ static void jsonObjectFinal(sqlite3_context *ctx){ /**************************************************************************** ** The json_each virtual table ****************************************************************************/ +typedef struct JsonParent JsonParent; +struct JsonParent { + u32 iHead; /* Start of object or array */ + u32 iValue; /* Start of the value */ + u32 iEnd; /* First byte past the end */ + u32 nPath; /* Length of path */ + i64 iKey; /* Key for JSONB_ARRAY */ +}; + typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ u32 iRowid; /* The rowid */ - u32 iBegin; /* The first node of the scan */ - u32 i; /* Index in sParse.aNode[] of current row */ + u32 i; /* Index in sParse.aBlob[] of current row */ u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ + u32 nRoot; /* Size of the root path in bytes */ + u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ - char *zJson; /* Input JSON */ - char *zRoot; /* Path by which to filter zJson */ + u32 nParent; /* Current nesting depth */ + u32 nParentAlloc; /* Space allocated for aParent[] */ + JsonParent *aParent; /* Parent elements of i */ + sqlite3 *db; /* Database connection */ + JsonString path; /* Current path */ JsonParse sParse; /* Parse of the input JSON */ }; +typedef struct JsonEachConnection JsonEachConnection; +struct JsonEachConnection { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ +}; + /* Constructor for the json_each virtual table */ static int jsonEachConnect( @@ -194108,7 +211554,7 @@ static int jsonEachConnect( sqlite3_vtab **ppVtab, char **pzErr ){ - sqlite3_vtab *pNew; + JsonEachConnection *pNew; int rc; /* Column numbers */ @@ -194126,36 +211572,40 @@ static int jsonEachConnect( #define JEACH_JSON 8 #define JEACH_ROOT 9 - UNUSED_PARAM(pzErr); - UNUSED_PARAM(argv); - UNUSED_PARAM(argc); - UNUSED_PARAM(pAux); + UNUSED_PARAMETER(pzErr); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(pAux); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = (JsonEachConnection*)sqlite3DbMallocZero(db, sizeof(*pNew)); + *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + pNew->db = db; } return rc; } /* destructor for json_each virtual table */ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); + JsonEachConnection *p = (JsonEachConnection*)pVtab; + sqlite3DbFree(p->db, pVtab); return SQLITE_OK; } /* constructor for a JsonEachCursor object for json_each(). */ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachConnection *pVtab = (JsonEachConnection*)p; JsonEachCursor *pCur; - UNUSED_PARAM(p); - pCur = sqlite3_malloc( sizeof(*pCur) ); + UNUSED_PARAMETER(p); + pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); + pCur->db = pVtab->db; + jsonStringZero(&pCur->path); *ppCursor = &pCur->base; return SQLITE_OK; } @@ -194173,22 +211623,24 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); - sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); + jsonStringReset(&p->path); + sqlite3DbFree(p->db, p->aParent); p->iRowid = 0; p->i = 0; + p->aParent = 0; + p->nParent = 0; + p->nParentAlloc = 0; p->iEnd = 0; p->eType = 0; - p->zJson = 0; - p->zRoot = 0; } /* Destructor for a jsonEachCursor object */ static int jsonEachClose(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; jsonEachCursorReset(p); - sqlite3_free(cur); + + sqlite3DbFree(p->db, cur); return SQLITE_OK; } @@ -194199,175 +211651,233 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ return p->i >= p->iEnd; } +/* +** If the cursor is currently pointing at the label of a object entry, +** then return the index of the value. For all other cases, return the +** current pointer position, which is the value. +*/ +static int jsonSkipLabel(JsonEachCursor *p){ + if( p->eType==JSONB_OBJECT ){ + u32 sz = 0; + u32 n = jsonbPayloadSize(&p->sParse, p->i, &sz); + return p->i + n + sz; + }else{ + return p->i; + } +} + +/* +** Append the path name for the current element. +*/ +static void jsonAppendPathName(JsonEachCursor *p){ + assert( p->nParent>0 ); + assert( p->eType==JSONB_ARRAY || p->eType==JSONB_OBJECT ); + if( p->eType==JSONB_ARRAY ){ + jsonPrintf(30, &p->path, "[%lld]", p->aParent[p->nParent-1].iKey); + }else{ + u32 n, sz = 0, k, i; + const char *z; + int needQuote = 0; + n = jsonbPayloadSize(&p->sParse, p->i, &sz); + k = p->i + n; + z = (const char*)&p->sParse.aBlob[k]; + if( sz==0 || !sqlite3Isalpha(z[0]) ){ + needQuote = 1; + }else{ + for(i=0; ipath,".\"%.*s\"", sz, z); + }else{ + jsonPrintf(sz+2,&p->path,".%.*s", sz, z); + } + } +} + /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; + int rc = SQLITE_OK; if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==0 || pUp->eU==3 ); - json_testcase( pUp->eU==3 ); - VVA( pUp->eU = 3 ); - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; - } + u8 x; + u8 levelChange = 0; + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + x = p->sParse.aBlob[i] & 0x0f; + n = jsonbPayloadSize(&p->sParse, i, &sz); + if( x==JSONB_OBJECT || x==JSONB_ARRAY ){ + JsonParent *pParent; + if( p->nParent>=p->nParentAlloc ){ + JsonParent *pNew; + u64 nNew; + nNew = p->nParentAlloc*2 + 3; + pNew = sqlite3DbRealloc(p->db, p->aParent, sizeof(JsonParent)*nNew); + if( pNew==0 ) return SQLITE_NOMEM; + p->nParentAlloc = (u32)nNew; + p->aParent = pNew; + } + levelChange = 1; + pParent = &p->aParent[p->nParent]; + pParent->iHead = p->i; + pParent->iValue = i; + pParent->iEnd = i + n + sz; + pParent->iKey = -1; + pParent->nPath = (u32)p->path.nUsed; + if( p->eType && p->nParent ){ + jsonAppendPathName(p); + if( p->path.eErr ) rc = SQLITE_NOMEM; + } + p->nParent++; + p->i = i + n; + }else{ + p->i = i + n + sz; + } + while( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ + p->nParent--; + p->path.nUsed = p->aParent[p->nParent].nPath; + levelChange = 1; + } + if( levelChange ){ + if( p->nParent>0 ){ + JsonParent *pParent = &p->aParent[p->nParent-1]; + u32 iVal = pParent->iValue; + p->eType = p->sParse.aBlob[iVal] & 0x0f; + }else{ + p->eType = 0; } } }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } - } + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->i = i + n + sz; } - return SQLITE_OK; + if( p->eType==JSONB_ARRAY && p->nParent ){ + p->aParent[p->nParent-1].iKey++; + } + p->iRowid++; + return rc; } -/* Append the name of the path for element i to pStr +/* Length of the path for rowid==0 in bRecursive mode. */ -static void jsonEachComputePath( - JsonEachCursor *p, /* The cursor */ - JsonString *pStr, /* Write the path here */ - u32 i /* Path to this element */ -){ - JsonNode *pNode, *pUp; - u32 iUp; - if( i==0 ){ - jsonAppendChar(pStr, '$'); - return; - } - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==3 || (pUp->eU==0 && pUp->u.iKey==0) ); - testcase( pUp->eU==0 ); - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - assert( pNode->eU==1 ); - jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); +static int jsonEachPathLength(JsonEachCursor *p){ + u32 n = p->path.nUsed; + char *z = p->path.zBuf; + if( p->iRowid==0 && p->bRecursive && n>=2 ){ + while( n>1 ){ + n--; + if( z[n]=='[' || z[n]=='.' ){ + u32 x, sz = 0; + char cSaved = z[n]; + z[n] = 0; + assert( p->sParse.eEdit==0 ); + x = jsonLookupStep(&p->sParse, 0, z+1, 0); + z[n] = cSaved; + if( JSON_LOOKUP_ISERROR(x) ) continue; + if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; + } + } } + return n; } /* Return the value of a column */ static int jsonEachColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ + int iColumn /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( i ){ + switch( iColumn ){ case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + if( p->nParent==0 ){ + u32 n, j; + if( p->nRoot==1 ) break; + j = jsonEachPathLength(p); + n = p->nRoot - j; + if( n==0 ){ + break; + }else if( p->path.zBuf[j]=='[' ){ + i64 x; + sqlite3Atoi64(&p->path.zBuf[j+1], &x, n-1, SQLITE_UTF8); + sqlite3_result_int64(ctx, x); + }else if( p->path.zBuf[j+1]=='"' ){ + sqlite3_result_text(ctx, &p->path.zBuf[j+2], n-3, SQLITE_TRANSIENT); }else{ - iKey = p->iRowid; + sqlite3_result_text(ctx, &p->path.zBuf[j+1], n-1, SQLITE_TRANSIENT); } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + break; + } + if( p->eType==JSONB_OBJECT ){ + jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); + }else{ + assert( p->eType==JSONB_ARRAY ); + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); } break; } case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); + u32 i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + u32 i = jsonSkipLabel(p); + u8 eType = p->sParse.aBlob[i] & 0x0f; + sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); break; } case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); + u32 i = jsonSkipLabel(p); + if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); + } break; } case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + sqlite3_result_int64(ctx, (sqlite3_int64)p->i); break; } case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + if( p->nParent>0 && p->bRecursive ){ + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); } break; } case JEACH_FULLKEY: { - JsonString x; - jsonInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - assert( pThis->eU==1 ); - jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); - } - } - jsonResult(&x); + u64 nBase = p->path.nUsed; + if( p->nParent ) jsonAppendPathName(p); + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + p->path.nUsed = nBase; break; } case JEACH_PATH: { - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - /* no break */ deliberate_fall_through + u32 n = jsonEachPathLength(p); + sqlite3_result_text64(ctx, p->path.zBuf, n, + SQLITE_TRANSIENT, SQLITE_UTF8); + break; } default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_STATIC); break; } case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + if( p->sParse.zJson==0 ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); + } break; } } @@ -194399,7 +211909,7 @@ static int jsonEachBestIndex( /* This implementation assumes that JSON and ROOT are the last two ** columns in the table */ assert( JEACH_ROOT == JEACH_JSON+1 ); - UNUSED_PARAM(tab); + UNUSED_PARAMETER(tab); aIdx[0] = aIdx[1] = -1; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ @@ -194417,6 +211927,13 @@ static int jsonEachBestIndex( idxMask |= iMask; } } + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + if( (unusableMask & ~idxMask)!=0 ){ /* If there are any unusable constraints on JSON or ROOT, then reject ** this entire plan */ @@ -194451,78 +211968,97 @@ static int jsonEachFilter( int argc, sqlite3_value **argv ){ JsonEachCursor *p = (JsonEachCursor*)cur; - const char *z; const char *zRoot = 0; - sqlite3_int64 n; + u32 i, n, sz; - UNUSED_PARAM(idxStr); - UNUSED_PARAM(argc); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); - if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ - int rc = SQLITE_NOMEM; - if( p->sParse.oom==0 ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + p->sParse.db = p->db; + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + }else{ + p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); + p->sParse.nJson = sqlite3_value_bytes(argv[0]); + if( p->sParse.zJson==0 ){ + p->i = p->iEnd = 0; + return SQLITE_OK; } - jsonEachCursorReset(p); - return rc; - }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ - jsonEachCursorReset(p); - return SQLITE_NOMEM; - }else{ - JsonNode *pNode = 0; - if( idxNum==3 ){ - const char *zErr = 0; - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); - if( zRoot[0]!='$' ){ - zErr = zRoot; - }else{ - pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + if( jsonConvertTextToBlob(&p->sParse, 0) ){ + if( p->sParse.oom ){ + return SQLITE_NOMEM; } - if( zErr ){ + goto json_each_malformed_input; + } + } + if( idxNum==3 ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + p->nRoot = sqlite3Strlen30(zRoot); + if( zRoot[1]==0 ){ + i = p->i = 0; + p->eType = 0; + }else{ + i = jsonLookupStep(&p->sParse, 0, zRoot+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; + return SQLITE_OK; + } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; - }else if( pNode==0 ){ - return SQLITE_OK; } - }else{ - pNode = p->sParse.aNode; - } - p->iBegin = p->i = (int)(pNode - p->sParse.aNode); - p->eType = pNode->eType; - if( p->eType>=JSON_ARRAY ){ - assert( pNode->eU==0 ); - VVA( pNode->eU = 3 ); - pNode->u.iKey = 0; - p->iEnd = p->i + pNode->n + 1; - if( p->bRecursive ){ - p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; - if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ - p->i--; - } + if( p->sParse.iLabel ){ + p->i = p->sParse.iLabel; + p->eType = JSONB_OBJECT; }else{ - p->i++; - } - }else{ - p->iEnd = p->i+1; - } + p->i = i; + p->eType = JSONB_ARRAY; + } + } + jsonAppendRaw(&p->path, zRoot, p->nRoot); + }else{ + i = p->i = 0; + p->eType = 0; + p->nRoot = 1; + jsonAppendRaw(&p->path, "$", 1); + } + p->nParent = 0; + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->iEnd = i+n+sz; + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ + p->i = i + n; + p->eType = p->sParse.aBlob[i] & 0x0f; + p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); + if( p->aParent==0 ) return SQLITE_NOMEM; + p->nParent = 1; + p->nParentAlloc = 1; + p->aParent[0].iKey = 0; + p->aParent[0].iEnd = p->iEnd; + p->aParent[0].iHead = p->i; + p->aParent[0].iValue = i; } return SQLITE_OK; + +json_each_malformed_input: + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } /* The methods of the json_each virtual table */ @@ -194550,7 +212086,8 @@ static sqlite3_module jsonEachModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; /* The methods of the json_tree virtual table. */ @@ -194578,111 +212115,98 @@ static sqlite3_module jsonTreeModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/**************************************************************************** -** The following routines are the only publically visible identifiers in this -** file. Call the following routines in order to register the various SQL -** functions and the virtual table implemented by this file. -****************************************************************************/ - -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ - int rc = SQLITE_OK; - unsigned int i; - static const struct { - const char *zName; - int nArg; - int flag; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFunc[] = { - { "json", 1, 0, jsonRemoveFunc }, - { "json_array", -1, 0, jsonArrayFunc }, - { "json_array_length", 1, 0, jsonArrayLengthFunc }, - { "json_array_length", 2, 0, jsonArrayLengthFunc }, - { "json_extract", -1, 0, jsonExtractFunc }, - { "json_insert", -1, 0, jsonSetFunc }, - { "json_object", -1, 0, jsonObjectFunc }, - { "json_patch", 2, 0, jsonPatchFunc }, - { "json_quote", 1, 0, jsonQuoteFunc }, - { "json_remove", -1, 0, jsonRemoveFunc }, - { "json_replace", -1, 0, jsonReplaceFunc }, - { "json_set", -1, 1, jsonSetFunc }, - { "json_type", 1, 0, jsonTypeFunc }, - { "json_type", 2, 0, jsonTypeFunc }, - { "json_valid", 1, 0, jsonValidFunc }, - +#endif /* !defined(SQLITE_OMIT_JSON) */ + +/* +** Register JSON functions. +*/ +SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ +#ifndef SQLITE_OMIT_JSON + static FuncDef aJsonFunc[] = { + /* sqlite3_result_subtype() ----, ,--- sqlite3_value_subtype() */ + /* | | */ + /* Uses cache ------, | | ,---- Returns JSONB */ + /* | | | | */ + /* Number of arguments ---, | | | | ,--- Flags */ + /* | | | | | | */ + JFUNCTION(json, 1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), + JFUNCTION(json_extract, -1,1,1, 0,0,0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1,1,0, 0,1,0, jsonExtractFunc), + JFUNCTION(->, 2,1,1, 0,0,JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2,1,0, 0,0,JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1,1,1, 1,0,0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1,1,0, 1,1,0, jsonSetFunc), + JFUNCTION(json_object, -1,0,1, 1,0,0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), + JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), + JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_replace, -1,1,1, 1,0,0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1,1,0, 1,1,0, jsonReplaceFunc), + JFUNCTION(json_set, -1,1,1, 1,0,JSON_ISSET, jsonSetFunc), + JFUNCTION(jsonb_set, -1,1,0, 1,1,JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_type, 2,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_valid, 1,1,0, 0,0,0, jsonValidFunc), + JFUNCTION(json_valid, 2,1,0, 0,0,0, jsonValidFunc), #if SQLITE_DEBUG - /* DEBUG and TESTING functions */ - { "json_parse", 1, 0, jsonParseFunc }, - { "json_test1", 1, 0, jsonTest1Func }, -#endif + JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), +#endif + WAGGREGATE(json_group_array, 1, 0, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(json_group_object, 2, 0, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC) }; + sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); +#endif +} + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) +/* +** Register the JSON table-valued functions +*/ +SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3 *db){ + int rc = SQLITE_OK; static const struct { - const char *zName; - int nArg; - void (*xStep)(sqlite3_context*,int,sqlite3_value**); - void (*xFinal)(sqlite3_context*); - void (*xValue)(sqlite3_context*); - } aAgg[] = { - { "json_group_array", 1, - jsonArrayStep, jsonArrayFinal, jsonArrayValue }, - { "json_group_object", 2, - jsonObjectStep, jsonObjectFinal, jsonObjectValue }, - }; -#ifndef SQLITE_OMIT_VIRTUALTABLE - static const struct { - const char *zName; - sqlite3_module *pModule; + const char *zName; + sqlite3_module *pModule; } aMod[] = { { "json_each", &jsonEachModule }, { "json_tree", &jsonTreeModule }, }; -#endif - static const int enc = - SQLITE_UTF8 | - SQLITE_DETERMINISTIC | - SQLITE_INNOCUOUS; - for(i=0; i */ /* #include */ /* #include */ @@ -194847,6 +212376,7 @@ struct Rtree { int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ + char *zNodeName; /* Name of the %_node table */ u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ u32 nCursor; /* Number of open cursors */ @@ -194859,7 +212389,6 @@ struct Rtree { ** headed by the node (leaf nodes have RtreeNode.iNode==0). */ RtreeNode *pDeleted; - int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ /* Blob I/O on xxx_node */ sqlite3_blob *pNodeBlob; @@ -195156,17 +212685,23 @@ struct RtreeMatchArg { ** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined ** at run-time. */ -#ifndef SQLITE_BYTEORDER -#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SQLITE_BYTEORDER 1234 -#elif defined(sparc) || defined(__ppc__) -# define SQLITE_BYTEORDER 4321 -#else -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ -#endif +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif @@ -195187,7 +212722,7 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ - assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(p) ); #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 pCoord->u = _byteswap_ulong(*(u32*)p); #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -195241,7 +212776,7 @@ static void writeInt16(u8 *p, int i){ } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; - assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(p) ); assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); #if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -195369,11 +212904,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob *pBlob = pRtree->pNodeBlob; - pRtree->pNodeBlob = 0; - sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } /* @@ -195392,7 +212925,7 @@ static int nodeAcquire( ** increase its reference count and return it. */ if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ - if( pParent && pParent!=pNode->pParent ){ + if( pParent && ALWAYS(pParent!=pNode->pParent) ){ RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -195412,14 +212945,11 @@ static int nodeAcquire( } } if( pRtree->pNodeBlob==0 ){ - char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); - if( zTab==0 ) return SQLITE_NOMEM; - rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, pRtree->zNodeName, + "data", iNode, 0, &pRtree->pNodeBlob); - sqlite3_free(zTab); } if( rc ){ - nodeBlobReset(pRtree); *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ @@ -195479,6 +213009,7 @@ static int nodeAcquire( } *ppNode = pNode; }else{ + nodeBlobReset(pRtree); if( pNode ){ pRtree->nNodeRef--; sqlite3_free(pNode); @@ -195623,6 +213154,7 @@ static void nodeGetCoord( int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -195812,7 +213344,9 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); pRtree->nCursor--; - nodeBlobReset(pRtree); + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } @@ -195969,7 +213503,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(pCellData) ); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ case RTREE_FALSE: break; /* Never satisfied */ @@ -196022,7 +213556,7 @@ static void rtreeLeafConstraint( || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; - assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(pCellData) ); RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ @@ -196397,7 +213931,11 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc==SQLITE_OK && ALWAYS(p) ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } @@ -196415,6 +213953,7 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( rc ) return rc; if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ @@ -196512,6 +214051,8 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); + /* ** Rtree virtual table module xFilter method. */ @@ -196541,7 +214082,8 @@ static int rtreeFilter( i64 iNode = 0; int eType = sqlite3_value_numeric_type(argv[0]); if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ @@ -196592,7 +214134,20 @@ static int rtreeFilter( p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; - }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + }else if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_value_int64(argv[ii]); +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = iVal; +#else + p->u.rValue = (double)iVal; + if( iVal>=((sqlite3_int64)1)<<48 + || iVal<=-(((sqlite3_int64)1)<<48) + ){ + if( p->op==RTREE_LT ) p->op = RTREE_LE; + if( p->op==RTREE_GT ) p->op = RTREE_GE; + } +#endif + }else if( eType==SQLITE_FLOAT ){ #ifdef SQLITE_RTREE_INT_ONLY p->u.rValue = sqlite3_value_int64(argv[ii]); #else @@ -196723,11 +214278,12 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ u8 op; + u8 doOmit = 1; switch( p->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; - case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; doOmit = 0; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; doOmit = 0; break; case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; - case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; doOmit = 0; break; case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; default: op = 0; break; @@ -196736,15 +214292,19 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ zIdxStr[iIdx++] = op; zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; + pIdxInfo->aConstraintUsage[ii].omit = doOmit; } } } pIdxInfo->idxNum = 2; pIdxInfo->needToFreeIdxStr = 1; - if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ - return SQLITE_NOMEM; + if( iIdx>0 ){ + pIdxInfo->idxStr = sqlite3_malloc( iIdx+1 ); + if( pIdxInfo->idxStr==0 ){ + return SQLITE_NOMEM; + } + memcpy(pIdxInfo->idxStr, zIdxStr, iIdx+1); } nRow = pRtree->nRowEst >> (iIdx/2); @@ -196823,31 +214383,22 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ */ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; - int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; iinDim2; ii+=2){ - RtreeCoord *a1 = &p1->aCoord[ii]; - RtreeCoord *a2 = &p2->aCoord[ii]; - if( (!isInt && (a2[0].fa1[1].f)) - || ( isInt && (a2[0].ia1[1].i)) - ){ - return 0; + if( pRtree->eCoordType==RTREE_COORD_INT32 ){ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].ia1[1].i ) return 0; + } + }else{ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].fa1[1].f ) return 0; } } return 1; } -/* -** Return the amount cell p would grow by if it were unioned with pCell. -*/ -static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - RtreeDValue area; - RtreeCell cell; - memcpy(&cell, p, sizeof(RtreeCell)); - area = cellArea(pRtree, &cell); - cellUnion(pRtree, &cell, pCell); - return (cellArea(pRtree, &cell)-area); -} - static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, @@ -196894,38 +214445,52 @@ static int ChooseLeaf( for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; sqlite3_int64 iBest = 0; - + int bFound = 0; RtreeDValue fMinGrowth = RTREE_ZERO; RtreeDValue fMinArea = RTREE_ZERO; - int nCell = NCELL(pNode); - RtreeCell cell; RtreeNode *pChild = 0; - RtreeCell *aCell = 0; - - /* Select the child node which will be enlarged the least if pCell - ** is inserted into it. Resolve ties by choosing the entry with - ** the smallest area. + /* First check to see if there is are any cells in pNode that completely + ** contains pCell. If two or more cells in pNode completely contain pCell + ** then pick the smallest. */ for(iCell=0; iCell1 ){ - int iLeft = 0; - int iRight = 0; - - int nLeft = nIdx/2; - int nRight = nIdx-nLeft; - int *aLeft = aIdx; - int *aRight = &aIdx[nLeft]; - - SortByDistance(aLeft, nLeft, aDistance, aSpare); - SortByDistance(aRight, nRight, aDistance, aSpare); - - memcpy(aSpare, aLeft, sizeof(int)*nLeft); - aLeft = aSpare; - - while( iLeftnDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); - } - } - for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); - } - - for(ii=0; iinDim; iDim++){ - RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); - aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); - } - } - - SortByDistance(aOrder, nCell, aDistance, aSpare); - nodeZero(pRtree, pNode); - - for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ - RtreeCell *p = &aCell[aOrder[ii]]; - nodeInsertCell(pRtree, pNode, p); - if( p->iRowid==pCell->iRowid ){ - if( iHeight==0 ){ - rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, p->iRowid, pNode->iNode); - } - } - } - if( rc==SQLITE_OK ){ - rc = fixBoundingBox(pRtree, pNode); - } - for(; rc==SQLITE_OK && iiiNode currently contains - ** the height of the sub-tree headed by the cell. - */ - RtreeNode *pInsert; - RtreeCell *p = &aCell[aOrder[ii]]; - rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); - if( rc==SQLITE_OK ){ - int rc2; - rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); - rc2 = nodeRelease(pRtree, pInsert); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - - sqlite3_free(aCell); - return rc; -} - /* ** Insert cell pCell into node pNode. Node pNode is the head of a ** subtree iHeight high (leaf nodes have iHeight==0). @@ -197674,12 +215067,7 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ - if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ - rc = SplitNode(pRtree, pNode, pCell, iHeight); - }else{ - pRtree->iReinsertHeight = iHeight; - rc = Reinsert(pRtree, pNode, pCell, iHeight); - } + rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); if( ALWAYS(rc==SQLITE_OK) ){ @@ -197921,7 +215309,7 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + memset(&cell, 0, sizeof(cell)); /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: @@ -198022,7 +215410,6 @@ static int rtreeUpdate( } if( rc==SQLITE_OK ){ int rc2; - pRtree->iReinsertHeight = -1; rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ @@ -198052,7 +215439,7 @@ constraint: static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; assert( pRtree->inWrTrans==0 ); - pRtree->inWrTrans++; + pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -198066,6 +215453,9 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ nodeBlobReset(pRtree); return SQLITE_OK; } +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); +} /* ** The xRename method for rtree module virtual tables. @@ -198163,8 +215553,11 @@ static int rtreeShadowName(const char *zName){ return 0; } +/* Forward declaration */ +static int rtreeIntegrity(sqlite3_vtab*, const char*, const char*, int, char**); + static sqlite3_module rtreeModule = { - 3, /* iVersion */ + 4, /* iVersion */ rtreeCreate, /* xCreate - create a table */ rtreeConnect, /* xConnect - connect to an existing table */ rtreeBestIndex, /* xBestIndex - Determine search strategy */ @@ -198181,13 +215574,14 @@ static sqlite3_module rtreeModule = { rtreeBeginTransaction, /* xBegin - begin transaction */ rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - rtreeShadowName /* xShadowName */ + rtreeShadowName, /* xShadowName */ + rtreeIntegrity /* xIntegrity */ }; static int rtreeSqlInit( @@ -198280,7 +215674,7 @@ static int rtreeSqlInit( } sqlite3_free(zSql); } - if( pRtree->nAux ){ + if( pRtree->nAux && rc!=SQLITE_NOMEM ){ pRtree->zReadAuxSql = sqlite3_mprintf( "SELECT * FROM \"%w\".\"%w_rowid\" WHERE rowid=?1", zDb, zPrefix); @@ -198443,22 +215837,27 @@ static int rtreeInit( } sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + /* Allocate the sqlite3_vtab structure */ nDb = (int)strlen(argv[1]); nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); if( !pRtree ){ return SQLITE_NOMEM; } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); pRtree->nBusy = 1; pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->zNodeName = &pRtree->zName[nName+1]; pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); /* Create/Connect to the underlying relational database schema. If @@ -198955,7 +216354,6 @@ static int rtreeCheckTable( ){ RtreeCheck check; /* Common context for various routines */ sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ - int bEnd = 0; /* True if transaction should be closed */ int nAux = 0; /* Number of extra columns. */ /* Initialize the context object */ @@ -198964,24 +216362,14 @@ static int rtreeCheckTable( check.zDb = zDb; check.zTab = zTab; - /* If there is not already an open transaction, open one now. This is - ** to ensure that the queries run as part of this integrity-check operate - ** on a consistent snapshot. */ - if( sqlite3_get_autocommit(db) ){ - check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); - bEnd = 1; - } - /* Find the number of auxiliary columns */ - if( check.rc==SQLITE_OK ){ - pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); - if( pStmt ){ - nAux = sqlite3_column_count(pStmt) - 2; - sqlite3_finalize(pStmt); - }else - if( check.rc!=SQLITE_NOMEM ){ - check.rc = SQLITE_OK; - } + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); + if( pStmt ){ + nAux = sqlite3_column_count(pStmt) - 2; + sqlite3_finalize(pStmt); + }else + if( check.rc!=SQLITE_NOMEM ){ + check.rc = SQLITE_OK; } /* Find number of dimensions in the rtree table. */ @@ -199012,15 +216400,35 @@ static int rtreeCheckTable( sqlite3_finalize(check.aCheckMapping[0]); sqlite3_finalize(check.aCheckMapping[1]); - /* If one was opened, close the transaction */ - if( bEnd ){ - int rc = sqlite3_exec(db, "END", 0, 0, 0); - if( check.rc==SQLITE_OK ) check.rc = rc; - } *pzReport = check.zReport; return check.rc; } +/* +** Implementation of the xIntegrity method for Rtree. +*/ +static int rtreeIntegrity( + sqlite3_vtab *pVtab, /* The virtual table to check */ + const char *zSchema, /* Schema in which the virtual table lives */ + const char *zName, /* Name of the virtual table */ + int isQuick, /* True for a quick_check */ + char **pzErr /* Write results here */ +){ + Rtree *pRtree = (Rtree*)pVtab; + int rc; + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAMETER(zSchema); + UNUSED_PARAMETER(zName); + UNUSED_PARAMETER(isQuick); + rc = rtreeCheckTable(pRtree->db, pRtree->zDb, pRtree->zName, pzErr); + if( rc==SQLITE_OK && *pzErr ){ + *pzErr = sqlite3_mprintf("In RTree %s.%s:\n%z", + pRtree->zDb, pRtree->zName, *pzErr); + if( (*pzErr)==0 ) rc = SQLITE_NOMEM; + } + return rc; +} + /* ** Usage: ** @@ -199116,11 +216524,7 @@ static void rtreecheck( # define GEODEBUG(X) #endif -#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ -/* -** Versions of isspace(), isalnum() and isdigit() to which it is safe -** to pass signed char values. -*/ +/* Character class routines */ #ifdef sqlite3Isdigit /* Use the SQLite core versions if this routine is part of the ** SQLite amalgamation */ @@ -199135,6 +216539,7 @@ static void rtreecheck( # define safe_isxdigit(x) isxdigit((unsigned char)(x)) #endif +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function. @@ -199157,7 +216562,7 @@ static const char geopolyIsSpace[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#define fast_isspace(x) (geopolyIsSpace[(unsigned char)x]) #endif /* JSON NULL - back to original code */ /* Compiler and version */ @@ -199246,7 +216651,7 @@ static void geopolySwab32(unsigned char *a){ /* Skip whitespace. Return the next non-whitespace character. */ static char geopolySkipSpace(GeoParse *p){ - while( safe_isspace(p->z[0]) ) p->z++; + while( fast_isspace(p->z[0]) ) p->z++; return p->z[0]; } @@ -199397,7 +216802,7 @@ static GeoPoly *geopolyFuncParam( int nByte; testcase( pCtx==0 ); if( sqlite3_value_type(pVal)==SQLITE_BLOB - && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + && (nByte = sqlite3_value_bytes(pVal))>=(int)(4+6*sizeof(GeoCoord)) ){ const unsigned char *a = sqlite3_value_blob(pVal); int nVertex; @@ -199455,6 +216860,7 @@ static void geopolyBlobFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); @@ -199474,6 +216880,7 @@ static void geopolyJsonFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3 *db = sqlite3_context_db_handle(context); sqlite3_str *x = sqlite3_str_new(db); @@ -199555,6 +216962,7 @@ static void geopolyXformFunc( double F = sqlite3_value_double(argv[6]); GeoCoord x1, y1, x0, y0; int ii; + (void)argc; if( p ){ for(ii=0; iinVertex; ii++){ x0 = GeoX(p,ii); @@ -199605,6 +217013,7 @@ static void geopolyAreaFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ sqlite3_result_double(context, geopolyArea(p)); sqlite3_free(p); @@ -199630,6 +217039,7 @@ static void geopolyCcwFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; if( p ){ if( geopolyArea(p)<0.0 ){ int ii, jj; @@ -199684,6 +217094,7 @@ static void geopolyRegularFunc( int n = sqlite3_value_int(argv[3]); int i; GeoPoly *p; + (void)argc; if( n<3 || r<=0.0 ) return; if( n>1000 ) n = 1000; @@ -199793,6 +217204,7 @@ static void geopolyBBoxFunc( sqlite3_value **argv ){ GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + (void)argc; if( p ){ sqlite3_result_blob(context, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); @@ -199820,6 +217232,7 @@ static void geopolyBBoxStep( ){ RtreeCoord a[4]; int rc = SQLITE_OK; + (void)argc; (void)geopolyBBox(context, argv[0], a, &rc); if( rc==SQLITE_OK ){ GeoBBox *pBBox; @@ -199908,6 +217321,8 @@ static void geopolyContainsPointFunc( int v = 0; int cnt = 0; int ii; + (void)argc; + if( p1==0 ) return; for(ii=0; iinVertex-1; ii++){ v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), @@ -199947,6 +217362,7 @@ static void geopolyWithinFunc( ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ @@ -200277,6 +217693,7 @@ static void geopolyOverlapFunc( ){ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; if( p1 && p2 ){ int x = geopolyOverlap(p1, p2); if( x<0 ){ @@ -200297,8 +217714,12 @@ static void geopolyDebugFunc( int argc, sqlite3_value **argv ){ + (void)context; + (void)argc; #ifdef GEOPOLY_ENABLE_DEBUG geo_debug = sqlite3_value_int(argv[0]); +#else + (void)argv; #endif } @@ -200326,26 +217747,31 @@ static int geopolyInit( sqlite3_str *pSql; char *zSql; int ii; + (void)pAux; sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); if( !pRtree ){ return SQLITE_NOMEM; } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); pRtree->nBusy = 1; pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->zNodeName = &pRtree->zName[nName+1]; pRtree->eCoordType = RTREE_COORD_REAL32; pRtree->nDim = 2; pRtree->nDim2 = 4; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); /* Create/Connect to the underlying relational database schema. If @@ -200442,6 +217868,7 @@ static int geopolyFilter( RtreeNode *pRoot = 0; int rc = SQLITE_OK; int iCell = 0; + (void)idxStr; rtreeReference(pRtree); @@ -200568,6 +217995,7 @@ static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iRowidTerm = -1; int iFuncTerm = -1; int idxNum = 0; + (void)tab; for(ii=0; iinConstraint; ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; @@ -200757,7 +218185,6 @@ static int geopolyUpdate( } if( rc==SQLITE_OK ){ int rc2; - pRtree->iReinsertHeight = -1; rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ @@ -200788,7 +218215,7 @@ static int geopolyUpdate( sqlite3_free(p); nChange = 1; } - for(jj=1; jjnAux; jj++){ + for(jj=1; jj -#include -#include -#include - -/* #include */ - -#ifndef SQLITE_CORE -/* #include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#else -/* #include "sqlite3.h" */ -#endif - -/* -** This function is called when an ICU function called from within -** the implementation of an SQL scalar function returns an error. -** -** The scalar function context passed as the first argument is -** loaded with an error message based on the following two args. -*/ -static void icuFunctionError( - sqlite3_context *pCtx, /* SQLite scalar function context */ - const char *zName, /* Name of ICU function that failed */ - UErrorCode e /* Error code returned by ICU function */ -){ - char zBuf[128]; - sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); - zBuf[127] = '\0'; - sqlite3_result_error(pCtx, zBuf, -1); -} - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) - -/* -** Maximum length (in bytes) of the pattern in a LIKE or GLOB -** operator. -*/ -#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH -# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 -#endif - -/* -** Version of sqlite3_free() that is always a function, never a macro. -*/ -static void xFree(void *p){ - sqlite3_free(p); -} - -/* -** This lookup table is used to help decode the first byte of -** a multi-byte UTF8 character. It is copied here from SQLite source -** code file utf8.c. -*/ -static const unsigned char icuUtf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, -}; - -#define SQLITE_ICU_READ_UTF8(zIn, c) \ - c = *(zIn++); \ - if( c>=0xc0 ){ \ - c = icuUtf8Trans1[c-0xc0]; \ - while( (*zIn & 0xc0)==0x80 ){ \ - c = (c<<6) + (0x3f & *(zIn++)); \ - } \ - } - -#define SQLITE_ICU_SKIP_UTF8(zIn) \ - assert( *zIn ); \ - if( *(zIn++)>=0xc0 ){ \ - while( (*zIn & 0xc0)==0x80 ){zIn++;} \ - } - - -/* -** Compare two UTF-8 strings for equality where the first string is -** a "LIKE" expression. Return true (1) if they are the same and -** false (0) if they are different. -*/ -static int icuLikeCompare( - const uint8_t *zPattern, /* LIKE pattern */ - const uint8_t *zString, /* The UTF-8 string to compare against */ - const UChar32 uEsc /* The escape character */ -){ - static const uint32_t MATCH_ONE = (uint32_t)'_'; - static const uint32_t MATCH_ALL = (uint32_t)'%'; - - int prevEscape = 0; /* True if the previous character was uEsc */ - - while( 1 ){ - - /* Read (and consume) the next character from the input pattern. */ - uint32_t uPattern; - SQLITE_ICU_READ_UTF8(zPattern, uPattern); - if( uPattern==0 ) break; - - /* There are now 4 possibilities: - ** - ** 1. uPattern is an unescaped match-all character "%", - ** 2. uPattern is an unescaped match-one character "_", - ** 3. uPattern is an unescaped escape character, or - ** 4. uPattern is to be handled as an ordinary character - */ - if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ - /* Case 1. */ - uint8_t c; - - /* Skip any MATCH_ALL or MATCH_ONE characters that follow a - ** MATCH_ALL. For each MATCH_ONE, skip one character in the - ** test string. - */ - while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ - if( c==MATCH_ONE ){ - if( *zString==0 ) return 0; - SQLITE_ICU_SKIP_UTF8(zString); - } - zPattern++; - } - - if( *zPattern==0 ) return 1; - - while( *zString ){ - if( icuLikeCompare(zPattern, zString, uEsc) ){ - return 1; - } - SQLITE_ICU_SKIP_UTF8(zString); - } - return 0; - - }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ - /* Case 2. */ - if( *zString==0 ) return 0; - SQLITE_ICU_SKIP_UTF8(zString); - - }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ - /* Case 3. */ - prevEscape = 1; - - }else{ - /* Case 4. */ - uint32_t uString; - SQLITE_ICU_READ_UTF8(zString, uString); - uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); - uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); - if( uString!=uPattern ){ - return 0; - } - prevEscape = 0; - } - } - - return *zString==0; -} - -/* -** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the -** pattern and the second argument is the string. So, the SQL statements: -** -** A LIKE B -** -** is implemented as like(B, A). If there is an escape character E, -** -** A LIKE B ESCAPE E -** -** is mapped to like(B, A, E). -*/ -static void icuLikeFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *zA = sqlite3_value_text(argv[0]); - const unsigned char *zB = sqlite3_value_text(argv[1]); - UChar32 uEsc = 0; - - /* Limit the length of the LIKE or GLOB pattern to avoid problems - ** of deep recursion and N*N behavior in patternCompare(). - */ - if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ - sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); - return; - } - - - if( argc==3 ){ - /* The escape character string must consist of a single UTF-8 character. - ** Otherwise, return an error. - */ - int nE= sqlite3_value_bytes(argv[2]); - const unsigned char *zE = sqlite3_value_text(argv[2]); - int i = 0; - if( zE==0 ) return; - U8_NEXT(zE, i, nE, uEsc); - if( i!=nE){ - sqlite3_result_error(context, - "ESCAPE expression must be a single character", -1); - return; - } - } - - if( zA && zB ){ - sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); - } -} - -/* -** Function to delete compiled regexp objects. Registered as -** a destructor function with sqlite3_set_auxdata(). -*/ -static void icuRegexpDelete(void *p){ - URegularExpression *pExpr = (URegularExpression *)p; - uregex_close(pExpr); -} - -/* -** Implementation of SQLite REGEXP operator. This scalar function takes -** two arguments. The first is a regular expression pattern to compile -** the second is a string to match against that pattern. If either -** argument is an SQL NULL, then NULL Is returned. Otherwise, the result -** is 1 if the string matches the pattern, or 0 otherwise. -** -** SQLite maps the regexp() function to the regexp() operator such -** that the following two are equivalent: -** -** zString REGEXP zPattern -** regexp(zPattern, zString) -** -** Uses the following ICU regexp APIs: -** -** uregex_open() -** uregex_matches() -** uregex_close() -*/ -static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - UErrorCode status = U_ZERO_ERROR; - URegularExpression *pExpr; - UBool res; - const UChar *zString = sqlite3_value_text16(apArg[1]); - - (void)nArg; /* Unused parameter */ - - /* If the left hand side of the regexp operator is NULL, - ** then the result is also NULL. - */ - if( !zString ){ - return; - } - - pExpr = sqlite3_get_auxdata(p, 0); - if( !pExpr ){ - const UChar *zPattern = sqlite3_value_text16(apArg[0]); - if( !zPattern ){ - return; - } - pExpr = uregex_open(zPattern, -1, 0, 0, &status); - - if( U_SUCCESS(status) ){ - sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); - }else{ - assert(!pExpr); - icuFunctionError(p, "uregex_open", status); - return; - } - } - - /* Configure the text that the regular expression operates on. */ - uregex_setText(pExpr, zString, -1, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "uregex_setText", status); - return; - } - - /* Attempt the match */ - res = uregex_matches(pExpr, 0, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "uregex_matches", status); - return; - } - - /* Set the text that the regular expression operates on to a NULL - ** pointer. This is not really necessary, but it is tidier than - ** leaving the regular expression object configured with an invalid - ** pointer after this function returns. - */ - uregex_setText(pExpr, 0, 0, &status); - - /* Return 1 or 0. */ - sqlite3_result_int(p, res ? 1 : 0); -} - -/* -** Implementations of scalar functions for case mapping - upper() and -** lower(). Function upper() converts its input to upper-case (ABC). -** Function lower() converts to lower-case (abc). -** -** ICU provides two types of case mapping, "general" case mapping and -** "language specific". Refer to ICU documentation for the differences -** between the two. -** -** To utilise "general" case mapping, the upper() or lower() scalar -** functions are invoked with one argument: -** -** upper('ABC') -> 'abc' -** lower('abc') -> 'ABC' -** -** To access ICU "language specific" case mapping, upper() or lower() -** should be invoked with two arguments. The second argument is the name -** of the locale to use. Passing an empty string ("") or SQL NULL value -** as the second argument is the same as invoking the 1 argument version -** of upper() or lower(). -** -** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> '\u131' (small dotless i) -** -** http://www.icu-project.org/userguide/posix.html#case_mappings -*/ -static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - const UChar *zInput; /* Pointer to input string */ - UChar *zOutput = 0; /* Pointer to output buffer */ - int nInput; /* Size of utf-16 input string in bytes */ - int nOut; /* Size of output buffer in bytes */ - int cnt; - int bToUpper; /* True for toupper(), false for tolower() */ - UErrorCode status; - const char *zLocale = 0; - - assert(nArg==1 || nArg==2); - bToUpper = (sqlite3_user_data(p)!=0); - if( nArg==2 ){ - zLocale = (const char *)sqlite3_value_text(apArg[1]); - } - - zInput = sqlite3_value_text16(apArg[0]); - if( !zInput ){ - return; - } - nOut = nInput = sqlite3_value_bytes16(apArg[0]); - if( nOut==0 ){ - sqlite3_result_text16(p, "", 0, SQLITE_STATIC); - return; - } - - for(cnt=0; cnt<2; cnt++){ - UChar *zNew = sqlite3_realloc(zOutput, nOut); - if( zNew==0 ){ - sqlite3_free(zOutput); - sqlite3_result_error_nomem(p); - return; - } - zOutput = zNew; - status = U_ZERO_ERROR; - if( bToUpper ){ - nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); - }else{ - nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); - } - - if( U_SUCCESS(status) ){ - sqlite3_result_text16(p, zOutput, nOut, xFree); - }else if( status==U_BUFFER_OVERFLOW_ERROR ){ - assert( cnt==0 ); - continue; - }else{ - icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); - } - return; - } - assert( 0 ); /* Unreachable */ -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ - -/* -** Collation sequence destructor function. The pCtx argument points to -** a UCollator structure previously allocated using ucol_open(). -*/ -static void icuCollationDel(void *pCtx){ - UCollator *p = (UCollator *)pCtx; - ucol_close(p); -} - -/* -** Collation sequence comparison function. The pCtx argument points to -** a UCollator structure previously allocated using ucol_open(). -*/ -static int icuCollationColl( - void *pCtx, - int nLeft, - const void *zLeft, - int nRight, - const void *zRight -){ - UCollationResult res; - UCollator *p = (UCollator *)pCtx; - res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); - switch( res ){ - case UCOL_LESS: return -1; - case UCOL_GREATER: return +1; - case UCOL_EQUAL: return 0; - } - assert(!"Unexpected return value from ucol_strcoll()"); - return 0; -} - -/* -** Implementation of the scalar function icu_load_collation(). -** -** This scalar function is used to add ICU collation based collation -** types to an SQLite database connection. It is intended to be called -** as follows: -** -** SELECT icu_load_collation(, ); -** -** Where is a string containing an ICU locale identifier (i.e. -** "en_AU", "tr_TR" etc.) and is the name of the -** collation sequence to create. -*/ -static void icuLoadCollation( - sqlite3_context *p, - int nArg, - sqlite3_value **apArg -){ - sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); - UErrorCode status = U_ZERO_ERROR; - const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ - const char *zName; /* SQL Collation sequence name (eg. "japanese") */ - UCollator *pUCollator; /* ICU library collation object */ - int rc; /* Return code from sqlite3_create_collation_x() */ - - assert(nArg==2); - (void)nArg; /* Unused parameter */ - zLocale = (const char *)sqlite3_value_text(apArg[0]); - zName = (const char *)sqlite3_value_text(apArg[1]); - - if( !zLocale || !zName ){ - return; - } - - pUCollator = ucol_open(zLocale, &status); - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "ucol_open", status); - return; - } - assert(p); - - rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, - icuCollationColl, icuCollationDel - ); - if( rc!=SQLITE_OK ){ - ucol_close(pUCollator); - sqlite3_result_error(p, "Error registering collation function", -1); - } -} - -/* -** Register the ICU extension functions with database db. -*/ -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ -# define SQLITEICU_EXTRAFLAGS (SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) - static const struct IcuScalar { - const char *zName; /* Function name */ - unsigned char nArg; /* Number of arguments */ - unsigned int enc; /* Optimal text encoding */ - unsigned char iContext; /* sqlite3_user_data() context */ - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } scalars[] = { - {"icu_load_collation",2,SQLITE_UTF8|SQLITE_DIRECTONLY,1, icuLoadCollation}, -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) - {"regexp", 2, SQLITE_ANY|SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, - {"lower", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, - {"lower", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, - {"like", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ - }; - int rc = SQLITE_OK; - int i; - - for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ - const struct IcuScalar *p = &scalars[i]; - rc = sqlite3_create_function( - db, p->zName, p->nArg, p->enc, - p->iContext ? (void*)db : (void*)0, - p->xFunc, 0, 0 - ); - } - - return rc; -} - -#if !SQLITE_CORE -#ifdef _WIN32 -__declspec(dllexport) -#endif -SQLITE_API int sqlite3_icu_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3IcuInit(db); -} -#endif - -#endif - -/************** End of icu.c *************************************************/ -/************** Begin file fts3_icu.c ****************************************/ -/* -** 2007 June 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file implements a tokenizer for fts3 based on the ICU library. -*/ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#ifdef SQLITE_ENABLE_ICU - -/* #include */ -/* #include */ -/* #include "fts3_tokenizer.h" */ - -#include -/* #include */ -/* #include */ -#include - -typedef struct IcuTokenizer IcuTokenizer; -typedef struct IcuCursor IcuCursor; - -struct IcuTokenizer { - sqlite3_tokenizer base; - char *zLocale; -}; - -struct IcuCursor { - sqlite3_tokenizer_cursor base; - - UBreakIterator *pIter; /* ICU break-iterator object */ - int nChar; /* Number of UChar elements in pInput */ - UChar *aChar; /* Copy of input using utf-16 encoding */ - int *aOffset; /* Offsets of each character in utf-8 input */ - - int nBuffer; - char *zBuffer; - - int iToken; -}; - -/* -** Create a new tokenizer instance. -*/ -static int icuCreate( - int argc, /* Number of entries in argv[] */ - const char * const *argv, /* Tokenizer creation arguments */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ -){ - IcuTokenizer *p; - int n = 0; - - if( argc>0 ){ - n = strlen(argv[0])+1; - } - p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); - if( !p ){ - return SQLITE_NOMEM; - } - memset(p, 0, sizeof(IcuTokenizer)); - - if( n ){ - p->zLocale = (char *)&p[1]; - memcpy(p->zLocale, argv[0], n); - } - - *ppTokenizer = (sqlite3_tokenizer *)p; - - return SQLITE_OK; -} - -/* -** Destroy a tokenizer -*/ -static int icuDestroy(sqlite3_tokenizer *pTokenizer){ - IcuTokenizer *p = (IcuTokenizer *)pTokenizer; - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. -*/ -static int icuOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *zInput, /* Input string */ - int nInput, /* Length of zInput in bytes */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ -){ - IcuTokenizer *p = (IcuTokenizer *)pTokenizer; - IcuCursor *pCsr; - - const int32_t opt = U_FOLD_CASE_DEFAULT; - UErrorCode status = U_ZERO_ERROR; - int nChar; - - UChar32 c; - int iInput = 0; - int iOut = 0; - - *ppCursor = 0; - - if( zInput==0 ){ - nInput = 0; - zInput = ""; - }else if( nInput<0 ){ - nInput = strlen(zInput); - } - nChar = nInput+1; - pCsr = (IcuCursor *)sqlite3_malloc64( - sizeof(IcuCursor) + /* IcuCursor */ - ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ - (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ - ); - if( !pCsr ){ - return SQLITE_NOMEM; - } - memset(pCsr, 0, sizeof(IcuCursor)); - pCsr->aChar = (UChar *)&pCsr[1]; - pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; - - pCsr->aOffset[iOut] = iInput; - U8_NEXT(zInput, iInput, nInput, c); - while( c>0 ){ - int isError = 0; - c = u_foldCase(c, opt); - U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); - if( isError ){ - sqlite3_free(pCsr); - return SQLITE_ERROR; - } - pCsr->aOffset[iOut] = iInput; - - if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); - if( !U_SUCCESS(status) ){ - sqlite3_free(pCsr); - return SQLITE_ERROR; - } - pCsr->nChar = iOut; - - ubrk_first(pCsr->pIter); - *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; - return SQLITE_OK; -} - -/* -** Close a tokenization cursor previously opened by a call to icuOpen(). -*/ -static int icuClose(sqlite3_tokenizer_cursor *pCursor){ - IcuCursor *pCsr = (IcuCursor *)pCursor; - ubrk_close(pCsr->pIter); - sqlite3_free(pCsr->zBuffer); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** Extract the next token from a tokenization cursor. -*/ -static int icuNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ - const char **ppToken, /* OUT: *ppToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ -){ - IcuCursor *pCsr = (IcuCursor *)pCursor; - - int iStart = 0; - int iEnd = 0; - int nByte = 0; - - while( iStart==iEnd ){ - UChar32 c; - - iStart = ubrk_current(pCsr->pIter); - iEnd = ubrk_next(pCsr->pIter); - if( iEnd==UBRK_DONE ){ - return SQLITE_DONE; - } - - while( iStartaChar, iWhite, pCsr->nChar, c); - if( u_isspace(c) ){ - iStart = iWhite; - }else{ - break; - } - } - assert(iStart<=iEnd); - } - - do { - UErrorCode status = U_ZERO_ERROR; - if( nByte ){ - char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); - if( !zNew ){ - return SQLITE_NOMEM; - } - pCsr->zBuffer = zNew; - pCsr->nBuffer = nByte; - } - - u_strToUTF8( - pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ - &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ - &status /* Output success/failure */ - ); - } while( nByte>pCsr->nBuffer ); - - *ppToken = pCsr->zBuffer; - *pnBytes = nByte; - *piStartOffset = pCsr->aOffset[iStart]; - *piEndOffset = pCsr->aOffset[iEnd]; - *piPosition = pCsr->iToken++; - - return SQLITE_OK; -} - -/* -** The set of routines that implement the simple tokenizer -*/ -static const sqlite3_tokenizer_module icuTokenizerModule = { - 0, /* iVersion */ - icuCreate, /* xCreate */ - icuDestroy, /* xCreate */ - icuOpen, /* xOpen */ - icuClose, /* xClose */ - icuNext, /* xNext */ - 0, /* xLanguageid */ -}; - -/* -** Set *ppModule to point at the implementation of the ICU tokenizer. -*/ -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ - *ppModule = &icuTokenizerModule; -} - -#endif /* defined(SQLITE_ENABLE_ICU) */ -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_icu.c ********************************************/ /************** Begin file sqlite3rbu.c **************************************/ /* ** 2014 August 30 @@ -202103,7 +218711,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** The order of the columns in the data_% table does not matter. ** ** Instead of a regular table, the RBU database may also contain virtual -** tables or view named using the data_ naming scheme. +** tables or views named using the data_ naming scheme. ** ** Instead of the plain data_ naming scheme, RBU database tables ** may also be named data_, where is any sequence @@ -202116,7 +218724,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** If the target database table is a virtual table or a table that has no ** PRIMARY KEY declaration, the data_% table must also contain a column -** named "rbu_rowid". This column is mapped to the tables implicit primary +** named "rbu_rowid". This column is mapped to the table's implicit primary ** key column - "rowid". Virtual tables for which the "rowid" column does ** not function like a primary key value cannot be updated using RBU. For ** example, if the target db contains either of the following: @@ -202549,6 +219157,34 @@ SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); +/* +** As part of applying an RBU update or performing an RBU vacuum operation, +** the system must at one point move the *-oal file to the equivalent *-wal +** path. Normally, it does this by invoking POSIX function rename(2) directly. +** Except on WINCE platforms, where it uses win32 API MoveFileW(). This +** function may be used to register a callback that the RBU module will invoke +** instead of one of these APIs. +** +** If a callback is registered with an RBU handle, it invokes it instead +** of rename(2) when it needs to move a file within the file-system. The +** first argument passed to the xRename() callback is a copy of the second +** argument (pArg) passed to this function. The second is the full path +** to the file to move and the third the full path to which it should be +** moved. The callback function should return SQLITE_OK to indicate +** success. If an error occurs, it should return an SQLite error code. +** In this case the RBU operation will be abandoned and the error returned +** to the RBU user. +** +** Passing a NULL pointer in place of the xRename argument to this function +** restores the default behaviour. +*/ +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +); + + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, @@ -202722,6 +219358,7 @@ typedef unsigned int u32; typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; #endif /* @@ -202916,6 +219553,8 @@ struct sqlite3rbu { int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; + void *pRenameArg; + int (*xRename)(void*, const char*, const char*); /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -203406,6 +220045,7 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ if( rc!=SQLITE_ROW ){ rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -204086,7 +220726,7 @@ static char *rbuVacuumTableStart( ** the caller has to use an OFFSET clause to extract only the required ** rows from the sourct table, just as it does for an RBU update operation. */ -char *rbuVacuumIndexStart( +static char *rbuVacuumIndexStart( sqlite3rbu *p, /* RBU handle */ RbuObjIter *pIter /* RBU iterator object */ ){ @@ -205304,7 +221944,7 @@ static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->zState==0 ){ const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); } } @@ -205500,7 +222140,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; @@ -205552,11 +222192,11 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ ** no-ops. These locks will not be released until the connection ** is closed. ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** * Attempting to xSync() the database file causes an SQLITE_NOTICE ** error. ** ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[] ** array populated with a set of (frame -> page) mappings. Because the ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy ** data from the wal file into the database file according to the @@ -205566,7 +222206,7 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ int rc2; p->eStage = RBU_STAGE_CAPTURE; rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + if( rc2!=SQLITE_NOTICE ) p->rc = rc2; } if( p->rc==SQLITE_OK && p->nFrame>0 ){ @@ -205612,7 +222252,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->mLock!=mReq ){ pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } pRbu->pgsz = iAmt; @@ -205662,6 +222302,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if @@ -205670,9 +222315,20 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ static int rbuLockDatabase(sqlite3 *db){ int rc = SQLITE_OK; sqlite3_file *fd = 0; - sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); - if( fd->pMethods ){ + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE); + } + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + }else{ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + } + + if( rc==SQLITE_OK && fd->pMethods ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); if( rc==SQLITE_OK ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE); @@ -205764,32 +222420,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ -#if defined(_WIN32_WCE) - { - LPWSTR zWideOal; - LPWSTR zWideWal; - - zWideOal = rbuWinUtf8ToUnicode(zOal); - if( zWideOal ){ - zWideWal = rbuWinUtf8ToUnicode(zWal); - if( zWideWal ){ - if( MoveFileW(zWideOal, zWideWal) ){ - p->rc = SQLITE_OK; - }else{ - p->rc = SQLITE_IOERR; - } - sqlite3_free(zWideWal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - sqlite3_free(zWideOal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - } -#else - p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; -#endif + p->rc = p->xRename(p->pRenameArg, zOal, zWal); } if( p->rc!=SQLITE_OK @@ -206376,7 +223007,8 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ static void rbuDeleteOalFile(sqlite3rbu *p){ char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget); if( zOal ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + sqlite3_vfs *pVfs = 0; + sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 ); pVfs->xDelete(pVfs, zOal, 0); sqlite3_free(zOal); @@ -206528,6 +223160,7 @@ static sqlite3rbu *openRbuHandle( /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); + sqlite3rbu_rename_handler(p, 0, 0); rbuCreateVfs(p); /* Open the target, RBU and state databases */ @@ -206919,6 +223552,54 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ return rc; } +/* +** Default xRename callback for RBU. +*/ +static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; +#if defined(_WIN32_WCE) + { + LPWSTR zWideOld; + LPWSTR zWideNew; + + zWideOld = rbuWinUtf8ToUnicode(zOld); + if( zWideOld ){ + zWideNew = rbuWinUtf8ToUnicode(zNew); + if( zWideNew ){ + if( MoveFileW(zWideOld, zWideNew) ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR; + } + sqlite3_free(zWideNew); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOld); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + } +#else + rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK; +#endif + return rc; +} + +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +){ + if( xRename ){ + pRbu->xRename = xRename; + pRbu->pRenameArg = pArg; + }else{ + pRbu->xRename = xDefaultRename; + pRbu->pRenameArg = 0; + } +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: @@ -206975,7 +223656,7 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ ** database file are recorded. xShmLock() calls to unlock the same ** locks are no-ops (so that once obtained, these locks are never ** relinquished). Finally, calls to xSync() on the target database -** file fail with SQLITE_INTERNAL errors. +** file fail with SQLITE_NOTICE errors. */ static void rbuUnlockShm(rbu_file *p){ @@ -207084,9 +223765,12 @@ static int rbuVfsClose(sqlite3_file *pFile){ sqlite3_free(p->zDel); if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ + const sqlite3_io_methods *pMeth = p->pReal->pMethods; rbuMainlistRemove(p); rbuUnlockShm(p); - p->pReal->pMethods->xShmUnmap(p->pReal, 0); + if( pMeth->iVersion>1 && pMeth->xShmUnmap ){ + pMeth->xShmUnmap(p->pReal, 0); + } } else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ rbuUpdateTempSize(p, 0); @@ -207254,7 +223938,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ rbu_file *p = (rbu_file *)pFile; if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } return SQLITE_OK; } @@ -207545,6 +224229,25 @@ static int rbuVfsOpen( rbuVfsShmUnmap, /* xShmUnmap */ 0, 0 /* xFetch, xUnfetch */ }; + static sqlite3_io_methods rbuvfs_io_methods1 = { + 1, /* iVersion */ + rbuVfsClose, /* xClose */ + rbuVfsRead, /* xRead */ + rbuVfsWrite, /* xWrite */ + rbuVfsTruncate, /* xTruncate */ + rbuVfsSync, /* xSync */ + rbuVfsFileSize, /* xFileSize */ + rbuVfsLock, /* xLock */ + rbuVfsUnlock, /* xUnlock */ + rbuVfsCheckReservedLock, /* xCheckReservedLock */ + rbuVfsFileControl, /* xFileControl */ + rbuVfsSectorSize, /* xSectorSize */ + rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, 0, 0, 0, 0, 0 + }; + + + rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs; rbu_file *pFd = (rbu_file *)pFile; @@ -207599,10 +224302,15 @@ static int rbuVfsOpen( rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ + const sqlite3_io_methods *pMeth = pFd->pReal->pMethods; /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods ** pointer and, if the file is a main database file, link it into the ** mutex protected linked list of all such files. */ - pFile->pMethods = &rbuvfs_io_methods; + if( pMeth->iVersion<2 || pMeth->xShmLock==0 ){ + pFile->pMethods = &rbuvfs_io_methods1; + }else{ + pFile->pMethods = &rbuvfs_io_methods; + } if( flags & SQLITE_OPEN_MAIN_DB ){ rbuMainlistAdd(pFd); } @@ -208035,6 +224743,7 @@ static int statConnect( StatTable *pTab = 0; int rc = SQLITE_OK; int iDb; + (void)pAux; if( argc>=4 ){ Token nm; @@ -208088,6 +224797,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iSchema = -1; int iName = -1; int iAgg = -1; + (void)tab; /* Look for a valid schema=? constraint. If found, change the idxNum to ** 1 and request the value of that constraint be sent to xFilter. And @@ -208613,6 +225323,8 @@ static int statFilter( int iArg = 0; /* Count of argv[] parameters used so far */ int rc = SQLITE_OK; /* Result of this operation */ const char *zName = 0; /* Only provide analysis of this table */ + (void)argc; + (void)idxStr; statResetCsr(pCsr); sqlite3_finalize(pCsr->pStmt); @@ -208696,16 +225408,16 @@ static int statColumn( } break; case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); + sqlite3_result_int64(ctx, pCsr->nCell); break; case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); + sqlite3_result_int64(ctx, pCsr->nPayload); break; case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); + sqlite3_result_int64(ctx, pCsr->nUnused); break; case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); + sqlite3_result_int64(ctx, pCsr->nMxPayload); break; case 8: /* pgoffset */ if( !pCsr->isAgg ){ @@ -208713,7 +225425,7 @@ static int statColumn( } break; case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); + sqlite3_result_int64(ctx, pCsr->szPage); break; case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -208763,7 +225475,8 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); } @@ -208847,8 +225560,13 @@ static int dbpageConnect( ){ DbpageTable *pTab = 0; int rc = SQLITE_OK; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -208885,6 +225603,7 @@ static int dbpageDisconnect(sqlite3_vtab *pVtab){ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; int iPlan = 0; + (void)tab; /* If there is a schema= constraint, it must be honored. Report a ** ridiculously large estimated cost if the schema= constraint is @@ -208999,6 +225718,8 @@ static int dbpageFilter( sqlite3 *db = pTab->db; Btree *pBt; + (void)idxStr; + /* Default setting is no rows of result */ pCsr->pgno = 1; pCsr->mxPgno = 0; @@ -209013,7 +225734,7 @@ static int dbpageFilter( pCsr->iDb = 0; } pBt = db->aDb[pCsr->iDb].pBt; - if( pBt==0 ) return SQLITE_OK; + if( NEVER(pBt==0) ) return SQLITE_OK; pCsr->pPager = sqlite3BtreePager(pBt); pCsr->szPage = sqlite3BtreeGetPageSize(pBt); pCsr->mxPgno = sqlite3BtreeLastPage(pBt); @@ -209048,12 +225769,18 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, - SQLITE_TRANSIENT); + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); } - sqlite3PagerUnref(pDbPage); break; } default: { /* schema */ @@ -209062,7 +225789,7 @@ static int dbpageColumn( break; } } - return SQLITE_OK; + return rc; } static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -209088,6 +225815,7 @@ static int dbpageUpdate( Pager *pPager; int szPage; + (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ zErr = "read-only"; goto update_fail; @@ -209097,18 +225825,20 @@ static int dbpageUpdate( goto update_fail; } pgno = sqlite3_value_int(argv[0]); - if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + if( sqlite3_value_type(argv[0])==SQLITE_NULL + || (Pgno)sqlite3_value_int(argv[1])!=pgno + ){ zErr = "cannot insert"; goto update_fail; } zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( iDb<0 ){ + iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; + if( NEVER(iDb<0) ){ zErr = "no such schema"; goto update_fail; } pBt = pTab->db->aDb[iDb].pBt; - if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){ + if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ zErr = "bad page number"; goto update_fail; } @@ -209122,11 +225852,12 @@ static int dbpageUpdate( pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pDbPage); - if( rc==SQLITE_OK ){ - memcpy(sqlite3PagerGetData(pDbPage), - sqlite3_value_blob(argv[3]), - szPage); + const void *pData = sqlite3_value_blob(argv[3]); + assert( pData!=0 || pTab->db->mallocFailed ); + if( pData + && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK + ){ + memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -209148,7 +225879,7 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ int i; for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); + if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); } return SQLITE_OK; } @@ -209182,7 +225913,8 @@ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); } @@ -209219,6 +225951,8 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; typedef struct SessionHook SessionHook; @@ -209240,6 +225974,7 @@ struct sqlite3_session { int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); @@ -209310,17 +226045,32 @@ struct sqlite3_changeset_iter { ** The data associated with each hash-table entry is a structure containing ** a subset of the initial values that the modified row contained at the ** start of the session. Or no initial values if the row was inserted. +** +** pDfltStmt: +** This is only used by the sqlite3changegroup_xxx() APIs, not by +** regular sqlite3_session objects. It is a SELECT statement that +** selects the default value for each table column. For example, +** if the table is +** +** CREATE TABLE xx(a DEFAULT 1, b, c DEFAULT 'abc') +** +** then this variable is the compiled version of: +** +** SELECT 1, NULL, 'abc' */ struct SessionTable { SessionTable *pNext; char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ + const char **azDflt; /* Default value expressions */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ int nChange; /* Size of apChange[] array */ SessionChange **apChange; /* Hash table buckets */ + sqlite3_stmt *pDfltStmt; }; /* @@ -209489,6 +226239,7 @@ struct SessionTable { struct SessionChange { u8 op; /* One of UPDATE, DELETE, INSERT */ u8 bIndirect; /* True if this change is "indirect" */ + u16 nRecordField; /* Number of fields in aRecord[] */ int nMaxSize; /* Max size of eventual changeset record */ int nRecord; /* Number of bytes in buffer aRecord[] */ u8 *aRecord; /* Buffer containing old.* record */ @@ -209514,7 +226265,7 @@ static int sessionVarintLen(int iVal){ ** Read a varint value from aBuf[] into *piVal. Return the number of ** bytes read. */ -static int sessionVarintGet(u8 *aBuf, int *piVal){ +static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } @@ -209708,6 +226459,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -209716,48 +226468,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; inCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; - - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); + if( rc!=SQLITE_OK ) return rc; + + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - z = (const u8 *)sqlite3_value_blob(pVal); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - assert( pTab->bStat1==0 || i!=1 ); - *pbNullPK = 1; } } } @@ -209771,9 +226528,11 @@ static int sessionPreupdateHash( ** Return the number of bytes of space occupied by the value (including ** the type byte). */ -static int sessionSerialLen(u8 *a){ - int e = *a; +static int sessionSerialLen(const u8 *a){ + int e; int n; + assert( a!=0 ); + e = *a; if( e==0 || e==0xFF ) return 1; if( e==SQLITE_NULL ) return 1; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; @@ -210040,6 +226799,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -210047,6 +226807,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -210069,6 +226834,7 @@ static int sessionPreupdateEqual( rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); } assert( rc==SQLITE_OK ); + (void)rc; /* Suppress warning about unused variable */ if( sqlite3_value_type(pVal)!=eType ) return 0; /* A SessionChange object never has a NULL value in a PK column */ @@ -210171,13 +226937,14 @@ static int sessionGrowHash( ** ** For example, if the table is declared as: ** -** CREATE TABLE tbl1(w, x, y, z, PRIMARY KEY(w, z)); +** CREATE TABLE tbl1(w, x DEFAULT 'abc', y, z, PRIMARY KEY(w, z)); ** -** Then the four output variables are populated as follows: +** Then the five output variables are populated as follows: ** ** *pnCol = 4 ** *pzTab = "tbl1" ** *pazCol = {"w", "x", "y", "z"} +** *pazDflt = {NULL, 'abc', NULL, NULL} ** *pabPK = {1, 0, 0, 1} ** ** All returned buffers are part of the same single allocation, which must @@ -210191,7 +226958,9 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + const char ***pazDflt, /* OUT: Array of default value expressions */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; @@ -210202,10 +226971,18 @@ static int sessionTableInfo( int i; u8 *pAlloc = 0; char **azCol = 0; + char **azDflt = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); + *pazCol = 0; + *pabPK = 0; + *pnCol = 0; + if( pzTab ) *pzTab = 0; + if( pazDflt ) *pazDflt = 0; + nThis = sqlite3Strlen30(zThis); if( nThis==12 && 0==sqlite3_stricmp("sqlite_stat1", zThis) ){ rc = sqlite3_table_column_metadata(db, zDb, zThis, 0, 0, 0, 0, 0, 0); @@ -210219,50 +226996,47 @@ static int sessionTableInfo( }else if( rc==SQLITE_ERROR ){ zPragma = sqlite3_mprintf(""); }else{ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return rc; } }else{ zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); } if( !zPragma ){ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return SQLITE_NOMEM; } rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0); sqlite3_free(zPragma); if( rc!=SQLITE_OK ){ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return rc; } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ - nByte += sqlite3_column_bytes(pStmt, 1); + nByte += sqlite3_column_bytes(pStmt, 1); /* name */ + nByte += sqlite3_column_bytes(pStmt, 4); /* dflt_value */ nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; /* pk */ } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ - nByte += nDbCol * (sizeof(const char *) + sizeof(u8) + 1); + nByte += nDbCol * (sizeof(const char *)*2 + sizeof(u8) + 1 + 1); pAlloc = sessionMalloc64(pSession, nByte); if( pAlloc==0 ){ rc = SQLITE_NOMEM; + }else{ + memset(pAlloc, 0, nByte); } } if( rc==SQLITE_OK ){ azCol = (char **)pAlloc; - pAlloc = (u8 *)&azCol[nDbCol]; + azDflt = (char**)&azCol[nDbCol]; + pAlloc = (u8 *)&azDflt[nDbCol]; abPK = (u8 *)pAlloc; pAlloc = &abPK[nDbCol]; if( pzTab ){ @@ -210272,43 +227046,57 @@ static int sessionTableInfo( } i = 0; + if( bRowid ){ + size_t nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); + int nDflt = sqlite3_column_bytes(pStmt, 4); const unsigned char *zName = sqlite3_column_text(pStmt, 1); + const unsigned char *zDflt = sqlite3_column_text(pStmt, 4); + if( zName==0 ) break; memcpy(pAlloc, zName, nName+1); azCol[i] = (char *)pAlloc; pAlloc += nName+1; + if( zDflt ){ + memcpy(pAlloc, zDflt, nDflt+1); + azDflt[i] = (char *)pAlloc; + pAlloc += nDflt+1; + }else{ + azDflt[i] = 0; + } abPK[i] = sqlite3_column_int(pStmt, 5); i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and ** free any allocation made. An error code will be returned in this case. */ if( rc==SQLITE_OK ){ - *pazCol = (const char **)azCol; + *pazCol = (const char**)azCol; + if( pazDflt ) *pazDflt = (const char**)azDflt; *pabPK = abPK; *pnCol = nDbCol; }else{ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } /* -** This function is only called from within a pre-update handler for a -** write to table pTab, part of session pSession. If this is the first -** write to this table, initalize the SessionTable.nCol, azCol[] and -** abPK[] arrays accordingly. +** This function is called to initialize the SessionTable.nCol, azCol[] +** abPK[] and azDflt[] members of SessionTable object pTab. If these +** fields are already initilialized, this function is a no-op. ** ** If an error occurs, an error code is stored in sqlite3_session.rc and ** non-zero returned. Or, if no error occurs but the table has no primary @@ -210316,14 +227104,22 @@ static int sessionTableInfo( ** indicate that updates on this table should be ignored. SessionTable.abPK ** is set to NULL in this case. */ -static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ +static int sessionInitTable( + sqlite3_session *pSession, /* Optional session handle */ + SessionTable *pTab, /* Table object to initialize */ + sqlite3 *db, /* Database handle to read schema from */ + const char *zDb /* Name of db - "main", "temp" etc. */ +){ + int rc = SQLITE_OK; + if( pTab->nCol==0 ){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); - pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + rc = sessionTableInfo(pSession, db, zDb, + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &pTab->azDflt, &abPK, + ((pSession==0 || pSession->bImplicitPK) ? &pTab->bRowid : 0) ); - if( pSession->rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ int i; for(i=0; inCol; i++){ if( abPK[i] ){ @@ -210335,14 +227131,321 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ pTab->bStat1 = 1; } - if( pSession->bEnableSize ){ + if( pSession && pSession->bEnableSize ){ pSession->nMaxChangesetSize += ( 1 + sessionVarintLen(pTab->nCol) + pTab->nCol + strlen(pTab->zName)+1 ); } } } - return (pSession->rc || pTab->abPK==0); + + if( pSession ){ + pSession->rc = rc; + return (rc || pTab->abPK==0); + } + return rc; +} + +/* +** Re-initialize table object pTab. +*/ +static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ + int nCol = 0; + const char **azCol = 0; + const char **azDflt = 0; + u8 *abPK = 0; + int bRowid = 0; + + assert( pSession->rc==SQLITE_OK ); + + pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, + pTab->zName, &nCol, 0, &azCol, &azDflt, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( pSession->rc==SQLITE_OK ){ + if( pTab->nCol>nCol || pTab->bRowid!=bRowid ){ + pSession->rc = SQLITE_SCHEMA; + }else{ + int ii; + int nOldCol = pTab->nCol; + for(ii=0; iinCol ){ + if( pTab->abPK[ii]!=abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + }else if( abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + } + + if( pSession->rc==SQLITE_OK ){ + const char **a = pTab->azCol; + pTab->azCol = azCol; + pTab->nCol = nCol; + pTab->azDflt = azDflt; + pTab->abPK = abPK; + azCol = a; + } + if( pSession->bEnableSize ){ + pSession->nMaxChangesetSize += (nCol - nOldCol); + pSession->nMaxChangesetSize += sessionVarintLen(nCol); + pSession->nMaxChangesetSize -= sessionVarintLen(nOldCol); + } + } + } + + sqlite3_free((char*)azCol); + return pSession->rc; +} + +/* +** Session-change object (*pp) contains an old.* record with fewer than +** nCol fields. This function updates it with the default values for +** the missing fields. +*/ +static void sessionUpdateOneChange( + sqlite3_session *pSession, /* For memory accounting */ + int *pRc, /* IN/OUT: Error code */ + SessionChange **pp, /* IN/OUT: Change object to update */ + int nCol, /* Number of columns now in table */ + sqlite3_stmt *pDflt /* SELECT */ +){ + SessionChange *pOld = *pp; + + while( pOld->nRecordFieldnRecordField; + int eType = sqlite3_column_type(pDflt, iField); + switch( eType ){ + case SQLITE_NULL: + nIncr = 1; + break; + case SQLITE_INTEGER: + case SQLITE_FLOAT: + nIncr = 9; + break; + default: { + int n = sqlite3_column_bytes(pDflt, iField); + nIncr = 1 + sessionVarintLen(n) + n; + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + break; + } + } + + nByte = nIncr + (sizeof(SessionChange) + pOld->nRecord); + pNew = sessionMalloc64(pSession, nByte); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return; + }else{ + memcpy(pNew, pOld, sizeof(SessionChange)); + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, pOld->aRecord, pOld->nRecord); + pNew->aRecord[pNew->nRecord++] = (u8)eType; + switch( eType ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_column_int64(pDflt, iField); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_FLOAT: { + double rVal = sqlite3_column_double(pDflt, iField); + i64 iVal = 0; + memcpy(&iVal, &rVal, sizeof(rVal)); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pDflt, iField); + const char *z = (const char*)sqlite3_column_text(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pDflt, iField); + const u8 *z = (const u8*)sqlite3_column_blob(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + + sessionFree(pSession, pOld); + *pp = pOld = pNew; + pNew->nRecordField++; + pNew->nMaxSize += nIncr; + if( pSession ){ + pSession->nMaxChangesetSize += nIncr; + } + } + } +} + +/* +** Ensure that there is room in the buffer to append nByte bytes of data. +** If not, use sqlite3_realloc() to grow the buffer so that there is. +** +** If successful, return zero. Otherwise, if an OOM condition is encountered, +** set *pRc to SQLITE_NOMEM and return non-zero. +*/ +static int sessionBufferGrow(SessionBuffer *p, i64 nByte, int *pRc){ +#define SESSION_MAX_BUFFER_SZ (0x7FFFFF00 - 1) + i64 nReq = p->nBuf + nByte; + if( *pRc==SQLITE_OK && nReq>p->nAlloc ){ + u8 *aNew; + i64 nNew = p->nAlloc ? p->nAlloc : 128; + + do { + nNew = nNew*2; + }while( nNewSESSION_MAX_BUFFER_SZ ){ + nNew = SESSION_MAX_BUFFER_SZ; + if( nNewaBuf, nNew); + if( 0==aNew ){ + *pRc = SQLITE_NOMEM; + }else{ + p->aBuf = aNew; + p->nAlloc = nNew; + } + } + return (*pRc!=SQLITE_OK); +} + + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a string to the buffer. All bytes in the string +** up to (but not including) the nul-terminator are written to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendStr( + SessionBuffer *p, + const char *zStr, + int *pRc +){ + int nStr = sqlite3Strlen30(zStr); + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ + memcpy(&p->aBuf[p->nBuf], zStr, nStr); + p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; + } +} + +/* +** Format a string using printf() style formatting and then append it to the +** buffer using sessionAppendString(). +*/ +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + +/* +** Prepare a statement against database handle db that SELECTs a single +** row containing the default values for each column in table pTab. For +** example, if pTab is declared as: +** +** CREATE TABLE pTab(a PRIMARY KEY, b DEFAULT 123, c DEFAULT 'abcd'); +** +** Then this function prepares and returns the SQL statement: +** +** SELECT NULL, 123, 'abcd'; +*/ +static int sessionPrepareDfltStmt( + sqlite3 *db, /* Database handle */ + SessionTable *pTab, /* Table to prepare statement for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + SessionBuffer sql = {0,0,0}; + int rc = SQLITE_OK; + const char *zSep = " "; + int ii = 0; + + *ppStmt = 0; + sessionAppendPrintf(&sql, &rc, "SELECT"); + for(ii=0; iinCol; ii++){ + const char *zDflt = pTab->azDflt[ii] ? pTab->azDflt[ii] : "NULL"; + sessionAppendPrintf(&sql, &rc, "%s%s", zSep, zDflt); + zSep = ", "; + } + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (const char*)sql.aBuf, -1, ppStmt, 0); + } + sqlite3_free(sql.aBuf); + + return rc; +} + +/* +** Table pTab has one or more existing change-records with old.* records +** with fewer than pTab->nCol columns. This function updates all such +** change-records with the default values for the missing columns. +*/ +static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ + sqlite3_stmt *pStmt = 0; + int rc = pSession->rc; + + rc = sessionPrepareDfltStmt(pSession->db, pTab, &pStmt); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii = 0; + SessionChange **pp = 0; + for(ii=0; iinChange; ii++){ + for(pp=&pTab->apChange[ii]; *pp; pp=&((*pp)->pNext)){ + if( (*pp)->nRecordField!=pTab->nCol ){ + sessionUpdateOneChange(pSession, &rc, pp, pTab->nCol, pStmt); + } + } + } + } + + pSession->rc = rc; + rc = sqlite3_finalize(pStmt); + if( pSession->rc==SQLITE_OK ) pSession->rc = rc; + return pSession->rc; } /* @@ -210393,6 +227496,7 @@ static int sessionUpdateMaxSize( ){ i64 nNew = 2; if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; if( op!=SQLITE_DELETE ){ int ii; for(ii=0; iinCol; ii++){ @@ -210409,12 +227513,16 @@ static int sessionUpdateMaxSize( }else{ int ii; u8 *pCsr = pC->aRecord; - for(ii=0; iinCol; ii++){ + if( pTab->bRowid ){ + nNew += 9 + 1; + pCsr += 9; + } + for(ii=pTab->bRowid; iinCol; ii++){ int bChanged = 1; int nOld = 0; int eType; sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); if( p==0 ){ return SQLITE_NOMEM; } @@ -210493,22 +227601,29 @@ static int sessionUpdateMaxSize( */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ int iHash; int bNull = 0; int rc = SQLITE_OK; + int nExpect = 0; SessionStat1Ctx stat1 = {{0,0,0,0,0},0}; if( pSession->rc ) return; /* Load table details if required */ - if( sessionInitTable(pSession, pTab) ) return; + if( sessionInitTable(pSession, pTab, pSession->db, pSession->zDb) ) return; /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + nExpect = pSession->hook.xCount(pSession->hook.pCtx); + if( (pTab->nCol-pTab->bRowid)nCol-pTab->bRowid)!=nExpect ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -210541,14 +227656,16 @@ static void sessionPreupdateOneChange( /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ @@ -210563,7 +227680,7 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; inCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -210578,9 +227695,12 @@ static void sessionPreupdateOneChange( rc = sessionSerializeValue(0, p, &nByte); if( rc!=SQLITE_OK ) goto error_out; } + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } /* Allocate the change object */ - pC = (SessionChange *)sessionMalloc64(pSession, nByte); + pC = (SessionChange*)sessionMalloc64(pSession, nByte); if( !pC ){ rc = SQLITE_NOMEM; goto error_out; @@ -210594,7 +227714,12 @@ static void sessionPreupdateOneChange( ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; inCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -210608,6 +227733,7 @@ static void sessionPreupdateOneChange( if( pSession->bIndirect || pSession->hook.xDepth(pSession->hook.pCtx) ){ pC->bIndirect = 1; } + pC->nRecordField = pTab->nCol; pC->nRecord = nByte; pC->op = op; pC->pNext = pTab->apChange[iHash]; @@ -210693,6 +227819,8 @@ static void xPreUpdate( int nDb = sqlite3Strlen30(zDb); assert( sqlite3_mutex_held(db->mutex) ); + (void)iKey1; + (void)iKey2; for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ SessionTable *pTab; @@ -210707,9 +227835,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -210748,6 +227877,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -210756,19 +227886,20 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ + (void)pCtx; return 0; } @@ -210842,17 +227973,18 @@ static char *sessionExprCompareOther( } static char *sessionSelectFindNew( - int nCol, const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -210866,7 +227998,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(pTab->nCol, zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -210877,8 +228011,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -210888,6 +228024,27 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; iinCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( sqlite3_session *pSession, SessionTable *pTab, @@ -210902,11 +228059,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -210917,12 +228076,15 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; @@ -210951,7 +228113,7 @@ SQLITE_API int sqlite3session_diff( /* Locate and if necessary initialize the target table object */ rc = sessionFindTable(pSession, zTbl, &pTo); if( pTo==0 ) goto diff_out; - if( sessionInitTable(pSession, pTo) ){ + if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){ rc = pSession->rc; goto diff_out; } @@ -210961,9 +228123,12 @@ SQLITE_API int sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, 0, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -211076,6 +228241,7 @@ static void sessionDeleteTable(sqlite3_session *pSession, SessionTable *pList){ sessionFree(pSession, p); } } + sqlite3_finalize(pTab->pDfltStmt); sessionFree(pSession, (char*)pTab->azCol); /* cast works around VC++ bug */ sessionFree(pSession, pTab->apChange); sessionFree(pSession, pTab); @@ -211108,9 +228274,7 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession){ ** associated hash-tables. */ sessionDeleteTable(pSession, pSession->pTable); - /* Assert that all allocations have been freed and then free the - ** session object itself. */ - assert( pSession->nMalloc==0 ); + /* Free the session object. */ sqlite3_free(pSession); } @@ -211181,48 +228345,6 @@ SQLITE_API int sqlite3session_attach( return rc; } -/* -** Ensure that there is room in the buffer to append nByte bytes of data. -** If not, use sqlite3_realloc() to grow the buffer so that there is. -** -** If successful, return zero. Otherwise, if an OOM condition is encountered, -** set *pRc to SQLITE_NOMEM and return non-zero. -*/ -static int sessionBufferGrow(SessionBuffer *p, i64 nByte, int *pRc){ -#define SESSION_MAX_BUFFER_SZ (0x7FFFFF00 - 1) - i64 nReq = p->nBuf + nByte; - if( *pRc==SQLITE_OK && nReq>p->nAlloc ){ - u8 *aNew; - i64 nNew = p->nAlloc ? p->nAlloc : 128; - - do { - nNew = nNew*2; - }while( nNewSESSION_MAX_BUFFER_SZ ){ - nNew = SESSION_MAX_BUFFER_SZ; - if( nNewaBuf, nNew); - if( 0==aNew ){ - *pRc = SQLITE_NOMEM; - }else{ - p->aBuf = aNew; - p->nAlloc = nNew; - } - } - return (*pRc!=SQLITE_OK); -} - /* ** Append the value passed as the second argument to the buffer passed ** as the first. @@ -211291,26 +228413,6 @@ static void sessionAppendBlob( } } -/* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a string to the buffer. All bytes in the string -** up to (but not including) the nul-terminator are written to the buffer. -** -** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before -** returning. -*/ -static void sessionAppendStr( - SessionBuffer *p, - const char *zStr, - int *pRc -){ - int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ - memcpy(&p->aBuf[p->nBuf], zStr, nStr); - p->nBuf += nStr; - } -} - /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string representation of integer iVal @@ -211343,7 +228445,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -211354,6 +228456,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -211489,7 +228592,7 @@ static int sessionAppendUpdate( /* If at least one field has been modified, this is not a no-op. */ if( bChanged ) bNoop = 0; - /* Add a field to the old.* record. This is omitted if this modules is + /* Add a field to the old.* record. This is omitted if this module is ** currently generating a patchset. */ if( bPatchset==0 ){ if( bChanged || abPK[i] ){ @@ -211578,12 +228681,20 @@ static int sessionAppendDelete( ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. ** -** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +** SELECT *, FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where is: +** +** 1 AND (?A OR ?1 IS ) AND ... +** +** for each non-pk . */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ @@ -211591,8 +228702,50 @@ static int sessionSelectStmt( ){ int rc = SQLITE_OK; char *zSql = 0; + const char *zSep = ""; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; int nSql = -1; + int i; + + SessionBuffer nooptest = {0, 0, 0}; + SessionBuffer pkfield = {0, 0, 0}; + SessionBuffer pkvar = {0, 0, 0}; + sessionAppendStr(&nooptest, ", 1", &rc); + + if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ + sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc); + sessionAppendStr(&pkfield, "tbl, idx", &rc); + sessionAppendStr(&pkvar, + "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc + ); + zCols = "tbl, ?2, stat"; + }else{ + for(i=0; ipTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ const char *zName = pTab->zName; - int nCol = 0; /* Number of columns in table */ - u8 *abPK = 0; /* Primary key array */ - const char **azCol = 0; /* Table columns */ int i; /* Used to iterate through hash buckets */ sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ int nRewind = buf.nBuf; /* Initial size of write buffer */ int nNoop; /* Size of buffer after writing tbl header */ + int nOldCol = pTab->nCol; /* Check the table schema is still Ok. */ - rc = sessionTableInfo(0, db, pSession->zDb, zName, &nCol, 0,&azCol,&abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ - rc = SQLITE_SCHEMA; + rc = sessionReinitTable(pSession, pTab); + if( rc==SQLITE_OK && pTab->nCol!=nOldCol ){ + rc = sessionUpdateChanges(pSession, pTab); } /* Write a table header */ @@ -211784,8 +228938,9 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ - rc = sessionSelectStmt( - db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + rc = sessionSelectStmt(db, 0, pSession->zDb, + zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel + ); } nNoop = buf.nBuf; @@ -211793,22 +228948,22 @@ static int sessionGenerateChangeset( SessionChange *p; /* Used to iterate through changes */ for(p=pTab->apChange[i]; rc==SQLITE_OK && p; p=p->pNext){ - rc = sessionSelectBind(pSel, nCol, abPK, p); + rc = sessionSelectBind(pSel, pTab->nCol, pTab->abPK, p); if( rc!=SQLITE_OK ) continue; if( sqlite3_step(pSel)==SQLITE_ROW ){ if( p->op==SQLITE_INSERT ){ int iCol; sessionAppendByte(&buf, SQLITE_INSERT, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); - for(iCol=0; iColnCol; iCol++){ sessionAppendCol(&buf, pSel, iCol, &rc); } }else{ - assert( abPK!=0 ); /* Because sessionSelectStmt() returned ok */ - rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK); + assert( pTab->abPK!=0 ); + rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, pTab->abPK); } }else if( p->op!=SQLITE_INSERT ){ - rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + rc = sessionAppendDelete(&buf, bPatchset, p, pTab->nCol,pTab->abPK); } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); @@ -211833,7 +228988,6 @@ static int sessionGenerateChangeset( if( buf.nBuf==nNoop ){ buf.nBuf = nRewind; } - sqlite3_free((char*)azCol); /* cast works around VC++ bug */ } } @@ -211868,7 +229022,7 @@ SQLITE_API int sqlite3session_changeset( int rc; if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; - rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset); + rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); assert( rc || pnChangeset==0 || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize ); @@ -211986,6 +229140,19 @@ SQLITE_API int sqlite3session_object_config(sqlite3_session *pSession, int op, v break; } + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + default: rc = SQLITE_MISUSE; } @@ -212244,15 +229411,19 @@ static int sessionReadRecord( } } if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 v = sessionGetI64(aVal); - if( eType==SQLITE_INTEGER ){ - sqlite3VdbeMemSetInt64(apOut[i], v); + if( (pIn->nData-pIn->iNext)<8 ){ + rc = SQLITE_CORRUPT_BKPT; }else{ - double d; - memcpy(&d, &v, 8); - sqlite3VdbeMemSetDouble(apOut[i], d); + sqlite3_int64 v = sessionGetI64(aVal); + if( eType==SQLITE_INTEGER ){ + sqlite3VdbeMemSetInt64(apOut[i], v); + }else{ + double d; + memcpy(&d, &v, 8); + sqlite3VdbeMemSetDouble(apOut[i], d); + } + pIn->iNext += 8; } - pIn->iNext += 8; } } } @@ -212440,14 +229611,14 @@ static int sessionChangesetNextOne( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; while( op=='T' || op=='P' ){ if( pbNew ) *pbNew = 1; @@ -212521,6 +229692,22 @@ static int sessionChangesetNextOne( if( p->op==SQLITE_INSERT ) p->op = SQLITE_DELETE; else if( p->op==SQLITE_DELETE ) p->op = SQLITE_INSERT; } + + /* If this is an UPDATE that is part of a changeset, then check that + ** there are no fields in the old.* record that are not (a) PK fields, + ** or (b) also present in the new.* record. + ** + ** Such records are technically corrupt, but the rebaser was at one + ** point generating them. Under most circumstances this is benign, but + ** can cause spurious SQLITE_RANGE errors when applying the changeset. */ + if( p->bPatchset==0 && p->op==SQLITE_UPDATE){ + for(i=0; inCol; i++){ + if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){ + sqlite3ValueFree(p->apValue[i]); + p->apValue[i] = 0; + } + } + } } return SQLITE_ROW; @@ -212958,6 +230145,8 @@ struct SessionApplyCtx { SessionBuffer rebase; /* Rebase information (if any) here */ u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; /* Number of prepared UPDATE statements to cache. */ @@ -213208,8 +230397,10 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + /* TODO */ + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -213367,22 +230558,34 @@ static int sessionBindRow( ** UPDATE, bind values from the old.* record. */ static int sessionSeekToRow( - sqlite3 *db, /* Database handle */ sqlite3_changeset_iter *pIter, /* Changeset iterator */ - u8 *abPK, /* Primary key flags array */ - sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ + SessionApplyCtx *p ){ + sqlite3_stmt *pSelect = p->pSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && iiabPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -213497,16 +230700,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -213614,7 +230823,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -213671,7 +230880,7 @@ static int sessionApplyOneOp( /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); @@ -213848,6 +231057,7 @@ static int sessionChangesetApply( memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); @@ -213885,6 +231095,7 @@ static int sessionChangesetApply( sApply.bStat1 = 0; sApply.bDeferConstraints = 1; sApply.bRebaseStarted = 0; + sApply.bRowid = 0; memset(&sApply.constraints, 0, sizeof(SessionBuffer)); /* If an xFilter() callback was specified, invoke it now. If the @@ -213904,8 +231115,8 @@ static int sessionChangesetApply( int i; sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo(0, - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, 0, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iflags & SQLITE_FkNoAction; + + if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ + db->flags |= ((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + if( rc==SQLITE_OK ){ rc = sessionChangesetApply( db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags ); } + + if( (flags & SQLITE_CHANGESETAPPLY_FKNOACTION) && savedFlag==0 ){ + assert( db->flags & SQLITE_FkNoAction ); + db->flags &= ~((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } return rc; } @@ -214129,6 +231353,10 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; + + sqlite3 *db; /* Configured by changegroup_schema() */ + char *zDb; /* Configured by changegroup_schema() */ }; /* @@ -214149,6 +231377,7 @@ static int sessionChangeMerge( ){ SessionChange *pNew = 0; int rc = SQLITE_OK; + assert( aRec!=0 ); if( !pExist ){ pNew = (SessionChange *)sqlite3_malloc64(sizeof(SessionChange) + nRec); @@ -214315,84 +231544,236 @@ static int sessionChangeMerge( } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Check if a changeset entry with nCol columns and the PK array passed +** as the final argument to this function is compatible with SessionTable +** pTab. If so, return 1. Otherwise, if they are incompatible in some way, +** return 0. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ - int bRebase /* True if hash table is for rebasing */ +static int sessionChangesetCheckCompat( + SessionTable *pTab, + int nCol, + u8 *abPK ){ - u8 *aRec; - int nRec; - int rc = SQLITE_OK; - SessionTable *pTab = 0; - - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; - - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; + if( pTab->azCol && nColnCol ){ + int ii; + for(ii=0; iinCol; ii++){ + u8 bPK = (ii < nCol) ? abPK[ii] : 0; + if( pTab->abPK[ii]!=bPK ) return 0; } + return 1; + } + return (pTab->nCol==nCol && 0==memcmp(abPK, pTab->abPK, nCol)); +} - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; +static int sessionChangesetExtendRecord( + sqlite3_changegroup *pGrp, + SessionTable *pTab, + int nCol, + int op, + const u8 *aRec, + int nRec, + SessionBuffer *pOut +){ + int rc = SQLITE_OK; + int ii = 0; - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; - } - if( !pTab ){ - SessionTable **ppTab; + assert( pTab->azCol ); + assert( nColnCol ); - pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; + pOut->nBuf = 0; + if( op==SQLITE_INSERT || (op==SQLITE_DELETE && pGrp->bPatch==0) ){ + /* Append the missing default column values to the record. */ + sessionAppendBlob(pOut, aRec, nRec, &rc); + if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){ + rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + } + for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ + int eType = sqlite3_column_type(pTab->pDfltStmt, ii); + sessionAppendByte(pOut, eType, &rc); + switch( eType ){ + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + }else{ + double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + memcpy(&iVal, &rVal, sizeof(i64)); + } + if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + } break; } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ - rc = SQLITE_SCHEMA; - break; + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pTab->pDfltStmt, ii); + sessionAppendVarint(pOut, n, &rc); + if( eType==SQLITE_TEXT ){ + const u8 *z = (const u8*)sqlite3_column_text(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + }else{ + const u8 *z = (const u8*)sqlite3_column_blob(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + } + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + } + }else if( op==SQLITE_UPDATE ){ + /* Append missing "undefined" entries to the old.* record. And, if this + ** is an UPDATE, to the new.* record as well. */ + int iOff = 0; + if( pGrp->bPatch==0 ){ + for(ii=0; iinCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); } } - if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; + sessionAppendBlob(pOut, &aRec[iOff], nRec-iOff, &rc); + for(ii=0; ii<(pTab->nCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); + } + }else{ + assert( op==SQLITE_DELETE && pGrp->bPatch ); + sessionAppendBlob(pOut, aRec, nRec, &rc); + } + + return rc; +} + +/* +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. +*/ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab +){ + int rc = SQLITE_OK; + SessionTable *pTab = 0; + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; + + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); + + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } + + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; + + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); + + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; + } } + + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } + + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } + + *ppTab = pTab; + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nColnCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -214407,16 +231788,39 @@ static int sessionChangesetToHash( break; } } + } + if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } if( rc==SQLITE_OK ) rc = pIter->rc; @@ -214505,6 +231909,31 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +/* +** Provide a database schema to the changegroup object. +*/ +SQLITE_API int sqlite3changegroup_schema( + sqlite3_changegroup *pGrp, + sqlite3 *db, + const char *zDb +){ + int rc = SQLITE_OK; + + if( pGrp->pList || pGrp->db ){ + /* Cannot add a schema after one or more calls to sqlite3changegroup_add(), + ** or after sqlite3changegroup_schema() has already been called. */ + rc = SQLITE_MISUSE; + }else{ + pGrp->zDb = sqlite3_mprintf("%s", zDb); + if( pGrp->zDb==0 ){ + rc = SQLITE_NOMEM; + }else{ + pGrp->db = db; + } + } + return rc; +} + /* ** Add the changeset currently stored in buffer pData, size nData bytes, ** to changeset-group p. @@ -214521,6 +231950,23 @@ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void return rc; } +/* +** Add a single change to a changeset-group. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + return SQLITE_ERROR; + } + return sessionOneChangeToHash(pGrp, pIter, 0); +} + /* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. @@ -214568,7 +232014,9 @@ SQLITE_API int sqlite3changegroup_output_strm( */ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ + sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } @@ -214717,7 +232165,7 @@ static void sessionAppendPartialUpdate( if( !pIter->abPK[i] && a1[0] ) bData = 1; memcpy(pOut, a1, n1); pOut += n1; - }else if( a2[0]!=0xFF ){ + }else if( a2[0]!=0xFF && a1[0] ){ bData = 1; memcpy(pOut, a2, n2); pOut += n2; @@ -214970,6 +232418,7 @@ SQLITE_API int sqlite3rebaser_rebase_strm( SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ if( p ){ sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); sqlite3_free(p); } } @@ -215067,8 +232516,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -215100,8 +232549,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -215111,8 +232563,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -215128,12 +232582,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -215159,6 +232614,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -215273,6 +232732,39 @@ struct Fts5PhraseIter { ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ @@ -215310,6 +232802,13 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); }; /* @@ -215504,8 +233003,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -215553,7 +233052,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -215562,7 +233061,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -215570,7 +233069,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); @@ -215742,6 +233241,10 @@ typedef struct Fts5Config Fts5Config; ** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. ** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** ** zContent: ** ** zContentRowid: @@ -215776,9 +233279,11 @@ struct Fts5Config { int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bTokendata; /* "tokendata=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; @@ -215787,6 +233292,7 @@ struct Fts5Config { int ePattern; /* FTS_PATTERN_XXX constant */ /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ @@ -215795,6 +233301,8 @@ struct Fts5Config { int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -215804,8 +233312,11 @@ struct Fts5Config { #endif }; -/* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 4 +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -215874,7 +233385,7 @@ static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) @@ -215961,16 +233472,19 @@ struct Fts5IndexIter { /* ** Values used as part of the flags argument passed to IndexQuery(). */ -#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ -#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ -#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ -#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ -#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 -#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 +#define FTS5INDEX_QUERY_SCANONETERM 0x0100 /* ** Create/destroy an Fts5Index object. @@ -216039,6 +233553,10 @@ static void *sqlite3Fts5StructureRef(Fts5Index*); static void sqlite3Fts5StructureRelease(void*); static int sqlite3Fts5StructureTest(Fts5Index*, void*); +/* +** Used by xInstToken(): +*/ +static int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); /* ** Insert or remove data to or from the index. Each time a document is @@ -216113,6 +233631,16 @@ static int sqlite3Fts5IndexReset(Fts5Index *p); static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + +static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); + +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ +static int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff +); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -216125,7 +233653,7 @@ static int sqlite3Fts5GetVarintLen(u32 iVal); static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); -#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b)) #define fts5GetVarint sqlite3Fts5GetVarint #define fts5FastGetVarint32(a, iOff, nVal) { \ @@ -216197,6 +233725,11 @@ static int sqlite3Fts5HashWrite( */ static void sqlite3Fts5HashClear(Fts5Hash*); +/* +** Return true if the hash is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash*); + static int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ int nPre, @@ -216213,11 +233746,13 @@ static void sqlite3Fts5HashScanNext(Fts5Hash*); static int sqlite3Fts5HashScanEof(Fts5Hash*); static void sqlite3Fts5HashScanEntry(Fts5Hash *, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ); + /* ** End of interface to code in fts5_hash.c. **************************************************************************/ @@ -216338,6 +233873,10 @@ static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); +static int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); +static int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*); +static void sqlite3Fts5ExprClearTokens(Fts5Expr*); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by @@ -216461,7 +234000,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); #define FTS5_STAR 15 /* This file is automatically generated by Lemon from input grammar -** source file "fts5parse.y". */ +** source file "fts5parse.y". +*/ /* ** 2000-05-29 ** @@ -216573,6 +234113,9 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser ** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser ** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYREALLOC Name of the realloc() function to use +** fts5YYFREE Name of the free() function to use +** fts5YYDYNSTACK True if stack space should be extended on heap ** fts5YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** fts5YYNSTATE the combined number of states. @@ -216586,6 +234129,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** fts5YY_NO_ACTION The fts5yy_action[] code for no-op ** fts5YY_MIN_REDUCE Minimum value for reduce actions ** fts5YY_MAX_REDUCE Maximum value for reduce actions +** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -216612,6 +234157,9 @@ typedef union { #define sqlite3Fts5ParserARG_PARAM ,pParse #define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; #define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define fts5YYREALLOC realloc +#define fts5YYFREE free +#define fts5YYDYNSTACK 0 #define sqlite3Fts5ParserCTX_SDECL #define sqlite3Fts5ParserCTX_PDECL #define sqlite3Fts5ParserCTX_PARAM @@ -216629,6 +234177,8 @@ typedef union { #define fts5YY_NO_ACTION 82 #define fts5YY_MIN_REDUCE 83 #define fts5YY_MAX_REDUCE 110 +#define fts5YY_MIN_DSTRCTR 16 +#define fts5YY_MAX_DSTRCTR 24 /************* End control #defines *******************************************/ #define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) @@ -216644,6 +234194,22 @@ typedef union { # define fts5yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if fts5YYSTACKDEPTH<=0 || fts5YYDYNSTACK +# define fts5YYGROWABLESTACK 1 +#else +# define fts5YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if fts5YYSTACKDEPTH<=0 +# undef fts5YYSTACKDEPTH +# define fts5YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -216804,14 +234370,9 @@ struct fts5yyParser { #endif sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ -#if fts5YYSTACKDEPTH<=0 - int fts5yystksz; /* Current side of the stack */ - fts5yyStackEntry *fts5yystack; /* The parser's stack */ - fts5yyStackEntry fts5yystk0; /* First stack entry */ -#else - fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ - fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ -#endif + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ + fts5yyStackEntry *fts5yystack; /* The parser stack */ + fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct fts5yyParser fts5yyParser; @@ -216918,37 +234479,45 @@ static const char *const fts5yyRuleName[] = { #endif /* NDEBUG */ -#if fts5YYSTACKDEPTH<=0 +#if fts5YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int fts5yyGrowStack(fts5yyParser *p){ + int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack); int newSize; int idx; fts5yyStackEntry *pNew; - newSize = p->fts5yystksz*2 + 100; - idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; - if( p->fts5yystack==&p->fts5yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->fts5yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->fts5yytos - p->fts5yystack); + if( p->fts5yystack==p->fts5yystk0 ){ + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->fts5yystack = pNew; - p->fts5yytos = &p->fts5yystack[idx]; + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; #ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", - fts5yyTracePrompt, p->fts5yystksz, newSize); - } -#endif - p->fts5yystksz = newSize; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->fts5yystackEnd = &p->fts5yystack[newSize-1]; + return 0; } +#endif /* fts5YYGROWABLESTACK */ + +#if !fts5YYGROWABLESTACK +/* For builds that do no have a growable stack, fts5yyGrowStack always +** returns an error. +*/ +# define fts5yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -216968,24 +234537,14 @@ static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PD #ifdef fts5YYTRACKMAXSTACKDEPTH fts5yypParser->fts5yyhwm = 0; #endif -#if fts5YYSTACKDEPTH<=0 - fts5yypParser->fts5yytos = NULL; - fts5yypParser->fts5yystack = NULL; - fts5yypParser->fts5yystksz = 0; - if( fts5yyGrowStack(fts5yypParser) ){ - fts5yypParser->fts5yystack = &fts5yypParser->fts5yystk0; - fts5yypParser->fts5yystksz = 1; - } -#endif + fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt = -1; #endif fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; fts5yypParser->fts5yystack[0].stateno = 0; fts5yypParser->fts5yystack[0].major = 0; -#if fts5YYSTACKDEPTH>0 - fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK @@ -217099,9 +234658,26 @@ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ */ static void sqlite3Fts5ParserFinalize(void *p){ fts5yyParser *pParser = (fts5yyParser*)p; - while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); -#if fts5YYSTACKDEPTH<=0 - if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); + + /* In-lined version of calling fts5yy_pop_parser_stack() for each + ** element left in the stack */ + fts5yyStackEntry *fts5yytos = pParser->fts5yytos; + while( fts5yytos>pParser->fts5yystack ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + if( fts5yytos->major>=fts5YY_MIN_DSTRCTR ){ + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); + } + fts5yytos--; + } + +#if fts5YYGROWABLESTACK + if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); #endif } @@ -217328,25 +234904,19 @@ static void fts5yy_shift( assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); } #endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yytos>fts5yypParser->fts5yystackEnd ){ - fts5yypParser->fts5yytos--; - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ + fts5yytos = fts5yypParser->fts5yytos; + if( fts5yytos>fts5yypParser->fts5yystackEnd ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yypParser->fts5yytos--; fts5yyStackOverflow(fts5yypParser); return; } + fts5yytos = fts5yypParser->fts5yytos; + assert( fts5yytos <= fts5yypParser->fts5yystackEnd ); } -#endif if( fts5yyNewState > fts5YY_MAX_SHIFT ){ fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; } - fts5yytos = fts5yypParser->fts5yytos; fts5yytos->stateno = fts5yyNewState; fts5yytos->major = fts5yyMajor; fts5yytos->minor.fts5yy0 = fts5yyMinor; @@ -217783,19 +235353,12 @@ static void sqlite3Fts5Parser( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); } #endif -#if fts5YYSTACKDEPTH>0 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ - fts5yyStackOverflow(fts5yypParser); - break; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yyStackOverflow(fts5yypParser); break; } } -#endif } fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ @@ -218051,15 +235614,19 @@ static int fts5CInstIterInit( */ typedef struct HighlightContext HighlightContext; struct HighlightContext { - CInstIter iter; /* Coalesced Instance Iterator */ - int iPos; /* Current token offset in zIn[] */ + /* Constant parameters to fts5HighlightCb() */ int iRangeStart; /* First token to include */ int iRangeEnd; /* If non-zero, last token to include */ const char *zOpen; /* Opening highlight */ const char *zClose; /* Closing highlight */ const char *zIn; /* Input text */ int nIn; /* Size of input text in bytes */ - int iOff; /* Current offset within zIn[] */ + + /* Variables modified by fts5HighlightCb() */ + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iOff; /* Have copied up to this offset in zIn[] */ + int bOpen; /* True if highlight is open */ char *zOut; /* Output value */ }; @@ -218092,8 +235659,8 @@ static int fts5HighlightCb( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iStartOff, /* Start byte offset of token */ + int iEndOff /* End byte offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; @@ -218104,35 +235671,60 @@ static int fts5HighlightCb( if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } - if( iPos==p->iter.iStart ){ + /* If the parenthesis is open, and this token is not part of the current + ** phrase, and the starting byte offset of this token is past the point + ** that has currently been copied into the output buffer, close the + ** parenthesis. */ + if( p->bOpen + && (iPos<=p->iter.iStart || p->iter.iStart<0) + && iStartOff>p->iOff + ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + + /* If this is the start of a new phrase, and the highlight is not open: + ** + ** * copy text from the input up to the start of the phrase, and + ** * open the highlight. + */ + if( iPos==p->iter.iStart && p->bOpen==0 ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); fts5HighlightAppend(&rc, p, p->zOpen, -1); p->iOff = iStartOff; + p->bOpen = 1; } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + if( p->bOpen==0 ){ + assert( p->iRangeEnd>=0 ); fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->bOpen = 1; } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - fts5HighlightAppend(&rc, p, p->zClose, -1); p->iOff = iEndOff; + if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ - fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - p->iOff = iEndOff; - if( iPos>=p->iter.iStart && iPositer.iEnd ){ + if( iPos==p->iRangeEnd ){ + if( p->bOpen ){ + if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + } fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; } return rc; @@ -218162,9 +235754,12 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); - - if( ctx.zIn ){ + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } @@ -218172,6 +235767,9 @@ static void fts5HighlightFunction( if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); if( rc==SQLITE_OK ){ @@ -218347,6 +235945,7 @@ static void fts5SnippetFunction( iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); @@ -218449,6 +236048,9 @@ static void fts5SnippetFunction( if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } if( ctx.iRangeEnd>=(nColSize-1) ){ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); }else{ @@ -218724,6 +236326,7 @@ static void sqlite3Fts5BufferAppendBlob( ){ if( nData ){ if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -218825,6 +236428,7 @@ static int sqlite3Fts5PoslistNext64( i64 *piOff /* IN/OUT: Current offset */ ){ int i = *pi; + assert( a!=0 || i==0 ); if( i>=n ){ /* EOF */ *piOff = -1; @@ -218832,6 +236436,7 @@ static int sqlite3Fts5PoslistNext64( }else{ i64 iOff = *piOff; u32 iVal; + assert( a!=0 ); fts5FastGetVarint32(a, i, iVal); if( iVal<=1 ){ if( iVal==0 ){ @@ -219087,6 +236692,8 @@ static void sqlite3Fts5TermsetFree(Fts5Termset *p){ #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (64*1024) @@ -219417,6 +237024,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -219451,6 +237068,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed tokendata=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bTokendata = (zArg[0]=='1'); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } @@ -219615,6 +237242,7 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } + assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK ); for(i=3; rc==SQLITE_OK && ibContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + /* If a tokenizer= option was successfully parsed, the tokenizer has ** already been allocated. Otherwise, allocate an instance of the default ** tokenizer (unicode61) now. */ @@ -219954,6 +237604,18 @@ static int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -219968,6 +237630,18 @@ static int sqlite3Fts5ConfigSetValue( rc = SQLITE_OK; *pbBadkey = 1; } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -219990,6 +237664,7 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ @@ -220012,15 +237687,20 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = sqlite3_finalize(p); } - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION + *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); } + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ @@ -220048,6 +237728,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* #include "fts5Int.h" */ /* #include "fts5parse.h" */ +#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH +# define SQLITE_FTS5_MAX_EXPR_DEPTH 256 +#endif + /* ** All token types in the generated fts5parse.h file are greater than 0. */ @@ -220088,11 +237772,17 @@ struct Fts5Expr { ** FTS5_NOT (nChild, apChild valid) ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) +** +** iHeight: +** Distance from this node to furthest leaf. This is always 0 for nodes +** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one +** greater than the largest child value. */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ + int iHeight; /* Distance to tree leaf nodes */ /* Next method for this node. */ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); @@ -220121,7 +237811,9 @@ struct Fts5ExprNode { struct Fts5ExprTerm { u8 bPrefix; /* True for a prefix term */ u8 bFirst; /* True if token must be first in column */ - char *zTerm; /* nul-terminated term */ + char *pTerm; /* Term data */ + int nQueryTerm; /* Effective size of term in bytes */ + int nFullTerm; /* Size of term in bytes incl. tokendata */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; @@ -220162,6 +237854,31 @@ struct Fts5Parse { int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ }; +/* +** Check that the Fts5ExprNode.iHeight variables are set correctly in +** the expression tree passed as the only argument. +*/ +#ifndef NDEBUG +static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){ + if( rc==SQLITE_OK ){ + if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){ + assert( p->iHeight==0 ); + }else{ + int ii; + int iMaxChild = 0; + for(ii=0; iinChild; ii++){ + Fts5ExprNode *pChild = p->apChild[ii]; + iMaxChild = MAX(iMaxChild, pChild->iHeight); + assert_expr_depth_ok(SQLITE_OK, pChild); + } + assert( p->iHeight==iMaxChild+1 ); + } + } +} +#else +# define assert_expr_depth_ok(rc, p) +#endif + static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); @@ -220276,6 +237993,8 @@ static int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert_expr_depth_ok(sParse.rc, sParse.pExpr); + /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ @@ -220316,10 +238035,27 @@ static int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } +/* +** Assuming that buffer z is at least nByte bytes in size and contains a +** valid utf-8 string, return the number of characters in the string. +*/ +static int fts5ExprCountChar(const char *z, int nByte){ + int nRet = 0; + int ii; + for(ii=0; ii=3 ){ + + if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){ int jj; zExpr[iOut++] = '"'; for(jj=iFirst; jjnPhrase + p2->nPhrase; @@ -220449,7 +238186,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ } sqlite3_free(p2->apExprPhrase); sqlite3_free(p2); - }else{ + }else if( p2 ){ *pp1 = p2; } @@ -220947,7 +238684,7 @@ static int fts5ExprNearInitAll( p->pIter = 0; } rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + pExpr->pIndex, p->pTerm, p->nQueryTerm, (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), pNear->pColset, @@ -221584,7 +239321,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; - sqlite3_free(pTerm->zTerm); + sqlite3_free(pTerm->pTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; @@ -221658,6 +239395,9 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( }else{ if( pRet->nPhrase>0 ){ Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pParse!=0 ); + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>=2 ); assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); if( pPhrase->nTerm==0 ){ fts5ExprPhraseFree(pPhrase); @@ -221679,6 +239419,7 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + Fts5Config *pConfig; int rc; }; @@ -221712,8 +239453,12 @@ static int fts5ParseTokenize( rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, (size_t)nByte); - pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); - memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + pSyn->nFullTerm = pSyn->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } + memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } @@ -221738,7 +239483,11 @@ static int fts5ParseTokenize( if( rc==SQLITE_OK ){ pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->nFullTerm = pTerm->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); + } } } @@ -221805,6 +239554,7 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( memset(&sCtx, 0, sizeof(TokenCtx)); sCtx.pPhrase = pAppend; + sCtx.pConfig = pConfig; rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ @@ -221852,12 +239602,15 @@ static int sqlite3Fts5ExprClonePhrase( Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); @@ -221870,7 +239623,7 @@ static int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ sqlite3_int64 nByte; @@ -221884,26 +239637,27 @@ static int sqlite3Fts5ExprClonePhrase( } } - if( pOrig->nTerm ){ - int i; /* Used to iterate through phrase terms */ - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } - }else{ - /* This happens when parsing a token or quoted phrase that contains - ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ @@ -222220,6 +239974,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); @@ -222228,6 +239983,9 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ }else{ p->apChild[p->nChild++] = pSub; } + for( ; iinChild; ii++){ + p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1); + } } /* @@ -222258,6 +240016,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( if( pRet ){ pRet->eType = FTS5_AND; pRet->nChild = nTerm; + pRet->iHeight = 1; fts5ExprAssignXNext(pRet); pParse->nPhrase--; for(ii=0; iiapPhrase[0]->aTerm[ii]; + Fts5ExprTerm *pTo = &pPhrase->aTerm[0]; pParse->apPhrase[pParse->nPhrase++] = pPhrase; pPhrase->nTerm = 1; - pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup( - &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1 - ); + pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm); + pTo->nQueryTerm = p->nQueryTerm; + pTo->nFullTerm = p->nFullTerm; pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) ); @@ -222363,6 +240124,14 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); + if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ + sqlite3Fts5ParseError(pParse, + "fts5 expression tree is too large (maximum depth %d)", + SQLITE_FTS5_MAX_EXPR_DEPTH + ); + sqlite3_free(pRet); + pRet = 0; + } } } } @@ -222398,6 +240167,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -222441,7 +240211,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( return pRet; } -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ sqlite3_int64 nByte = 0; Fts5ExprTerm *p; @@ -222449,16 +240219,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + nByte += pTerm->nQueryTerm * 2 + 3 + 2; } zQuoted = sqlite3_malloc64(nByte); if( zQuoted ){ int i = 0; for(p=pTerm; p; p=p->pSynonym){ - char *zIn = p->zTerm; + char *zIn = p->pTerm; + char *zEnd = &zIn[p->nQueryTerm]; zQuoted[i++] = '"'; - while( *zIn ){ + while( zInnTerm; iTerm++){ - char *zTerm = pPhrase->aTerm[iTerm].zTerm; - zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; + zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", + p->nQueryTerm, p->pTerm + ); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } @@ -222547,6 +240320,8 @@ static char *fts5ExprPrintTcl( if( zRet==0 ) return 0; } + }else if( pExpr->eType==0 ){ + zRet = sqlite3_mprintf("{}"); }else{ char const *zOp = 0; int i; @@ -222808,14 +240583,14 @@ static void fts5ExprFold( sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); } } -#endif /* ifdef SQLITE_TEST */ +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called during initialization to register the fts5_expr() scalar ** UDF with the SQLite handle passed as the only argument. */ static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) struct Fts5ExprFunc { const char *z; void (*x)(sqlite3_context*,int,sqlite3_value**); @@ -222936,6 +240711,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +/* +** pToken is a buffer nToken bytes in size that may or may not contain +** an embedded 0x00 byte. If it does, return the number of bytes in +** the buffer before the 0x00. If it does not, return nToken. +*/ +static int fts5QueryTerm(const char *pToken, int nToken){ + int ii; + for(ii=0; iipExpr; int i; + int nQuery = nToken; + i64 iRowid = pExpr->pRoot->iRowid; UNUSED_PARAM2(iUnused1, iUnused2); - if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE; + if( pExpr->pConfig->bTokendata ){ + nQuery = fts5QueryTerm(pToken, nQuery); + } if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ - Fts5ExprTerm *pTerm; + Fts5ExprTerm *pT; if( p->aPopulator[i].bOk==0 ) continue; - for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = (int)strlen(pTerm->zTerm); - if( (nTerm==nToken || (nTermbPrefix)) - && memcmp(pTerm->zTerm, pToken, nTerm)==0 + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + if( (pT->nQueryTerm==nQuery || (pT->nQueryTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); + if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){ + int iCol = p->iOff>>32; + int iTokOff = p->iOff & 0x7FFFFFFF; + rc = sqlite3Fts5IndexIterWriteTokendata( + pT->pIter, pToken, nToken, iRowid, iCol, iTokOff + ); + } if( rc ) return rc; break; } @@ -223098,6 +240895,83 @@ static int sqlite3Fts5ExprPhraseCollist( return rc; } +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +static int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + +/* +** Does the work of the fts5_api.xInstToken() API method. +*/ +static int sqlite3Fts5ExprInstToken( + Fts5Expr *pExpr, + i64 iRowid, + int iPhrase, + int iCol, + int iOff, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + Fts5ExprTerm *pTerm = 0; + int rc = SQLITE_OK; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + pTerm = &pPhrase->aTerm[iToken]; + if( pTerm->bPrefix==0 ){ + if( pExpr->pConfig->bTokendata ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; + } + } + return rc; +} + +/* +** Clear the token mappings for all Fts5IndexIter objects mannaged by +** the expression passed as the only argument. +*/ +static void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ + int ii; + for(ii=0; iinPhrase; ii++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } +} + /* ** 2014 August 11 ** @@ -223136,10 +241010,15 @@ struct Fts5Hash { /* ** Each entry in the hash table is represented by an object of the -** following type. Each object, its key (a nul-terminated string) and -** its current data are stored in a single memory allocation. The -** key immediately follows the object in memory. The position list -** data immediately follows the key data in memory. +** following type. Each object, its key, and its current data are stored +** in a single memory allocation. The key immediately follows the object +** in memory. The position list data immediately follows the key data +** in memory. +** +** The key is Fts5HashEntry.nKey bytes in size. It consists of a single +** byte identifying the index (either the main term index or a prefix-index), +** followed by the term data. For example: "0token". There is no +** nul-terminator - in this case nKey=6. ** ** The data that follows the key is in a similar, but not identical format ** to the doclist data stored in the database. It is: @@ -223274,8 +241153,7 @@ static int fts5HashResize(Fts5Hash *pHash){ unsigned int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), - (int)strlen(fts5EntryKey(p))); + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -223359,7 +241237,7 @@ static int sqlite3Fts5HashWrite( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ char *zKey = fts5EntryKey(p); if( zKey[0]==bByte - && p->nKey==nToken + && p->nKey==nToken+1 && memcmp(&zKey[1], pToken, nToken)==0 ){ break; @@ -223389,9 +241267,9 @@ static int sqlite3Fts5HashWrite( zKey[0] = bByte; memcpy(&zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); - p->nKey = nToken; + p->nKey = nToken+1; zKey[nToken+1] = '\0'; - p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); + p->nData = nToken+1 + sizeof(Fts5HashEntry); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -223508,12 +241386,17 @@ static Fts5HashEntry *fts5HashEntryMerge( *ppOut = p1; p1 = 0; }else{ - int i = 0; char *zKey1 = fts5EntryKey(p1); char *zKey2 = fts5EntryKey(p2); - while( zKey1[i]==zKey2[i] ) i++; + int nMin = MIN(p1->nKey, p2->nKey); + + int cmp = memcmp(zKey1, zKey2, nMin); + if( cmp==0 ){ + cmp = p1->nKey - p2->nKey; + } + assert( cmp!=0 ); - if( ((u8)zKey1[i])>((u8)zKey2[i]) ){ + if( cmp>0 ){ /* p2 is smaller */ *ppOut = p2; ppOut = &p2->pScanNext; @@ -223532,10 +241415,8 @@ static Fts5HashEntry *fts5HashEntryMerge( } /* -** Extract all tokens from hash table iHash and link them into a list -** in sorted order. The hash table is cleared before returning. It is -** the responsibility of the caller to free the elements of the returned -** list. +** Link all tokens from hash table iHash into a list in sorted order. The +** tokens are not removed from the hash table. */ static int fts5HashEntrySort( Fts5Hash *pHash, @@ -223557,7 +241438,7 @@ static int fts5HashEntrySort( Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ if( pTerm==0 - || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; @@ -223575,7 +241456,6 @@ static int fts5HashEntrySort( pList = fts5HashEntryMerge(pList, ap[i]); } - pHash->nEntry = 0; sqlite3_free(ap); *ppSorted = pList; return SQLITE_OK; @@ -223597,12 +241477,11 @@ static int sqlite3Fts5HashQuery( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ zKey = fts5EntryKey(p); - assert( p->nKey+1==(int)strlen(zKey) ); - if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break; + if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break; } if( p ){ - int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1; + int nHashPre = sizeof(Fts5HashEntry) + nTerm; int nList = p->nData - nHashPre; u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); if( pRet ){ @@ -223629,6 +241508,28 @@ static int sqlite3Fts5HashScanInit( return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); } +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + static void sqlite3Fts5HashScanNext(Fts5Hash *p){ assert( !sqlite3Fts5HashScanEof(p) ); p->pScan = p->pScan->pScanNext; @@ -223641,19 +241542,22 @@ static int sqlite3Fts5HashScanEof(Fts5Hash *p){ static void sqlite3Fts5HashScanEntry( Fts5Hash *pHash, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ char *zKey = fts5EntryKey(p); - int nTerm = (int)strlen(zKey); + int nTerm = p->nKey; fts5HashAddPoslistSize(pHash, p, 0); *pzTerm = zKey; - *ppDoclist = (const u8*)&zKey[nTerm+1]; - *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); + *pnTerm = nTerm; + *ppDoclist = (const u8*)&zKey[nTerm]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm); }else{ *pzTerm = 0; + *pnTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } @@ -223715,6 +241619,26 @@ static void sqlite3Fts5HashScanEntry( # error "FTS5_MAX_PREFIX_INDEXES is too large" #endif +#define FTS5_MAX_LEVEL 64 + +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + /* ** Details: ** @@ -223722,7 +241646,7 @@ static void sqlite3Fts5HashScanEntry( ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding +** , contains the following 6 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** @@ -223731,12 +241655,12 @@ static void sqlite3Fts5HashScanEntry( ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists ** of a single 32-bit configuration cookie value followed by a list of -** SQLite varints. If the FTS table features more than one index (because -** there are one or more prefix indexes), it is guaranteed that all share -** the same cookie value. +** SQLite varints. ** -** Immediately following the configuration cookie, the record begins with -** three varints: +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. +** +** Next, the record continues with three varints: ** ** + number of levels, ** + total number of segments on all levels, @@ -223751,6 +241675,12 @@ static void sqlite3Fts5HashScanEntry( ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** ** 2. The Averages Record: ** ** A single record within the %_data table. The data is a list of varints. @@ -223866,6 +241796,38 @@ static void sqlite3Fts5HashScanEntry( ** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size */ /* @@ -223899,6 +241861,7 @@ static void sqlite3Fts5HashScanEntry( #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) #ifdef SQLITE_DEBUG static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } @@ -223925,6 +241888,9 @@ typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; +typedef struct Fts5TokenDataIter Fts5TokenDataIter; +typedef struct Fts5TokenDataMap Fts5TokenDataMap; +typedef struct Fts5TombstoneArray Fts5TombstoneArray; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ @@ -223934,6 +241900,12 @@ struct Fts5Data { /* ** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ @@ -223948,19 +241920,25 @@ struct Fts5Index { int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ /* Error state. */ int rc; /* Current error code */ + int flushRc; /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ - sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxNextSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -223980,11 +241958,23 @@ struct Fts5DoclistIter { ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ @@ -223994,6 +241984,7 @@ struct Fts5StructureLevel { struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ @@ -224053,9 +242044,6 @@ struct Fts5CResult { ** iLeafOffset: ** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -224085,6 +242073,13 @@ struct Fts5CResult { ** ** iTermIdx: ** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -224093,6 +242088,7 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); @@ -224119,6 +242115,15 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +/* +** Array of tombstone pages. Reference counted. +*/ +struct Fts5TombstoneArray { + int nRef; /* Number of pointers to this object */ + int nTombstone; + Fts5Data *apTombstone[1]; /* Array of tombstone pages */ +}; + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -224163,9 +242168,16 @@ struct Fts5SegIter { ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. +** +** pColset: +** If not NULL, points to an object containing a set of column indices. +** Only matches that occur in one of these columns will be returned. +** The Fts5Iter does not own the Fts5Colset object, and so it is not +** freed when the iterator is closed - it is owned by the upper layer. */ struct Fts5Iter { Fts5IndexIter base; /* Base class containing output vars */ + Fts5TokenDataIter *pTokenDataIter; Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ @@ -224183,7 +242195,6 @@ struct Fts5Iter { Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -224222,6 +242233,60 @@ static u16 fts5GetU16(const u8 *aIn){ return ((u16)aIn[0] << 8) + aIn[1]; } +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} + /* ** Allocate and return a buffer at least nByte bytes in size. ** @@ -224449,10 +242514,17 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ /* ** Remove all records associated with segment iSegid. */ -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( @@ -224563,11 +242635,19 @@ static int fts5StructureDecode( int nSegment = 0; sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); @@ -224614,9 +242694,18 @@ static int fts5StructureDecode( rc = FTS5_CORRUPT; break; } + assert( pSeg!=0 ); i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } if( pSeg->pgnoLastpgnoFirst ){ rc = FTS5_CORRUPT; break; @@ -224627,6 +242716,9 @@ static int fts5StructureDecode( } } if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); @@ -224644,6 +242736,7 @@ static int fts5StructureDecode( */ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; @@ -224838,6 +242931,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); @@ -224846,9 +242940,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ sqlite3Fts5Put32(buf.p, iCookie); buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); @@ -224862,9 +242959,17 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ assert( pLvl->nMerge<=pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } } } @@ -225007,9 +243112,9 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ } if( iOffnn ){ - i64 iVal; + u64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; - iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; }else{ @@ -225102,42 +243207,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } - - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; + + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -225333,7 +243421,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -225404,6 +243492,25 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ } } +/* +** Allocate a tombstone hash page array object (pIter->pTombArray) for +** the iterator passed as the second argument. If an OOM error occurs, +** leave an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const int nTomb = pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + Fts5TombstoneArray *pNew; + pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ + pNew->nTombstone = nTomb; + pNew->nRef = 1; + pIter->pTombArray = pNew; + } + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -225432,10 +243539,12 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); @@ -225443,6 +243552,7 @@ static void fts5SegIterInit( pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); } } @@ -225629,7 +243739,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -225653,15 +243763,16 @@ static void fts5SegIterNext_None( }else{ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; - sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -225727,11 +243838,12 @@ static void fts5SegIterNext( }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList = 0; assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); @@ -225741,8 +243853,7 @@ static void fts5SegIterNext( pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); *pbNewTerm = 1; } @@ -225822,7 +243933,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); @@ -226128,7 +244239,7 @@ static void fts5SegIterSeekInit( fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); } - if( p->rc==SQLITE_OK && bGe==0 ){ + if( p->rc==SQLITE_OK && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM)) ){ pIter->flags |= FTS5_SEGITER_ONETERM; if( pIter->pLeaf ){ if( flags & FTS5INDEX_QUERY_DESC ){ @@ -226144,6 +244255,9 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); + if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM) ){ + fts5SegIterAllocTombstone(p, pIter); + } /* Either: ** @@ -226160,6 +244274,79 @@ static void fts5SegIterSeekInit( ); } + +/* +** SQL used by fts5SegIterNextInit() to find the page to open. +*/ +static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){ + if( p->pIdxNextSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? ORDER BY term ASC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + + } + return p->pIdxNextSelect; +} + +/* +** This is similar to fts5SegIterSeekInit(), except that it initializes +** the segment iterator to point to the first term following the page +** with pToken/nToken on it. +*/ +static void fts5SegIterNextInit( + Fts5Index *p, + const char *pTerm, int nTerm, + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = -1; /* Page of segment to open */ + int bDlidx = 0; + sqlite3_stmt *pSel = 0; /* SELECT to find iPg */ + + pSel = fts5IdxNextStmt(p); + if( pSel ){ + assert( p->rc==SQLITE_OK ); + sqlite3_bind_int(pSel, 1, pSeg->iSegid); + sqlite3_bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC); + + if( sqlite3_step(pSel)==SQLITE_ROW ){ + i64 val = sqlite3_column_int64(pSel, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pSel); + sqlite3_bind_null(pSel, 2); + if( p->rc ) return; + } + + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + pIter->flags |= FTS5_SEGITER_ONETERM; + if( iPg>=0 ){ + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + fts5SegIterSetNext(p, pIter); + } + if( pIter->pLeaf ){ + const u8 *a = pIter->pLeaf->p; + int iTermOff = 0; + + pIter->iPgidxOff = pIter->pLeaf->szLeaf; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff); + pIter->iLeafOffset = iTermOff; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); + + assert( p->rc!=SQLITE_OK || + fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0 + ); + } +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within the ** in-memory hash table. If there is no such term in the hash-table, the @@ -226186,14 +244373,21 @@ static void fts5SegIterHashInit( const u8 *pList = 0; p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); - sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); - n = (z ? (int)strlen((const char*)z) : 0); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList); if( pList ){ pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf ){ pLeaf->p = (u8*)pList; } } + + /* The call to sqlite3Fts5HashScanInit() causes the hash table to + ** fill the size field of all existing position lists. This means they + ** can no longer be appended to. Since the only scenario in which they + ** can be appended to is if the previous operation on this table was + ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this + ** possibility altogether. */ + p->bDelete = 0; }else{ p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), (const char*)pTerm, nTerm, (void**)&pLeaf, &nList @@ -226224,6 +244418,37 @@ static void fts5SegIterHashInit( fts5SegIterSetNext(p, pIter); } +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iinRef--; + if( p->nRef<=0 ){ + int ii; + for(ii=0; iinTombstone; ii++){ + fts5DataRelease(p->apTombstone[ii]); + } + sqlite3_free(p); + } + } +} + /* ** Zero the iterator passed as the only argument. */ @@ -226231,6 +244456,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); + fts5TombstoneArrayDelete(pIter->pTombArray); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -226364,7 +244590,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ - p1->bDel = p2->bDel; return i2; } res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; @@ -226383,7 +244608,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -226398,21 +244624,23 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } @@ -226473,7 +244701,6 @@ static void fts5SegIterNextFrom( }while( p->rc==SQLITE_OK ); } - /* ** Free the iterator object passed as the second argument. */ @@ -226565,6 +244792,85 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){ pIter->iSwitchRowid = pSeg->iRowid; } +/* +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + Fts5TombstoneArray *pArray = pSeg->pTombArray; + + if( pSeg->pLeaf && pArray ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pArray->apTombstone[iPg]==0 ){ + pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pArray->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pArray->apTombstone[iPg], + pArray->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + /* ** Move the iterator to the next entry. ** @@ -226602,7 +244908,9 @@ static void fts5MultiIterNext( fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ pIter->xSetOutputs(pIter, pSeg); return; } @@ -226634,7 +244942,9 @@ static void fts5MultiIterNext2( } fts5AssertMultiIterSetup(p, pIter); - }while( fts5MultiIterIsEmpty(p, pIter) ); + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); } } @@ -226647,7 +244957,7 @@ static Fts5Iter *fts5MultiIterAlloc( int nSeg ){ Fts5Iter *pNew; - int nSlot; /* Power of two >= nSeg */ + i64 nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlotnSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pIter, iEq, iIter); + } + } + fts5MultiIterSetEof(pIter); + fts5AssertMultiIterSetup(p, pIter); + + if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter)) + || fts5MultiIterIsDeleted(pIter) + ){ + fts5MultiIterNext(p, pIter, 0, 0); + }else if( pIter->base.bEof==0 ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + pIter->xSetOutputs(pIter, pSeg); + } +} /* ** Allocate a new Fts5Iter object. @@ -227127,7 +245463,7 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -227148,7 +245484,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -227173,29 +245509,12 @@ static void fts5MultiIterNew( assert( iIter==nSeg ); } - /* If the above was successful, each component iterators now points + /* If the above was successful, each component iterator now points ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); - } - } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ - fts5MultiIterNext(p, pNew, 0, 0); - }else if( pNew->base.bEof==0 ){ - Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; - pNew->xSetOutputs(pNew, pSeg); - } - + fts5MultiIterFinishSetup(p, pNew); }else{ fts5MultiIterFree(pNew); *ppOut = 0; @@ -227220,7 +245539,6 @@ static void fts5MultiIterNew2( pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -227367,7 +245685,10 @@ static void fts5IndexDiscardData(Fts5Index *p){ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; + p->nPendingRow = 0; + p->flushRc = SQLITE_OK; } + p->nContentlessDelete = 0; } /* @@ -227581,7 +245902,7 @@ static void fts5WriteDlidxAppend( } if( pDlidx->bPrevValid ){ - iVal = iRowid - pDlidx->iPrev; + iVal = (u64)iRowid - (u64)pDlidx->iPrev; }else{ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); @@ -227748,7 +246069,9 @@ static void fts5WriteAppendRowid( fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); - fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; @@ -227766,7 +246089,7 @@ static void fts5WriteAppendPoslistData( const u8 *a = aData; int n = nData; - assert( p->pConfig->pgsz>0 ); + assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK ); while( p->rc==SQLITE_OK && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz ){ @@ -227901,7 +246224,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); if( p->rc==SQLITE_OK ){ /* Set the szLeaf field */ fts5PutU16(&buf.p[2], (u16)buf.n); @@ -228002,6 +246325,12 @@ static void fts5IndexMergeLevel( /* Read input from all segments in the input level */ nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); @@ -228061,8 +246390,11 @@ static void fts5IndexMergeLevel( int i; /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); for(i=0; iaSeg[i].iSegid); + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); } /* Remove the redundant segments from the input level */ @@ -228088,6 +246420,43 @@ static void fts5IndexMergeLevel( if( pnRem ) *pnRem -= writer.nLeafWritten; } +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert_nc( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + } + } + return iRet; +} + /* ** Do up to nPg pages of automerge work on the index. ** @@ -228107,14 +246476,15 @@ static int fts5IndexMerge( int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ - /* Set iBestLvl to the level to read input segments from. */ + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ assert( pStruct->nLevel>0 ); for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; if( pLvl->nMerge ){ if( pLvl->nMerge>nBest ){ iBestLvl = iLvl; - nBest = pLvl->nMerge; + nBest = nMin; } break; } @@ -228123,22 +246493,18 @@ static int fts5IndexMerge( iBestLvl = iLvl; } } - - /* If nBest is still 0, then the index must be empty. */ -#ifdef SQLITE_DEBUG - for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ - assert( pStruct->aLevel[iLvl].nSeg==0 ); + if( nBestaLevel[iBestLvl].nMerge==0 ){ - break; - } + if( iBestLvl<0 ) break; bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } + + if( nMin==1 ) nMin = 2; } *ppStruct = pStruct; return bRet; @@ -228179,16 +246545,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -228222,6 +246588,469 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } +/* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + i1 += fts5GetVarint32(&aPg[i1], iFirst); + if( iFirstrc = FTS5_CORRUPT; + break; + } + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** + ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. + */ + { + int iSOP; /* Start-Of-Position-list */ + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + + /* If the position-list for the entry being removed flows over past + ** the end of this page, delete the portion of the position-list on the + ** next page and beyond. + ** + ** Set variable bLastInDoclist to true if this entry happens + ** to be the last rowid in the doclist for its term. */ + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + } + + if( pSeg->bDel==0 ){ + if( iNextOff!=iPgIdx ){ + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last + ** in its doclist. */ + int iKeyOff = 0; + for(iIdx=0; iIdxbDel ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); + aPg[iOff++] = 0x01; + }else if( bLastInDoclist==0 ){ + if( iNextOff!=iPgIdx ){ + u64 iNextDelta = 0; + iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); + } + }else if( + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + int iKeyOff = 0; + + /* Set iKeyOff to the offset of the term that will be removed - the + ** last offset in the footer that is not greater than iStart. */ + for(iIdx=0; iIdx(u32)iStart ) break; + iKeyOff += iVal; + } + assert_nc( iKey>=1 ); + + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ + iDelKeyOff = iOff = iKeyOff; + + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>pSeg->term.n ){ + p->rc = FTS5_CORRUPT; + }else if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + /* Assuming no error has occurred, this block does final edits to the + ** leaf page before writing it back to disk. Input variables are: + ** + ** nPg: Total initial size of leaf page. + ** iPgIdx: Initial offset of page footer. + ** + ** iOff: Offset to move data to + ** iNextOff: Offset to move data from + */ + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ + + int iPrevKeyOut = 0; + int iKeyIn = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + for(iIdx=0; iIdxiOff ? nShift : 0)); + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + iPrevKeyOut = iKeyOut; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + int nTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -228238,143 +247067,197 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); - iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); - if( iSegid ){ - const int pgsz = p->pConfig->pgsz; - int eDetail = p->pConfig->eDetail; - Fts5StructureSegment *pSeg; /* New segment within pStruct */ - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ + iSegid = fts5AllocateSegid(p, pStruct); + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ - Fts5SegWriter writer; - fts5WriteInit(p, &writer, iSegid); + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - pBuf = &writer.writer.buf; - pPgidx = &writer.writer.pgidx; + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } - /* fts5WriteInit() should have initialized the buffers to (most likely) - ** the maximum space required. */ - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } - /* Begin scanning through hash table entries. This loop runs once for each - ** term/doclist currently stored within the hash table. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); - } - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ - const char *zTerm; /* Buffer containing term */ - const u8 *pDoclist; /* Pointer to doclist for this term */ - int nDoclist; /* Size of doclist in bytes */ - - /* Write the term for this entry to disk. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; - - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ - /* The entire doclist will fit on the current leaf. */ - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); - }else{ - i64 iRowid = 0; - u64 iDelta = 0; - int iOff = 0; - - /* The entire doclist will not fit on this leaf. The following - ** loop iterates through the poslists that make up the current - ** doclist. */ - while( p->rc==SQLITE_OK && iOffp[0], (u16)pBuf->n); /* first rowid on page */ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); - writer.bFirstRowidInPage = 0; - fts5WriteDlidxAppend(p, &writer, iRowid); + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + u64 iRowidDelta = (u64)iRowid - (u64)iPrev; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta); + } if( p->rc!=SQLITE_OK ) break; - }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); - } - assert( pBuf->n<=pBuf->nSpace ); + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; - if( eDetail==FTS5_DETAIL_NONE ){ - if( iOffp[pBuf->n++] = 0; - iOff++; + if( eDetail==FTS5_DETAIL_NONE ){ if( iOffp[pBuf->n++] = 0; iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); } - } - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - }else{ - int bDummy; - int nPos; - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); - nCopy += nPos; - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); - } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); + int bDel = 0; + int nPos = 0; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel); + if( bDel && bSecureDelete ){ + fts5BufferAppendVarint(&p->rc, pBuf, nPos*2); + iOff += nCopy; + nCopy = nPos; + }else{ + nCopy += nPos; + } + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; } - if( iPos>=nCopy ) break; } + iOff += nCopy; } - iOff += nCopy; } } - } - /* TODO2: Doclist terminator written here. */ - /* pBuf->p[pBuf->n++] = '\0'; */ - assert( pBuf->n<=pBuf->nSpace ); - if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); - } - sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &pgnoLast); + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); + } + fts5WriteFinish(p, &writer, &pgnoLast); - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); + } } - fts5StructurePromote(p, 0, pStruct); } - fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); @@ -228385,10 +247268,21 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->nPendingData ){ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } + if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); - p->nPendingData = 0; fts5FlushOneHash(p); + if( p->rc==SQLITE_OK ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; + } } } @@ -228404,17 +247298,22 @@ static Fts5Structure *fts5IndexOptimizeStruct( /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** - ** + it consists of fewer than two segments, or - ** + all segments are on the same level, or - ** + all segments except one are currently inputs to a merge operation. + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. ** - ** In the first case, return NULL. In the second, increment the ref-count - ** on *pStruct and return a copy of the pointer to it. + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. */ - if( nSeg<2 ) return 0; + if( nSeg==0 ) return 0; for(i=0; inLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } fts5StructureRef(pStruct); return pStruct; } @@ -228427,10 +247326,11 @@ static Fts5Structure *fts5IndexOptimizeStruct( if( pNew ){ Fts5StructureLevel *pLvl; nByte = nSeg * sizeof(Fts5StructureSegment); - pNew->nLevel = pStruct->nLevel+1; + pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; - pLvl = &pNew->aLevel[pStruct->nLevel]; + pNew->nOriginCntr = pStruct->nOriginCntr; + pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; @@ -228460,7 +247360,9 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); + assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || pStruct!=0 ); fts5StructureInvalidate(p); if( pStruct ){ @@ -228489,7 +247391,10 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ ** INSERT command. */ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct = fts5StructureRead(p); + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); @@ -228497,7 +247402,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; - nMin = 2; + nMin = 1; nMerge = nMerge*-1; } if( pStruct && pStruct->nLevel ){ @@ -228512,7 +247417,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ static void fts5AppendRowid( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ @@ -228522,7 +247427,7 @@ static void fts5AppendRowid( static void fts5AppendPoslist( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ @@ -228597,10 +247502,10 @@ static void fts5MergeAppendDocid( } #endif -#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ - assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ - fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ - (iLastRowid) = (iRowid); \ +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ } /* @@ -228732,7 +247637,7 @@ static void fts5MergePrefixLists( /* Initialize a doclist-iterator for each input buffer. Arrange them in ** a linked-list starting at pHead in ascending order of rowid. Avoid ** linking any iterators already at EOF into the linked list at all. */ - assert( nBuf+1<=sizeof(aMerger)/sizeof(aMerger[0]) ); + assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) ); memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); pHead = &aMerger[nBuf]; fts5DoclistIterInit(p1, &pHead->iter); @@ -228863,7 +247768,7 @@ static void fts5SetupPrefixIter( u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ - Fts5Iter **ppIter /* OUT: New iterator */ + Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; @@ -228871,7 +247776,7 @@ static void fts5SetupPrefixIter( int nMerge = 1; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -228884,8 +247789,9 @@ static void fts5SetupPrefixIter( aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); - if( aBuf && pStruct ){ + if( p->rc==SQLITE_OK ){ const int flags = FTS5INDEX_QUERY_SCAN | FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_NOOUTPUT; @@ -228897,6 +247803,12 @@ static void fts5SetupPrefixIter( int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); + + /* If iIdx is non-zero, then it is the number of a prefix-index for + ** prefixes 1 character longer than the prefix being queried for. That + ** index contains all the doclists required, except for the one + ** corresponding to the prefix itself. That one is extracted from the + ** main term index here. */ if( iIdx!=0 ){ int dummy = 0; const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; @@ -228910,7 +247822,7 @@ static void fts5SetupPrefixIter( Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; p1->xSetOutputs(p1, pSeg); if( p1->base.nData ){ - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } } @@ -228920,6 +247832,7 @@ static void fts5SetupPrefixIter( pToken[0] = FTS5_MAIN_PREFIX + iIdx; fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) @@ -228935,7 +247848,6 @@ static void fts5SetupPrefixIter( } if( p1->base.nData==0 ) continue; - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ int i1 = i*nMerge; @@ -228958,7 +247870,7 @@ static void fts5SetupPrefixIter( iLastRowid = 0; } - xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } @@ -228974,7 +247886,7 @@ static void fts5SetupPrefixIter( } fts5MultiIterFree(p1); - pData = fts5IdxMalloc(p, sizeof(Fts5Data)+doclist.n+FTS5_DATA_ZERO_PADDING); + pData = fts5IdxMalloc(p, sizeof(*pData)+doclist.n+FTS5_DATA_ZERO_PADDING); if( pData ){ pData->p = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; @@ -229011,6 +247923,9 @@ static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ p->iWriteRowid = iRowid; p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } return fts5IndexReturn(p); } @@ -229048,6 +247963,9 @@ static int sqlite3Fts5IndexReinit(Fts5Index *p){ fts5StructureInvalidate(p); fts5IndexDiscardData(p); memset(&s, 0, sizeof(Fts5Structure)); + if( p->pConfig->bContentlessDelete ){ + s.nOriginCntr = 1; + } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); @@ -229111,7 +248029,9 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxNextSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -229205,6 +248125,457 @@ static int sqlite3Fts5IndexWrite( return rc; } +/* +** pToken points to a buffer of size nToken bytes containing a search +** term, including the index number at the start, used on a tokendata=1 +** table. This function returns true if the term in buffer pBuf matches +** token pToken/nToken. +*/ +static int fts5IsTokendataPrefix( + Fts5Buffer *pBuf, + const u8 *pToken, + int nToken +){ + return ( + pBuf->n>=nToken + && 0==memcmp(pBuf->p, pToken, nToken) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + ); +} + +/* +** Ensure the segment-iterator passed as the only argument points to EOF. +*/ +static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; +} + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits. Or, for an iterator used by an +** "ORDER BY rank" query, it accumulates an array of these for the entire +** query. +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +*/ +struct Fts5TokenDataIter { + int nIter; + int nIterAlloc; + + int nMap; + int nMapAlloc; + Fts5TokenDataMap *aMap; + + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[1]; +}; + +/* +** This function appends iterator pAppend to Fts5TokenDataIter pIn and +** returns the result. +*/ +static Fts5TokenDataIter *fts5AppendTokendataIter( + Fts5Index *p, /* Index object (for error code) */ + Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */ + Fts5Iter *pAppend /* Append this iterator */ +){ + Fts5TokenDataIter *pRet = pIn; + + if( p->rc==SQLITE_OK ){ + if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + if( pIn==0 ) memset(pNew, 0, nByte); + pRet = pNew; + pNew->nIterAlloc = nAlloc; + } + } + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pAppend); + }else{ + pRet->apIter[pRet->nIter++] = pAppend; + } + assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc ); + + return pRet; +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + +/* +** Append a mapping to the token-map belonging to object pT. +*/ +static void fts5TokendataIterAppendMap( + Fts5Index *p, + Fts5TokenDataIter *pT, + int iIter, + i64 iRowid, + i64 iPos +){ + if( p->rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nByte = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->nMap++; + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function sets the iterator output +** variables (pIter->base.*) according to the contents of the current +** row. +*/ +static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ + int ii; + int nHit = 0; + i64 iRowid = SMALLEST_INT64; + int iMin = 0; + + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + pIter->base.nData = 0; + pIter->base.pData = 0; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 ){ + if( nHit==0 || p->base.iRowidbase.iRowid; + nHit = 1; + pIter->base.pData = p->base.pData; + pIter->base.nData = p->base.nData; + iMin = ii; + }else if( p->base.iRowid==iRowid ){ + nHit++; + } + } + } + + if( nHit==0 ){ + pIter->base.bEof = 1; + }else{ + int eDetail = pIter->pIndex->pConfig->eDetail; + pIter->base.bEof = 0; + pIter->base.iRowid = iRowid; + + if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1); + }else + if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ + int nReader = 0; + int nByte = 0; + i64 iPrev = 0; + + /* Allocate array of iterators if they are not already allocated. */ + if( pT->aPoslistReader==0 ){ + pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( + &pIter->pIndex->rc, + pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)) + ); + if( pT->aPoslistReader==0 ) return; + pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; + } + + /* Populate an iterator for each poslist that will be merged */ + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( iRowid==p->base.iRowid ){ + pT->aPoslistToIter[nReader] = ii; + sqlite3Fts5PoslistReaderInit( + p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] + ); + nByte += p->base.nData; + } + } + + /* Ensure the output buffer is large enough */ + if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10) ){ + return; + } + + /* Ensure the token-mapping is large enough */ + if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + pT->aMap, nNew*sizeof(Fts5TokenDataMap) + ); + if( aNew==0 ){ + pIter->pIndex->rc = SQLITE_NOMEM; + return; + } + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pIter->poslist.n = 0; + + while( 1 ){ + i64 iMinPos = LARGEST_INT64; + + /* Find smallest position */ + iMin = 0; + for(ii=0; iiaPoslistReader[ii]; + if( pReader->bEof==0 ){ + if( pReader->iPosiPos; + iMin = ii; + } + } + } + + /* If all readers were at EOF, break out of the loop. */ + if( iMinPos==LARGEST_INT64 ) break; + + sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos); + sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]); + + if( eDetail==FTS5_DETAIL_FULL ){ + pT->aMap[pT->nMap].iPos = iMinPos; + pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin]; + pT->aMap[pT->nMap].iRowid = iRowid; + pT->nMap++; + } + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function advances the iterator. If +** argument bFrom is false, then the iterator is advanced to the next +** entry. Or, if bFrom is true, it is advanced to the first entry with +** a rowid of iFrom or greater. +*/ +static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ + int ii; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 + && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidbase.bEof==0 + && p->base.iRowidrc==SQLITE_OK + ){ + fts5MultiIterNext(pIndex, p, 0, 0); + } + } + } + + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } +} + +/* +** If the segment-iterator passed as the first argument is at EOF, then +** set pIter->term to a copy of buffer pTerm. +*/ +static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ + if( pIter && pIter->aSeg[0].pLeaf==0 ){ + fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); + } +} + +/* +** This function sets up an iterator to use for a non-prefix query on a +** tokendata=1 table. +*/ +static Fts5Iter *fts5SetupTokendataIter( + Fts5Index *p, /* FTS index to query */ + const u8 *pToken, /* Buffer containing query term */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset /* Colset to filter on */ +){ + Fts5Iter *pRet = 0; + Fts5TokenDataIter *pSet = 0; + Fts5Structure *pStruct = 0; + const int flags = FTS5INDEX_QUERY_SCANONETERM | FTS5INDEX_QUERY_SCAN; + + Fts5Buffer bSeek = {0, 0, 0}; + Fts5Buffer *pSmall = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + + while( p->rc==SQLITE_OK ){ + Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; + Fts5Iter *pNew = 0; + Fts5SegIter *pNewIter = 0; + Fts5SegIter *pPrevIter = 0; + + int iLvl, iSeg, ii; + + pNew = fts5MultiIterAlloc(p, pStruct->nSegment); + if( pSmall ){ + fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); + fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); + }else{ + fts5BufferSet(&p->rc, &bSeek, nToken, pToken); + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + pNewIter = &pNew->aSeg[0]; + pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + int bDone = 0; + + if( pPrevIter ){ + if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ + memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); + memset(pPrevIter, 0, sizeof(Fts5SegIter)); + bDone = 1; + }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){ + fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); + bDone = 1; + } + } + + if( bDone==0 ){ + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + } + + if( pPrevIter ){ + if( pPrevIter->pTombArray ){ + pNewIter->pTombArray = pPrevIter->pTombArray; + pNewIter->pTombArray->nRef++; + } + }else{ + fts5SegIterAllocTombstone(p, pNewIter); + } + + pNewIter++; + if( pPrevIter ) pPrevIter++; + if( p->rc ) break; + } + } + fts5TokendataSetTermIfEof(pPrev, pSmall); + + pNew->bSkipEmpty = 1; + pNew->pColset = pColset; + fts5IterSetOutputCb(&p->rc, pNew); + + /* Loop through all segments in the new iterator. Find the smallest + ** term that any segment-iterator points to. Iterator pNew will be + ** used for this term. Also, set any iterator that points to a term that + ** does not match pToken/nToken to point to EOF */ + pSmall = 0; + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pII = &pNew->aSeg[ii]; + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + fts5SegIterSetEOF(pII); + } + if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ + pSmall = &pII->term; + } + } + + /* If pSmall is still NULL at this point, then the new iterator does + ** not point to any terms that match the query. So delete it and break + ** out of the loop - all required iterators have been collected. */ + if( pSmall==0 ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + /* Append this iterator to the set and continue. */ + pSet = fts5AppendTokendataIter(p, pSet, pNew); + } + + if( p->rc==SQLITE_OK && pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Iter *pIter = pSet->apIter[ii]; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM; + } + fts5MultiIterFinishSetup(p, pIter); + } + } + + if( p->rc==SQLITE_OK ){ + pRet = fts5MultiIterAlloc(p, 0); + } + if( pRet ){ + pRet->pTokenDataIter = pSet; + if( pSet ){ + fts5IterSetOutputsTokendata(pRet); + }else{ + pRet->base.bEof = 1; + } + }else{ + fts5TokendataIterDelete(pSet); + } + + fts5StructureRelease(pStruct); + fts5BufferFree(&bSeek); + return pRet; +} + + /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -229226,7 +248597,12 @@ static int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ - if( nToken ) memcpy(&buf.p[1], pToken, nToken); + int bTokendata = pConfig->bTokendata; + if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to @@ -229253,7 +248629,10 @@ static int sqlite3Fts5IndexQuery( } } - if( iIdx<=pConfig->nPrefix ){ + if( bTokendata && iIdx==0 ){ + buf.p[0] = '0'; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); @@ -229300,7 +248679,11 @@ static int sqlite3Fts5IndexQuery( static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 0, 0); + }else{ + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + } return fts5IndexReturn(pIter->pIndex); } @@ -229333,7 +248716,11 @@ static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 1, iMatch); + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + } return fts5IndexReturn(pIter->pIndex); } @@ -229348,6 +248735,99 @@ static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +/* +** This is used by xInstToken() to access the token at offset iOff, column +** iCol of row iRowid. The token is returned via output variables *ppOut +** and *pnOut. The iterator passed as the first argument must be a tokendata=1 +** iterator (pIter->pTokenDataIter!=0). +*/ +static int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5TokenDataMap *aMap = pT->aMap; + i64 iPos = (((i64)iCol)<<32) + iOff; + + int i1 = 0; + int i2 = pT->nMap; + int iTest = 0; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + } + + return SQLITE_OK; +} + +/* +** Clear any existing entries from the token-map associated with the +** iterator passed as the only argument. +*/ +static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter && pIter->pTokenDataIter ){ + pIter->pTokenDataIter->nMap = 0; + } +} + +/* +** Set a token-mapping for the iterator passed as the first argument. This +** is used in detail=column or detail=none mode when a token is requested +** using the xInstToken() API. In this case the caller tokenizers the +** current row and configures the token-mapping via multiple calls to this +** function. +*/ +static int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, int iCol, int iOff +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *p = pIter->pIndex; + int ii; + + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); + assert( pIter->pTokenDataIter ); + + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); + } + return fts5IndexReturn(p); +} + /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ @@ -229355,6 +248835,7 @@ static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); fts5MultiIterFree(pIter); sqlite3Fts5IndexCloseReader(pIndex); } @@ -229438,6 +248919,347 @@ static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ return fts5IndexReturn(p); } +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} /************************************************************************* ************************************************************************** @@ -229521,7 +249343,9 @@ static int fts5QueryCksum( int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIter = 0; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + int rc = sqlite3Fts5IndexQuery( + p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA), 0, &pIter + ); while( rc==SQLITE_OK && ALWAYS(pIter!=0) && 0==sqlite3Fts5IterEof(pIter) ){ i64 rowid = pIter->iRowid; @@ -229688,7 +249512,7 @@ static void fts5IndexIntegrityCheckEmpty( } static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int iTermOff = 0; + i64 iTermOff = 0; int ii; Fts5Buffer buf1 = {0,0,0}; @@ -229697,7 +249521,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ ii = pLeaf->szLeaf; while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff; + i64 iOff; int nIncr; ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); @@ -229742,6 +249566,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -229777,7 +249602,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -229841,9 +249678,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowidrc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } @@ -229937,6 +249777,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ @@ -229972,13 +249813,14 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum ** function only. */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ int *piSegid, /* OUT: Segment id */ int *pbDlidx, /* OUT: Dlidx flag */ int *piHeight, /* OUT: Height */ @@ -229994,13 +249836,16 @@ static void fts5DecodeRowid( iRowid >>= FTS5_DATA_DLI_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); + iRowid >>= FTS5_DATA_ID_B; + + *pbTombstone = (int)(iRowid & 0x0001); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ @@ -230010,14 +249855,16 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno ); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, @@ -230032,16 +249879,22 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -230066,9 +249919,9 @@ static void fts5DecodeStructure( fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -230091,9 +249944,9 @@ static void fts5DecodeAverages( zSpace = " "; } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Buffer (a/n) is assumed to contain a list of serialized varints. Read ** each varint and append its string representation to buffer pBuf. Return @@ -230110,9 +249963,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The start of buffer (a/n) contains the start of a doclist. The doclist ** may or may not finish within the buffer. This function appends a text @@ -230145,9 +249998,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. @@ -230188,9 +250041,27 @@ static void fts5DecodeRowidList( sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){ + int ii; + fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1); + if( *pRc==SQLITE_OK ){ + for(ii=0; iin; ii++){ + if( pTerm->p[ii]==0x00 ){ + pBuf->p[pBuf->n++] = '\\'; + pBuf->p[pBuf->n++] = '0'; + }else{ + pBuf->p[pBuf->n++] = pTerm->p[ii]; + } + } + pBuf->p[pBuf->n] = 0x00; + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). */ @@ -230201,6 +250072,7 @@ static void fts5DecodeFunction( ){ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ @@ -230223,7 +250095,7 @@ static void fts5DecodeFunction( if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( bDlidx ){ @@ -230242,6 +250114,28 @@ static void fts5DecodeFunction( " %d(%lld)", lvl.iLeafPgno, lvl.iRowid ); } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iiestimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iLevelnLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=p->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called as part of registering the FTS5 module with database @@ -230446,7 +250569,7 @@ static void fts5RowidFunction( ** SQLite error code is returned instead. */ static int sqlite3Fts5IndexInit(sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); @@ -230463,6 +250586,37 @@ static int sqlite3Fts5IndexInit(sqlite3 *db){ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } return rc; #else return SQLITE_OK; @@ -230598,6 +250752,8 @@ struct Fts5FullTable { Fts5Storage *pStorage; /* Document store */ Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ + int iSavepoint; /* Successful xSavepoint()+1 */ + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -230741,7 +250897,7 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ break; case FTS5_SYNC: - assert( p->ts.eState==1 ); + assert( p->ts.eState==1 || p->ts.eState==2 ); p->ts.eState = 2; break; @@ -230756,21 +250912,21 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ break; case FTS5_SAVEPOINT: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); assert( iSavepoint>=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint; break; case FTS5_RELEASE: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); assert( iSavepoint<=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint-1; break; case FTS5_ROLLBACKTO: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=-1 ); /* The following assert() can fail if another vtab strikes an error ** within an xSavepoint() call then SQLite calls xRollbackTo() - without @@ -230886,6 +251042,13 @@ static int fts5InitVtab( pConfig->pzErrmsg = 0; } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -231128,12 +251291,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } idxStr[iIdxStr] = '\0'; - /* Set idxFlags flags for the ORDER BY clause */ + /* Set idxFlags flags for the ORDER BY clause + ** + ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC". + */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; if( iSort==(pConfig->nCol+1) && bSeenMatch ){ idxFlags |= FTS5_BI_ORDER_RANK; - }else if( iSort==-1 ){ + }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; } if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ @@ -231285,7 +251451,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; - CsrFlagSet(pCsr, FTS5CSR_EOF); + CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT); }else if( rc==SQLITE_ROW ){ const u8 *a; const u8 *aBlob; @@ -231385,6 +251551,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table, + ** clear any token mappings accumulated at the fts5_index.c level. In + ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH, + ** we need to retain the mappings for the entire query. */ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata + ){ + sqlite3Fts5ExprClearTokens(pCsr->pExpr); + } + if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; @@ -231810,6 +251986,9 @@ static int fts5FilterMethod( pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); } + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) goto filter_out; + if( pTab->pSortCsr ){ /* If pSortCsr is non-NULL, then this call is being made as part of ** processing for a "... MATCH ORDER BY rank" query (ePlan is @@ -231832,6 +252011,7 @@ static int fts5FilterMethod( pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); }else if( pCsr->pExpr ){ + assert( rc==SQLITE_OK ); rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( bOrderByRank ){ @@ -232003,6 +252183,7 @@ static int fts5SpecialInsert( Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; int bError = 0; + int bLoadConfig = 0; if( 0==sqlite3_stricmp("delete-all", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -232014,6 +252195,7 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NONE ){ fts5SetVtabError(pTab, @@ -232023,6 +252205,7 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); }else if( 0==sqlite3_stricmp("merge", zCmd) ){ @@ -232035,8 +252218,13 @@ static int fts5SpecialInsert( }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif + }else if( 0==sqlite3_stricmp("flush", zCmd) ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); }else{ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); } @@ -232048,6 +252236,12 @@ static int fts5SpecialInsert( } } } + + if( rc==SQLITE_OK && bLoadConfig ){ + pTab->p.pConfig->iCookie--; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + return rc; } @@ -232104,9 +252298,10 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; /* A transaction must be open when this is called. */ - assert( pTab->ts.eState==1 ); + assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); assert( pVtab->zErrMsg==0 ); assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); @@ -232114,6 +252309,11 @@ static int fts5UpdateMethod( || sqlite3_value_type(apVal[0])==SQLITE_NULL ); assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ @@ -232128,7 +252328,15 @@ static int fts5UpdateMethod( if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ - rc = fts5SpecialDelete(pTab, apVal); + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; + } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } @@ -232145,7 +252353,7 @@ static int fts5UpdateMethod( ** Cases 3 and 4 may violate the rowid constraint. */ int eConflict = SQLITE_ABORT; - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ eConflict = sqlite3_vtab_on_conflict(pConfig->db); } @@ -232153,8 +252361,12 @@ static int fts5UpdateMethod( assert( nArg!=1 || eType0==SQLITE_INTEGER ); /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. */ - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ + ** This is not suported. Except - they are both supported if the CREATE + ** VIRTUAL TABLE statement contained "contentless_delete=1". */ + if( eType0==SQLITE_INTEGER + && pConfig->eContent==FTS5_CONTENT_NONE + && pConfig->bContentlessDelete==0 + ){ pTab->p.base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName @@ -232166,6 +252378,7 @@ static int fts5UpdateMethod( else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + bUpdateOrDelete = 1; } /* INSERT or UPDATE */ @@ -232177,10 +252390,12 @@ static int fts5UpdateMethod( } else if( eType0!=SQLITE_INTEGER ){ - /* If this is a REPLACE, first remove the current entry (if any) */ + /* An INSERT statement. If the conflict-mode is REPLACE, first remove + ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); } @@ -232209,10 +252424,24 @@ static int fts5UpdateMethod( rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } + bUpdateOrDelete = 1; } } } + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -232225,8 +252454,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_SYNC, 0); pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - fts5TripCursors(pTab); - rc = sqlite3Fts5StorageSync(pTab->pStorage); + rc = sqlite3Fts5FlushToDisk(&pTab->p); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -232322,7 +252550,10 @@ static int fts5ApiColumnText( ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) || pCsr->ePlan==FTS5_PLAN_SPECIAL ){ *pz = 0; @@ -232347,8 +252578,9 @@ static int fts5CsrPoslist( int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; @@ -232372,15 +252604,21 @@ static int fts5CsrPoslist( CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ - Fts5Sorter *pSorter = pCsr->pSorter; - int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - *pn = pSorter->aIdx[iPhrase] - i1; - *pa = &pSorter->aPoslist[i1]; + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } }else{ - *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pa = 0; + *pn = 0; } + return rc; } @@ -232487,12 +252725,6 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; -#if 0 - }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ - *piPhrase = pCsr->aInst[iIdx*3]; - *piCol = pCsr->aInst[iIdx*3 + 2]; - *piOff = -1; -#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -232747,13 +252979,56 @@ static int fts5ApiPhraseFirstColumn( return rc; } +/* +** xQueryToken() API implemenetation. +*/ +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} + +/* +** xInstToken() API implemenetation. +*/ +static int fts5ApiInstToken( + Fts5Context *pCtx, + int iIdx, + int iToken, + const char **ppOut, int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + int iPhrase = pCsr->aInst[iIdx*3]; + int iCol = pCsr->aInst[iIdx*3 + 1]; + int iOff = pCsr->aInst[iIdx*3 + 2]; + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5ExprInstToken( + pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut + ); + } + } + return rc; +} + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { - 2, /* iVersion */ + 3, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -232773,6 +253048,8 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + fts5ApiInstToken }; /* @@ -232993,6 +253270,12 @@ static int fts5ColumnMethod( sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); } pConfig->pzErrmsg = 0; + }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ + char *zErr = sqlite3_mprintf("cannot UPDATE a subset of " + "columns on fts5 contentless-delete table: %s", pConfig->zName + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); } return rc; } @@ -233031,8 +253314,10 @@ static int fts5RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ + int rc; Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - return sqlite3Fts5StorageRename(pTab->pStorage, zName); + rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); + return rc; } static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ @@ -233046,9 +253331,15 @@ static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + return rc; } /* @@ -233057,9 +253348,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); + if( (iSavepoint+1)iSavepoint ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint; + } + } + return rc; } /* @@ -233069,10 +253367,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); - return sqlite3Fts5StorageRollback(pTab->pStorage); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + pTab->p.pConfig->pgsz = 0; + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + } + return rc; } /* @@ -233208,14 +253510,16 @@ static int sqlite3Fts5GetTokenizer( if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate( pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok ); pConfig->pTokApi = &pMod->x; if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + if( pzErr && rc!=SQLITE_NOMEM ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } }else{ pConfig->ePattern = sqlite3Fts5TokenizerPattern( pMod->x.xCreate, pConfig->pTok @@ -233274,7 +253578,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2022-01-06 13:25:41 872ba256cbf61d9290b571c0e6d82a20c224ca3ad82971edc46b29818d5d17a0", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-08-13 09:16:08 c9c2ab54ba1f5f46360f1b4f35d849cd3f080e6fc2b6c60e91b16c63f69a1e33", -1, SQLITE_TRANSIENT); } /* @@ -233292,9 +253596,47 @@ static int fts5ShadowName(const char *zName){ return 0; } +/* +** Run an integrity check on the FTS5 data structures. Return a string +** if anything is found amiss. Return a NULL pointer if everything is +** OK. +*/ +static int fts5IntegrityMethod( + sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */ + const char *zSchema, /* Name of schema in which this table lives */ + const char *zTabname, /* Name of the table itself */ + int isQuick, /* True if this is a quick-check */ + char **pzErr /* Write error message here */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc; + + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; +} + static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts5CreateMethod, /* xConnect */ fts5ConnectMethod, /* xBestIndex */ fts5BestIndexMethod, @@ -233317,7 +253659,8 @@ static int fts5Init(sqlite3 *db){ /* xSavepoint */ fts5SavepointMethod, /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, - /* xShadowName */ fts5ShadowName + /* xShadowName */ fts5ShadowName, + /* xIntegrity */ fts5IntegrityMethod }; int rc; @@ -233347,7 +253690,9 @@ static int fts5Init(sqlite3 *db){ } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( - db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 + db, "fts5_source_id", 0, + SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS, + p, fts5SourceIdFunc, 0, 0 ); } } @@ -233485,10 +253830,10 @@ static int fts5StorageGetStmt( "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ "SELECT %s FROM %s AS T", /* SCAN */ @@ -233536,6 +253881,19 @@ static int fts5StorageGetStmt( break; } + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + default: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); break; @@ -233725,9 +254083,11 @@ static int sqlite3Fts5StorageOpen( } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = sqlite3Fts5CreateTable( - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr - ); + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( @@ -233804,7 +254164,7 @@ static int fts5StorageDeleteFromIndex( ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; @@ -233820,7 +254180,6 @@ static int fts5StorageDeleteFromIndex( ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ const char *zText; @@ -233857,6 +254216,37 @@ static int fts5StorageDeleteFromIndex( return rc; } +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} /* ** Insert a record into the %_docsize table. Specifically, do: @@ -233877,10 +254267,17 @@ static int fts5StorageInsertDocsize( rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pReplace, 1, iRowid); - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } } } return rc; @@ -233944,7 +254341,15 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + } } /* Delete the %_docsize record */ @@ -234021,7 +254426,7 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ @@ -234532,7 +254937,9 @@ static int sqlite3Fts5StorageSync(Fts5Storage *p){ i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); if( p->bTotalsValid ){ rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; + if( rc==SQLITE_OK ){ + p->bTotalsValid = 0; + } } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexSync(p->pIndex); @@ -234656,7 +255063,7 @@ static int fts5AsciiCreate( int i; memset(p, 0, sizeof(AsciiTokenizer)); memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - for(i=0; rc==SQLITE_OK && i=0xc0 ){ \ + while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + typedef struct Unicode61Tokenizer Unicode61Tokenizer; struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ @@ -234952,17 +255366,16 @@ static int fts5UnicodeCreate( } /* Search for a "categories" argument */ - for(i=0; rc==SQLITE_OK && ibFold = 1; - for(i=0; rc==SQLITE_OK && iiFoldParam = 0; + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } }else{ rc = SQLITE_ERROR; } } + if( iiFoldParam!=0 && pNew->bFold==0 ){ + rc = SQLITE_ERROR; + } + if( rc!=SQLITE_OK ){ fts5TriDelete((Fts5Tokenizer*)pNew); pNew = 0; @@ -235901,40 +256329,62 @@ static int fts5TriTokenize( TrigramTokenizer *p = (TrigramTokenizer*)pTok; int rc = SQLITE_OK; char aBuf[32]; + char *zOut = aBuf; + int ii; const unsigned char *zIn = (const unsigned char*)pText; const unsigned char *zEof = &zIn[nText]; u32 iCode; + int aStart[3]; /* Input offset of each character in aBuf[] */ UNUSED_PARAM(unusedFlags); - while( 1 ){ - char *zOut = aBuf; - int iStart = zIn - (const unsigned char*)pText; - const unsigned char *zNext; - - READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; - zNext = zIn; - if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); + + /* Populate aBuf[] with the characters for the first trigram. */ + for(ii=0; ii<3; ii++){ + do { + aStart[ii] = zIn - (const unsigned char*)pText; READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; - }else{ - break; - } - if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); + if( iCode==0 ) return SQLITE_OK; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + WRITE_UTF8(zOut, iCode); + } + + /* At the start of each iteration of this loop: + ** + ** aBuf: Contains 3 characters. The 3 characters of the next trigram. + ** zOut: Points to the byte following the last character in aBuf. + ** aStart[3]: Contains the byte offset in the input text corresponding + ** to the start of each of the three characters in the buffer. + */ + assert( zIn<=zEof ); + while( 1 ){ + int iNext; /* Start of character following current tri */ + const char *z1; + + /* Read characters from the input up until the first non-diacritic */ + do { + iNext = zIn - (const unsigned char*)pText; READ_UTF8(zIn, zEof, iCode); if( iCode==0 ) break; - if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); - }else{ - break; - } - rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf); - if( rc!=SQLITE_OK ) break; - zIn = zNext; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + + /* Pass the current trigram back to fts5 */ + rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext); + if( iCode==0 || rc!=SQLITE_OK ) break; + + /* Remove the first character from buffer aBuf[]. Append the character + ** with codepoint iCode. */ + z1 = aBuf; + FTS5_SKIP_UTF8(z1); + memmove(aBuf, z1, zOut - z1); + zOut -= (z1 - aBuf); + WRITE_UTF8(zOut, iCode); + + /* Update the aStart[] array */ + aStart[0] = aStart[1]; + aStart[1] = aStart[2]; + aStart[2] = iNext; } return rc; @@ -235957,7 +256407,9 @@ static int sqlite3Fts5TokenizerPattern( ){ if( xCreate==fts5TriCreate ){ TrigramTokenizer *p = (TrigramTokenizer*)pTok; - return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + if( p->iFoldParam==0 ){ + return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + } } return FTS5_PATTERN_NONE; } @@ -237746,7 +258198,7 @@ static int fts5VocabFilterMethod( if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); nTerm = sqlite3_value_bytes(pEq); - f = 0; + f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ zTerm = (const char *)sqlite3_value_text(pGe); @@ -237900,7 +258352,8 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; void *p = (void*)pGlobal; @@ -237945,6 +258398,16 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE + +#define STMT_NUM_INTEGER_COLUMN 10 +typedef struct StmtRow StmtRow; +struct StmtRow { + sqlite3_int64 iRowid; /* Rowid value */ + char *zSql; /* column "sql" */ + int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */ + StmtRow *pNext; /* Next row to return */ +}; + /* stmt_vtab is a subclass of sqlite3_vtab which will ** serve as the underlying representation of a stmt virtual table */ @@ -237962,8 +258425,7 @@ typedef struct stmt_cursor stmt_cursor; struct stmt_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ sqlite3 *db; /* Database connection for this cursor */ - sqlite3_stmt *pStmt; /* Statement cursor is currently pointing at */ - sqlite3_int64 iRowid; /* The rowid */ + StmtRow *pRow; /* Current row */ }; /* @@ -238003,11 +258465,15 @@ static int stmtConnect( #define STMT_COLUMN_MEM 10 /* SQLITE_STMTSTATUS_MEMUSED */ + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; rc = sqlite3_declare_vtab(db, "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep," "reprep,run,mem)"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -238029,7 +258495,7 @@ static int stmtDisconnect(sqlite3_vtab *pVtab){ */ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ stmt_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((stmt_vtab*)p)->db; @@ -238037,10 +258503,21 @@ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +static void stmtCsrReset(stmt_cursor *pCur){ + StmtRow *pRow = 0; + StmtRow *pNext = 0; + for(pRow=pCur->pRow; pRow; pRow=pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + pCur->pRow = 0; +} + /* ** Destructor for a stmt_cursor. */ static int stmtClose(sqlite3_vtab_cursor *cur){ + stmtCsrReset((stmt_cursor*)cur); sqlite3_free(cur); return SQLITE_OK; } @@ -238051,8 +258528,9 @@ static int stmtClose(sqlite3_vtab_cursor *cur){ */ static int stmtNext(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - pCur->iRowid++; - pCur->pStmt = sqlite3_next_stmt(pCur->db, pCur->pStmt); + StmtRow *pNext = pCur->pRow->pNext; + sqlite3_free(pCur->pRow); + pCur->pRow = pNext; return SQLITE_OK; } @@ -238066,39 +258544,11 @@ static int stmtColumn( int i /* Which column to return */ ){ stmt_cursor *pCur = (stmt_cursor*)cur; - switch( i ){ - case STMT_COLUMN_SQL: { - sqlite3_result_text(ctx, sqlite3_sql(pCur->pStmt), -1, SQLITE_TRANSIENT); - break; - } - case STMT_COLUMN_NCOL: { - sqlite3_result_int(ctx, sqlite3_column_count(pCur->pStmt)); - break; - } - case STMT_COLUMN_RO: { - sqlite3_result_int(ctx, sqlite3_stmt_readonly(pCur->pStmt)); - break; - } - case STMT_COLUMN_BUSY: { - sqlite3_result_int(ctx, sqlite3_stmt_busy(pCur->pStmt)); - break; - } - default: { - assert( i==STMT_COLUMN_MEM ); - i = SQLITE_STMTSTATUS_MEMUSED + - STMT_COLUMN_NSCAN - SQLITE_STMTSTATUS_FULLSCAN_STEP; - /* Fall thru */ - } - case STMT_COLUMN_NSCAN: - case STMT_COLUMN_NSORT: - case STMT_COLUMN_NAIDX: - case STMT_COLUMN_NSTEP: - case STMT_COLUMN_REPREP: - case STMT_COLUMN_RUN: { - sqlite3_result_int(ctx, sqlite3_stmt_status(pCur->pStmt, - i-STMT_COLUMN_NSCAN+SQLITE_STMTSTATUS_FULLSCAN_STEP, 0)); - break; - } + StmtRow *pRow = pCur->pRow; + if( i==STMT_COLUMN_SQL ){ + sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_int(ctx, pRow->aCol[i]); } return SQLITE_OK; } @@ -238109,7 +258559,7 @@ static int stmtColumn( */ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ stmt_cursor *pCur = (stmt_cursor*)cur; - *pRowid = pCur->iRowid; + *pRowid = pCur->pRow->iRowid; return SQLITE_OK; } @@ -238119,7 +258569,7 @@ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int stmtEof(sqlite3_vtab_cursor *cur){ stmt_cursor *pCur = (stmt_cursor*)cur; - return pCur->pStmt==0; + return pCur->pRow==0; } /* @@ -238134,9 +258584,57 @@ static int stmtFilter( int argc, sqlite3_value **argv ){ stmt_cursor *pCur = (stmt_cursor *)pVtabCursor; - pCur->pStmt = 0; - pCur->iRowid = 0; - return stmtNext(pVtabCursor); + sqlite3_stmt *p = 0; + sqlite3_int64 iRowid = 1; + StmtRow **ppRow = 0; + + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; + stmtCsrReset(pCur); + ppRow = &pCur->pRow; + for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ + const char *zSql = sqlite3_sql(p); + sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; + StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql); + + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(StmtRow)); + if( zSql ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, zSql, nSql); + } + pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p); + pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p); + pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p); + pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_SORT, 0 + ); + pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_AUTOINDEX, 0 + ); + pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_VM_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_REPREPARE, 0 + ); + pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_RUN, 0 + ); + pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_MEMUSED, 0 + ); + pNew->iRowid = iRowid++; + *ppRow = pNew; + ppRow = &pNew->pNext; + } + + return SQLITE_OK; } /* @@ -238149,6 +258647,7 @@ static int stmtBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ + (void)tab; pIdxInfo->estimatedCost = (double)500; pIdxInfo->estimatedRows = 500; return SQLITE_OK; @@ -238183,6 +258682,7 @@ static sqlite3_module stmtModule = { 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -238218,6 +258718,7 @@ SQLITE_API int sqlite3_stmt_init( /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } /************************** End of sqlite3.c ******************************/ + #ifdef SQLITE_HAS_CODEC /************** Begin file hw_codec_openssl.h *******************************/ #ifndef EXPOSE_INTERNAL_FUNC @@ -238253,10 +258754,12 @@ typedef struct{ void *cipher; int keySize; int keyInfoSize; + int cipherPageSize; int initVectorSize; int hmacSize; int reserveSize; int hmacAlgo; + int kdfAlgo; int rekeyHmacAlgo; }CodecConstant; @@ -238360,11 +258863,24 @@ CODEC_STATIC int opensslGetInitVectorSize(void *cipher){ #define CIPHER_HMAC_ALGORITHM_SHA1 1 #define CIPHER_HMAC_ALGORITHM_SHA256 2 +#define CIPHER_HMAC_ALGORITHM_SHA512 3 #define DEFAULT_HMAC_ALGORITHM CIPHER_HMAC_ALGORITHM_SHA1 #define CIPHER_HMAC_ALGORITHM_NAME_SHA1 "SHA1" #define CIPHER_HMAC_ALGORITHM_NAME_SHA256 "SHA256" +#define CIPHER_HMAC_ALGORITHM_NAME_SHA512 "SHA512" + +#define CIPHER_KDF_ALGORITHM_SHA1 1 +#define CIPHER_KDF_ALGORITHM_SHA256 2 +#define CIPHER_KDF_ALGORITHM_SHA512 3 + +#define DEFAULT_KDF_ALGORITHM CIPHER_KDF_ALGORITHM_SHA1 + +#define CIPHER_KDF_ALGORITHM_NAME_SHA1 "KDF_SHA1" +#define CIPHER_KDF_ALGORITHM_NAME_SHA256 "KDF_SHA256" +#define CIPHER_KDF_ALGORITHM_NAME_SHA512 "KDF_SHA512" + CODEC_STATIC int opensslGetHmacSize(KeyContext *keyCtx){ if( keyCtx==NULL ){ @@ -238374,6 +258890,8 @@ CODEC_STATIC int opensslGetHmacSize(KeyContext *keyCtx){ return EVP_MD_size(EVP_sha1()); }else if( keyCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ return EVP_MD_size(EVP_sha256()); + }else if( keyCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ + return EVP_MD_size(EVP_sha512()); } return 0; } @@ -238421,6 +258939,8 @@ CODEC_STATIC int opensslHmac(Buffer *key, Buffer *input1, Buffer *input2, Buffer HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha1(), NULL); }else if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha256(), NULL); + }else if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ + HMAC_Init_ex(ctx, key->buffer, key->bufferSize, EVP_sha512(), NULL); } HMAC_Update(ctx, input1->buffer, input1->bufferSize); HMAC_Update(ctx, input2->buffer, input2->bufferSize); @@ -238430,13 +258950,16 @@ CODEC_STATIC int opensslHmac(Buffer *key, Buffer *input1, Buffer *input2, Buffer return SQLITE_OK; } -CODEC_STATIC void opensslKdf(Buffer *password, Buffer *salt, int workfactor, Buffer *key, int hmacAlgo){ - if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ +CODEC_STATIC void opensslKdf(Buffer *password, Buffer *salt, int workfactor, Buffer *key, int kdfAlgo){ + if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA1 ){ PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, workfactor, EVP_sha1(), key->bufferSize, key->buffer); - }else if( hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ + }else if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA256 ){ PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, workfactor, EVP_sha256(), key->bufferSize, key->buffer); + }else if( kdfAlgo==CIPHER_KDF_ALGORITHM_SHA512 ){ + PKCS5_PBKDF2_HMAC((const char *)(password->buffer), password->bufferSize, salt->buffer, salt->bufferSize, + workfactor, EVP_sha512(), key->bufferSize, key->buffer); } } @@ -238638,12 +259161,22 @@ CODEC_STATIC int sqlite3CodecDeriveKey(CodecContext *ctx, OperateContext whichKe if(keyCtx->deriveFlag){ return SQLITE_OK; } + errno_t memcpyRc = EOK; + unsigned char salt[SALT_SIZE]; + if (ctx->pBt != NULL && sqlite3OsRead(ctx->pBt->pBt->pPager->fd, salt, SALT_SIZE, 0) == SQLITE_OK) { + assert(SALT_SIZE == FILE_HEADER_SIZE); + if (memcmp(SQLITE_FILE_HEADER, salt, SALT_SIZE) != 0 && memcmp(ctx->salt, salt, SALT_SIZE) != 0) { + memcpyRc = memcpy_s(ctx->salt, FILE_HEADER_SIZE, salt, SALT_SIZE); + if(memcpyRc != EOK){ + return SQLITE_ERROR; + } + } + } sqlite3CodecInitDeriveKeyMemory(keyCtx); if(keyCtx->key == NULL || keyCtx->hmacKey == NULL || keyCtx->keyInfo == NULL){ sqlite3CodecClearDeriveKey(keyCtx); return SQLITE_NOMEM; } - errno_t memcpyRc = EOK; if((keyCtx->passwordSize == keyCtx->codecConst.keyInfoSize) && (sqlite3CodecIsKeyInfoFormat(keyCtx->password, keyCtx->passwordSize))){ sqlite3CodecHex2Bin(keyCtx->password + 2, keyCtx->codecConst.keySize * 2, keyCtx->key); @@ -238671,7 +259204,7 @@ CODEC_STATIC int sqlite3CodecDeriveKey(CodecContext *ctx, OperateContext whichKe salt.bufferSize = SALT_SIZE; key.buffer = keyCtx->key; key.bufferSize = keyCtx->codecConst.keySize; - opensslKdf(&password, &salt, keyCtx->iter, &key, keyCtx->codecConst.hmacAlgo); + opensslKdf(&password, &salt, keyCtx->iter, &key, keyCtx->codecConst.kdfAlgo); keyCtx->keyInfo[0] = 'x'; keyCtx->keyInfo[1] = '\''; sqlite3CodecBin2Hex(keyCtx->key, keyCtx->codecConst.keySize, keyCtx->keyInfo + 2); @@ -238691,7 +259224,7 @@ CODEC_STATIC int sqlite3CodecDeriveKey(CodecContext *ctx, OperateContext whichKe hmacSalt.bufferSize = SALT_SIZE; hmacKey.buffer = keyCtx->hmacKey; hmacKey.bufferSize = keyCtx->codecConst.keySize; - opensslKdf(&hmacPassword, &hmacSalt, HMAC_ITER, &hmacKey, keyCtx->codecConst.hmacAlgo); + opensslKdf(&hmacPassword, &hmacSalt, HMAC_ITER, &hmacKey, keyCtx->codecConst.kdfAlgo); keyCtx->deriveFlag = 1; if(sqlite3CodecKeyCtxCmp(keyCtx, secondKeyCtx)){ sqlite3CodecClearDeriveKey(secondKeyCtx); @@ -238746,26 +259279,31 @@ CODEC_STATIC int sqlite3CodecSetIter(KeyContext *keyCtx, int iter){ #define CIPHER_NAME_AES_256_CBC "aes-256-cbc" #define CIPHER_NAME_AES_256_GCM "aes-256-gcm" -static int g_defaultAttachKdfIter = 10000; -static int g_defaultAttachCipher = CIPHER_ID_AES_256_GCM; -static int g_defaultAttachHmacAlgo = DEFAULT_HMAC_ALGORITHM; - struct CodecCipherNameId { int cipherId; const char *cipherName; }; -struct CodecCipherNameId g_cipherNameIdMap[CIPHER_TOTAL_NUM] = { +static const struct CodecCipherNameId g_cipherNameIdMap[CIPHER_TOTAL_NUM] = { { CIPHER_ID_AES_256_CBC, CIPHER_NAME_AES_256_CBC }, { CIPHER_ID_AES_256_GCM, CIPHER_NAME_AES_256_GCM } }; -CODEC_STATIC void sqlite3CodecSetDefaultAttachCipher(const char *cipherName){ +SQLITE_PRIVATE void sqlite3CodecResetParameters(CodecParameter *p) +{ + p->kdfIter = DEFAULT_ITER; + p->pageSize = DEFAULT_PAGE_SIZE; + p->cipher = CIPHER_ID_AES_256_GCM; + p->hmacAlgo = DEFAULT_HMAC_ALGORITHM; + p->kdfAlgo = DEFAULT_KDF_ALGORITHM; +} + +CODEC_STATIC void sqlite3CodecSetDefaultAttachCipher(CodecParameter *parm, const char *cipherName){ int i; for( i=0; icipher = g_cipherNameIdMap[i].cipherId; sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); return; } @@ -238773,46 +259311,72 @@ CODEC_STATIC void sqlite3CodecSetDefaultAttachCipher(const char *cipherName){ sqlite3_log(SQLITE_WARNING, "invalid attach cipher algorithm"); } -CODEC_STATIC const char *sqlite3CodecGetDefaultAttachCipher(){ +CODEC_STATIC const char *sqlite3CodecGetDefaultAttachCipher(CodecParameter *parm){ const char *attachedCipher = CIPHER_NAME_AES_256_GCM; sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - if( (g_defaultAttachCipher>=0) && (g_defaultAttachCiphercipher>=0) && (parm->ciphercipher].cipherName; } sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); return attachedCipher; } -CODEC_STATIC void sqlite3CodecSetDefaultAttachKdfIter(int iter){ +CODEC_STATIC void sqlite3CodecSetDefaultAttachKdfIter(CodecParameter *parm, int iter){ if( iter<=0 ){ return; } sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - g_defaultAttachKdfIter = iter; + parm->kdfIter = iter; sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } -CODEC_STATIC int sqlite3CodecGetDefaultAttachKdfIter(){ +CODEC_STATIC int sqlite3CodecGetDefaultAttachKdfIter(CodecParameter *parm){ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - int iterNum = g_defaultAttachKdfIter; + int iterNum = parm->kdfIter; sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); return iterNum; } -CODEC_STATIC void sqlite3CodecSetDefaultAttachHmacAlgo(int hmacAlgo){ +CODEC_STATIC void sqlite3CodecSetDefaultAttachHmacAlgo(CodecParameter *parm, int hmacAlgo){ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - g_defaultAttachHmacAlgo = hmacAlgo; + parm->hmacAlgo = hmacAlgo; sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); } -CODEC_STATIC int sqlite3CodecGetDefaultAttachHmacAlgo(){ +CODEC_STATIC int sqlite3CodecGetDefaultAttachHmacAlgo(CodecParameter *parm){ sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - int hmacAlgo = g_defaultAttachHmacAlgo; + int hmacAlgo = parm->hmacAlgo; sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); return hmacAlgo; } -#endif +CODEC_STATIC void sqlite3CodecSetDefaultAttachKdfAlgo(CodecParameter *parm, int kdfAlgo){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + parm->kdfAlgo = kdfAlgo; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +CODEC_STATIC int sqlite3CodecGetDefaultAttachKdfAlgo(CodecParameter *parm){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + int kdfAlgo = parm->kdfAlgo; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return kdfAlgo; +} + +CODEC_STATIC void sqlite3CodecSetDefaultAttachPageSize(CodecParameter *parm, int pageSize){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + parm->pageSize = pageSize; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +} + +CODEC_STATIC int sqlite3CodecGetDefaultAttachPageSize(CodecParameter *parm){ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + int pageSize = parm->pageSize; + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + return pageSize; +} + +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ // You should clear key derive infos and password infos before you call this function CODEC_STATIC int sqlite3CodecSetPassword(KeyContext *keyCtx, const void *zKey, int nKey){ @@ -238847,6 +259411,30 @@ CODEC_STATIC int sqlite3CodecSetHmacAlgorithm(KeyContext *keyCtx, int hmacAlgo){ return SQLITE_OK; } +CODEC_STATIC int sqlite3CodecSetKdfAlgorithm(KeyContext *keyCtx, int kdfAlgo){ + keyCtx->codecConst.kdfAlgo = kdfAlgo; + return SQLITE_OK; +} + +CODEC_STATIC int sqlite3CodecSetCipherPageSize(CodecContext *ctx, int size){ + if(!((size != 0) && ((size & (size - 1)) == 0)) || size < 512 || size > 65536) { + sqlite3_log(SQLITE_ERROR, "codec: cipher_page_size not a power of 2 and between 512 and 65536 inclusive(%d).", size); + return SQLITE_ERROR; + } + int cipherPageSize = ctx->readCtx->codecConst.cipherPageSize; + (void)memset_s(ctx->buffer, cipherPageSize, 0, cipherPageSize); + sqlite3_free(ctx->buffer); + ctx->readCtx->codecConst.cipherPageSize = size; + ctx->writeCtx->codecConst.cipherPageSize = size; + + ctx->buffer = (unsigned char *)sqlite3Malloc(size); + if (ctx->buffer == NULL) { + sqlite3_log(SQLITE_NOMEM, "codec: alloc mem failed when set cipher page size(%d).", size); + return SQLITE_NOMEM; + } + return SQLITE_OK; +} + // You should clear output before you call this function CODEC_STATIC int sqlite3CodecCopyKeyContext(KeyContext *input, KeyContext *output){ errno_t rc = memcpy_s(output, sizeof(KeyContext), input, KEY_CONTEXT_HEAD_SIZE); @@ -238870,29 +259458,46 @@ CODEC_STATIC int sqlite3CodecCopyKeyContext(KeyContext *input, KeyContext *outpu // You should clear key context before you call this function #ifdef SQLITE_CODEC_ATTACH_CHANGED -CODEC_STATIC int sqlite3CodecInitKeyContext(KeyContext *keyCtx, const void *zKey, int nKey, int attachFlag, - int hmacAlgo){ +CODEC_STATIC int sqlite3CodecInitKeyContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey, int attachFlag){ #else -CODEC_STATIC int sqlite3CodecInitKeyContext(KeyContext *keyCtx, const void *zKey, int nKey){ -#endif +CODEC_STATIC int sqlite3CodecInitKeyContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey){ +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ int rc = SQLITE_OK; + KeyContext *keyCtx = ctx->readCtx; #ifdef SQLITE_CODEC_ATTACH_CHANGED if( attachFlag!=0 ){ - rc = sqlite3CodecSetCodecConstant(keyCtx, sqlite3CodecGetDefaultAttachCipher()); - rc += sqlite3CodecSetIter(keyCtx, sqlite3CodecGetDefaultAttachKdfIter()); + CodecParameter *parm = &p->db->codecParm; + int hmacAlgo = sqlite3CodecGetDefaultAttachHmacAlgo(parm); + rc = sqlite3CodecSetCodecConstant(keyCtx, sqlite3CodecGetDefaultAttachCipher(parm)); + rc += sqlite3CodecSetIter(keyCtx, sqlite3CodecGetDefaultAttachKdfIter(parm)); if( hmacAlgo!=0 ){ rc += sqlite3CodecSetHmacAlgorithm(keyCtx, hmacAlgo); } + int attachKdfAlgo = sqlite3CodecGetDefaultAttachKdfAlgo(parm); + if( attachKdfAlgo!=0 ){ + rc += sqlite3CodecSetKdfAlgorithm(keyCtx, attachKdfAlgo); + } + int cipherPageSize = sqlite3CodecGetDefaultAttachPageSize(parm); + if( cipherPageSize!=0 ){ + rc += sqlite3CodecSetCipherPageSize(ctx, cipherPageSize); + if ( rc != SQLITE_OK ) { + sqlite3CodecFreeKeyContext(keyCtx); + return SQLITE_ERROR; + } + rc += sqlite3BtreeSetPageSize(p, cipherPageSize, keyCtx->codecConst.reserveSize, 0); + } }else{ rc = sqlite3CodecSetCodecConstant(keyCtx, DEFAULT_CIPHER); rc += sqlite3CodecSetIter(keyCtx, DEFAULT_ITER); rc += sqlite3CodecSetHmacAlgorithm(keyCtx, DEFAULT_HMAC_ALGORITHM); + rc += sqlite3CodecSetKdfAlgorithm(keyCtx, DEFAULT_KDF_ALGORITHM); } #else rc = sqlite3CodecSetCodecConstant(keyCtx, DEFAULT_CIPHER); rc += sqlite3CodecSetIter(keyCtx, DEFAULT_ITER); rc += sqlite3CodecSetHmacAlgorithm(keyCtx, DEFAULT_HMAC_ALGORITHM); -#endif + rc += sqlite3CodecSetKdfAlgorithm(keyCtx, DEFAULT_KDF_ALGORITHM); +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ keyCtx->codecConst.rekeyHmacAlgo = DEFAULT_HMAC_ALGORITHM; rc += sqlite3CodecSetPassword(keyCtx, zKey, nKey); if(rc != SQLITE_OK){ @@ -238905,7 +259510,8 @@ CODEC_STATIC int sqlite3CodecInitKeyContext(KeyContext *keyCtx, const void *zKey // This function will free all resources of codec context, except it self. CODEC_STATIC void sqlite3CodecFreeContext(CodecContext *ctx){ if(ctx->buffer){ - (void)memset_s(ctx->buffer, DEFAULT_PAGE_SIZE, 0, DEFAULT_PAGE_SIZE); + int cipherPageSize = ctx->readCtx->codecConst.cipherPageSize; + (void)memset_s(ctx->buffer, cipherPageSize, 0, cipherPageSize); sqlite3_free(ctx->buffer); ctx->buffer = NULL; } @@ -238925,32 +259531,35 @@ CODEC_STATIC void sqlite3CodecFreeContext(CodecContext *ctx){ #ifdef SQLITE_CODEC_ATTACH_CHANGED CODEC_STATIC int sqlite3CodecInitContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey, int nDb){ int attachFlag = (nDb > 1) ? 1 : 0; + int defaultPageSz = attachFlag ? p->db->codecParm.pageSize : DEFAULT_PAGE_SIZE; #else CODEC_STATIC int sqlite3CodecInitContext(CodecContext *ctx, Btree *p, const void *zKey, int nKey){ -#endif + int defaultPageSz = DEFAULT_PAGE_SIZE; +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ sqlite3_file *fd = p->pBt->pPager->fd; ctx->pBt = p; ctx->savePassword = 0; - ctx->buffer = (unsigned char *)sqlite3Malloc(DEFAULT_PAGE_SIZE); + ctx->buffer = (unsigned char *)sqlite3Malloc(defaultPageSz); ctx->readCtx = (KeyContext *)sqlite3Malloc(sizeof(KeyContext)); ctx->writeCtx = (KeyContext *)sqlite3Malloc(sizeof(KeyContext)); if(ctx->buffer == NULL || ctx->readCtx == NULL || ctx->writeCtx == NULL){ sqlite3CodecFreeContext(ctx); return SQLITE_NOMEM; } - errno_t memsetRc = memset_s(ctx->buffer, DEFAULT_PAGE_SIZE, 0, DEFAULT_PAGE_SIZE); + errno_t memsetRc = memset_s(ctx->buffer, defaultPageSz, 0, defaultPageSz); memsetRc += memset_s(ctx->readCtx, sizeof(KeyContext), 0, sizeof(KeyContext)); memsetRc += memset_s(ctx->writeCtx, sizeof(KeyContext), 0, sizeof(KeyContext)); if(memsetRc != EOK){ sqlite3CodecFreeContext(ctx); return SQLITE_ERROR; } + ctx->readCtx->codecConst.cipherPageSize = defaultPageSz; + ctx->writeCtx->codecConst.cipherPageSize = defaultPageSz; #ifdef SQLITE_CODEC_ATTACH_CHANGED - int attachHmacAlgo = sqlite3CodecGetDefaultAttachHmacAlgo(); - int rc = sqlite3CodecInitKeyContext(ctx->readCtx, zKey, nKey, attachFlag, attachHmacAlgo); + int rc = sqlite3CodecInitKeyContext(ctx, p, zKey, nKey, attachFlag); #else - int rc = sqlite3CodecInitKeyContext(ctx->readCtx, zKey, nKey); -#endif + int rc = sqlite3CodecInitKeyContext(ctx, p, zKey, nKey); +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ if(rc != SQLITE_OK){ sqlite3CodecFreeContext(ctx); return SQLITE_ERROR; @@ -239129,7 +259738,8 @@ CODEC_STATIC int sqlite3CodecDecryptData(CodecContext *ctx, OperateContext which inputBuffer.buffer = input; inputBuffer.bufferSize = bufferSize - keyCtx->codecConst.reserveSize; if(sqlite3CodecCheckHmac(keyCtx, pgno, inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize, input, input + inputBuffer.bufferSize + keyCtx->codecConst.initVectorSize)){ - sqlite3_log(SQLITE_WARNING, "codec: check hmac error at page %d when decrypt data.", pgno); + sqlite3_log(SQLITE_ERROR, "codec: check hmac error at page %d, hmac %d, kdf %d, pageSize %d, iter %d.", + pgno, keyCtx->codecConst.hmacAlgo, keyCtx->codecConst.kdfAlgo, keyCtx->codecConst.cipherPageSize, keyCtx->iter); return SQLITE_ERROR; } unsigned char *initVector = input + inputBuffer.bufferSize; @@ -239149,7 +259759,6 @@ CODEC_STATIC int sqlite3CodecDecryptData(CodecContext *ctx, OperateContext which void* sqlite3Codec(void *ctx, void *data, Pgno pgno, int mode){ CodecContext *pCtx = (CodecContext *)ctx; unsigned char *pData = (unsigned char *)data; - int offset = 0; int rc = SQLITE_OK; errno_t memcpyRc = EOK; @@ -239159,33 +259768,34 @@ void* sqlite3Codec(void *ctx, void *data, Pgno pgno, int mode){ if(pgno == 1){ offset = FILE_HEADER_SIZE; } + int cipherPageSize = pCtx->readCtx->codecConst.cipherPageSize; switch(mode){ case 0: case 2: case 3: if(pgno == 1){ - memcpyRc = memcpy_s(pCtx->buffer, DEFAULT_PAGE_SIZE, SQLITE_FILE_HEADER, FILE_HEADER_SIZE); + memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, SQLITE_FILE_HEADER, FILE_HEADER_SIZE); if(memcpyRc != EOK){ sqlite3CodecSetError(pCtx, SQLITE_ERROR); return pData; } } - rc = sqlite3CodecDecryptData(pCtx, OPERATE_CONTEXT_READ, pgno, DEFAULT_PAGE_SIZE - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); + rc = sqlite3CodecDecryptData(pCtx, OPERATE_CONTEXT_READ, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); if(rc != SQLITE_OK){ sqlite3CodecSetError(pCtx, rc); } - (void)memcpy_s(pData, DEFAULT_PAGE_SIZE, pCtx->buffer, DEFAULT_PAGE_SIZE); + (void)memcpy_s(pData, cipherPageSize, pCtx->buffer, cipherPageSize); return pData; break; case 6: if(pgno == 1){ - memcpyRc = memcpy_s(pCtx->buffer, DEFAULT_PAGE_SIZE, pCtx->salt, FILE_HEADER_SIZE); + memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, pCtx->salt, FILE_HEADER_SIZE); if(memcpyRc != EOK){ sqlite3CodecSetError(pCtx, SQLITE_ERROR); return pData; } } - rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_WRITE, pgno, DEFAULT_PAGE_SIZE - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); + rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_WRITE, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); if(rc != SQLITE_OK){ sqlite3CodecSetError(pCtx, rc); return pData; @@ -239194,13 +259804,13 @@ void* sqlite3Codec(void *ctx, void *data, Pgno pgno, int mode){ break; case 7: if(pgno == 1){ - memcpyRc = memcpy_s(pCtx->buffer, DEFAULT_PAGE_SIZE, pCtx->salt, FILE_HEADER_SIZE); + memcpyRc = memcpy_s(pCtx->buffer, cipherPageSize, pCtx->salt, FILE_HEADER_SIZE); if(memcpyRc != EOK){ sqlite3CodecSetError(pCtx, SQLITE_ERROR); return pData; } } - rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_READ, pgno, DEFAULT_PAGE_SIZE - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); + rc = sqlite3CodecEncryptData(pCtx, OPERATE_CONTEXT_READ, pgno, cipherPageSize - offset, (unsigned char *)(pData + offset), pCtx->buffer + offset); if(rc != SQLITE_OK){ sqlite3CodecSetError(pCtx, rc); return pData; @@ -239245,16 +259855,16 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); #else int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey); -#endif +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ if(rc != SQLITE_OK){ sqlite3_free(ctx); return rc; } sqlite3PagerSetCodec(sqlite3BtreePager(p), sqlite3Codec, NULL, sqlite3CodecDetach, (void *)ctx); - db->nextPagesize = DEFAULT_PAGE_SIZE; + db->nextPagesize = ctx->readCtx->codecConst.cipherPageSize; p->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; - sqlite3BtreeSetPageSize(p, DEFAULT_PAGE_SIZE, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); sqlite3BtreeSecureDelete(p, 1); if(isOpen(p->pBt->pPager->fd)){ sqlite3BtreeSetAutoVacuum(p, SQLITE_DEFAULT_AUTOVACUUM); @@ -239333,10 +259943,11 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ int oldHmacAlgo = ctx->writeCtx->codecConst.hmacAlgo; if( ctx->writeCtx->codecConst.rekeyHmacAlgo!=ctx->writeCtx->codecConst.hmacAlgo ){ sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, ctx->writeCtx->codecConst.rekeyHmacAlgo); + sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, ctx->writeCtx->codecConst.rekeyHmacAlgo); } for(pgno = 1; pgno <= (unsigned int)pageCount; pgno++){ - if(PAGER_MJ_PGNO(pPager) != pgno){ + if(PAGER_SJ_PGNO(pPager) != pgno){ rc = sqlite3PagerGet(pPager, pgno, &page, 0); if(rc == SQLITE_OK){ rc = sqlite3PagerWrite(page); @@ -239357,6 +259968,7 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ }else{ if( ctx->writeCtx->codecConst.rekeyHmacAlgo!=oldHmacAlgo ){ sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, oldHmacAlgo); + sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, oldHmacAlgo); } (void)sqlite3BtreeRollback(p, SQLITE_ABORT_ROLLBACK, 0); } @@ -239386,23 +259998,47 @@ int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, co } CodecContext *ctx = (CodecContext *)(p->pBt->pPager->pCodec); #ifdef SQLITE_CODEC_ATTACH_CHANGED + CodecParameter *parm = &db->codecParm; if(sqlite3StrICmp(zLeft, "cipher_default_attach_cipher") == 0 && zRight != NULL){ - (void)sqlite3CodecSetDefaultAttachCipher(zRight); + (void)sqlite3CodecSetDefaultAttachCipher(parm, zRight); return 1; }else if(sqlite3StrICmp(zLeft, "cipher_default_attach_kdf_iter") == 0 && zRight != NULL){ - (void)sqlite3CodecSetDefaultAttachKdfIter(atoi(zRight)); + (void)sqlite3CodecSetDefaultAttachKdfIter(parm, atoi(zRight)); return 1; }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_hmac_algo")==0 && zRight!=NULL ){ + /* + ** Make sure to set the Kdf algorithm after setting the Hmac algorithm, or it will not take effect. + ** This behavior is to ensure backward compatible. + */ if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA1)==0 ){ - sqlite3CodecSetDefaultAttachHmacAlgo(CIPHER_HMAC_ALGORITHM_SHA1); + sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA1); + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA1); }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ - sqlite3CodecSetDefaultAttachHmacAlgo(CIPHER_HMAC_ALGORITHM_SHA256); + sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA256); + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA256); + }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ + sqlite3CodecSetDefaultAttachHmacAlgo(parm, CIPHER_HMAC_ALGORITHM_SHA512); + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA512); + }else{ + return 0; + } + return 1; + }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_kdf_algo")==0 && zRight!=NULL ){ + if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA1)==0 ){ + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA1); + }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA256)==0 ){ + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA256); + }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA512)==0 ){ + sqlite3CodecSetDefaultAttachKdfAlgo(parm, CIPHER_KDF_ALGORITHM_SHA512); }else{ return 0; } return 1; + }else if( sqlite3StrICmp(zLeft, "cipher_default_attach_page_size")==0 && zRight!=NULL ){ + (void)sqlite3CodecSetDefaultAttachPageSize(parm, atoi(zRight)); + return 1; } -#endif +#endif /* SQLITE_CODEC_ATTACH_CHANGED */ if(ctx == NULL){ return 0; } @@ -239411,9 +260047,10 @@ int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, co sqlite3_mutex_enter(db->mutex); (void)sqlite3CodecSetCodecConstant(ctx->readCtx, zRight); (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, ctx->readCtx->codecConst.hmacAlgo); + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, ctx->readCtx->codecConst.hmacAlgo); sqlite3CodecFreeKeyContext(ctx->writeCtx); (void)sqlite3CodecCopyKeyContext(ctx->readCtx, ctx->writeCtx); - sqlite3BtreeSetPageSize(p, DEFAULT_PAGE_SIZE, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); sqlite3_mutex_leave(db->mutex); }else{ sqlite3CodecReturnPragmaResult(parse, "codec_cipher", opensslGetCipherName(ctx->writeCtx->codecConst.cipher)); @@ -239430,25 +260067,84 @@ int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, co } } }else if( sqlite3StrICmp(zLeft, "codec_hmac_algo")==0 ){ + /* + ** Make sure to set the Kdf algorithm after setting the Hmac algorithm, or it will not take effect. + ** This behavior is to ensure backward compatible. + */ if(zRight){ sqlite3_mutex_enter(db->mutex); if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA1)==0 ){ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA1); (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA1); + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA1); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA1); }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA256); (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA256); + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA256); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA256); + }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ + (void)sqlite3CodecSetHmacAlgorithm(ctx->readCtx, CIPHER_HMAC_ALGORITHM_SHA512); + (void)sqlite3CodecSetHmacAlgorithm(ctx->writeCtx, CIPHER_HMAC_ALGORITHM_SHA512); + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA512); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA512); }else{ sqlite3_mutex_leave(db->mutex); return 0; } - sqlite3BtreeSetPageSize(p, DEFAULT_PAGE_SIZE, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); sqlite3_mutex_leave(db->mutex); }else{ if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA1 ){ sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA1); }else if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA256); + }else if( ctx->writeCtx->codecConst.hmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ + sqlite3CodecReturnPragmaResult(parse, "codec_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA512); + } + } + }else if( sqlite3StrICmp(zLeft, "codec_kdf_algo")==0 ){ + if(zRight){ + sqlite3_mutex_enter(db->mutex); + if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA1)==0 ){ + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA1); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA1); + }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA256)==0 ){ + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA256); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA256); + }else if( sqlite3_stricmp(zRight, CIPHER_KDF_ALGORITHM_NAME_SHA512)==0 ){ + (void)sqlite3CodecSetKdfAlgorithm(ctx->readCtx, CIPHER_KDF_ALGORITHM_SHA512); + (void)sqlite3CodecSetKdfAlgorithm(ctx->writeCtx, CIPHER_KDF_ALGORITHM_SHA512); + }else{ + sqlite3_mutex_leave(db->mutex); + return 0; + } + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3_mutex_leave(db->mutex); + }else{ + if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA1 ){ + sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA1); + }else if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA256 ){ + sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA256); + }else if( ctx->writeCtx->codecConst.kdfAlgo==CIPHER_KDF_ALGORITHM_SHA512 ){ + sqlite3CodecReturnPragmaResult(parse, "codec_kdf_algo", CIPHER_KDF_ALGORITHM_NAME_SHA512); + } + } + }else if( sqlite3StrICmp(zLeft, "codec_page_size")==0 ){ + if(zRight){ + sqlite3_mutex_enter(db->mutex); + int rc = sqlite3CodecSetCipherPageSize(ctx, atoi(zRight)); + if (rc != SQLITE_OK){ + sqlite3_mutex_leave(db->mutex); + return 0; + } + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3_mutex_leave(db->mutex); + } else { + char *pageSize = sqlite3_mprintf("%d", ctx->readCtx->codecConst.cipherPageSize); + if (pageSize != NULL) { + sqlite3CodecReturnPragmaResult(parse, "codec_page_size", pageSize); + sqlite3_free(pageSize); } } }else if(sqlite3StrICmp(zLeft, "codec_rekey_hmac_algo") == 0){ @@ -239458,6 +260154,8 @@ int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, co ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA1; }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA256)==0 ){ ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA256; + }else if( sqlite3_stricmp(zRight, CIPHER_HMAC_ALGORITHM_NAME_SHA512)==0 ){ + ctx->writeCtx->codecConst.rekeyHmacAlgo = CIPHER_HMAC_ALGORITHM_SHA512; }else{ sqlite3_mutex_leave(db->mutex); return 0; @@ -239468,6 +260166,8 @@ int sqlite3CodecPragma(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, co sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA1); }else if( ctx->writeCtx->codecConst.rekeyHmacAlgo==CIPHER_HMAC_ALGORITHM_SHA256 ){ sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA256); + }else if( ctx->writeCtx->codecConst.rekeyHmacAlgo==CIPHER_HMAC_ALGORITHM_SHA512 ){ + sqlite3CodecReturnPragmaResult(parse, "codec_rekey_hmac_algo", CIPHER_HMAC_ALGORITHM_NAME_SHA512); } } }else{ @@ -239640,16 +260340,1311 @@ export_finish: } /************** End file hw_codec.c *****************************************/ #endif +#ifdef SQLITE_META_DWR +#define META_DWR_MAX_PAGES 500 +#define META_DWR_MAGIC 0x234A86D9 +#define META_DWR_VERSION 0x00000001 +#define META_DWR_INVALID_ZONE 0x55 +#define META_DWR_HEADER_PAGE_SIZE 4096 +#define META_DWR_HEADER_DEFAULT_PAGE_CNT 8 +typedef struct ScanPages { + u32 pageCnt; + u32 pageBufSize; + u32 maxPageNo; + Pgno *pages; +} ScanPages; + +typedef struct MetaDwrHdr { + u32 magic; + u32 version; + u32 dbSize; + u32 mxFrameInWal; + u32 freeListPageNo; + u32 freeListPageCnt; + u32 schemaCookie; + u32 pageSz; + u32 pageCnt; + u64 dbFileInode; + u32 reserved[12]; + u32 checkSum; + u8 *zones; + Pgno *pages; + u32 pageBufSize; + u8 hdrValid; + u8 checkFileId; + u16 needSync; + i64 lastSyncTime; +} MetaDwrHdr; + +#define META_VERIFIED_HDR_LEN (offsetof(MetaDwrHdr, zones)) +#define META_ZONES_LENGTH (META_DWR_MAX_PAGES * sizeof(u8)) +#define META_PAGE_NO_OFFSET (META_VERIFIED_HDR_LEN + META_ZONES_LENGTH) +#define META_FILE_UPDATE_TIMES_PER_SYNC 100 // sync once for every 100 update +#define META_FILE_SYNC_TIMEOUT_MS 30000 // 30 seconds + +static int MetaDwrHeaderSimpleCheck(Pager *pPager, MetaDwrHdr *hdr) { +#if SQLITE_OS_UNIX + if (hdr->checkFileId) { + unixFile *fd = (unixFile *)pPager->fd; + if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL) { + return SQLITE_INTERNAL; + } + if (fd->pInode->fileId.ino != hdr->dbFileInode) { + sqlite3_log(SQLITE_IOERR_DATA, "Ino mismatch file %llu dwr file %llu", + fd->pInode->fileId.ino, hdr->dbFileInode); + return SQLITE_IOERR_DATA; + } + } +#endif + if (hdr->pageCnt > META_DWR_MAX_PAGES || hdr->version != META_DWR_VERSION || + hdr->magic != META_DWR_MAGIC || hdr->checkSum != META_DWR_MAGIC) { + sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file check wrong pageCnt %u, version %u, magic %u, checkSum %u", + hdr->pageCnt, hdr->version, hdr->magic, hdr->checkSum); + return SQLITE_IOERR_DATA; + } + if (hdr->pageSz != pPager->pageSize) { + sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file check wrong pageSz %u-%u", hdr->pageSz, pPager->pageSize); + return SQLITE_IOERR_DATA; + } + return SQLITE_OK; +} + +static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char *zLeft, const char *zRight) { + Btree *pBt = db->aDb[iDb].pBt; + if (pBt == NULL || zLeft == NULL || sqlite3StrICmp(zLeft, "meta_double_write") != 0) { + return 0; + } + Pager *pPager = pBt->pBt->pPager; + if (pPager == NULL) { + sqlite3_log(SQLITE_WARNING_DUMP, "Invalid pager handle"); + return 1; + } + if (zRight == NULL) { + Vdbe *v = sqlite3GetVdbe(parse); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "meta_double_write", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, pPager->metaFd ? "enabled" : "disabled", 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } else if (strncmp(zRight, "enabled", 7) == 0) { + sqlite3_mutex_enter(db->mutex); + // only support enabled meta double write + int rc = MetaDwrOpenAndCheck(pBt); + if (rc != SQLITE_OK) { + parse->nErr++; + parse->rc = rc; + } + sqlite3_mutex_leave(db->mutex); + } else if (strncmp(zRight, "disabled", 8) == 0) { + sqlite3_mutex_enter(db->mutex); + MetaDwrDisable(pBt); + sqlite3_mutex_leave(db->mutex); + } + return 1; +} + +static int GetBtreePageNo(MemPage *pPage, void *args) { + ScanPages *pInfo = (ScanPages *)args; + // realloc buffer to store pages + u32 pageCnt = pInfo->pageCnt; + if (pageCnt == pInfo->pageBufSize) { + u32 memSz = sizeof(Pgno) * ROUND8(pageCnt + 1); + Pgno *pages = sqlite3Malloc(memSz); + if (pages == NULL) { + sqlite3_log(SQLITE_NOMEM, "GetPages alloc buffer go wrong %u", memSz); + return SQLITE_NOMEM; + } + if (pageCnt != 0) { + memcpy(pages, pInfo->pages, pageCnt * sizeof(Pgno)); + } + sqlite3_free(pInfo->pages); + pInfo->pageBufSize = ROUND8(pageCnt + 1); + pInfo->pages = pages; + } + pInfo->pages[pageCnt] = pPage->pgno; + pInfo->pageCnt++; + if (pInfo->maxPageNo < pPage->pgno) { + pInfo->maxPageNo = pPage->pgno; + } + return 0; +} + +typedef int (*ScanFn)(MemPage *pPage, void *args); +static SQLITE_NOINLINE int ScanOverflowPages( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + CellInfo *pInfo, /* Size information about the cell */ + ScanFn fn, /* Scan pages function */ + void *args) { + BtShared *pBt; + Pgno ovflPgno; + int rc; + int nOvfl; + u32 ovflPageSize; + + if (pCell + pInfo->nSize > pPage->aDataEnd) { + /* Cell extends past end of page */ + return SQLITE_CORRUPT_BKPT; + } + ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; + assert(pBt->usableSize > 4); + ovflPageSize = pBt->usableSize - 4; + nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1) / ovflPageSize; + assert(nOvfl > 0 || + (CORRUPT_DB && (pInfo->nPayload + ovflPageSize) < ovflPageSize)); + while (nOvfl--) { + Pgno iNext = 0; + MemPage *pOvfl = 0; + if (ovflPgno < 2 || ovflPgno > btreePagecount(pBt)) { + return SQLITE_CORRUPT_BKPT; + } + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); + if (rc) + return rc; + if (pOvfl) { + rc = fn(pOvfl, args); + if (rc) { + return rc; + } + sqlite3PagerUnref(pOvfl->pDbPage); + } + ovflPgno = iNext; + } + return SQLITE_OK; +} + +static int ScanBtreePage( + BtShared *pBt, /* The BTree that contains the table */ + Pgno pgno, /* Page number to clear */ + ScanFn fn, /* Scan pages function */ + void *args) { /* Scan pages args */ + MemPage *pPage; + int rc; + unsigned char *pCell; + int i; + int hdr; + CellInfo info; + + assert(sqlite3_mutex_held(pBt->mutex)); + if (pgno > btreePagecount(pBt)) { + return SQLITE_OK; + } + rc = getAndInitPage(pBt, pgno, &pPage, 0); + if (rc) { + return rc; + } + rc = fn(pPage, args); + if (rc) { + goto SCAN_PAGE_OUT; + } + hdr = pPage->hdrOffset; + for (i = pPage->nCell - 1; i >= 0; i--) { + pCell = findCell(pPage, i); + if (!pPage->leaf) { + rc = ScanBtreePage(pBt, get4byte(pCell), fn, args); + if (rc) { + goto SCAN_PAGE_OUT; + } + } + pPage->xParseCell(pPage, pCell, &info); + if (info.nLocal != info.nPayload) { + rc = ScanOverflowPages(pPage, pCell, &info, fn, args); + if (rc) { + goto SCAN_PAGE_OUT; + } + } + } + if (!pPage->leaf) { + rc = ScanBtreePage(pBt, get4byte(&pPage->aData[hdr + 8]), fn, args); + if (rc) { + goto SCAN_PAGE_OUT; + } + } +SCAN_PAGE_OUT: + releasePage(pPage); + return rc; +} + +static inline int ScanMetaPages(Btree *pBt, ScanPages *pages) { + return ScanBtreePage(pBt->pBt, 1, GetBtreePageNo, pages); +} + +static int ReleaseMetaPages(ScanPages *pages) { + sqlite3_free(pages->pages); + pages->pages = NULL; + return SQLITE_OK; +} + +static void InitMetaHeader(MetaDwrHdr *hdr) { + (void)memset(hdr, 0, META_VERIFIED_HDR_LEN); + hdr->magic = META_DWR_MAGIC; + hdr->version = META_DWR_VERSION; + hdr->checkSum = META_DWR_MAGIC; +} + +static void MetaDwrReleaseHdr(MetaDwrHdr *hdr) { + if (!hdr) { + return; + } + sqlite3_free(hdr->zones); + sqlite3_free(hdr); +} + +static int ExpandMetaPageBuf(MetaDwrHdr *hdr, u32 minimalPageCnt, u32 bufHasData) { + if (minimalPageCnt < hdr->pageBufSize && hdr->zones != NULL) { + return SQLITE_OK; + } + int pageBufSz = ROUND8(MAX(hdr->pageCnt, minimalPageCnt)); + u8 *zones = (u8 *)sqlite3Malloc(pageBufSz * (sizeof(u8) + sizeof(Pgno))); + if (zones == NULL) { + return SQLITE_NOMEM_BKPT; + } + Pgno *pgnos = (Pgno *)(zones + pageBufSz); + if (hdr->zones != NULL) { + if (bufHasData && hdr->pageCnt > 0) { + (void)memcpy(zones, hdr->zones, hdr->pageCnt * sizeof(u8)); + (void)memcpy(pgnos, hdr->pages, hdr->pageCnt * sizeof(Pgno)); + } + sqlite3_free(hdr->zones); + } + hdr->pageBufSize = pageBufSz; + hdr->zones = zones; + hdr->pages = pgnos; + return SQLITE_OK; +} + +static MetaDwrHdr *AllocInitMetaHeaderDwr(Pager *pPager) { + MetaDwrHdr *hdr = sqlite3MallocZero(sizeof(MetaDwrHdr)); + if (hdr == NULL) { + return NULL; + } + InitMetaHeader(hdr); + int rc = ExpandMetaPageBuf(hdr, META_DWR_HEADER_DEFAULT_PAGE_CNT, 0); + if (rc != SQLITE_OK) { + MetaDwrReleaseHdr(hdr); + return NULL; + } + hdr->checkFileId = (pPager->pVfs != NULL && sqlite3_stricmp(pPager->pVfs->zName, "unix") == 0); + return hdr; +} + +static void MetaDwrCloseFile(Pager *pPager) { + if (!pPager->metaFd) { + return; + } +#if SQLITE_OS_UNIX + if (pPager->metaMapPage) { + osMunmap(pPager->metaMapPage, META_DWR_HEADER_PAGE_SIZE); + pPager->metaMapPage = NULL; + } +#endif + if (pPager->metaHdr && pPager->metaHdr->needSync > 0) { + (void)sqlite3OsSync(pPager->metaFd, SQLITE_SYNC_NORMAL); + } + sqlite3OsClose(pPager->metaFd); +} + +static void MetaDwrPagerRelease(Pager *pPager) { + MetaDwrCloseFile(pPager); + MetaDwrReleaseHdr(pPager->metaHdr); + pPager->metaHdr = NULL; + if (pPager->metaFd) { + sqlite3_free(pPager->metaFd); + pPager->metaFd = NULL; + } +} + +static inline int ReadFromHdrPage(Pager *pPager, void *data, int amt, i64 offset) { + if (pPager->metaMapPage) { + (void)memcpy(data, (u8 *)pPager->metaMapPage + offset, amt); + return SQLITE_OK; + } + return sqlite3OsRead(pPager->metaFd, data, amt, offset); +} + +static int MetaDwrReadHeader(Pager *pPager, MetaDwrHdr *hdr) { + i64 sz = 0; + int rc = sqlite3OsFileSize(pPager->metaFd, &sz); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "Meta dwr file size go wrong"); + return rc; + } + if (sz <= META_DWR_HEADER_PAGE_SIZE) { + rc = SQLITE_IOERR_DATA; + goto READ_META_OUT; + } + rc = ReadFromHdrPage(pPager, hdr, META_VERIFIED_HDR_LEN, 0); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "Meta dwr file header read wrong"); + goto READ_META_OUT; + } + rc = MetaDwrHeaderSimpleCheck(pPager, hdr); + if (rc != SQLITE_OK) { + goto READ_META_OUT; + } + // avoid realloc buffer if buf can't hold all pages + rc = ExpandMetaPageBuf(hdr, hdr->pageCnt, 0); + if (rc != SQLITE_OK) { + goto READ_META_OUT; + } + int zoneSize = hdr->pageCnt * sizeof(u8); + rc = ReadFromHdrPage(pPager, hdr->zones, zoneSize, META_VERIFIED_HDR_LEN); + if (rc != SQLITE_OK) { + goto READ_META_OUT; + } + rc = ReadFromHdrPage(pPager, hdr->pages, hdr->pageCnt * sizeof(Pgno), META_PAGE_NO_OFFSET); + if (rc != SQLITE_OK) { + goto READ_META_OUT; + } + for (u32 i = 0; i < hdr->pageCnt; i++) { + u8 zoneIdx = hdr->zones[i]; + if (zoneIdx != 0 && zoneIdx != 1 && zoneIdx != META_DWR_INVALID_ZONE) { + sqlite3_log(SQLITE_IOERR_DATA, "Invalid zoneIdx %d", zoneIdx); + rc = SQLITE_IOERR_DATA; + break; + } + } +READ_META_OUT: + if (rc == SQLITE_IOERR_DATA) { + InitMetaHeader(hdr); + rc = SQLITE_OK; + } + return rc; +} + +static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) { + // 1 header page, idx correspond 2 zone pages + return META_DWR_HEADER_PAGE_SIZE + pageSz * (idx * 2 + zone); +} + +static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) { + MetaDwrHdr *hdr = pBt->pPager->metaHdr; + // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie + const u8 *dbHdrInfo = &pBt->pPage1->aData[28]; + hdr->dbSize = sqlite3Get4byte(dbHdrInfo); +#ifndef SQLITE_OMIT_WAL + if (pagerUseWal(pBt->pPager)) { + WalIndexHdr *pWalHdr = &pBt->pPager->pWal->hdr; + if (pWalHdr->isInit) { + hdr->mxFrameInWal = pWalHdr->mxFrame; + hdr->dbSize = pWalHdr->nPage; + } + } else { + hdr->mxFrameInWal = 0; + } +#endif +#if SQLITE_OS_UNIX + if (hdr->checkFileId) { + unixFile *fd = (unixFile *)pBt->pPager->fd; + if (fd == NULL || fd->pInode == NULL) { + sqlite3_log(SQLITE_WARNING_DUMP, "update meta header invalid fd"); + hdr->hdrValid = 0; + return; + } + hdr->dbFileInode = fd->pInode->fileId.ino; + } +#endif + hdr->freeListPageNo = sqlite3Get4byte(dbHdrInfo + 4); + hdr->freeListPageCnt = sqlite3Get4byte(dbHdrInfo + 8); + hdr->schemaCookie = sqlite3Get4byte(dbHdrInfo + 12); + hdr->hdrValid = 1; +} + +static inline int WriteToHdrPage(Pager *pPager, const void *data, int amt, i64 offset) { + if (pPager->metaMapPage) { + (void)memcpy((u8 *)pPager->metaMapPage + offset, data, amt); + return SQLITE_OK; + } + return sqlite3OsWrite(pPager->metaFd, data, amt, offset); +} + +static int MetaDwrWriteHeader(Pager *pPager, MetaDwrHdr *hdr) { + if (pPager->metaChanged == 0 || hdr == NULL || hdr->pageCnt == 0 || hdr->hdrValid == 0) { + return SQLITE_OK; + } + hdr->hdrValid = 0; + hdr->pageSz = pPager->pageSize; + hdr->dbSize = pPager->dbSize; + int rc = WriteToHdrPage(pPager, hdr, META_VERIFIED_HDR_LEN, 0); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "update meta header write hdr %u wrong", META_VERIFIED_HDR_LEN); + return rc; + } + if (hdr->zones) { + int zoneSize = hdr->pageCnt * sizeof(u8); + rc = WriteToHdrPage(pPager, hdr->zones, zoneSize, META_VERIFIED_HDR_LEN); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "update meta header write zonebuf %u wrong", zoneSize); + return rc; + } + } + if (hdr->pages) { + int pageBufSz = hdr->pageCnt * sizeof(Pgno); + rc = WriteToHdrPage(pPager, hdr->pages, pageBufSz, META_PAGE_NO_OFFSET); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "update meta header write pagebuf %u wrong", pageBufSz); + } + } + if (rc == SQLITE_OK) { + u64 size = CaculateMetaDwrWriteOffset((int)pPager->pageSize, hdr->pageCnt, 0); + rc = sqlite3OsTruncate(pPager->metaFd, size); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "update meta header truncate filesz %lu wrong", size); + } + i64 timeMs = 0; + sqlite3OsCurrentTimeInt64(pPager->pVfs, &timeMs); + if ((timeMs - hdr->lastSyncTime) > META_FILE_SYNC_TIMEOUT_MS || + hdr->needSync >= META_FILE_UPDATE_TIMES_PER_SYNC) { + rc = sqlite3OsSync(pPager->metaFd, SQLITE_SYNC_NORMAL); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "update meta header sync filesz %lu wrong", size); + } + hdr->lastSyncTime = timeMs; + hdr->needSync = 0; + } else { + hdr->needSync++; + } + } + return rc; +} + +static int MetaDwrFindPageIdx(MetaDwrHdr *hdr, u32 pgno, u32 *idx) { + for (u32 i = 0; i < hdr->pageCnt && i < META_DWR_MAX_PAGES; i++) { + if (pgno == hdr->pages[i]) { + *idx = i; + return 1; + } + } + return 0; +} + +static int MetaDwrWriteOnePage(Btree *pBt, PgHdr *pPage, MetaDwrHdr *hdr, u8 curZones, u32 idx) { + int rc = SQLITE_OK; + u8 pageExpand = 0; + if (hdr->pageCnt <= idx) { + rc = ExpandMetaPageBuf(hdr, idx + 1, 1); + if (rc != SQLITE_OK) { + return rc; + } + pageExpand = 1; + } + Pager *pPager = pBt->pBt->pPager; + // asume zone 0 or 1 + u8 zone = 1 - curZones; + int pageSz = sqlite3BtreeGetPageSize(pBt); + u64 ofs = CaculateMetaDwrWriteOffset(pageSz, idx, zone); + void *pData; +#if defined(SQLITE_HAS_CODEC) + if ((pData = sqlite3PagerCodec(pPage)) == 0) + return SQLITE_NOMEM; +#else + pData = pPage->pData; +#endif + rc = sqlite3OsWrite(pPager->metaFd, pData, pageSz, ofs); + if (rc != SQLITE_OK) { + return rc; + } + hdr->zones[idx] = zone; + hdr->pages[idx] = pPage->pgno; + if (pageExpand) { + hdr->pageCnt++; + } + return SQLITE_OK; +} + +static int MetaDwrRestoreAllPages(Btree *pBt, const ScanPages *metaPages, MetaDwrHdr *hdr) { + u32 i = 0; + PgHdr *p = NULL; + int rc = SQLITE_OK; + for (i = 0; i < metaPages->pageCnt && i < META_DWR_MAX_PAGES; i++) { + Pgno pgno = metaPages->pages[i]; + if (pgno > btreePagecount(pBt->pBt)) { + sqlite3_log(SQLITE_WARNING_DUMP, "pageno %d overlimit", pgno); + return SQLITE_CORRUPT_BKPT; + } + rc = sqlite3PagerGet(pBt->pBt->pPager, pgno, &p, 0); + if (rc) { + return rc; + } + rc = MetaDwrWriteOnePage(pBt, p, hdr, 1, i); + sqlite3PagerUnref(p); + if (rc) { + return rc; + } + } + MetaDwrUpdateHeaderDbInfo(pBt->pBt); + return rc; +} + +static inline const char *GetMetaFilePath(Pager *pPager) +{ + return pPager->metaFd == NULL ? NULL : ((const char *)pPager->metaFd + ROUND8(pPager->pVfs->szOsFile)); +} + +static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { + if (pPager->metaFd || pPager->zFilename == NULL) { + return SQLITE_OK; + } + sqlite3BeginBenignMalloc(); + sqlite3_vfs *pVfs = pPager->pVfs; + int size = strlen(pPager->zFilename) + sizeof("-dwr"); + int szOsFile = ROUND8(pVfs->szOsFile); + sqlite3_file *metaFd = (sqlite3_file *)sqlite3MallocZero(szOsFile + size); + char *metaPath = (char *)metaFd + szOsFile; + if (metaFd == NULL) { + sqlite3EndBenignMalloc(); + sqlite3_log(SQLITE_NOMEM_BKPT, "sqlite alloc memsize %d go wrong", szOsFile + size); + return SQLITE_NOMEM_BKPT; + } + sqlite3_snprintf(size, metaPath, "%s-dwr", pPager->zFilename); + int exists = 0; + int rc = sqlite3OsAccess(pVfs, metaPath, SQLITE_ACCESS_EXISTS, &exists); + if (rc != SQLITE_OK) { + goto INIT_META_OUT; + } + if (!exists && !openCreate) { + sqlite3_free(metaFd); + goto INIT_META_OUT; + } + u32 flags = (SQLITE_OPEN_READWRITE | SQLITE_OPEN_SUPER_JOURNAL); + if (openCreate) { + flags |= SQLITE_OPEN_CREATE; + } + rc = sqlite3OsOpen(pVfs, metaPath, metaFd, (int)flags, 0); + if (rc != SQLITE_OK) { + goto INIT_META_OUT; + } +#if SQLITE_OS_UNIX + if (pPager->metaMapPage == NULL) { + sqlite3_int64 sz = META_DWR_HEADER_PAGE_SIZE; + sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_CHUNK_SIZE, &sz); + sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz); + void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, + ((unixFile *)metaFd)->h, 0); + if (page != MAP_FAILED) { + pPager->metaMapPage = page; + } + } +#endif + pPager->metaFd = metaFd; +INIT_META_OUT: + sqlite3EndBenignMalloc(); + if (rc != SQLITE_OK && metaFd != NULL) { + sqlite3_free(metaFd); + } + return rc; +} + +void MetaDwrCheckVacuum(BtShared *pBt) { + if (!pBt || !pBt->pPager->metaFd) { + return; + } + if (pBt->nPage < pBt->maxMetaPage) { + pBt->pPager->metaChanged = META_SCHEMA_CHANGED; + } +} + +static inline u8 LocalMetaHdrValid(Pager *pPager) { + return pPager->metaMapPage != NULL && memcmp(pPager->metaMapPage, pPager->metaHdr, + META_VERIFIED_HDR_LEN) == 0; +} + +static int MetaDwrLoadHdr(Pager *pPager) { + if (!pPager->metaHdr) { + pPager->metaHdr = AllocInitMetaHeaderDwr(pPager); + if (pPager->metaHdr == NULL) { + return SQLITE_NOMEM_BKPT; + } + } + if (LocalMetaHdrValid(pPager)) { + return SQLITE_OK; + } + return MetaDwrReadHeader(pPager, pPager->metaHdr); +} + +static int MetaDwrLoadAndCheckMetaFile(BtShared *pBt, u8 reportErr) { + int rc = MetaDwrLoadHdr(pBt->pPager); + if (rc != SQLITE_OK) { + return rc; + } + MetaDwrHdr *hdr = pBt->pPager->metaHdr; + if (hdr->pageCnt == 0) { + return reportErr ? SQLITE_IOERR_DATA : SQLITE_OK; + } + // 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie + u8 *dbHdrInfo = &pBt->pPage1->aData[28]; + if (hdr->dbSize != pBt->pPager->dbSize || hdr->dbSize != sqlite3Get4byte(dbHdrInfo) || + hdr->freeListPageNo != sqlite3Get4byte(dbHdrInfo + 4) || + hdr->freeListPageCnt != sqlite3Get4byte(dbHdrInfo + 8) || + hdr->schemaCookie != sqlite3Get4byte(dbHdrInfo + 12)) { + sqlite3_log(SQLITE_IOERR_DATA, "Meta dwr file expect %u-%u-%u-%u-%u but gotton %u-%u-%u-%u-%u", + pBt->pPager->dbSize, hdr->dbSize, sqlite3Get4byte(dbHdrInfo), sqlite3Get4byte(dbHdrInfo + 4), + sqlite3Get4byte(dbHdrInfo + 8), sqlite3Get4byte(dbHdrInfo + 12), hdr->dbSize, hdr->freeListPageNo, + hdr->freeListPageCnt, hdr->schemaCookie); + // reinit + InitMetaHeader(hdr); + if (reportErr) { + return SQLITE_IOERR_DATA; + } + } + return SQLITE_OK; +} + +static int MetaDwrReadOnePage(Pager *pPager, MetaDwrHdr *hdr, int idx, u8 *pData) { + u64 ofs = CaculateMetaDwrWriteOffset(pPager->pageSize, idx, hdr->zones[idx]); + int rc = sqlite3OsRead(pPager->metaFd, pData, pPager->pageSize, ofs); + CODEC1(pPager, pData, hdr->pages[idx], 3, rc = SQLITE_NOMEM_BKPT); + return rc; +} + +static int MetaDwrRecoverHeadPage( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **pDbPage, + int flag) { + if (pPager->metaFd == NULL) { + return pgno == 1 ? SQLITE_NOTADB : SQLITE_CORRUPT; + } + sqlite3_pcache_page *pCachePage = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if (pCachePage == NULL) { + sqlite3_log(SQLITE_NOMEM_BKPT, "Get meta page wrong %d", pgno); + return SQLITE_NOMEM_BKPT; + } + DbPage *pPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pCachePage); + pPage->pPager = pPager; + assert(pCachePage != 0); + int rc = MetaDwrLoadHdr(pPager); + if (rc != SQLITE_OK) { + goto RELEASE_OUT; + } + MetaDwrHdr *hdr = pPager->metaHdr; + u8 walChecked = 0; +#ifndef SQLITE_OMIT_WAL + if (pagerUseWal(pPager)) { + WalIndexHdr *pWalHdr = &pPager->pWal->hdr; + if (pWalHdr->isInit && pWalHdr->mxFrame != 0) { + if (hdr->mxFrameInWal != pWalHdr->mxFrame || hdr->dbSize != pWalHdr->nPage) { + rc = SQLITE_NOTADB; + sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr wal hdr expect %u-%u but gotten %u-%u", + hdr->mxFrameInWal, hdr->dbSize, pWalHdr->mxFrame, pWalHdr->nPage); + goto RELEASE_OUT; + } else { + walChecked = 1; + } + } + } + if (walChecked == 0) { + i64 size = 0; + rc = sqlite3OsFileSize(pPager->fd, &size); + if (rc != SQLITE_OK) { + rc = SQLITE_NOTADB; + sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr get db file size go wrong"); + goto RELEASE_OUT; + } + i64 expectSz = (i64)hdr->dbSize * (i64)hdr->pageSz; + if (size != expectSz) { + rc = SQLITE_NOTADB; + sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr expect file size %lu but gotten size %llu", + expectSz, size); + goto RELEASE_OUT; + } + } +#endif + rc = SQLITE_NOTADB; + for (u32 i = 0; i < hdr->pageCnt; i++) { + if (hdr->pages[i] != pgno) { + continue; + } + rc = MetaDwrReadOnePage(pPager, hdr, i, sqlite3PagerGetData(pPage)); + if (rc == SQLITE_OK) { + *pDbPage = pPage; + if (pPage->pgno == 1) { + memcpy(&pPager->dbFileVers, &((u8 *)pPage->pData)[24], sizeof(pPager->dbFileVers)); + } + pager_set_pagehash(pPage); + } + break; + } +RELEASE_OUT: + if (rc != SQLITE_OK && pPage != NULL) { + sqlite3PcacheDrop(pPage); + } + return rc; +} + +static int MetaDwrRestoreChangedPages(Btree *pBt) { + Pager *pPager = pBt->pBt->pPager; + MetaDwrHdr *hdr = pPager->metaHdr; + u8 *zones = sqlite3MallocZero(hdr->pageBufSize * sizeof(u8)); + if (zones == NULL) { + sqlite3_log(SQLITE_NOMEM_BKPT, "Alloc zones buffer size %u go wrong", hdr->pageBufSize * sizeof(u8)); + return SQLITE_NOMEM_BKPT; + } + if (hdr->pageCnt > 0) { + memcpy(zones, hdr->zones, hdr->pageCnt * sizeof(u8)); + } + u32 idx = 0; + PgHdr *p = 0; + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + int rc = SQLITE_OK; + for (p = pList; p; p = p->pDirty) { + if (MetaDwrFindPageIdx(hdr, p->pgno, &idx) == 0) { + continue; + } + rc = MetaDwrWriteOnePage(pBt, p, hdr, zones[idx], idx); + if (rc != SQLITE_OK) { + break; + } + } + if (rc == SQLITE_OK) { + MetaDwrUpdateHeaderDbInfo(pBt->pBt); + } + sqlite3_free(zones); + return rc; +} + +static int MetaDwrUpdateMetaPages(Btree *pBt) { + Pager *pPager = pBt->pBt->pPager; + if (!pPager || !pPager->metaFd || pPager->memDb || pPager->readOnly || pBt->pBt->pPage1 == NULL) { + return SQLITE_OK; + } + if (pPager->metaChanged == 0) { + if ((pBt->pBt->pPage1->pDbPage->flags & PGHDR_DIRTY) == 0) { + return SQLITE_OK; + } + pPager->metaChanged = META_HEADER_CHANGED; + } + sqlite3BeginBenignMalloc(); + int rc = MetaDwrLoadHdr(pPager); + if (rc != SQLITE_OK) { + goto UPDATE_OUT; + } + // only update header page + if (pPager->metaChanged == META_HEADER_CHANGED) { + rc = MetaDwrRestoreChangedPages(pBt); + goto UPDATE_OUT; + } + // update schema pages + ScanPages metaPages = {0}; + rc = ScanMetaPages(pBt, &metaPages); + if (rc != SQLITE_OK) { + goto UPDATE_OUT; + } + MetaDwrHdr *hdr = pPager->metaHdr; + // rewrite + if (hdr->pageCnt == 0 || hdr->pageCnt != metaPages.pageCnt || pBt->pBt->nPage > pBt->pBt->maxMetaPage || + memcmp(hdr->pages, metaPages.pages, hdr->pageCnt * sizeof(Pgno))) { + // if page numbers unorderred, restore all pages + rc = MetaDwrRestoreAllPages(pBt, &metaPages, hdr); + } else { + rc = MetaDwrRestoreChangedPages(pBt); + } + if (rc == SQLITE_OK) { + pBt->pBt->maxMetaPage = metaPages.maxPageNo; + } + ReleaseMetaPages(&metaPages); +UPDATE_OUT: + sqlite3EndBenignMalloc(); + return rc; +} + +static int MetaDwrRecoverSinglePage(Btree *pBt, int pgno, u8 *pData) { + if (pgno < 1 || pBt == NULL) { + return SQLITE_CORRUPT_BKPT; + } + Pager *pPager = sqlite3BtreePager(pBt); + DbPage *pDbPage = NULL; + int rc = sqlite3PagerGet(pPager, pgno, &pDbPage, 0); + if (rc) { + return rc; + } + if ((rc = sqlite3PagerWrite(pDbPage)) == SQLITE_OK) { + memcpy(sqlite3PagerGetData(pDbPage), pData, pPager->pageSize); + } else { + sqlite3_log(rc, "Dwr recoverwrite meta page %d failed", pgno); + } + sqlite3PagerUnref(pDbPage); + return rc; +} + +static int MetaDwrCheckMeta(Btree *pBt) { + int nErr = 0; + Pgno aRoot[2] = {0, 1}; // quick check and only check root btree + sqlite3_value aMem[2] = {0}; + char *errStr = NULL; + int rc = sqlite3BtreeIntegrityCheck(pBt->db, pBt, &aRoot[0], &aMem[0], 2, SQLITE_INTEGRITY_CHECK_ERROR_MAX, + &nErr, &errStr); + if (nErr == 0) { + assert(errStr == 0); + return SQLITE_OK; + } + if (rc != SQLITE_OK) { + sqlite3_free(errStr); + return rc; + } + sqlite3_log(SQLITE_WARNING_DUMP, "Integrity check %s", errStr); + sqlite3_free(errStr); + return SQLITE_CORRUPT; +} + +static int MetaDwrBeginTrans(Btree *pBt, int wrflag) { + pBt->pBt->btsFlags &= ~BTS_READ_ONLY; + Pager *pPager = pBt->pBt->pPager; + void *xGetMethod = pPager->xGet; + pPager->xGetMethod = MetaDwrRecoverHeadPage; + pPager->xGet = MetaDwrRecoverHeadPage; + int rc = sqlite3BtreeBeginTrans(pBt, wrflag, 0); + pPager->xGet = xGetMethod; + pPager->xGetMethod = 0; + if (rc == SQLITE_OK) { + sqlite3PagerWrite(pBt->pBt->pPage1->pDbPage); + sqlite3_log(rc, "sqlite fix meta header"); + } + return rc; +} + +static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion) +{ + Pager *pPager = pBt->pBt->pPager; + assert(sqlite3_mutex_held(pBt->pBt->mutex)); + if (!pPager->metaFd || pBt->pBt->metaRecoverStatus || pPager->readOnly || pPager->memDb) { + return SQLITE_NOTADB; + } + int rc = MetaDwrLoadHdr(pPager); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "MetaDwr load header failed"); + return rc; + } + pBt->pBt->metaRecoverStatus = META_IN_RECOVERY; + rc = MetaDwrBeginTrans(pBt, 2); + if (rc != SQLITE_OK) { + return rc; + } + void *pData = NULL; + pPager->metaChanged = META_HEADER_CHANGED; + MetaDwrHdr *hdr = pPager->metaHdr; + sqlite3_log(SQLITE_WARNING_DUMP, "sqlite meta recover %u frames", hdr->pageCnt); + int szPage = sqlite3BtreeGetPageSize(pBt); + pData = sqlite3Malloc(szPage); + if (pData == NULL) { + rc = SQLITE_NOMEM; + sqlite3_log(rc, "Dwr malloc mem size %d failed", szPage); + goto DWR_RECOVER_OUT; + } + for (u32 i = 0; i < hdr->pageCnt; i++) { + rc = MetaDwrReadOnePage(pPager, hdr, i, pData); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "Dwr read %d meta page failed ", i); + break; + } + rc = MetaDwrRecoverSinglePage(pBt, hdr->pages[i], pData); + if (rc != SQLITE_OK) { + sqlite3_log(rc, "Dwr recover %d meta page failed ", i); + break; + } + } +DWR_RECOVER_OUT: + /* Close the transaction, if one was opened. */ + if (rc == SQLITE_OK) { + sqlite3BtreeCommit(pBt); + } else { + (void)sqlite3BtreeRollback(pBt, SQLITE_ABORT_ROLLBACK, 0); + } + if (rc == SQLITE_OK) { + rc = sqlite3BtreeBeginTrans(pBt, wrflag, pSchemaVersion); + } + if (rc == SQLITE_OK) { + pBt->pBt->metaRecoverStatus = META_RECOVER_SUCCESS; + } + if (pData) { + sqlite3_free(pData); + } + return rc; +} + +static int Sqlite3MetaDwrCheckRestore(Btree *pBt) { + Pager *pPager = pBt->pBt->pPager; + int rc = MetaDwrOpenFile(pPager, 1); + if (rc != SQLITE_OK) { + return rc; + } + ScanPages metaPages = {0}; + rc = ScanMetaPages(pBt, &metaPages); + if (rc != SQLITE_OK || metaPages.pageCnt == 0) { + goto CHK_RESTORE_OUT; + } + rc = MetaDwrLoadAndCheckMetaFile(pBt->pBt, 0); + if (rc != SQLITE_OK) { + goto CHK_RESTORE_OUT; + } + MetaDwrHdr *hdr = pPager->metaHdr; + if (hdr->pageCnt == 0 || hdr->pageCnt != metaPages.pageCnt || + memcmp(hdr->pages, metaPages.pages, hdr->pageCnt * sizeof(Pgno))) { + sqlite3_log(SQLITE_WARNING_DUMP, "sqlite meta restore all"); + rc = MetaDwrRestoreAllPages(pBt, &metaPages, hdr); + if (rc == SQLITE_OK) { + pPager->metaChanged = META_SCHEMA_CHANGED; + rc = MetaDwrWriteHeader(pPager, hdr); + pPager->metaChanged = 0; + } + } + if (rc == SQLITE_OK) { + pBt->pBt->maxMetaPage = metaPages.maxPageNo; + } +CHK_RESTORE_OUT: + ReleaseMetaPages(&metaPages); + return rc; +} + +static inline u8 IsConnectionValidForCheck(Pager *pPager) +{ +#if SQLITE_OS_UNIX + unixFile *fd = (unixFile *)pPager->fd; + // unix and only one connection exist + if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL || + sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0 || fd->pInode->nRef != 1) { + return 0; + } + return 1; +#else + return 0; +#endif +} + +static int MetaDwrOpenAndCheck(Btree *pBt) +{ + Pager *pPager = pBt->pBt->pPager; + if (pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { + return SQLITE_OK; + } +#ifdef SQLITE_HAS_CODEC + // not support codec right now + if (pPager->xCodec) { + return SQLITE_OK; + } +#endif + sqlite3BtreeEnter(pBt); + int rc = SQLITE_OK; + int openedTransaction = 0; + int tnxState = sqlite3BtreeTxnState(pBt); + if (tnxState == SQLITE_TXN_NONE) { + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if (rc != SQLITE_OK) { + goto DWR_OPEN_OUT; + } + openedTransaction = 1; + } + rc = MetaDwrCheckMeta(pBt); + if (rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB) { + // keep txn status after recover + rc = MetaDwrRecoverAndBeginTran(pBt, tnxState == SQLITE_TXN_WRITE ? 1 : 0, 0); + goto DWR_OPEN_OUT; + } + rc = Sqlite3MetaDwrCheckRestore(pBt); +DWR_OPEN_OUT: + if (rc == SQLITE_OK && pBt->pBt->metaRecoverStatus == META_RECOVER_SUCCESS) { + rc = MetaDwrCheckMeta(pBt); + if (rc == SQLITE_OK) { + rc = SQLITE_META_RECOVERED; + } + } + /* Close the transaction, if one was opened. */ + if (openedTransaction) { + sqlite3BtreeCommit(pBt); + } + sqlite3BtreeLeave(pBt); + return rc; +} + +static void MetaDwrDisable(Btree *pBt) +{ + Pager *pPager = pBt->pBt->pPager; + if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { + return; + } +#ifdef SQLITE_HAS_CODEC + // not support codec right now + if (pPager->xCodec) { + return; + } +#endif + sqlite3BtreeEnter(pBt); + MetaDwrCloseFile(pPager); + MetaDwrReleaseHdr(pPager->metaHdr); + pPager->metaHdr = NULL; + const char *metaPath = GetMetaFilePath(pPager); + if (metaPath != NULL) { + // (void)osUnlink(metaPath); + } + if (pPager->metaFd) { + sqlite3_free(pPager->metaFd); + pPager->metaFd = NULL; + } + sqlite3BtreeLeave(pBt); +} +#endif +#if SQLITE_OS_UNIX +#include +#include +static inline int OsGetTid(void) +{ +#if defined(__linux__) + return (int)syscall(__NR_gettid); +#elif defined(__APPLE__) + return (int)syscall(SYS_thread_selfid); +#else + return 0; +#endif +} + +static void ResetLockStatus(void) +{ + (void)memset(&g_lockStatus, 0, sizeof(g_lockStatus)); + g_lockStatus.curTid = OsGetTid(); +} +/* +** Record lock info, correspond wal aLock buf, 1 aLock: 1 +*/ +static inline void TryRecordTid(int *tidBuf, int ofs, int lockLen) +{ + int lockOfs = ofs + lockLen; + for (int i = ofs; i < lockOfs; i++) { + if (tidBuf[i] == 0) { + tidBuf[i] = g_lockStatus.curTid; + } + } +} +/* +** Clear locks info. +*/ +static inline void TryClearTid(int *tidBuf, int ofs, int lockLen) +{ + int lockOfs = ofs + lockLen; + for (int i = ofs; i < lockOfs; i++) { + if (tidBuf[i] == g_lockStatus.curTid) { + tidBuf[i] = 0; + } + } +} + +static inline void MarkLockBusy(u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess) +{ + g_lockStatus.busyLockIdx = lockIdx; + g_lockStatus.busyLockType = lockType; + g_lockStatus.lockByProcess = lockByProcess; + g_lockStatus.lockLen = lockLen; +} + +static void MarkLockStatus(u32 lockIdx, u32 lockLen, u8 lockType) +{ + if (lockLen == 0 || (lockIdx + lockLen) > MAX_LOCK_NUM) { + sqlite3_log(SQLITE_ERROR, "Unexpect lock index %u lockLen %d!", lockIdx, lockLen); + return; + } + // only busy error code need record + if (g_lockStatus.lockLen != 0 && lockIdx == g_lockStatus.busyLockIdx) { + g_lockStatus.busyLockIdx = 0; + g_lockStatus.busyLockType = NO_LOCK; + g_lockStatus.lockLen = 0; + } + if (lockLen == 1) { + g_lockStatus.lockStatus[lockIdx] = lockType; + } else { + size_t len = sizeof(u8) * lockLen; + (void)memset(&g_lockStatus.lockStatus[lockIdx], lockType, len); + } +} + +static inline void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockType, u8 lockByProcess) +{ + if (rc == SQLITE_OK) { + MarkLockStatus(lockIdx, lockLen, lockType); + } else if (rc == SQLITE_BUSY) { + MarkLockBusy(lockIdx, lockLen, lockType, lockByProcess); + } +} + +static inline const char *TrxLockName(int eLock) +{ + return eLock == NO_LOCK ? "NO_LOCK" : + eLock == RESERVED_LOCK ? "RESERVED" : + eLock == EXCLUSIVE_LOCK ? "EXCLUSIVE" : + eLock == SHARED_LOCK ? "SHARED" : + eLock == PENDING_LOCK ? "PENDING": + eLock == UNKNOWN_LOCK ? "UNKNOWN" : "UNKNOWN_LOCK"; +} + +static inline const char *IdxToLockName(u32 idx) +{ + const char *lockName[MAX_LOCK_NUM] = {"write", "ckpt", "recover", "read0", + "read1", "read2", "read3", "read4", "wal_dms", "trxLock"}; + return (idx < MAX_LOCK_NUM) ? lockName[idx] : "errLock"; +} + +static void DumpHandleLock(char *dumpBuf, int dumpBufLen) +{ + char *tmp = dumpBuf; + u8 *lockStatus = g_lockStatus.lockStatus; + int availLen = dumpBufLen - 1; + dumpBuf[availLen] = '\0'; + for (int i = 0; i < MAX_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { + if (lockStatus[i] != NO_LOCK) { + tmp[0] = '\0'; + sqlite3_snprintf(availLen, tmp, "<%s, %s>", IdxToLockName((u32)i), TrxLockName(lockStatus[i])); + int len = strlen(tmp); + tmp += len; + availLen -= len; + } + } + sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]BusyLine:%d, idx:%d, type:%d, fileLock:%d, len:%d, handleLocks:%s", + g_lockStatus.busyLine, g_lockStatus.busyLockIdx, g_lockStatus.busyLockType, g_lockStatus.lockByProcess, + g_lockStatus.lockLen, tmp != dumpBuf ? dumpBuf : "none"); +} + +static inline const char *FlockToName(int l_type) +{ + return l_type == F_RDLCK ? "F_RDLCK" : + l_type == F_WRLCK ? "F_WRLCK" : + l_type == F_UNLCK ? "F_UNLCK" : "F_UNKNOWN"; +} + +static int DumpProcessLocks(int fd, struct flock *lock, const char *lockName, char *dumpBuf, int bufLen) +{ + dumpBuf[0] = '\0'; + if (osFcntl(fd, F_GETLK, lock) != SQLITE_OK) { + sqlite3_log(SQLITE_ERROR, "[SQLite]Get wal file lock ofs %u failed, errno: %d", lock->l_start, errno); + return 0; + } + if (lock->l_type != F_UNLCK) { + sqlite3_snprintf(bufLen, dumpBuf, "<%s, pid:%u, %s>", lockName, lock->l_pid, FlockToName(lock->l_type)); + return strlen(dumpBuf); + } + return 0; +} + +static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) +{ + unixInodeInfo *inode = file->pInode; + if (inode == NULL) { + sqlite3_log(SQLITE_ERROR, "[SQLite]Inode is null!"); + return; + } + sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]acqLock:%s, dbRef:%d, lockCnt:%d, curLock:%s, processLock:%d", + TrxLockName(file->eFileLock), inode->nRef, inode->nLock, TrxLockName(inode->eFileLock), inode->bProcessLock); + const char *lockName[DB_LOCK_NUM] = {"pending", "reserved", "shared_first"}; + char *tmp = dumpBuf; + int availLen = dumpBufLen - 1; + dumpBuf[availLen] = '\0'; + for (int i = 0; i < DB_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { + off_t ofs = i + PENDING_BYTE; + off_t lockLen = (ofs == SHARED_FIRST) ? SHARED_SIZE : 1; + struct flock lock = {.l_type = F_WRLCK, .l_start = ofs, .l_len = lockLen, .l_whence = SEEK_SET}; + int lockBufLen = DumpProcessLocks(file->h, &lock, lockName[i], tmp, availLen); + tmp += lockBufLen; + availLen -= lockBufLen; + } + if (tmp != dumpBuf) { + sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]Trx locks: %s", dumpBuf); + } +} + +static void DumpWalLocks(unixFile *file, u8 walEnabled, char *dumpBuf, int dumpBufLen) +{ + if (!walEnabled || file->pShm == NULL || file->pShm->pShmNode == NULL) { + sqlite3_log(SQLITE_ERROR, "[SQLite]Wal mode disabled!"); + return; + } + unixShmNode *pShmNode = file->pShm->pShmNode; + char *tmp = dumpBuf; + int availLen = dumpBufLen - 1; + dumpBuf[availLen] = '\0'; + for (int i = 0; i < WAL_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { + if (i < SQLITE_SHM_NLOCK && pShmNode->aLock[i]) { + tmp[0] = '\0'; + sqlite3_snprintf(availLen, tmp, "<%s, %d, tid:%d>", IdxToLockName((u32)i), pShmNode->aLock[i], + pShmNode->aLockTid[i]); + int strLen = strlen(tmp); + tmp += strLen; + availLen -= strLen; + } + off_t ofs = i + WALINDEX_LOCK_OFFSET; + struct flock lock = {.l_type = F_WRLCK, .l_start = ofs, .l_len = 1, .l_whence = SEEK_SET}; + int bufLen = DumpProcessLocks(pShmNode->hShm, &lock, IdxToLockName((u32)i), tmp, availLen); + tmp += bufLen; + availLen -= bufLen; + } + if (tmp != dumpBuf) { + sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]Wal locks: %s", dumpBuf); + } +} + +static void DumpLocksInfo(unixFile *file, int walEnabled) +{ + char *dumpBuf = sqlite3Malloc(DUMP_BUF_MAX_LEN); + if (dumpBuf == NULL) { + sqlite3_log(SQLITE_ERROR, "[SQLite]Can't alloc bufferSz %d for dump!", DUMP_BUF_MAX_LEN); + return; + } + DumpHandleLock(dumpBuf, DUMP_BUF_MAX_LEN); + DumpTrxProcessLocks(file, dumpBuf, DUMP_BUF_MAX_LEN); + DumpWalLocks(file, walEnabled, dumpBuf, DUMP_BUF_MAX_LEN); + sqlite3_free(dumpBuf); +} + +#ifndef SQLITE_OMIT_WAL +static void DumpLocksByWal(Wal *pWal) +{ + if (pWal == NULL) { + sqlite3_log(SQLITE_ERROR, "Wal ptr is NULL!"); + return; + } + if (pWal->pVfs == NULL || sqlite3_stricmp(pWal->pVfs->zName, "unix") != 0) { + return; + } + DumpLocksInfo((unixFile *)(pWal->pDbFd), 1); +} +#endif /* #ifndef SQLITE_OMIT_WAL */ + +static void DumpLocksByPager(Pager *pPager) +{ + if (pPager == NULL) { + sqlite3_log(SQLITE_ERROR, "Pager ptr is NULL!"); + return; + } + if (pPager->pVfs == NULL || sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0) { + return; + } +#ifndef SQLITE_OMIT_WAL + DumpLocksInfo((unixFile *)(pPager->fd), pPager->pWal != NULL); +#else /* #ifndef SQLITE_OMIT_WAL */ + DumpLocksInfo((unixFile *)(pPager->fd), 0); +#endif /* #ifndef SQLITE_OMIT_WAL */ +} +#endif /* SQLITE_OS_UNIX */ // hw export the symbols #ifdef SQLITE_EXPORT_SYMBOLS -#if defined(__GNUC__) -# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) -#elif defined(_MSC_VER) -# define EXPORT_SYMBOLS __declspec(dllexport) -#else -# define EXPORT_SYMBOLS +#ifndef SQLITE_CKSUMVFS_STATIC +int sqlite3_register_cksumvfs(const char *NotUsed){ + return SQLITE_MISUSE; +} +int sqlite3_unregister_cksumvfs(void){ + return SQLITE_MISUSE; +} + +struct sqlite3_api_routines_cksumvfs { + int (*register_cksumvfs)(const char *); + int (*unregister_cksumvfs)(); +}; +typedef struct sqlite3_api_routines_cksumvfs sqlite3_api_routines_cksumvfs; +static const sqlite3_api_routines_cksumvfs sqlite3CksumvfsApis = { + sqlite3_register_cksumvfs, + sqlite3_unregister_cksumvfs +}; + +EXPORT_SYMBOLS const sqlite3_api_routines_cksumvfs *sqlite3_export_cksumvfs_symbols = &sqlite3CksumvfsApis; #endif + struct sqlite3_api_routines_hw { int (*initialize)(); int (*config)(int,...); @@ -239673,9 +261668,10 @@ static const sqlite3_api_routines_hw sqlite3HwApis = { 0, 0, 0 -#endif +#endif /* SQLITE_HAS_CODEC */ }; EXPORT_SYMBOLS const sqlite3_api_routines *sqlite3_export_symbols = &sqlite3Apis; EXPORT_SYMBOLS const sqlite3_api_routines_hw *sqlite3_export_hw_symbols = &sqlite3HwApis; -#endif \ No newline at end of file +/************** End hw export the symbols *****************************************/ +#endif /* SQLITE_EXPORT_SYMBOLS */ diff --git a/mock/sqlite/src/sqlite3icu.c b/mock/sqlite/src/sqlite3icu.c new file mode 100644 index 00000000..75aa78cc --- /dev/null +++ b/mock/sqlite/src/sqlite3icu.c @@ -0,0 +1,925 @@ +/****************************************************************************** +** This file is an amalgamation of many separate C source files from SQLite +** version 3.40.1. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when SQLite is compiled as a single +** translation unit. +** +** This file is all you need to compile SQLite. To use SQLite in other +** programs, you need this file and the "sqlite3.h" header file that defines +** the programming interface to the SQLite library. (If you do not have +** the "sqlite3.h" header file at hand, you will find a copy embedded within +** the text of this file. Search for "Begin file sqlite3.h" to find the start +** of the embedded sqlite3.h header file.) Additional code files may be needed +** if you want a wrapper to interface SQLite with your choice of programming +** language. The code for the "sqlite3" command-line shell is also in a +** separate file. This file contains only code for the core SQLite library. +*/ +/* +** 2019.09.02-Complete codec logic for encryption and decryption. +** Huawei Technologies Co, Ltd. +*/ +/************** Begin file icu.c *********************************************/ +/* +** 2007 May 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. +*/ +#include +#include +#include +#include +#include + +#include "sqlite3icu.h" +#include "sqlite3.h" + +#ifdef HARMONY_OS +#include "common/unicode/putil.h" +#endif + +#if !defined(SQLITE_CORE) \ + || defined(SQLITE_ENABLE_ICU) \ + || defined(SQLITE_ENABLE_ICU_COLLATIONS) + +/* Include ICU headers */ +#include +#include +#include +#include + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif + +/* #include */ + +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif + +// hw export the symbols +#ifdef SQLITE_EXPORT_SYMBOLS +#if defined(__GNUC__) +# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) +#elif defined(_MSC_VER) +# define EXPORT_SYMBOLS __declspec(dllexport) +#else +# define EXPORT_SYMBOLS +#endif +#endif + +EXPORT_SYMBOLS SQLITE_API int sqlite3IcuInit(sqlite3 *db); +#ifdef SQLITE_ENABLE_ICU +EXPORT_SYMBOLS SQLITE_API void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); +} + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while( 1 ){ + + /* Read (and consume) the next character from the input pattern. */ + uint32_t uPattern; + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + } + zPattern++; + } + + if( *zPattern==0 ) return 1; + + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ + return 1; + } + SQLITE_ICU_SKIP_UTF8(zString); + } + return 0; + + }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 2. */ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + + }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + uint32_t uString; + SQLITE_ICU_READ_UTF8(zString, uString); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; + } + } + + return *zString==0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + pExpr = sqlite3_get_auxdata(p, 0); + } + if( !pExpr ){ + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> '\u131' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; + const char *zLocale = 0; + + assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); + return; + } + + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } + + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } + return; + } + assert( 0 ); /* Unreachable */ +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. +*/ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2 || nArg==3); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; izName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 + ); + } + + return rc; +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +#endif + +/************** End of icu.c *************************************************/ +/************** Begin file fts3_icu.c ****************************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU + +/* #include */ +/* #include */ +/* #include "fts3_tokenizer.h" */ + +#include +/* #include */ +/* #include */ +#include + +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; + +struct IcuCursor { + sqlite3_tokenizer_cursor base; + + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ + + int nBuffer; + char *zBuffer; + + int iToken; +}; + +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; + + if( argc>0 ){ + n = strlen(argv[0])+1; + } + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); + + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } + + *ppTokenizer = (sqlite3_tokenizer *)p; + + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; + + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; + + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc64( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; + + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; + + if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; + + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to icuOpen(). +*/ +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. +*/ +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + + int iStart = 0; + int iEnd = 0; + int nByte = 0; + + while( iStart==iEnd ){ + UChar32 c; + + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; + } + } + assert(iStart<=iEnd); + } + + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; + } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; + + return SQLITE_OK; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; + +/* +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +EXPORT_SYMBOLS SQLITE_API void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_icu.c ********************************************/ diff --git a/mock/src/mock_aa_fwk.cpp b/mock/src/mock_aa_fwk.cpp index 117f349c..bd1a01b8 100644 --- a/mock/src/mock_aa_fwk.cpp +++ b/mock/src/mock_aa_fwk.cpp @@ -231,6 +231,10 @@ Status DataObsMgrClient::NotifyChangeExt(const ChangeInfo &changeInfo) { return SUCCESS; } +Status DataObsMgrClient::NotifyProcessObserver(const std::string &key, const sptr &observer) +{ + return SUCCESS; +} DataAbilityObserverStub::DataAbilityObserverStub() {} DataAbilityObserverStub::~DataAbilityObserverStub() {} int DataAbilityObserverStub::OnRemoteRequest(uint32_t code, MessageParcel &data, MessageParcel &reply, diff --git a/mock/src/mock_account.cpp b/mock/src/mock_account.cpp index 8a1196ec..af02d5da 100644 --- a/mock/src/mock_account.cpp +++ b/mock/src/mock_account.cpp @@ -12,13 +12,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "account_info.h" #include "ohos_account_kits.h" -#include "os_account_subscribe_info.h" #include "os_account_manager.h" -#include "account_info.h" +#include "os_account_subscribe_info.h" namespace OHOS::AccountSA { -OsAccountSubscribeInfo::OsAccountSubscribeInfo() -{} +OsAccountSubscribeInfo::OsAccountSubscribeInfo() {} OsAccountSubscribeInfo::OsAccountSubscribeInfo(const std::set &states, bool withHandshake) {} OsAccountSubscribeInfo::OsAccountSubscribeInfo(const OS_ACCOUNT_SUBSCRIBE_TYPE &osAccountSubscribeType, const std::string &name) @@ -45,10 +44,10 @@ OsAccountSubscribeInfo *OsAccountSubscribeInfo::Unmarshalling(Parcel &parcel) class OhosAccountKitsImpl : public OHOS::AccountSA::OhosAccountKits { public: std::pair QueryOhosAccountInfo() override; - bool UpdateOhosAccountInfo(const std::string& accountName, const std::string& uid, - const std::string& eventStr) override; + bool UpdateOhosAccountInfo(const std::string &accountName, const std::string &uid, + const std::string &eventStr) override; int32_t QueryDeviceAccountId(std::int32_t &accountId) override; - int32_t GetDeviceAccountIdByUID(int32_t& uid) override; + int32_t GetDeviceAccountIdByUID(int32_t &uid) override; int32_t GetOhosAccountInfoByUserId(int32_t userId, OhosAccountInfo &accountInfo) override; @@ -60,34 +59,39 @@ public: int32_t SetOhosAccountInfoByUserId(const int32_t userId, const OhosAccountInfo &ohosAccountInfo, const std::string &eventStr) override; }; -OhosAccountKits& OhosAccountKits::GetInstance() +OhosAccountKits &OhosAccountKits::GetInstance() { static OhosAccountKitsImpl instance; return instance; } -ErrCode OsAccountManager::GetOsAccountLocalIdFromUid(const int uid, int& id) +ErrCode OsAccountManager::GetOsAccountLocalIdFromUid(const int uid, int &id) { int32_t localUid = uid; id = OhosAccountKits::GetInstance().GetDeviceAccountIdByUID(localUid); return ERR_OK; } -ErrCode OsAccountManager::QueryActiveOsAccountIds(std::vector& ids) +ErrCode OsAccountManager::QueryActiveOsAccountIds(std::vector &ids) { ids = { 0, 100, 101 }; return ERR_OK; } -ErrCode OsAccountManager::IsOsAccountActived(const int id, bool& isOsAccountActived) +ErrCode OsAccountManager::IsOsAccountActived(const int id, bool &isOsAccountActived) { isOsAccountActived = (id == 0 || id == 100); return ERR_OK; } -ErrCode OsAccountManager::IsOsAccountVerified(const int id, bool& isVerified) +ErrCode OsAccountManager::IsOsAccountVerified(const int id, bool &isVerified) { return ERR_OK; } -ErrCode OsAccountManager::GetForegroundOsAccounts(std::vector& accounts) + +ErrCode OsAccountManager::IsOsAccountForeground(const int32_t localId, bool &isForeground) +{ + return ERR_OK; +} +ErrCode OsAccountManager::GetForegroundOsAccounts(std::vector &accounts) { accounts = { { 100, 0 } }; return 0; @@ -112,12 +116,12 @@ std::pair OhosAccountKitsImpl::QueryOhos { return std::pair(); } -bool OhosAccountKitsImpl::UpdateOhosAccountInfo(const std::string& accountName, const std::string& uid, - const std::string& eventStr) +bool OhosAccountKitsImpl::UpdateOhosAccountInfo(const std::string &accountName, const std::string &uid, + const std::string &eventStr) { return false; } -int32_t OhosAccountKitsImpl::GetDeviceAccountIdByUID(int32_t& uid) +int32_t OhosAccountKitsImpl::GetDeviceAccountIdByUID(int32_t &uid) { return 0; } diff --git a/datamgr_service/services/distributeddataservice/framework/utils/time_statistic.cpp b/mock/src/mock_begetutil.cpp similarity index 79% rename from datamgr_service/services/distributeddataservice/framework/utils/time_statistic.cpp rename to mock/src/mock_begetutil.cpp index 971b4c07..89b42303 100644 --- a/datamgr_service/services/distributeddataservice/framework/utils/time_statistic.cpp +++ b/mock/src/mock_begetutil.cpp @@ -13,9 +13,13 @@ * limitations under the License. */ -#define LOG_TAG "TimeStatistic" -//#include "time_statistic.h" +#include "parameters.h" -namespace OHOS::DistributedData { - -} // namespace OHOS::DistributedData \ No newline at end of file +namespace OHOS { +namespace system { +bool GetBoolParameter(const std::string& key, bool def) +{ + return true; +} +} +} // namespace OHOS \ No newline at end of file diff --git a/mock/src/mock_dms.cpp b/mock/src/mock_dms.cpp index f69a756f..8c7d06a0 100644 --- a/mock/src/mock_dms.cpp +++ b/mock/src/mock_dms.cpp @@ -24,6 +24,7 @@ const std::string TAG = "DmsHandle"; int32_t DSchedEventListenerStub::OnRemoteRequest(uint32_t code, MessageParcel &data, MessageParcel &reply, MessageOption &option) { + return 0; } DSchedEventListenerStub::DSchedEventListenerStub() {} DSchedEventListenerStub::~DSchedEventListenerStub() {} diff --git a/mock/src/mock_huks.cpp b/mock/src/mock_huks.cpp index d37485ad..74650083 100644 --- a/mock/src/mock_huks.cpp +++ b/mock/src/mock_huks.cpp @@ -13,14 +13,14 @@ * limitations under the License. */ #include -#include -#include #include +#include #include #include -#include -#include #include +#include +#include +#include #include "securec.h" static std::set KEYS; @@ -186,8 +186,7 @@ static int32_t CheckBeforeAddParams(const struct HksParamSet *paramSet, const st } for (uint32_t i = 0; i < paramCnt; i++) { - if ((GetTagType((enum HksTag)(params[i].tag)) == HKS_TAG_TYPE_BYTES) && - (params[i].blob.data == NULL)) { + if ((GetTagType((enum HksTag)(params[i].tag)) == HKS_TAG_TYPE_BYTES) && (params[i].blob.data == NULL)) { HKS_LOG_E("invalid blob param!"); return HKS_ERROR_INVALID_ARGUMENT; } @@ -238,8 +237,8 @@ HKS_API_EXPORT int32_t HksFreshParamSet(struct HksParamSet *paramSet, bool isCop return HKS_ERROR_INVALID_ARGUMENT; } - if (isCopy && (memcpy_s((uint8_t *)paramSet + offset, size - offset, - paramSet->params[i].blob.data, paramSet->params[i].blob.size) != EOK)) { + if (isCopy && (memcpy_s((uint8_t *)paramSet + offset, size - offset, paramSet->params[i].blob.data, + paramSet->params[i].blob.size) != EOK)) { HKS_LOG_E("copy param blob failed!"); return HKS_ERROR_INSUFFICIENT_MEMORY; } @@ -259,8 +258,7 @@ HKS_API_EXPORT int32_t HksCheckParamSet(const struct HksParamSet *paramSet, uint { HKS_IF_NULL_RETURN(paramSet, HKS_ERROR_NULL_POINTER) - if ((size < sizeof(struct HksParamSet)) || (size > HKS_PARAM_SET_MAX_SIZE) || - (paramSet->paramSetSize != size) || + if ((size < sizeof(struct HksParamSet)) || (size > HKS_PARAM_SET_MAX_SIZE) || (paramSet->paramSetSize != size) || (paramSet->paramsCnt > ((size - sizeof(struct HksParamSet)) / sizeof(struct HksParam)))) { HKS_LOG_E("invalid param set!"); return HKS_ERROR_INVALID_ARGUMENT; @@ -280,8 +278,7 @@ HKS_API_EXPORT int32_t HksInitParamSet(struct HksParamSet **paramSet) return HKS_SUCCESS; } -HKS_API_EXPORT int32_t HksAddParams(struct HksParamSet *paramSet, - const struct HksParam *params, uint32_t paramCnt) +HKS_API_EXPORT int32_t HksAddParams(struct HksParamSet *paramSet, const struct HksParam *params, uint32_t paramCnt) { int32_t ret = CheckBeforeAddParams(paramSet, params, paramCnt); HKS_IF_NOT_SUCC_RETURN(ret, ret) @@ -338,8 +335,8 @@ static int32_t FreshParamSet(struct HksParamSet *paramSet, bool isCopy) HKS_LOG_E("blob size overflow!"); return HKS_ERROR_INVALID_ARGUMENT; } - if (isCopy && memcpy_s((uint8_t *)paramSet + offset, size - offset, - paramSet->params[i].blob.data, paramSet->params[i].blob.size) != EOK) { + if (isCopy && memcpy_s((uint8_t *)paramSet + offset, size - offset, paramSet->params[i].blob.data, + paramSet->params[i].blob.size) != EOK) { HKS_LOG_E("copy param blob failed!"); return HKS_ERROR_INSUFFICIENT_MEMORY; } @@ -362,8 +359,8 @@ HKS_API_EXPORT int32_t HksGetParam(const struct HksParamSet *paramSet, uint32_t return HKS_ERROR_INVALID_ARGUMENT; } - HKS_IF_NOT_SUCC_LOGE_RETURN(HksCheckParamSet(paramSet, paramSet->paramSetSize), - HKS_ERROR_INVALID_ARGUMENT, "invalid paramSet!") + HKS_IF_NOT_SUCC_LOGE_RETURN(HksCheckParamSet(paramSet, paramSet->paramSetSize), HKS_ERROR_INVALID_ARGUMENT, + "invalid paramSet!") for (uint32_t i = 0; i < paramSet->paramsCnt; i++) { if (tag == paramSet->params[i].tag) { @@ -375,8 +372,8 @@ HKS_API_EXPORT int32_t HksGetParam(const struct HksParamSet *paramSet, uint32_t return HKS_ERROR_PARAM_NOT_EXIST; } -HKS_API_EXPORT int32_t HksGetParamSet(const struct HksParamSet *inParamSet, - uint32_t inParamSetSize, struct HksParamSet **outParamSet) +HKS_API_EXPORT int32_t HksGetParamSet(const struct HksParamSet *inParamSet, uint32_t inParamSetSize, + struct HksParamSet **outParamSet) { int32_t ret = HksCheckParamSet(inParamSet, inParamSetSize); HKS_IF_NOT_SUCC_RETURN(ret, ret) @@ -419,8 +416,7 @@ HKS_API_EXPORT int32_t HksCheckParamMatch(const struct HksParam *baseParam, cons case HKS_TAG_TYPE_BOOL: return (baseParam->boolParam == param->boolParam) ? HKS_SUCCESS : HKS_ERROR_INVALID_ARGUMENT; case HKS_TAG_TYPE_BYTES: - if (baseParam->blob.size != param->blob.size || - baseParam->blob.data == NULL ||(param->blob.data == NULL)) { + if (baseParam->blob.size != param->blob.size || baseParam->blob.data == NULL || (param->blob.data == NULL)) { HKS_LOG_E("unmatch byte type len!"); return HKS_ERROR_INVALID_ARGUMENT; } @@ -496,14 +492,14 @@ HKS_API_EXPORT int32_t HksDeleteTagsFromParamSet(const uint32_t *tag, uint32_t t return HKS_SUCCESS; } -int32_t HksInit(const struct HksBlob *keyAlias, const struct HksParamSet *paramSet, - struct HksBlob *handle, struct HksBlob *token) +int32_t HksInit(const struct HksBlob *keyAlias, const struct HksParamSet *paramSet, struct HksBlob *handle, + struct HksBlob *token) { return HKS_SUCCESS; } -int32_t HksUpdate(const struct HksBlob *handle, const struct HksParamSet *paramSet, - const struct HksBlob *inData, struct HksBlob *outData) +int32_t HksUpdate(const struct HksBlob *handle, const struct HksParamSet *paramSet, const struct HksBlob *inData, + struct HksBlob *outData) { size_t size = std::min(outData->size, inData->size); outData->size = size; @@ -511,8 +507,8 @@ int32_t HksUpdate(const struct HksBlob *handle, const struct HksParamSet *paramS return HKS_SUCCESS; } -int32_t HksFinish(const struct HksBlob *handle, const struct HksParamSet *paramSet, - const struct HksBlob *inData, struct HksBlob *outData) +int32_t HksFinish(const struct HksBlob *handle, const struct HksParamSet *paramSet, const struct HksBlob *inData, + struct HksBlob *outData) { size_t size = std::min(outData->size, inData->size); (void)memcpy_s(outData->data, outData->size, inData->data, size); @@ -526,21 +522,21 @@ int32_t HksFinish(const struct HksBlob *handle, const struct HksParamSet *paramS return HKS_SUCCESS; } -void HksFree(void *ptr) -{ +int32_t HksAbort(const struct HksBlob *handle, const struct HksParamSet *paramSet) {} -} +void HksFree(void *ptr) {} int32_t HksLocalGenerateKey(const struct HksParamSet *paramSetIn, struct HksParamSet *paramSetOut) { paramSetOut->paramSetSize = paramSetIn->paramSetSize; paramSetOut->paramsCnt = paramSetIn->paramsCnt; -// (void)memcpy_s(paramSetIn->params, paramSetIn->paramSetSize, paramSetOut->params, paramSetOut->paramSetSize); + // (void)memcpy_s(paramSetIn->params, paramSetIn->paramSetSize, paramSetOut->params, paramSetOut->paramSetSize); HksMemCmp(paramSetIn->params, paramSetOut->params, paramSetOut->paramSetSize); return 0; } -int32_t HksClientGenerateKey(const struct HksBlob *keyAlias, const struct HksParamSet *paramSetIn, struct HksParamSet *paramSetOut) +int32_t HksClientGenerateKey(const struct HksBlob *keyAlias, const struct HksParamSet *paramSetIn, + struct HksParamSet *paramSetOut) { std::string key((char *)keyAlias->data); KEYS.insert(key); diff --git a/mock/src/mock_resouce_manager.cpp b/mock/src/mock_resouce_manager.cpp index 0edef923..2512a931 100644 --- a/mock/src/mock_resouce_manager.cpp +++ b/mock/src/mock_resouce_manager.cpp @@ -18,11 +18,213 @@ ResourceManager::~ResourceManager() {} class ResManagerImpl : public ResourceManager { public: ~ResManagerImpl() override {} - bool AddResource(const char *path) override + bool AddResource(const char *path, const uint32_t &selectedTypes, bool forceReload) override { return false; } - RState UpdateResConfig(ResConfig &resConfig) override + RState UpdateResConfig(ResConfig &resConfig, bool isUpdateTheme) override + { + return LOCALEINFO_IS_NULL; + } + RState GetResConfigById(uint32_t resId, ResConfig &resConfig, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetResConfigByName(const std::string &name, const ResType type, ResConfig &resConfig, + uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaById(uint32_t id, std::string &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaByName(const char *name, std::string &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaDataById(uint32_t id, size_t &len, std::unique_ptr &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaDataByName(const char *name, size_t &len, std::unique_ptr &outValue, + uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaBase64DataById(uint32_t id, std::string &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetMediaBase64DataByName(const char *name, std::string &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState IsLoadHap(std::string &hapPath) override + { + return LOCALEINFO_IS_NULL; + } + RState GetRawFileList(const std::string &rawDirPath, std::vector &rawfileList) override + { + return LOCALEINFO_IS_NULL; + } + RState GetDrawableInfoById(uint32_t id, std::string &type, size_t &len, std::unique_ptr &outValue, + uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetDrawableInfoByName(const char *name, std::string &type, size_t &len, + std::unique_ptr &outValue, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + bool AddResource(const std::string &path, const std::vector &overlayPaths) override + { + return false; + } + bool RemoveResource(const std::string &path, const std::vector &overlayPaths) override + { + return false; + } + RState GetStringFormatById(uint32_t id, std::string &outValue, + std::vector> &jsParams) override + { + return LOCALEINFO_IS_NULL; + } + RState GetStringFormatByName(const char *name, std::string &outValue, + std::vector> &jsParams) override + { + return LOCALEINFO_IS_NULL; + } + uint32_t GetResourceLimitKeys() override + { + return 0; + } + bool AddAppOverlay(const std::string &path) override + { + return false; + } + bool RemoveAppOverlay(const std::string &path) override + { + return false; + } + RState GetRawFdNdkFromHap(const std::string &rawFileName, RawFileDescriptor &descriptor) override + { + return LOCALEINFO_IS_NULL; + } + RState GetResId(const std::string &resTypeName, uint32_t &resId) override + { + return LOCALEINFO_IS_NULL; + } + void GetLocales(std::vector &outValue, bool includeSystem) override {} + RState GetDrawableInfoById(uint32_t id, std::tuple &drawableInfo, + std::unique_ptr &outValue, uint32_t iconType, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetDrawableInfoByName(const char *name, std::tuple &drawableInfo, + std::unique_ptr &outValue, uint32_t iconType, uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetSymbolById(uint32_t id, uint32_t &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetSymbolByName(const char *name, uint32_t &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetThemeIcons(uint32_t resId, std::pair, size_t> &foregroundInfo, + std::pair, size_t> &backgroundInfo, uint32_t density, + const std::string &abilityName) override + { + return LOCALEINFO_IS_NULL; + } + std::string GetThemeMask() override + { + return std::string(); + } + bool HasIconInTheme(const std::string &bundleName) override + { + return false; + } + RState GetOtherIconsInfo(const std::string &iconName, std::unique_ptr &outValue, size_t &len, + bool isGlobalMask) override + { + return LOCALEINFO_IS_NULL; + } + RState IsRawDirFromHap(const std::string &pathName, bool &outValue) override + { + return LOCALEINFO_IS_NULL; + } + std::shared_ptr GetOverrideResourceManager(std::shared_ptr overrideResConfig) override + { + return std::shared_ptr(); + } + RState UpdateOverrideResConfig(ResConfig &resConfig) override + { + return LOCALEINFO_IS_NULL; + } + void GetOverrideResConfig(ResConfig &resConfig) override {} + RState GetDynamicIcon(const std::string &resName, std::pair, size_t> &iconInfo, + uint32_t density) override + { + return LOCALEINFO_IS_NULL; + } + RState GetStringFormatById(std::string &outValue, uint32_t id, va_list args) override + { + return LOCALEINFO_IS_NULL; + } + RState GetStringFormatByName(std::string &outValue, const char *name, va_list args) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringById(std::string &outValue, uint32_t id, int quantity, + std::vector> &jsParams) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringByName(std::string &outValue, const char *name, int quantity, + std::vector> &jsParams) override + { + return LOCALEINFO_IS_NULL; + } + bool AddPatchResource(const char *path, const char *patchPath) override + { + return false; + } + RState GetThemeDataByName(const char *name, std::map &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetThemeDataById(uint32_t id, std::map &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetPatternDataById(uint32_t id, std::map &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetPatternDataByName(const char *name, std::map &outValue) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringById(std::string &outValue, uint32_t id, Quantity quantity, va_list args) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringById(std::string &outValue, uint32_t id, Quantity quantity, + std::vector> &jsParams) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringByName(std::string &outValue, const char *name, Quantity quantity, va_list args) override + { + return LOCALEINFO_IS_NULL; + } + RState GetFormatPluralStringByName(std::string &outValue, const char *name, Quantity quantity, + std::vector> &jsParams) override { return LOCALEINFO_IS_NULL; } @@ -139,35 +341,31 @@ public: { return LOCALEINFO_IS_NULL; } - RState GetProfileDataByName(const char *name, size_t len, std::unique_ptr &outValue) override - { - return LOCALEINFO_IS_NULL; - } - RState GetMediaById(uint32_t id, std::string &outValue) override + RState GetRawFilePathByName(const std::string &name, std::string &outValue) override { return LOCALEINFO_IS_NULL; } - RState GetMediaById(uint32_t id, uint32_t density, std::string &outValue) override + RState GetRawFileDescriptor(const std::string &name, RawFileDescriptor &descriptor) override { return LOCALEINFO_IS_NULL; } - RState GetMediaByName(const char *name, std::string &outValue) override + RState CloseRawFileDescriptor(const std::string &name) override { return LOCALEINFO_IS_NULL; } - RState GetMediaByName(const char *name, uint32_t density, std::string &outValue) override + RState GetRawFileFromHap(const std::string &rawFileName, size_t &len, std::unique_ptr &outValue) override { return LOCALEINFO_IS_NULL; } - RState GetRawFilePathByName(const std::string &name, std::string &outValue) override + RState GetRawFileDescriptorFromHap(const std::string &rawFileName, RawFileDescriptor &descriptor) override { return LOCALEINFO_IS_NULL; } - RState GetRawFileDescriptor(const std::string &name, RawFileDescriptor &descriptor) override + RState GetProfileDataById(uint32_t id, size_t &len, std::unique_ptr &outValue) override { return LOCALEINFO_IS_NULL; } - RState CloseRawFileDescriptor(const std::string &name) override + RState GetProfileDataByName(const char *name, size_t &len, std::unique_ptr &outValue) override { return LOCALEINFO_IS_NULL; } diff --git a/preferences/CMakeLists.txt b/preferences/CMakeLists.txt index 439ca095..17cc6ce5 100644 --- a/preferences/CMakeLists.txt +++ b/preferences/CMakeLists.txt @@ -1,11 +1,11 @@ cmake_minimum_required(VERSION 3.11.2) project(preferences) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "-std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -Wpragmas") set(KV_STORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kv_store) set(MOCK_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../mock) diff --git a/preferences/frameworks/common/include/log_print.h b/preferences/frameworks/common/include/log_print.h index 84ac7ce3..a15db7a1 100644 --- a/preferences/frameworks/common/include/log_print.h +++ b/preferences/frameworks/common/include/log_print.h @@ -65,14 +65,6 @@ static inline OHOS::HiviewDFX::HiLogLabel LogLabel() } \ } while (0) -#define LOG_RECORD_FILE_NAME(message) \ - do { \ - const char *name = fileName.data(); \ - auto pos = fileName.rfind('/'); \ - pos = (pos != std::string::npos) ? pos + 1 : 0; \ - LOG_DEBUG(message " fileName is %{private}s.", name + pos); \ - } while (0) - #define LOG_INFO(fmt, ...) \ do { \ auto lable = LogLabel(); \ diff --git a/preferences/frameworks/js/napi/common/include/uv_queue.h b/preferences/frameworks/js/napi/common/include/uv_queue.h index 87b04cde..8ebc2823 100644 --- a/preferences/frameworks/js/napi/common/include/uv_queue.h +++ b/preferences/frameworks/js/napi/common/include/uv_queue.h @@ -16,9 +16,6 @@ #define OHOS_UV_QUEUE_H #include #include "napi/native_api.h" -#include "napi/native_common.h" -#include "napi/native_node_api.h" -#include "uv.h" namespace OHOS::PreferencesJsKit { class UvQueue final { @@ -31,15 +28,7 @@ public: napi_env GetEnv(); void AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs = NapiArgsGenerator(), bool sendable = false); private: - static void Work(uv_work_t* work, int uvstatus); - struct UvEntry { - napi_env env; - NapiCallbackGetter callback; - NapiArgsGenerator args; - bool sendable; - }; napi_env env_ = nullptr; - uv_loop_s* loop_ = nullptr; }; } // namespace OHOS::PreferencesJsKit #endif // OHOS_UV_QUEUE_H diff --git a/preferences/frameworks/js/napi/common/src/napi_async_call.cpp b/preferences/frameworks/js/napi/common/src/napi_async_call.cpp index 5b3d57d1..7ce96222 100644 --- a/preferences/frameworks/js/napi/common/src/napi_async_call.cpp +++ b/preferences/frameworks/js/napi/common/src/napi_async_call.cpp @@ -114,8 +114,7 @@ napi_value AsyncCall::Async(napi_env env, std::shared_ptr context, } context->keep_ = context; napi_value resource = nullptr; - const std::string name_resource = "Preferences" + name; - napi_create_string_utf8(env, name_resource.c_str(), NAPI_AUTO_LENGTH, &resource); + napi_create_string_utf8(env, name.c_str(), NAPI_AUTO_LENGTH, &resource); // create async work, execute function is OnExecute, complete function is OnComplete napi_create_async_work(env, nullptr, resource, AsyncCall::OnExecute, AsyncCall::OnComplete, reinterpret_cast(context.get()), &context->work_); diff --git a/preferences/frameworks/js/napi/common/src/uv_queue.cpp b/preferences/frameworks/js/napi/common/src/uv_queue.cpp index bcd070a5..f0b4fbdd 100644 --- a/preferences/frameworks/js/napi/common/src/uv_queue.cpp +++ b/preferences/frameworks/js/napi/common/src/uv_queue.cpp @@ -12,20 +12,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "uv_queue.h" - -#include #include "log_print.h" +#include +#include "uv_queue.h" +#include "napi/native_common.h" +#include "napi/native_node_api.h" namespace OHOS::PreferencesJsKit { constexpr size_t MAX_CALLBACK_ARG_NUM = 6; UvQueue::UvQueue(napi_env env) : env_(env) { - if (env != nullptr) { - napi_get_uv_event_loop(env, &loop_); - } } UvQueue::~UvQueue() @@ -36,60 +34,47 @@ UvQueue::~UvQueue() void UvQueue::AsyncCall(NapiCallbackGetter getter, NapiArgsGenerator genArgs, bool sendable) { - if (loop_ == nullptr || !getter) { - LOG_ERROR("loop_ or callback is nullptr"); - return; - } - - uv_work_t* work = new (std::nothrow) uv_work_t; - if (work == nullptr) { + if (!getter) { + LOG_ERROR("callback is nullptr"); return; } - work->data = new (std::nothrow) UvEntry{ env_, getter, std::move(genArgs), sendable }; - if (work->data == nullptr) { - delete work; - return; - } - int ret = uv_queue_work(loop_, work, [](uv_work_t* work) {}, UvQueue::Work); - if (ret != 0) { - LOG_ERROR("Failed to uv_queue_work."); - delete static_cast(work->data); - delete work; - } -} -void UvQueue::Work(uv_work_t* work, int uvstatus) -{ - std::shared_ptr entry(static_cast(work->data), [work](UvEntry *data) { - delete data; - delete work; - }); - napi_handle_scope scope = nullptr; - napi_open_handle_scope(entry->env, &scope); - if (scope == nullptr) { - return; - } - napi_value method = entry->callback(entry->env); - if (method == nullptr) { - LOG_WARN("the callback is invalid, maybe is cleared!"); - napi_close_handle_scope(entry->env, scope); - return ; - } - int argc = 0; - napi_value argv[MAX_CALLBACK_ARG_NUM] = { nullptr }; - if (entry->args) { - argc = MAX_CALLBACK_ARG_NUM; - entry->args(entry->env, entry->sendable, argc, argv); - } - LOG_DEBUG("queue uv_after_work_cb"); - napi_value global = nullptr; - napi_get_global(entry->env, &global); - napi_value result; - napi_status status = napi_call_function(entry->env, global, method, argc, argv, &result); - if (status != napi_ok) { - LOG_ERROR("notify data change failed status:%{public}d.", status); + auto env = GetEnv(); + auto task = [env, getter, genArgs, sendable]() { + napi_handle_scope scope = nullptr; + napi_open_handle_scope(env, &scope); + if (scope == nullptr) { + return; + } + napi_value method = getter(env); + if (method == nullptr) { + LOG_WARN("the callback is invalid, maybe is cleared!"); + napi_close_handle_scope(env, scope); + return; + } + int argc = 0; + napi_value argv[MAX_CALLBACK_ARG_NUM] = { nullptr }; + if (genArgs) { + argc = MAX_CALLBACK_ARG_NUM; + genArgs(env, sendable, argc, argv); + } + napi_value global = nullptr; + napi_status status = napi_get_global(env, &global); + if (status != napi_ok) { + LOG_ERROR("get napi gloabl failed. status: %{public}d.", status); + napi_close_handle_scope(env, scope); + return; + } + napi_value result; + status = napi_call_function(env, global, method, argc, argv, &result); + if (status != napi_ok) { + LOG_ERROR("notify data change failed status: %{public}d.", status); + } + napi_close_handle_scope(env, scope); + }; + if (napi_ok != napi_send_event(env_, task, napi_eprio_immediate)) { + LOG_ERROR("Failed to napi_send_event."); } - napi_close_handle_scope(entry->env, scope); } napi_env UvQueue::GetEnv() diff --git a/preferences/frameworks/js/napi/preferences/src/napi_preferences.cpp b/preferences/frameworks/js/napi/preferences/src/napi_preferences.cpp index 339af282..a1230309 100644 --- a/preferences/frameworks/js/napi/preferences/src/napi_preferences.cpp +++ b/preferences/frameworks/js/napi/preferences/src/napi_preferences.cpp @@ -15,6 +15,8 @@ #include "napi_preferences.h" +#include +#include #include #include #include @@ -38,7 +40,7 @@ struct PreferencesAysncContext : public BaseContext { std::string key; PreferencesValue defValue = PreferencesValue(static_cast(0)); napi_ref inputValueRef = nullptr; - std::map allElements; + std::unordered_map allElements; bool hasKey = false; std::vector> preferencesObservers; @@ -163,11 +165,30 @@ int ParseDefValue(const napi_env env, const napi_value jsVal, std::shared_ptr context, napi_value &result) { - napi_create_object(env, &result); + std::vector descriptors; + descriptors.reserve(context->allElements.size()); + bool containsPureDigits = false; for (const auto &[key, value] : context->allElements) { - napi_set_named_property(env, result, key.c_str(), JSUtils::Convert2JSValue(env, value.value_)); + if (!containsPureDigits) { + if (IsPureDigits(key)) { + containsPureDigits = true; + } + } + descriptors.push_back(napi_property_descriptor( + DECLARE_NAPI_DEFAULT_PROPERTY(key.c_str(), JSUtils::Convert2JSValue(env, value.value_)))); + } + if (containsPureDigits) { + napi_create_object(env, &result); + napi_define_properties(env, result, descriptors.size(), descriptors.data()); + } else { + napi_create_object_with_properties(env, &result, descriptors.size(), descriptors.data()); } return OK; } @@ -186,7 +207,6 @@ std::pair> PreferencesProxy::GetS napi_value PreferencesProxy::GetAll(napi_env env, napi_callback_info info) { - LOG_DEBUG("GetAll start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 0, std::make_shared("0 or 1")); @@ -200,21 +220,20 @@ napi_value PreferencesProxy::GetAll(napi_env env, napi_callback_info info) LOG_ERROR("Failed to get instance when GetAll, The instance is nullptr."); return E_INNER_ERROR; } - context->allElements = instance->GetAll(); + context->allElements = instance->GetAllDatas(); return OK; }; auto output = [context](napi_env env, napi_value &result) { GetAllExecute(env, context, result); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetAll"); + return AsyncCall::Call(env, context, "PreferencesGetAll"); } napi_value PreferencesProxy::GetValue(napi_env env, napi_callback_info info) { - LOG_DEBUG("GetValue start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 2, std::make_shared("2 or 3")); @@ -235,7 +254,6 @@ napi_value PreferencesProxy::GetValue(napi_env env, napi_callback_info info) }; auto output = [context](napi_env env, napi_value &result) { if (context->defValue.IsLong()) { - LOG_DEBUG("GetValue get default value."); napi_get_reference_value(env, context->inputValueRef, &result); } else { result = JSUtils::Convert2JSValue(env, context->defValue.value_); @@ -245,14 +263,13 @@ napi_value PreferencesProxy::GetValue(napi_env env, napi_callback_info info) std::make_shared("Failed to delete reference when getting value.")); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetValue"); + return AsyncCall::Call(env, context, "PreferencesGetValue"); } napi_value PreferencesProxy::SetValue(napi_env env, napi_callback_info info) { - LOG_DEBUG("SetValue start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 2, std::make_shared("2 or 3")); @@ -265,7 +282,7 @@ napi_value PreferencesProxy::SetValue(napi_env env, napi_callback_info info) auto exec = [context]() -> int { auto instance = context->instance_.lock(); if (instance == nullptr) { - LOG_ERROR("Failed to get instance when GetValue, The instance is nullptr."); + LOG_ERROR("Failed to get instance when SetValue, The instance is nullptr."); return E_INNER_ERROR; } return instance->Put(context->key, context->defValue); @@ -274,17 +291,15 @@ napi_value PreferencesProxy::SetValue(napi_env env, napi_callback_info info) napi_status status = napi_get_undefined(env, &result); PRE_CHECK_RETURN_VOID_SET(status == napi_ok, std::make_shared("Failed to get undefined when setting value.")); - LOG_DEBUG("SetValue end."); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "SetValue"); + return AsyncCall::Call(env, context, "PreferencesSetValue"); } napi_value PreferencesProxy::Delete(napi_env env, napi_callback_info info) { - LOG_DEBUG("Delete start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 1, std::make_shared("1 or 2")); @@ -295,27 +310,25 @@ napi_value PreferencesProxy::Delete(napi_env env, napi_callback_info info) }; auto exec = [context]() -> int { auto instance = context->instance_.lock(); - if (instance == nullptr) { - LOG_ERROR("Failed to get instance when GetValue, The instance is nullptr."); - return E_INNER_ERROR; + if (instance != nullptr) { + return instance->Delete(context->key); } - return instance->Delete(context->key); + LOG_ERROR("Failed to get instance when Delete, The instance is nullptr."); + return E_INNER_ERROR; }; auto output = [context](napi_env env, napi_value &result) { napi_status status = napi_get_undefined(env, &result); PRE_CHECK_RETURN_VOID_SET(status == napi_ok, std::make_shared("Failed to get undefined when deleting value.")); - LOG_DEBUG("Delete end."); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Delete"); + return AsyncCall::Call(env, context, "PreferencesDelete"); } napi_value PreferencesProxy::HasKey(napi_env env, napi_callback_info info) { - LOG_DEBUG("HasKey start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 1, std::make_shared("1 or 2")); @@ -327,7 +340,7 @@ napi_value PreferencesProxy::HasKey(napi_env env, napi_callback_info info) auto exec = [context]() -> int { auto instance = context->instance_.lock(); if (instance == nullptr) { - LOG_ERROR("Failed to get instance when GetValue, The instance is nullptr."); + LOG_ERROR("Failed to get instance when HasKey, The instance is nullptr."); return E_INNER_ERROR; } context->hasKey = instance->HasKey(context->key); @@ -337,17 +350,15 @@ napi_value PreferencesProxy::HasKey(napi_env env, napi_callback_info info) napi_status status = napi_get_boolean(env, context->hasKey, &result); PRE_CHECK_RETURN_VOID_SET(status == napi_ok, std::make_shared("Failed to get boolean when having key.")); - LOG_DEBUG("HasKey end."); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "HasKey"); + return AsyncCall::Call(env, context, "PreferencesHasKey"); } napi_value PreferencesProxy::Flush(napi_env env, napi_callback_info info) { - LOG_DEBUG("Flush start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 0, std::make_shared("0 or 1")); @@ -358,7 +369,7 @@ napi_value PreferencesProxy::Flush(napi_env env, napi_callback_info info) auto exec = [context]() -> int { auto instance = context->instance_.lock(); if (instance == nullptr) { - LOG_ERROR("Failed to get instance when GetValue, The instance is nullptr."); + LOG_ERROR("Failed to get instance when Flush, The instance is nullptr."); return E_INNER_ERROR; } return instance->FlushSync(); @@ -367,17 +378,15 @@ napi_value PreferencesProxy::Flush(napi_env env, napi_callback_info info) napi_status status = napi_get_undefined(env, &result); PRE_CHECK_RETURN_VOID_SET(status == napi_ok, std::make_shared("Failed to get undefined when flushing.")); - LOG_DEBUG("Flush end."); }; context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Flush"); + return AsyncCall::Call(env, context, "PreferencesFlush"); } napi_value PreferencesProxy::Clear(napi_env env, napi_callback_info info) { - LOG_DEBUG("Clear start"); auto context = std::make_shared(); auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { PRE_CHECK_RETURN_VOID_SET(argc == 0, std::make_shared("0 or 1")); @@ -387,22 +396,21 @@ napi_value PreferencesProxy::Clear(napi_env env, napi_callback_info info) }; auto exec = [context]() -> int { auto instance = context->instance_.lock(); - if (instance == nullptr) { - LOG_ERROR("Failed to get instance when GetValue, The instance is nullptr."); - return E_INNER_ERROR; + if (instance != nullptr) { + return instance->Clear(); } - return instance->Clear(); + LOG_ERROR("Failed to get instance when Clear, The instance is nullptr."); + return E_INNER_ERROR; }; auto output = [context](napi_env env, napi_value &result) { napi_status status = napi_get_undefined(env, &result); PRE_CHECK_RETURN_VOID_SET(status == napi_ok, std::make_shared("Failed to get undefined when clearing.")); - LOG_DEBUG("Clear end."); }; context->SetAction(env, info, input, exec, output); - + PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Clear"); + return AsyncCall::Call(env, context, "PreferencesClear"); } napi_value PreferencesProxy::RegisterObserver(napi_env env, napi_callback_info info) diff --git a/preferences/frameworks/js/napi/preferences/src/napi_preferences_helper.cpp b/preferences/frameworks/js/napi/preferences/src/napi_preferences_helper.cpp index ec3056b0..48ad429a 100644 --- a/preferences/frameworks/js/napi/preferences/src/napi_preferences_helper.cpp +++ b/preferences/frameworks/js/napi/preferences/src/napi_preferences_helper.cpp @@ -78,10 +78,10 @@ int ParseOptionalParameters(const napi_env env, napi_value *argv, std::shared_pt PRE_CHECK_RETURN_ERR_SET(napi_get_value_int32(env, temp, &intVal) == napi_ok, std::make_shared("The storageType must be StorageType which is enum.")); bool isTypeValid = (intVal == static_cast(StorageType::XML) || - intVal == static_cast(StorageType::CLKV)); + intVal == static_cast(StorageType::GSKV)); PRE_CHECK_RETURN_ERR_SET(isTypeValid, std::make_shared("Storage type value invalid.")); context->storageType = (intVal == static_cast(StorageType::XML)) ? - StorageType::XML : StorageType::CLKV; + StorageType::XML : StorageType::GSKV; } PRE_CHECK_RETURN_ERR_SET(status == napi_ok, std::make_shared("parse storage type: type of api failed")); @@ -119,7 +119,7 @@ napi_value GetPreferences(napi_env env, napi_callback_info info) auto exec = [context]() -> int { int errCode = E_OK; Options options(context->path, context->bundleName, context->dataGroupId, - context->storageType == StorageType::CLKV); + context->storageType == StorageType::GSKV); context->proxy = PreferencesHelper::GetPreferences(options, errCode); return errCode; }; @@ -131,7 +131,7 @@ napi_value GetPreferences(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetPreferences"); + return AsyncCall::Call(env, context, "PreferencesGetPreferences"); } napi_value DeletePreferences(napi_env env, napi_callback_info info) @@ -152,7 +152,7 @@ napi_value DeletePreferences(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "DeletePreferences"); + return AsyncCall::Call(env, context, "PreferencesDeletePreferences"); } napi_value RemovePreferencesFromCache(napi_env env, napi_callback_info info) @@ -173,7 +173,7 @@ napi_value RemovePreferencesFromCache(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "RemovePreferencesFromCache"); + return AsyncCall::Call(env, context, "PreferencesRemovePreferencesFromCache"); } napi_value IsStorageTypeSupported(napi_env env, napi_callback_info info) @@ -187,10 +187,10 @@ napi_value IsStorageTypeSupported(napi_env env, napi_callback_info info) PRE_NAPI_ASSERT_RETURN_VOID(env, napi_get_value_int32(env, argv[0], &intVal) == napi_ok, std::make_shared("The storageType must be StorageType which is enum.")); bool isTypeValid = (intVal == static_cast(StorageType::XML) || - intVal == static_cast(StorageType::CLKV)); + intVal == static_cast(StorageType::GSKV)); PRE_NAPI_ASSERT_RETURN_VOID(env, isTypeValid, std::make_shared("Storage type value invalid.")); context->storageType = (intVal == static_cast(StorageType::XML)) ? - StorageType::XML : StorageType::CLKV; + StorageType::XML : StorageType::GSKV; }; auto exec = [context]() -> int { @@ -207,7 +207,7 @@ napi_value IsStorageTypeSupported(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "IsStorageTypeSupported"); + return AsyncCall::Call(env, context, "PreferencesIsStorageTypeSupported"); } static napi_status SetNamedProperty(napi_env env, napi_value& obj, const std::string& name, int32_t value) @@ -215,7 +215,7 @@ static napi_status SetNamedProperty(napi_env env, napi_value& obj, const std::st napi_value property = nullptr; napi_status status = napi_create_int32(env, value, &property); PRE_NAPI_ASSERT_BASE(env, status == napi_ok, std::make_shared("napi_create_int32 failed!"), status); - + status = napi_set_named_property(env, obj, name.c_str(), property); PRE_NAPI_ASSERT_BASE(env, status == napi_ok, std::make_shared("napi_set_named_property failed!"), status); @@ -227,7 +227,7 @@ static napi_value ExportStorageType(napi_env env) napi_value storageType = nullptr; napi_create_object(env, &storageType); SetNamedProperty(env, storageType, "XML", static_cast(StorageType::XML)); - SetNamedProperty(env, storageType, "CLKV", static_cast(StorageType::CLKV)); + SetNamedProperty(env, storageType, "GSKV", static_cast(StorageType::GSKV)); napi_object_freeze(env, storageType); return storageType; } diff --git a/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences.cpp b/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences.cpp index 2903786f..e4bc041f 100644 --- a/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences.cpp +++ b/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences.cpp @@ -39,7 +39,7 @@ struct PreferencesAysncContext : public BaseContext { std::string key; PreferencesValue defValue = PreferencesValue(static_cast(0)); napi_ref inputValueRef = nullptr; - std::map allElements; + std::unordered_map allElements; bool hasKey = false; std::vector> preferencesObservers; @@ -177,6 +177,7 @@ std::pair> PreferencesProxy::GetS int GetAllExecute(napi_env env, std::shared_ptr context, napi_value &result) { std::vector descriptors; + descriptors.reserve(context->allElements.size()); for (const auto &[key, value] : context->allElements) { descriptors.push_back(napi_property_descriptor( DECLARE_NAPI_DEFAULT_PROPERTY(key.c_str(), Utils::ConvertToSendable(env, value.value_)))); @@ -200,14 +201,14 @@ napi_value PreferencesProxy::GetAll(napi_env env, napi_callback_info info) if (instance == nullptr) { return E_INNER_ERROR; } - context->allElements = instance->GetAll(); + context->allElements = instance->GetAllDatas(); return OK; }; auto output = [context](napi_env env, napi_value &result) { GetAllExecute(env, context, result); }; context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetAll"); + return AsyncCall::Call(env, context, "SendablePreferencesGetAll"); } napi_value PreferencesProxy::GetValue(napi_env env, napi_callback_info info) @@ -244,7 +245,7 @@ napi_value PreferencesProxy::GetValue(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetValue"); + return AsyncCall::Call(env, context, "SendablePreferencesGetValue"); } napi_value PreferencesProxy::SetValue(napi_env env, napi_callback_info info) @@ -275,7 +276,7 @@ napi_value PreferencesProxy::SetValue(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "SetValue"); + return AsyncCall::Call(env, context, "SendablePreferencesSetValue"); } napi_value PreferencesProxy::Delete(napi_env env, napi_callback_info info) @@ -305,7 +306,7 @@ napi_value PreferencesProxy::Delete(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Delete"); + return AsyncCall::Call(env, context, "SendablePreferencesDelete"); } napi_value PreferencesProxy::HasKey(napi_env env, napi_callback_info info) @@ -336,7 +337,7 @@ napi_value PreferencesProxy::HasKey(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "HasKey"); + return AsyncCall::Call(env, context, "SendablePreferencesHasKey"); } napi_value PreferencesProxy::Flush(napi_env env, napi_callback_info info) @@ -365,7 +366,7 @@ napi_value PreferencesProxy::Flush(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Flush"); + return AsyncCall::Call(env, context, "SendablePreferencesFlush"); } napi_value PreferencesProxy::Clear(napi_env env, napi_callback_info info) @@ -394,7 +395,7 @@ napi_value PreferencesProxy::Clear(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "Clear"); + return AsyncCall::Call(env, context, "SendablePreferencesClear"); } napi_value PreferencesProxy::RegisterObserver(napi_env env, napi_callback_info info) diff --git a/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences_helper.cpp b/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences_helper.cpp index b8197bed..1f456631 100644 --- a/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences_helper.cpp +++ b/preferences/frameworks/js/napi/sendable_preferences/src/napi_preferences_helper.cpp @@ -95,7 +95,7 @@ napi_value GetPreferences(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "GetPreferences"); + return AsyncCall::Call(env, context, "SendablePreferencesGetPreferences"); } napi_value DeletePreferences(napi_env env, napi_callback_info info) @@ -114,7 +114,7 @@ napi_value DeletePreferences(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "DeletePreferences"); + return AsyncCall::Call(env, context, "SendablePreferencesDeletePreferences"); } napi_value RemovePreferencesFromCache(napi_env env, napi_callback_info info) @@ -133,7 +133,7 @@ napi_value RemovePreferencesFromCache(napi_env env, napi_callback_info info) context->SetAction(env, info, input, exec, output); PRE_CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); - return AsyncCall::Call(env, context, "RemovePreferencesFromCache"); + return AsyncCall::Call(env, context, "SendablePreferencesRemovePreferencesFromCache"); } napi_value InitPreferencesHelper(napi_env env, napi_value exports) diff --git a/preferences/frameworks/js/napi/storage/src/napi_storage.cpp b/preferences/frameworks/js/napi/storage/src/napi_storage.cpp index 37f5787e..8c77ba48 100644 --- a/preferences/frameworks/js/napi/storage/src/napi_storage.cpp +++ b/preferences/frameworks/js/napi/storage/src/napi_storage.cpp @@ -36,11 +36,11 @@ namespace StorageJsKit { struct StorageAysncContext : public BaseContext { std::string key; PreferencesValue defValue = PreferencesValue(static_cast(0)); - std::map allElements; + std::unordered_map allElements; bool hasKey; std::list keysModified; std::vector> preferencesObservers; - + StorageAysncContext() : hasKey(false) { } diff --git a/preferences/frameworks/native/include/concurrent_map.h b/preferences/frameworks/native/include/concurrent_map.h index 0948516f..869fd031 100644 --- a/preferences/frameworks/native/include/concurrent_map.h +++ b/preferences/frameworks/native/include/concurrent_map.h @@ -17,8 +17,8 @@ #define OHOS_DISTRIBUTED_DATA_FRAMEWORKS_COMMON_CONCURRENT_MAP_H #include #include -#include #include +#include namespace OHOS { template diff --git a/preferences/frameworks/native/include/preferences_base.h b/preferences/frameworks/native/include/preferences_base.h index 4a8bdff1..e517978b 100644 --- a/preferences/frameworks/native/include/preferences_base.h +++ b/preferences/frameworks/native/include/preferences_base.h @@ -29,7 +29,6 @@ #include "preferences.h" #include "preferences_observer.h" -#include "preferences_utils.h" namespace OHOS { template class sptr; @@ -106,7 +105,7 @@ public: return E_OK; } - virtual bool IsClose() + virtual bool IsClose(const std::string &name) { return false; } @@ -117,6 +116,8 @@ public: std::string GetBundleName() const override; + virtual std::unordered_map GetAllDatas() override; + protected: Uri MakeUri(const std::string &key = ""); struct WeakPtrCompare { diff --git a/preferences/frameworks/native/include/preferences_enhance_impl.h b/preferences/frameworks/native/include/preferences_enhance_impl.h index 819d8b95..0e766def 100644 --- a/preferences/frameworks/native/include/preferences_enhance_impl.h +++ b/preferences/frameworks/native/include/preferences_enhance_impl.h @@ -57,13 +57,15 @@ public: std::pair GetValue(const std::string &key, const PreferencesValue &defValue) override; std::pair> GetAllData() override; + + std::unordered_map GetAllDatas() override; private: explicit PreferencesEnhanceImpl(const Options &options); static void NotifyPreferencesObserver(std::shared_ptr pref, const std::string &key, const PreferencesValue &value); static void NotifyPreferencesObserverBatchKeys(std::shared_ptr pref, - const std::map &data); - std::pair> GetAllInner(); + const std::unordered_map &data); + std::pair> GetAllInner(); std::shared_mutex dbMutex_; std::shared_ptr db_; diff --git a/preferences/frameworks/native/include/preferences_impl.h b/preferences/frameworks/native/include/preferences_impl.h index 3457c9c2..b30a77ab 100644 --- a/preferences/frameworks/native/include/preferences_impl.h +++ b/preferences/frameworks/native/include/preferences_impl.h @@ -20,12 +20,12 @@ #include #include #include -#include +#include +#include #include #include #include #include "safe_block_queue.h" -#include "concurrent_map.h" #include "preferences_base.h" namespace OHOS { @@ -59,36 +59,42 @@ public: int Close() override; - bool IsClose() override; + bool IsClose(const std::string &name) override; std::pair GetValue(const std::string &key, const PreferencesValue &defValue) override; std::pair> GetAllData() override; + + std::unordered_map GetAllDatas() override; private: explicit PreferencesImpl(const Options &options); - void NotifyPreferencesObserver(const std::list &keysModified, - const std::map &writeToDiskMap); + static void NotifyPreferencesObserver(std::shared_ptr pref, + std::shared_ptr> keysModified, + std::shared_ptr> writeToDisk); bool StartLoadFromDisk(); bool PreLoad(); /* thread function */ static void LoadFromDisk(std::shared_ptr pref); bool ReloadFromDisk(); - void AwaitLoadFile(); - bool WriteSettingXml(const Options &options, const std::map &writeToDiskMap); + inline void AwaitLoadFile(); static int WriteToDiskFile(std::shared_ptr pref); - bool ReadSettingXml(ConcurrentMap &conMap); + bool ReadSettingXml(std::unordered_map &conMap); std::atomic loaded_; bool isNeverUnlock_; bool loadResult_; - std::list modifiedKeys_; + std::unordered_set modifiedKeys_; + + std::atomic isCleared_; std::atomic isActive_; - ConcurrentMap valuesCache_; + std::shared_mutex cacheMutex_; + + std::unordered_map valuesCache_; std::shared_ptr> queue_; }; diff --git a/preferences/frameworks/native/include/preferences_xml_utils.h b/preferences/frameworks/native/include/preferences_xml_utils.h index 274366e8..c4fc35e9 100644 --- a/preferences/frameworks/native/include/preferences_xml_utils.h +++ b/preferences/frameworks/native/include/preferences_xml_utils.h @@ -18,6 +18,9 @@ #include #include +#include + +#include "preferences_value.h" namespace OHOS { namespace NativePreferences { @@ -33,10 +36,9 @@ public: class PreferencesXmlUtils { public: static bool ReadSettingXml(const std::string &fileName, const std::string &bundleName, - std::vector &settings); + std::unordered_map &conMap); static bool WriteSettingXml(const std::string &fileName, const std::string &bundleName, - const std::vector &settings); - static void LimitXmlPermission(const std::string &fileName); + const std::unordered_map &writeToDiskMap); private: PreferencesXmlUtils() diff --git a/preferences/frameworks/native/include/visibility.h b/preferences/frameworks/native/include/visibility.h index 7c7726f4..26bf0fdb 100644 --- a/preferences/frameworks/native/include/visibility.h +++ b/preferences/frameworks/native/include/visibility.h @@ -22,5 +22,8 @@ #ifndef API_LOCAL #define API_LOCAL __attribute__((visibility("hidden"))) #endif +#ifndef UNUSED_FUNCTION +#define UNUSED_FUNCTION __attribute__((unused)) +#endif #endif // OHOS_PREFERENCES_FRAMEWORKS_COMMON_VISIBILITY_H diff --git a/preferences/frameworks/native/platform/include/preferences_db_adapter.h b/preferences/frameworks/native/platform/include/preferences_db_adapter.h index d00f6a77..c620f81b 100644 --- a/preferences/frameworks/native/platform/include/preferences_db_adapter.h +++ b/preferences/frameworks/native/platform/include/preferences_db_adapter.h @@ -16,7 +16,6 @@ #ifndef PREFERENCES_DB_ADAPTER_H #define PREFERENCES_DB_ADAPTER_H -#include #include #include #include @@ -84,6 +83,13 @@ typedef struct GRD_DbValueT { } value; } GRD_DbValueT; +typedef enum { + GRD_EVENT_CORRUPTION, + GRD_EVENT_BOTTOM, +} GRD_EventTypeE; + +typedef void (*GRD_EventCallbackT)(void *callbackContext, const char *eventInfo); + typedef int32_t (*DBOpen)(const char *dbPath, const char *configStr, uint32_t flags, GRD_DB **db); typedef int32_t (*DBClose)(GRD_DB *db, uint32_t flags); typedef int32_t (*DBCreateCollection)(GRD_DB *db, const char *tableName, const char *optionStr, uint32_t flags); @@ -104,6 +110,7 @@ typedef int32_t (*KVFreeItem)(GRD_KVItemT *item); typedef int32_t (*FreeResultSet)(GRD_ResultSet *resultSet); typedef int32_t (*DBRepair)(const char *dbFile, const char *configStr); typedef GRD_DbValueT (*DBGetConfig)(GRD_DB *db, GRD_ConfigTypeE type); +typedef int32_t (*DBSetEventCallback)(void *callbackContext, GRD_EventTypeE type, GRD_EventCallbackT callback); struct GRD_APIInfo { DBOpen DbOpenApi = nullptr; @@ -124,6 +131,7 @@ struct GRD_APIInfo { FreeResultSet FreeResultSetApi = nullptr; DBRepair DbRepairApi = nullptr; DBGetConfig DbGetConfigApi = nullptr; + DBSetEventCallback DbSetEventCallbackApi = nullptr; }; class PreferenceDbAdapter { @@ -131,11 +139,15 @@ public: static bool IsEnhandceDbEnable(); static GRD_APIInfo& GetApiInstance(); static void ApiInit(); + static std::string GetDbEventInfo(); + static void DbEventCallback(void *callbackContext, const char *eventInfo); + static void RegisterDbEventCallback(); static void *gLibrary_; static std::mutex apiMutex_; static GRD_APIInfo api_; static std::atomic isInit_; + static thread_local std::string eventInfo_; }; class PreferencesDb { diff --git a/preferences/frameworks/native/platform/include/preferences_dfx_adapter.h b/preferences/frameworks/native/platform/include/preferences_dfx_adapter.h index dd32d101..02c27520 100644 --- a/preferences/frameworks/native/platform/include/preferences_dfx_adapter.h +++ b/preferences/frameworks/native/platform/include/preferences_dfx_adapter.h @@ -63,7 +63,7 @@ public: static void ReportFault(const ReportFaultParam &reportParam); static std::string GetModuleName(); - static void ReportAbnormalOperation(const ReportParam &reportParam, ReportedFaultBitMap faultOffset); + static void ReportAbnormalOperation(const ReportFaultParam &reportParam, ReportedFaultBitMap faultOffset); private: static ConcurrentMap reportedFaults_; }; diff --git a/preferences/frameworks/native/platform/include/preferences_file_operation.h b/preferences/frameworks/native/platform/include/preferences_file_operation.h index 20127b3c..0d05ef64 100644 --- a/preferences/frameworks/native/platform/include/preferences_file_operation.h +++ b/preferences/frameworks/native/platform/include/preferences_file_operation.h @@ -32,6 +32,7 @@ #include #include +#include #else @@ -65,11 +66,7 @@ constexpr int32_t AREA_MINI_SIZE = 4; constexpr int32_t AREA_OFFSET_SIZE = 5; constexpr int32_t FILE_PATH_MINI_SIZE = 6; -#ifndef UNUSED_FUNCTION -#define UNUSED_FUNCTION __attribute__((unused)) -#endif - -static UNUSED_FUNCTION void *DBDlOpen() +static void *DBDlOpen() { #ifndef _WIN32 return dlopen("libarkdata_db_core.z.so", RTLD_LAZY); @@ -78,7 +75,7 @@ static UNUSED_FUNCTION void *DBDlOpen() #endif } -static UNUSED_FUNCTION int Mkdir(const std::string &filePath) +static int Mkdir(const std::string &filePath) { #if defined(WINDOWS_PLATFORM) return mkdir(filePath.c_str()); @@ -87,7 +84,7 @@ static UNUSED_FUNCTION int Mkdir(const std::string &filePath) #endif } -static UNUSED_FUNCTION int Access(const std::string &filePath) +static int Access(const std::string &filePath) { #if defined(WINDOWS_PLATFORM) return _access(filePath.c_str(), FILE_EXIST); @@ -96,34 +93,57 @@ static UNUSED_FUNCTION int Access(const std::string &filePath) #endif } -static UNUSED_FUNCTION bool Fsync(const std::string &filePath) +static int Open(const std::string &filePath) { #if defined(WINDOWS_PLATFORM) - int fd = _open(filePath.c_str(), _O_WRONLY, _S_IWRITE); - if (fd == -1) { - return false; + return _open(filePath.c_str(), _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE); +#else + return open(filePath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660); +#endif +} + +static int Write(int fd, const unsigned char *buffer, ssize_t count) +{ +#if defined(WINDOWS_PLATFORM) + HANDLE hFile = (HANDLE)_get_osfhandle(fd); // 转换文件描述符为 HANDLE + if (hFile == INVALID_HANDLE_VALUE) { + return -1; } + DWORD bytesWritten = 0; + return WriteFile(hFile, buffer, (DWORD)count, &bytesWritten, NULL) ? (int)bytesWritten : -1; +#else + return write(fd, buffer, count); +#endif +} + +static int Close(int fd) +{ +#if defined(WINDOWS_PLATFORM) + return _close(fd); +#else + return close(fd); +#endif +} + +static bool Fsync(int fd) +{ +#if defined(WINDOWS_PLATFORM) HANDLE handle = (HANDLE)_get_osfhandle(fd); if (handle == INVALID_HANDLE_VALUE || !FlushFileBuffers(handle)) { - _close(fd); return false; } - _close(fd); #else - int fd = open(filePath.c_str(), O_RDWR, S_IRUSR | S_IWUSR); if (fd == -1) { return false; } if (fsync(fd) == -1) { - close(fd); return false; } - close(fd); #endif return true; } -static UNUSED_FUNCTION std::string ExtractFileName(const std::string &path) +static std::string ExtractFileName(const std::string &path) { auto pre = path.find("/"); auto end = path.rfind("/"); diff --git a/preferences/frameworks/native/platform/src/preferences_db_adapter.cpp b/preferences/frameworks/native/platform/src/preferences_db_adapter.cpp index e3a10163..19d0d5d9 100644 --- a/preferences/frameworks/native/platform/src/preferences_db_adapter.cpp +++ b/preferences/frameworks/native/platform/src/preferences_db_adapter.cpp @@ -30,6 +30,8 @@ GRD_APIInfo PreferenceDbAdapter::api_; std::atomic PreferenceDbAdapter::isInit_ = false; +thread_local std::string PreferenceDbAdapter::eventInfo_; + #if !defined(WINDOWS_PLATFORM) static const std::chrono::milliseconds WAIT_REPAIRE_TIMEOUT(5); #endif @@ -65,6 +67,8 @@ void GRDDBApiInitEnhance(GRD_APIInfo &GRD_DBApiInfo) GRD_DBApiInfo.FreeResultSetApi = (FreeResultSet)dlsym(PreferenceDbAdapter::gLibrary_, "GRD_FreeResultSet"); GRD_DBApiInfo.DbRepairApi = (DBRepair)dlsym(PreferenceDbAdapter::gLibrary_, "GRD_DBRepair"); GRD_DBApiInfo.DbGetConfigApi = (DBGetConfig)dlsym(PreferenceDbAdapter::gLibrary_, "GRD_GetConfig"); + GRD_DBApiInfo.DbSetEventCallbackApi = (DBSetEventCallback)dlsym(PreferenceDbAdapter::gLibrary_, + "GRD_SetEventCallback"); #endif } @@ -123,9 +127,38 @@ void PreferenceDbAdapter::ApiInit() LOG_DEBUG("use default db kernel"); } PreferenceDbAdapter::isInit_ = true; + PreferenceDbAdapter::RegisterDbEventCallback(); return; } +std::string PreferenceDbAdapter::GetDbEventInfo() +{ + std::string info = PreferenceDbAdapter::eventInfo_; + eventInfo_ = ""; + return info; +} + +void PreferenceDbAdapter::DbEventCallback(void *callbackContext, const char *eventInfo) +{ + (void)callbackContext; + if (eventInfo != NULL) { + PreferenceDbAdapter::eventInfo_ = eventInfo; + } +} + +void PreferenceDbAdapter::RegisterDbEventCallback() +{ + if (PreferenceDbAdapter::GetApiInstance().DbSetEventCallbackApi == nullptr) { // LCOV_EXCL_BR_LINE + LOG_ERROR("api load failed:DbSetEventCallbackApi"); + return; + } + int errCode = PreferenceDbAdapter::GetApiInstance().DbSetEventCallbackApi(nullptr, GRD_EVENT_CORRUPTION, + &PreferenceDbAdapter::DbEventCallback); + if (errCode != GRD_OK) { // LCOV_EXCL_BR_LINE + LOG_ERROR("set event callback failed, errCode: %{public}d", errCode); + } +} + PreferencesDb::PreferencesDb() { } @@ -270,7 +303,9 @@ int PreferencesDb::Init(const std::string &dbPath, const std::string &bundleName bundleName_ = bundleName; int errCode = OpenDb(false); if (errCode == GRD_DATA_CORRUPTED) { - PreferencesDfxManager::Report(GetReportParam("db corrupted", errCode), EVENT_NAME_DB_CORRUPTED); + std::string info = PreferenceDbAdapter::GetDbEventInfo(); + PreferencesDfxManager::Report(GetReportParam((info.empty() ? "db corrupted" : info), errCode), + EVENT_NAME_DB_CORRUPTED); int innerErr = TryRepairAndRebuild(errCode); if (innerErr != GRD_OK) { // log inside diff --git a/preferences/frameworks/native/platform/src/preferences_dfx_adapter.cpp b/preferences/frameworks/native/platform/src/preferences_dfx_adapter.cpp index 64289ac5..a10a0cdc 100644 --- a/preferences/frameworks/native/platform/src/preferences_dfx_adapter.cpp +++ b/preferences/frameworks/native/platform/src/preferences_dfx_adapter.cpp @@ -20,8 +20,6 @@ #include #include -#include "log_print.h" - #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) #include @@ -72,7 +70,8 @@ std::string PreferencesDfxManager::GetModuleName() return moduleName; } -void PreferencesDfxManager::ReportAbnormalOperation(const ReportParam &reportParam, ReportedFaultBitMap faultOffset) +void PreferencesDfxManager::ReportAbnormalOperation( + const ReportFaultParam &reportParam, ReportedFaultBitMap faultOffset) { uint64_t offset = static_cast(faultOffset); PreferencesDfxManager::reportedFaults_.Compute( @@ -81,7 +80,7 @@ void PreferencesDfxManager::ReportAbnormalOperation(const ReportParam &reportPar if ((report >> offset) & mask) { return true; } - PreferencesDfxManager::Report(reportParam, EVENT_NAME_PREFERENCES_FAULT); + PreferencesDfxManager::ReportFault(reportParam); report |= (mask << offset); return true; }); @@ -194,8 +193,8 @@ void PreferencesDfxManager::ReportFault(const ReportFaultParam &reportParam) { } - -void PreferencesDfxManager::ReportAbnormalOperation(const ReportParam &reportParam, ReportedFaultBitMap faultOffset) +void PreferencesDfxManager::ReportAbnormalOperation( + const ReportFaultParam &reportParam, ReportedFaultBitMap faultOffset) { } diff --git a/preferences/frameworks/native/platform/src/preferences_file_lock.cpp b/preferences/frameworks/native/platform/src/preferences_file_lock.cpp index b2ebebe5..c406b972 100644 --- a/preferences/frameworks/native/platform/src/preferences_file_lock.cpp +++ b/preferences/frameworks/native/platform/src/preferences_file_lock.cpp @@ -44,7 +44,7 @@ std::shared_ptr PreferencesLockManager::Get(const std::string fileNa } #if !defined(WINDOWS_PLATFORM) -static const std::chrono::microseconds WAIT_CONNECT_TIMEOUT(20); +static const std::chrono::milliseconds WAIT_CONNECT_TIMEOUT(20); static const int ATTEMPTS = 50; PreferencesFileLock::PreferencesFileLock(const std::string &path) { diff --git a/preferences/frameworks/native/src/base64_helper.cpp b/preferences/frameworks/native/src/base64_helper.cpp index 61e3c690..e8004349 100644 --- a/preferences/frameworks/native/src/base64_helper.cpp +++ b/preferences/frameworks/native/src/base64_helper.cpp @@ -19,17 +19,18 @@ namespace OHOS { namespace NativePreferences { -static const uint8_t base64Encoder[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const uint8_t base64Encoder[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static const char base64Decoder[] = { +static const int8_t base64Decoder[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, diff --git a/preferences/frameworks/native/src/preferences_base.cpp b/preferences/frameworks/native/src/preferences_base.cpp index cbce9136..83307547 100644 --- a/preferences/frameworks/native/src/preferences_base.cpp +++ b/preferences/frameworks/native/src/preferences_base.cpp @@ -22,9 +22,9 @@ #include #include -#include "base64_helper.h" #include "executor_pool.h" #include "log_print.h" +#include "preferences_utils.h" #include "preferences_dfx_adapter.h" #include "preferences_file_operation.h" #include "preferences_observer_stub.h" @@ -163,7 +163,7 @@ int PreferencesBase::FlushSync() int PreferencesBase::RegisterObserver(std::shared_ptr preferencesObserver, RegisterMode mode) { - IsClose(); + IsClose(std::string(__FUNCTION__)); std::unique_lock writeLock(obseverMetux_); if (mode == RegisterMode::LOCAL_CHANGE) { std::weak_ptr weakPreferencesObserver = preferencesObserver; @@ -192,7 +192,7 @@ int PreferencesBase::RegisterObserver(std::shared_ptr prefe int PreferencesBase::UnRegisterDataObserver(std::shared_ptr preferencesObserver, const std::vector &keys) { - IsClose(); + IsClose(std::string(__FUNCTION__)); std::unique_lock writeLock(obseverMetux_); auto it = dataObserversMap_.find(preferencesObserver); if (it == dataObserversMap_.end()) { @@ -234,8 +234,7 @@ std::pair PreferencesBase::GetValue(const std::string &ke std::pair> PreferencesBase::GetAllData() { - std::map map; - return std::make_pair(E_OK, map); + return {}; } std::string PreferencesBase::GetBundleName() const @@ -246,7 +245,7 @@ std::string PreferencesBase::GetBundleName() const int PreferencesBase::RegisterDataObserver(std::shared_ptr preferencesObserver, const std::vector &keys) { - IsClose(); + IsClose(std::string(__FUNCTION__)); std::unique_lock writeLock(obseverMetux_); auto it = dataObserversMap_.find(preferencesObserver); if (it == dataObserversMap_.end()) { @@ -259,9 +258,14 @@ int PreferencesBase::RegisterDataObserver(std::shared_ptr p return E_OK; } +std::unordered_map PreferencesBase::GetAllDatas() +{ + return {}; +} + int PreferencesBase::UnRegisterObserver(std::shared_ptr preferencesObserver, RegisterMode mode) { - IsClose(); + IsClose(std::string(__FUNCTION__)); std::unique_lock writeLock(obseverMetux_); if (mode == RegisterMode::LOCAL_CHANGE) { for (auto it = localObservers_.begin(); it != localObservers_.end(); ++it) { diff --git a/preferences/frameworks/native/src/preferences_enhance_impl.cpp b/preferences/frameworks/native/src/preferences_enhance_impl.cpp index cddc7bb6..2834e044 100644 --- a/preferences/frameworks/native/src/preferences_enhance_impl.cpp +++ b/preferences/frameworks/native/src/preferences_enhance_impl.cpp @@ -27,6 +27,7 @@ #include "preferences_file_operation.h" #include "log_print.h" #include "preferences_observer_stub.h" +#include "preferences_utils.h" #include "preferences_value.h" #include "preferences_value_parcel.h" @@ -236,9 +237,9 @@ int PreferencesEnhanceImpl::Delete(const std::string &key) return E_OK; } -std::pair> PreferencesEnhanceImpl::GetAllInner() +std::pair> PreferencesEnhanceImpl::GetAllInner() { - std::map map; + std::unordered_map map; if (db_ == nullptr) { LOG_ERROR("PreferencesEnhanceImpl:GetAll failed, db has been closed."); return std::make_pair(E_ALREADY_CLOSED, map); @@ -250,7 +251,7 @@ std::pair> PreferencesEnhanceImpl:: return std::make_pair(E_ERROR, map); } - std::map result; + std::unordered_map result; std::list, std::vector>> data; int errCode = db_->GetAll(data); if (errCode != E_OK) { @@ -276,8 +277,12 @@ std::pair> PreferencesEnhanceImpl:: std::map PreferencesEnhanceImpl::GetAll() { std::unique_lock writeLock(dbMutex_); - std::pair> res = GetAllInner(); - return res.second; + std::pair> res = GetAllInner(); + std::map allDatas; + for (auto &it : res.second) { + allDatas.insert_or_assign(it.first, it.second); + } + return allDatas; } void PreferencesEnhanceImpl::NotifyPreferencesObserver(std::shared_ptr pref, @@ -309,7 +314,7 @@ void PreferencesEnhanceImpl::NotifyPreferencesObserver(std::shared_ptr pref, - const std::map &data) + const std::unordered_map &data) { for (const auto &[key, value] : data) { NotifyPreferencesObserver(pref, key, value); @@ -325,13 +330,13 @@ int PreferencesEnhanceImpl::Clear() return E_ERROR; } - std::pair> res = GetAllInner(); + std::pair> res = GetAllInner(); if (res.first != E_OK) { LOG_ERROR("get all failed when clear, errCode=%{public}d", res.first); return res.first; } - std::map allData = res.second; + std::unordered_map allData = res.second; int errCode = db_->DropCollection(); if (errCode != E_OK) { @@ -420,11 +425,22 @@ std::pair PreferencesEnhanceImpl::GetValue(const std::str } return std::make_pair(E_OK, item.second); } - + std::pair> PreferencesEnhanceImpl::GetAllData() { std::unique_lock writeLock(dbMutex_); - return GetAllInner(); + std::pair> res = GetAllInner(); + std::map allDatas; + for (auto &it : res.second) { + allDatas.insert_or_assign(it.first, it.second); + } + return {res.first, allDatas}; +} + +std::unordered_map PreferencesEnhanceImpl::GetAllDatas() +{ + std::unique_lock writeLock(dbMutex_); + return GetAllInner().second; } } // End of namespace NativePreferences } // End of namespace OHOS diff --git a/preferences/frameworks/native/src/preferences_helper.cpp b/preferences/frameworks/native/src/preferences_helper.cpp index 58949143..5ddf82e7 100644 --- a/preferences/frameworks/native/src/preferences_helper.cpp +++ b/preferences/frameworks/native/src/preferences_helper.cpp @@ -22,13 +22,13 @@ #include "log_print.h" #include "preferences.h" -#include "preferences_db_adapter.h" #include "preferences_errno.h" #include "preferences_file_lock.h" #include "preferences_file_operation.h" #include "preferences_dfx_adapter.h" #include "preferences_impl.h" #include "preferences_enhance_impl.h" +#include "preferences_utils.h" namespace OHOS { namespace NativePreferences { @@ -133,7 +133,7 @@ int PreferencesHelper::GetPreferencesInner(const Options &options, bool &isEnhan std::shared_ptr &pref) { if (IsInTrustList(options.bundleName)) { - if (!IsFileExist(options.filePath) && IsStorageTypeSupported(StorageType::CLKV)) { + if (!IsFileExist(options.filePath) && IsStorageTypeSupported(StorageType::GSKV)) { pref = PreferencesEnhanceImpl::GetPreferences(options); isEnhancePreferences = true; return std::static_pointer_cast(pref)->Init(); @@ -145,19 +145,19 @@ int PreferencesHelper::GetPreferencesInner(const Options &options, bool &isEnhan if (!options.isEnhance) { // xml if (IsFileExist(options.filePath + ".db")) { - LOG_ERROR("CLKV exists, failed to get preferences by XML."); + LOG_ERROR("GSKV exists, failed to get preferences by XML."); return E_NOT_SUPPORTED; } pref = PreferencesImpl::GetPreferences(options); isEnhancePreferences = false; return std::static_pointer_cast(pref)->Init(); } - // clkv + // GSKV if (IsFileExist(options.filePath)) { - LOG_ERROR("XML exists, failed to get preferences by CLKV."); + LOG_ERROR("XML exists, failed to get preferences by GSKV."); return E_NOT_SUPPORTED; } - if (!IsStorageTypeSupported(StorageType::CLKV)) { + if (!IsStorageTypeSupported(StorageType::GSKV)) { // log inside return E_NOT_SUPPORTED; } @@ -191,9 +191,9 @@ std::shared_ptr PreferencesHelper::GetPreferences(const Options &op if (Access(filePath.c_str()) != 0) { LOG_ERROR("The path is invalid, prefName is %{public}s.", ExtractFileName(filePath).c_str()); if (!PreferencesHelper::isReportFault_.exchange(true)) { - ReportParam param = { options.bundleName, NORMAL_DB, ExtractFileName(options.filePath), - E_INVALID_FILE_PATH, 2, "The path is invalid." }; - PreferencesDfxManager::Report(param, EVENT_NAME_PREFERENCES_FAULT); + ReportFaultParam param = { "GetPreferences error", options.bundleName, NORMAL_DB, + ExtractFileName(options.filePath), E_INVALID_FILE_PATH, "The path is invalid." }; + PreferencesDfxManager::ReportFault(param); } } bool isEnhancePreferences = false; @@ -260,9 +260,9 @@ int PreferencesHelper::DeletePreferences(const std::string &path) fileLock.WriteLock(isMultiProcessing); if (isMultiProcessing) { LOG_ERROR("The file has cross-process operations, fileName is %{public}s.", ExtractFileName(filePath).c_str()); - ReportParam param = { bundleName, NORMAL_DB, ExtractFileName(path), - E_OPERAT_IS_CROSS_PROESS, 0, "Cross-process operations exist during file deleting." }; - PreferencesDfxManager::Report(param, EVENT_NAME_PREFERENCES_FAULT); + ReportFaultParam param = { "DeletePreferences error", bundleName, NORMAL_DB, ExtractFileName(filePath), + E_OPERAT_IS_CROSS_PROESS, "Cross-process operations." }; + PreferencesDfxManager::ReportFault(param); } std::remove(filePath.c_str()); std::remove(backupPath.c_str()); @@ -316,11 +316,11 @@ bool PreferencesHelper::IsStorageTypeSupported(const StorageType &type) if (type == StorageType::XML) { return true; } - if (type == StorageType::CLKV) { + if (type == StorageType::GSKV) { #if !defined(CROSS_PLATFORM) && defined(ARKDATA_DATABASE_CORE_ENABLE) return true; #else - LOG_WARN("CLKV not support this platform."); + LOG_WARN("GSKV not support this platform."); return false; #endif } diff --git a/preferences/frameworks/native/src/preferences_impl.cpp b/preferences/frameworks/native/src/preferences_impl.cpp index 0e96f96b..85527068 100644 --- a/preferences/frameworks/native/src/preferences_impl.cpp +++ b/preferences/frameworks/native/src/preferences_impl.cpp @@ -20,12 +20,10 @@ #include #include #include -#include #include #include #include -#include "base64_helper.h" #include "executor_pool.h" #include "log_print.h" #include "preferences_observer_stub.h" @@ -33,6 +31,7 @@ #include "preferences_file_operation.h" #include "preferences_anonymous.h" #include "preferences_dfx_adapter.h" +#include "preferences_utils.h" namespace OHOS { namespace NativePreferences { @@ -42,91 +41,6 @@ using namespace std::chrono; constexpr int32_t WAIT_TIME = 2; constexpr int32_t TASK_EXEC_TIME = 100; constexpr int32_t LOAD_XML_LOG_TIME = 1000; - -template -std::string GetTypeName() -{ - return "unknown"; -} - -template<> -std::string GetTypeName() -{ - return "int"; -} - -template<> -std::string GetTypeName() -{ - return "bool"; -} - -template<> -std::string GetTypeName() -{ - return "long"; -} - -template<> -std::string GetTypeName() -{ - return "uint64_t"; -} - -template<> -std::string GetTypeName() -{ - return "float"; -} - -template<> -std::string GetTypeName() -{ - return "double"; -} - -template<> -std::string GetTypeName() -{ - return "string"; -} - -template<> -std::string GetTypeName>() -{ - return "stringArray"; -} - -template<> -std::string GetTypeName>() -{ - return "doubleArray"; -} - -template<> -std::string GetTypeName>() -{ - return "boolArray"; -} - -template<> -std::string GetTypeName>() -{ - return "uint8Array"; -} - -template<> -std::string GetTypeName() -{ - return "object"; -} - -template<> -std::string GetTypeName() -{ - return "BigInt"; -} - PreferencesImpl::PreferencesImpl(const Options &options) : PreferencesBase(options) { loaded_.store(false); @@ -134,6 +48,7 @@ PreferencesImpl::PreferencesImpl(const Options &options) : PreferencesBase(optio loadResult_= false; queue_ = std::make_shared>(1); isActive_.store(true); + isCleared_.store(false); } PreferencesImpl::~PreferencesImpl() @@ -172,11 +87,12 @@ void PreferencesImpl::LoadFromDisk(std::shared_ptr pref) if (Access(filePath) != 0) { pref->isNeverUnlock_ = true; } - ConcurrentMap values; + std::unordered_map values; bool loadResult = pref->ReadSettingXml(values); if (!loadResult) { LOG_WARN("The settingXml %{public}s load failed.", ExtractFileName(pref->options_.filePath).c_str()); } else { + std::unique_lockcacheMutex_)> lock(pref->cacheMutex_); pref->valuesCache_ = std::move(values); pref->loadResult_ = true; pref->isNeverUnlock_ = false; @@ -192,7 +108,8 @@ bool PreferencesImpl::ReloadFromDisk() return false; } - ConcurrentMap values = valuesCache_; + std::unique_lock lock(cacheMutex_); + std::unordered_map values = valuesCache_; bool loadResult = ReadSettingXml(values); LOG_WARN("The settingXml %{public}s reload result is %{public}d", ExtractFileName(options_.filePath).c_str(), loadResult); @@ -242,11 +159,14 @@ PreferencesValue PreferencesImpl::Get(const std::string &key, const PreferencesV } AwaitLoadFile(); - IsClose(); + IsClose(std::string(__FUNCTION__)); - auto it = valuesCache_.Find(key); - if (it.first) { - return it.second; + std::shared_lock lock(cacheMutex_); + if (!isCleared_.load()) { + auto iter = valuesCache_.find(key); + if (iter != valuesCache_.end()) { + return iter->second; + } } return defValue; } @@ -254,93 +174,15 @@ PreferencesValue PreferencesImpl::Get(const std::string &key, const PreferencesV std::map PreferencesImpl::GetAll() { AwaitLoadFile(); - IsClose(); - return valuesCache_.Clone(); -} - -template -static void Convert2PrefValue(const Element &element, T &value) -{ - if constexpr (std::is_same::value) { - value = (element.value_.compare("true") == 0) ? true : false; - } else if constexpr (std::is_same::value) { - value = element.value_; - } else if constexpr (std::is_same::value) { - value = std::monostate(); - } else { - std::stringstream ss; - ss << element.value_; - ss >> value; - } -} - -template -static void Convert2PrefValue(const Element &element, std::vector &values) -{ - for (const auto &child : element.children_) { - T value; - Convert2PrefValue(child, value); - values.push_back(value); - } -} - -static void Convert2PrefValue(const Element &element, BigInt &value) -{ - for (const auto &child : element.children_) { - uint64_t val; - Convert2PrefValue(child, val); - value.words_.push_back(val); - } - value.sign_ = 0; - if (!value.words_.empty()) { - value.sign_ = static_cast(value.words_[value.words_.size() - 1]); - value.words_.pop_back(); - } -} - -template -bool GetPrefValue(const Element &element, T &value) -{ - LOG_WARN("unknown element type. the key is %{public}s", Anonymous::ToBeAnonymous(element.key_).c_str()); - return false; -} - -static void Convert2PrefValue(const Element &element, std::vector &value) -{ - if (!Base64Helper::Decode(element.value_, value)) { - value.clear(); - } -} - -static void Convert2PrefValue(const Element &element, Object &value) -{ - value.valueStr = element.value_; -} - -template -bool GetPrefValue(const Element &element, T &value) -{ - if (element.tag_ == GetTypeName()) { - First val; - Convert2PrefValue(element, val); - value = val; - return true; - } - return GetPrefValue(element, value); -} - -template -bool Convert2PrefValue(const Element &element, std::variant &value) -{ - return GetPrefValue(element, value); -} - -void ReadXmlElement(const Element &element, ConcurrentMap &prefConMap) -{ - PreferencesValue value(static_cast(0)); - if (Convert2PrefValue(element, value.value_)) { - prefConMap.Insert(element.key_, value); + IsClose(std::string(__FUNCTION__)); + std::map allDatas; + std::shared_lock lock(cacheMutex_); + if (!isCleared_.load()) { + for (auto &it : valuesCache_) { + allDatas.insert_or_assign(it.first, it.second); + } } + return allDatas; } static int64_t GetFileSize(const std::string &path) @@ -353,11 +195,10 @@ static int64_t GetFileSize(const std::string &path) return fileSize; } -bool PreferencesImpl::ReadSettingXml(ConcurrentMap &conMap) +bool PreferencesImpl::ReadSettingXml(std::unordered_map &conMap) { - std::vector settings; auto begin = static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); - if (!PreferencesXmlUtils::ReadSettingXml(options_.filePath, options_.bundleName, settings)) { + if (!PreferencesXmlUtils::ReadSettingXml(options_.filePath, options_.bundleName, conMap)) { return false; } auto end = static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); @@ -365,126 +206,29 @@ bool PreferencesImpl::ReadSettingXml(ConcurrentMap -void Convert2Element(Element &elem, const T &value) -{ - elem.tag_ = GetTypeName(); - if constexpr (std::is_same::value) { - elem.value_ = ((bool)value) ? "true" : "false"; - } else if constexpr (std::is_same::value) { - elem.value_ = value; - } else if constexpr (std::is_same::value) { - elem.value_ = {}; - } else { - elem.value_ = std::to_string(value); - } -} - -template -void Convert2Element(Element &elem, const std::vector &value) -{ - elem.tag_ = GetTypeName>(); - for (const T &val : value) { - Element element; - Convert2Element(element, val); - elem.children_.push_back(element); - } -} - -void Convert2Element(Element &elem, const std::vector &value) -{ - elem.tag_ = GetTypeName>(); - elem.value_ = Base64Helper::Encode(value); -} - -void Convert2Element(Element &elem, const Object &value) -{ - elem.tag_ = GetTypeName(); - elem.value_ = value.valueStr; -} - -void Convert2Element(Element &elem, const BigInt &value) -{ - elem.tag_ = GetTypeName(); - for (const auto &val : value.words_) { - Element element; - Convert2Element(element, val); - elem.children_.push_back(element); - } - // place symbol at the end - Element symbolElement; - Convert2Element(symbolElement, static_cast(value.sign_)); - elem.children_.push_back(symbolElement); -} - -template void GetElement(Element &elem, const T &value) -{ - LOG_WARN("unknown element type. the key is %{public}s", Anonymous::ToBeAnonymous(elem.key_).c_str()); -} - -template void GetElement(Element &elem, const T &value) -{ - auto *val = std::get_if(&value); - if (val != nullptr) { - return Convert2Element(elem, *val); - } - return GetElement(elem, value); -} - -template void Convert2Element(Element &elem, const std::variant &value) -{ - return GetElement(elem, value); -} - -void WriteXmlElement(Element &elem, const PreferencesValue &value) -{ - Convert2Element(elem, value.value_); -} - int PreferencesImpl::Close() { isActive_.store(false); return E_OK; } -bool PreferencesImpl::IsClose() +bool PreferencesImpl::IsClose(const std::string &name) { if (isActive_.load()) { return false; } LOG_WARN("file %{public}s is inactive.", ExtractFileName(options_.filePath).c_str()); - std::string operationMsg = "operation: Invalid operation on the preference instance."; - ReportParam reportParam = { options_.bundleName, NORMAL_DB, ExtractFileName(options_.filePath), - E_OBJECT_NOT_ACTIVE, 0, operationMsg}; + std::string operationMsg = " use after close."; + ReportFaultParam reportParam = { "inactive object", options_.bundleName, NORMAL_DB, + ExtractFileName(options_.filePath), E_OBJECT_NOT_ACTIVE, name + operationMsg }; PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::OBJECT_IS_NOT_ACTIVE); return true; } -bool PreferencesImpl::WriteSettingXml( - const Options &options, const std::map &writeToDiskMap) -{ - std::vector settings; - for (auto it = writeToDiskMap.begin(); it != writeToDiskMap.end(); it++) { - Element elem; - elem.key_ = it->first; - PreferencesValue value = it->second; - WriteXmlElement(elem, value); - settings.push_back(elem); - } - - return PreferencesXmlUtils::WriteSettingXml(options.filePath, options.bundleName, settings); -} - - bool PreferencesImpl::HasKey(const std::string &key) { if (CheckKey(key) != E_OK) { @@ -492,8 +236,12 @@ bool PreferencesImpl::HasKey(const std::string &key) } AwaitLoadFile(); - IsClose(); - return valuesCache_.Contains(key); + IsClose(std::string(__FUNCTION__)); + std::shared_lock lock(cacheMutex_); + if (isCleared_.load()) { + return false; + } + return valuesCache_.find(key) != valuesCache_.end(); } int PreferencesImpl::Put(const std::string &key, const PreferencesValue &value) @@ -507,16 +255,28 @@ int PreferencesImpl::Put(const std::string &key, const PreferencesValue &value) return errCode; } AwaitLoadFile(); - IsClose(); + IsClose(std::string(__FUNCTION__)); - valuesCache_.Compute(key, [this, &value](auto &key, PreferencesValue &val) { - if (val == value) { - return true; + std::unique_lock lock(cacheMutex_); + if (isCleared_.load()) { // has cleared. + for (auto &it : valuesCache_) { + modifiedKeys_.emplace(it.first); } - val = value; - modifiedKeys_.push_back(key); - return true; - }); + valuesCache_.clear(); + valuesCache_.insert_or_assign(key, value); + modifiedKeys_.emplace(key); + isCleared_.store(false); + } else { + auto iter = valuesCache_.find(key); + if (iter != valuesCache_.end()) { + PreferencesValue &val = iter->second; + if (val == value) { + return E_OK; + } + } + valuesCache_.insert_or_assign(key, value); + modifiedKeys_.emplace(key); + } return E_OK; } @@ -527,43 +287,49 @@ int PreferencesImpl::Delete(const std::string &key) return errCode; } AwaitLoadFile(); - IsClose(); - valuesCache_.EraseIf(key, [this](auto &key, PreferencesValue &val) { - modifiedKeys_.push_back(key); - return true; - }); + IsClose(std::string(__FUNCTION__)); + std::unique_lock lock(cacheMutex_); + if (isCleared_.load()) { + return E_OK; + } + if (valuesCache_.find(key) != valuesCache_.end()) { + valuesCache_.erase(key); + modifiedKeys_.emplace(key); + } return E_OK; } int PreferencesImpl::Clear() { AwaitLoadFile(); - IsClose(); - valuesCache_.EraseIf([this](auto &key, PreferencesValue &val) { - modifiedKeys_.push_back(key); - return true; - }); + IsClose(std::string(__FUNCTION__)); + isCleared_.store(true); return E_OK; } int PreferencesImpl::WriteToDiskFile(std::shared_ptr pref) { - std::list keysModified; - std::map writeToDiskMap; - pref->valuesCache_.DoActionWhenClone( - [pref, &writeToDiskMap, &keysModified](const std::map &map) { + auto keysModified = std::make_shared>(); + auto writeToDiskMap = std::make_shared>(); + { + std::unique_lockcacheMutex_)> lock(pref->cacheMutex_); + if (pref->isCleared_.load()) { + for (auto &it : pref->valuesCache_) { + pref->modifiedKeys_.emplace(it.first); + } + pref->valuesCache_.clear(); + pref->isCleared_.store(false); + } if (!pref->modifiedKeys_.empty()) { - keysModified = std::move(pref->modifiedKeys_); + *keysModified = std::move(pref->modifiedKeys_); + *writeToDiskMap = pref->valuesCache_; + } else { + // Cache has not changed, Not need to write persistent files. + LOG_INFO("No data to update persistent file"); + return E_OK; } - writeToDiskMap = std::move(map); - }); - - // Cache has not changed, Not need to write persistent files. - if (keysModified.empty()) { - LOG_INFO("No data to update persistent file"); - return E_OK; } - if (!pref->WriteSettingXml(pref->options_, writeToDiskMap)) { + if (!PreferencesXmlUtils::WriteSettingXml(pref->options_.filePath, pref->options_.bundleName, *writeToDiskMap)) { return E_ERROR; } if (pref->isNeverUnlock_) { @@ -572,13 +338,14 @@ int PreferencesImpl::WriteToDiskFile(std::shared_ptr pref) if (!pref->loadResult_) { pref->loadResult_ = true; } - pref->NotifyPreferencesObserver(keysModified, writeToDiskMap); + + NotifyPreferencesObserver(pref, keysModified, writeToDiskMap); return E_OK; } void PreferencesImpl::Flush() { - IsClose(); + IsClose(std::string(__FUNCTION__)); auto success = queue_->PushNoWait(1); if (!success) { return; @@ -605,7 +372,7 @@ void PreferencesImpl::Flush() int PreferencesImpl::FlushSync() { - IsClose(); + IsClose(std::string(__FUNCTION__)); auto success = queue_->PushNoWait(1); if (success) { if (queue_ == nullptr) { @@ -632,10 +399,13 @@ std::pair PreferencesImpl::GetValue(const std::string &ke } AwaitLoadFile(); - IsClose(); - auto iter = valuesCache_.Find(key); - if (iter.first) { - return std::make_pair(E_OK, iter.second); + IsClose(std::string(__FUNCTION__)); + std::shared_lock lock(cacheMutex_); + if (!isCleared_.load()) { + auto iter = valuesCache_.find(key); + if (iter != valuesCache_.end()) { + return std::make_pair(E_OK, iter->second); + } } return std::make_pair(E_NO_DATA, defValue); } @@ -643,57 +413,79 @@ std::pair PreferencesImpl::GetValue(const std::string &ke std::pair> PreferencesImpl::GetAllData() { AwaitLoadFile(); - IsClose(); - return std::make_pair(E_OK, valuesCache_.Clone()); + IsClose(std::string(__FUNCTION__)); + std::map allDatas; + std::shared_lock lock(cacheMutex_); + if (!isCleared_.load()) { + for (auto &it : valuesCache_) { + allDatas.insert_or_assign(it.first, it.second); + } + } + return std::make_pair(E_OK, allDatas); } -void PreferencesImpl::NotifyPreferencesObserver(const std::list &keysModified, - const std::map &writeToDiskMap) +std::unordered_map PreferencesImpl::GetAllDatas() { - if (keysModified.empty()) { + AwaitLoadFile(); + IsClose(std::string(__FUNCTION__)); + std::shared_lock lock(cacheMutex_); + if (!isCleared_.load()) { + return valuesCache_; + } + return {}; +} + +void PreferencesImpl::NotifyPreferencesObserver(std::shared_ptr pref, + std::shared_ptr> keysModified, + std::shared_ptr> writeToDisk) +{ + if (keysModified->empty()) { return; } - LOG_DEBUG("notify observer size:%{public}zu", dataObserversMap_.size()); - std::shared_lock autoLock(obseverMetux_); - for (const auto &[weakPrt, keys] : dataObserversMap_) { + std::shared_lock autoLock(pref->obseverMetux_); + for (const auto &[weakPrt, keys] : pref->dataObserversMap_) { std::map records; - for (auto key = keysModified.begin(); key != keysModified.end(); ++key) { - auto itKey = keys.find(*key); + for (auto &key : *keysModified) { + auto itKey = keys.find(key); if (itKey == keys.end()) { continue; } PreferencesValue value; - auto dataIt = writeToDiskMap.find(*key); - if (dataIt != writeToDiskMap.end()) { + auto dataIt = writeToDisk->find(key); + if (dataIt != writeToDisk->end()) { value = dataIt->second; } - records.insert({*key, value}); + records.insert({key, value}); } if (records.empty()) { continue; } if (std::shared_ptr sharedPtr = weakPrt.lock()) { - LOG_DEBUG("dataChange observer call, resultSize:%{public}zu", records.size()); sharedPtr->OnChange(records); } } - auto dataObsMgrClient = DataObsMgrClient::GetInstance(); - for (auto key = keysModified.begin(); key != keysModified.end(); ++key) { - for (auto it = localObservers_.begin(); it != localObservers_.end(); ++it) { - std::weak_ptr weakPreferencesObserver = *it; + for (auto &it : pref->localObservers_) { + for (auto &key : *keysModified) { + std::weak_ptr weakPreferencesObserver = it; if (std::shared_ptr sharedPreferencesObserver = weakPreferencesObserver.lock()) { - sharedPreferencesObserver->OnChange(*key); + sharedPreferencesObserver->OnChange(key); } } + } + ExecutorPool::Task task = [pref, keysModified] { + auto dataObsMgrClient = DataObsMgrClient::GetInstance(); if (dataObsMgrClient == nullptr) { - continue; + return; } - LOG_INFO("The %{public}s is changed, the observer needs to be triggered.", - Anonymous::ToBeAnonymous(*key).c_str()); - dataObsMgrClient->NotifyChange(MakeUri(*key)); - } + for (auto &key : *keysModified) { + LOG_INFO("The %{public}s is changed, the observer needs to be triggered.", + Anonymous::ToBeAnonymous(key).c_str()); + dataObsMgrClient->NotifyChange(pref->MakeUri(key)); + } + }; + executorPool_.Execute(std::move(task)); } } // End of namespace NativePreferences } // End of namespace OHOS diff --git a/preferences/frameworks/native/src/preferences_xml_utils.cpp b/preferences/frameworks/native/src/preferences_xml_utils.cpp index 5204394e..cf8be878 100644 --- a/preferences/frameworks/native/src/preferences_xml_utils.cpp +++ b/preferences/frameworks/native/src/preferences_xml_utils.cpp @@ -14,31 +14,361 @@ */ #include "preferences_xml_utils.h" - +#include "base64_helper.h" #include #include #include +#include #include "libxml/parser.h" +#include #include "log_print.h" #include "preferences_dfx_adapter.h" #include "preferences_file_lock.h" #include "preferences_file_operation.h" -#include "preferences_impl.h" +#include "preferences_utils.h" namespace OHOS { namespace NativePreferences { +constexpr int PRE_ALLOCATE_BUFFER_SIZE = 128; +constexpr int NO_SPACE_LEFT_ON_DEVICE = 28; +constexpr int DISK_QUOTA_EXCEEDED = 122; constexpr int REQUIRED_KEY_NOT_AVAILABLE = 126; constexpr int REQUIRED_KEY_REVOKED = 128; + +constexpr const char *TAG_PREFERENCES = "preferences"; +constexpr const char *TAG_VERSION = "version"; +constexpr const char *VERSION_VALUE = "1.0"; +constexpr const char *ATTR_KEY = "key"; +constexpr const char *ATTR_VALUE = "value"; + static bool ParseNodeElement(const xmlNode *node, Element &element); static bool ParsePrimitiveNodeElement(const xmlNode *node, Element &element); static bool ParseStringNodeElement(const xmlNode *node, Element &element); static bool ParseArrayNodeElement(const xmlNode *node, Element &element); -static xmlNode *CreateElementNode(Element &element); -static xmlNode *CreatePrimitiveNode(Element &element); -static xmlNode *CreateStringNode(Element &element); -static xmlNode *CreateArrayNode(Element &element); + +struct ValueVisitor { + std::string_view typeTag; + std::string valueStr; + std::string_view childrenTag; + std::vector arrayValues; + + void operator()(int val) + { + typeTag = "int"; + valueStr = std::to_string(val); + } + + void operator()(bool val) + { + typeTag = "bool"; + valueStr = val ? "true" : "false"; + } + + void operator()(int64_t val) + { + typeTag = "long"; + valueStr = std::to_string(val); + } + + void operator()(uint64_t val) + { + typeTag = "uint64_t"; + valueStr = std::to_string(val); + } + + void operator()(float val) + { + typeTag = "float"; + valueStr = std::to_string(val); + } + + void operator()(double val) + { + typeTag = "double"; + valueStr = std::to_string(val); + } + + void operator()(const std::string& val) + { + typeTag = "string"; + valueStr = val; + } + + void operator()(const std::vector& val) + { + typeTag = "stringArray"; + childrenTag = "string"; + arrayValues.reserve(val.size()); + for (const auto& s : val) { + arrayValues.push_back(s); + } + } + + void operator()(const std::vector& val) + { + typeTag = "doubleArray"; + childrenTag = "double"; + arrayValues.reserve(val.size()); + for (const auto& d : val) { + arrayValues.push_back(std::to_string(d)); + } + } + + void operator()(const std::vector& val) + { + typeTag = "boolArray"; + childrenTag = "bool"; + arrayValues.reserve(val.size()); + for (const auto& b : val) { + arrayValues.push_back(b ? "true" : "false"); + } + } + + void operator()(const std::vector& val) + { + typeTag = "uint8Array"; + valueStr = Base64Helper::Encode(val); + } + + void operator()(const Object& val) + { + typeTag = "object"; + valueStr = val.valueStr; + } + + void operator()(const BigInt& val) + { + typeTag = "BigInt"; + childrenTag = "uint64_t"; + arrayValues.reserve(val.words_.size() + 1); + for (const auto& word : val.words_) { + arrayValues.push_back(std::to_string(word)); + } + arrayValues.push_back(std::to_string(static_cast(val.sign_))); + } + + void operator()(const std::monostate&) + { + typeTag = "unknown"; + LOG_ERROR("Encountered unknown type in PreferencesValue"); + } +}; + +class XmlWriterWrapper { +public: + explicit XmlWriterWrapper(xmlTextWriterPtr writer) : writer_(writer) {} + ~XmlWriterWrapper() + { + if (writer_) { + xmlFreeTextWriter(writer_); + } + } + + xmlTextWriterPtr get() const + { + return writer_; + } +private: + xmlTextWriterPtr writer_; +}; + +class XmlBufferWrapper { +public: + explicit XmlBufferWrapper(xmlBufferPtr buffer) : buffer_(buffer) {} + ~XmlBufferWrapper() + { + if (buffer_) { + xmlBufferFree(buffer_); + } + } + + xmlBufferPtr get() const + { + return buffer_; + } +private: + xmlBufferPtr buffer_; +}; + +#define XML_CHECK(expr, msg) \ + do { \ + if (!(expr)) { \ + xmlErrorPtr xmlErr = xmlGetLastError(); \ + LOG_ERROR("%{public}s. Error: %{public}s", msg, xmlErr ? xmlErr->message : "Unknown"); \ + return false; \ + } \ + } while (0) + +template +std::string GetTypeName() +{ + return "unknown"; +} + +template<> +std::string GetTypeName() +{ + return "int"; +} + +template<> +std::string GetTypeName() +{ + return "bool"; +} + +template<> +std::string GetTypeName() +{ + return "long"; +} + +template<> +std::string GetTypeName() +{ + return "uint64_t"; +} + +template<> +std::string GetTypeName() +{ + return "float"; +} + +template<> +std::string GetTypeName() +{ + return "double"; +} + +template<> +std::string GetTypeName() +{ + return "string"; +} + +template<> +std::string GetTypeName>() +{ + return "stringArray"; +} + +template<> +std::string GetTypeName>() +{ + return "doubleArray"; +} + +template<> +std::string GetTypeName>() +{ + return "boolArray"; +} + +template<> +std::string GetTypeName>() +{ + return "uint8Array"; +} + +template<> +std::string GetTypeName() +{ + return "object"; +} + +template<> +std::string GetTypeName() +{ + return "BigInt"; +} + +template +static void Convert2PrefValue(const Element &element, T &value) +{ + if constexpr (std::is_same::value) { + value = element.value_; + } else if constexpr (std::is_same::value) { + value = (element.value_.compare("true") == 0) ? true : false; + } else if constexpr (std::is_same::value) { + value = std::monostate(); + } else { + std::stringstream ss; + ss << element.value_; + ss >> value; + } +} + +template +static void Convert2PrefValue(const Element &element, std::vector &values) +{ + for (const auto &child : element.children_) { + T value; + Convert2PrefValue(child, value); + values.push_back(value); + } +} + +static void Convert2PrefValue(const Element &element, BigInt &value) +{ + for (const auto &child : element.children_) { + uint64_t val; + Convert2PrefValue(child, val); + value.words_.push_back(val); + } + value.sign_ = 0; + if (!value.words_.empty()) { + value.sign_ = static_cast(value.words_[value.words_.size() - 1]); + value.words_.pop_back(); + } +} + +template +bool GetPrefValue(const Element &element, T &value) +{ + LOG_WARN("unknown element type. the key is %{public}s", Anonymous::ToBeAnonymous(element.key_).c_str()); + return false; +} + +static void Convert2PrefValue(const Element &element, std::vector &value) +{ + if (!Base64Helper::Decode(element.value_, value)) { + value.clear(); + } +} + +static void Convert2PrefValue(const Element &element, Object &value) +{ + value.valueStr = element.value_; +} + +template +bool GetPrefValue(const Element &element, T &value) +{ + if (element.tag_ == GetTypeName()) { + First val; + Convert2PrefValue(element, val); + value = val; + return true; + } + return GetPrefValue(element, value); +} + +template +bool Convert2PrefValue(const Element &element, std::variant &value) +{ + return GetPrefValue(element, value); +} + +void ReadXmlElement(const Element &element, std::unordered_map &prefConMap) +{ + PreferencesValue value(static_cast(0)); + if (Convert2PrefValue(element, value.value_)) { + prefConMap.insert({element.key_, value}); + } +} static bool IsFileExist(const std::string &inputPath) { @@ -73,17 +403,38 @@ static void ReportXmlFileCorrupted(const std::string &fileName, const std::strin ReportParam succreportParam = reportParam; succreportParam.errCode = E_OK; succreportParam.errnoCode = 0; - succreportParam.appendix = "operation: restore success"; + succreportParam.appendix = "restore success"; PreferencesDfxManager::Report(succreportParam, EVENT_NAME_DB_CORRUPTED); } -static bool RenameFromBackupFile(const std::string &fileName, const std::string &bundleName, bool &isReportCorrupt) +static bool ReportNonCorruptError( + const std::string &faultType, const std::string &fileName, const std::string &bundleName, int errCode) +{ + if (errCode == REQUIRED_KEY_NOT_AVAILABLE || errCode == REQUIRED_KEY_REVOKED) { + ReportFaultParam reportParam = { faultType, bundleName, NORMAL_DB, ExtractFileName(fileName), + E_OPERAT_IS_LOCKED, faultType + " the screen is locked." }; + PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::USE_WHEN_SCREEN_LOCKED); + return true; + } + if (errCode == NO_SPACE_LEFT_ON_DEVICE || errCode == DISK_QUOTA_EXCEEDED) { + ReportFaultParam param = { faultType, bundleName, NORMAL_DB, ExtractFileName(fileName), + E_ERROR, faultType + " " + std::strerror(errCode)}; + PreferencesDfxManager::ReportFault(param); + return true; + } + return false; +} + +static bool RenameFromBackupFile( + const std::string &fileName, const std::string &bundleName, bool &isReportCorrupt, bool &isBakFileExist) { std::string backupFileName = MakeFilePath(fileName, STR_BACKUP); if (!IsFileExist(backupFileName)) { + isBakFileExist = false; LOG_DEBUG("the backup file does not exist."); return false; } + isBakFileExist = true; xmlResetLastError(); int errCode = 0; auto bakDoc = std::shared_ptr(ReadFile(backupFileName, errCode), @@ -94,11 +445,7 @@ static bool RenameFromBackupFile(const std::string &fileName, const std::string LOG_ERROR("restore XML file: %{public}s failed, errno is %{public}d, error is %{public}s.", ExtractFileName(fileName).c_str(), errCode, errMessage.c_str()); std::remove(backupFileName.c_str()); - if (errCode == REQUIRED_KEY_NOT_AVAILABLE || errCode == REQUIRED_KEY_REVOKED) { - std::string operationMsg = "Read bak file when the screen is locked."; - const ReportParam reportParam = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_OBJECT_NOT_ACTIVE, errCode, operationMsg}; - PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::USE_WHEN_SCREEN_LOCKED); + if (ReportNonCorruptError("read bak failed", fileName, bundleName, errCode)) { return false; } isReportCorrupt = true; @@ -114,8 +461,8 @@ static bool RenameFromBackupFile(const std::string &fileName, const std::string LOG_ERROR("failed to stat backup file."); } std::string appindex = "Restored from the backup. The file size is " + std::to_string(fileStats.st_size) + "."; - const ReportParam reportParam = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_XML_RESTORED_FROM_BACKUP_FILE, 0, appindex}; + ReportFaultParam reportParam = { "read failed", bundleName, NORMAL_DB, ExtractFileName(fileName), + E_XML_RESTORED_FROM_BACKUP_FILE, appindex }; PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::RESTORE_FROM_BAK); LOG_INFO("restore XML file %{public}s successfully.", ExtractFileName(fileName).c_str()); return true; @@ -161,11 +508,7 @@ static xmlDoc *XmlReadFile(const std::string &fileName, const std::string &bundl errMessage = (xmlErr != nullptr) ? xmlErr->message : "null"; LOG_ERROR("failed to read XML format file: %{public}s, errno is %{public}d, error is %{public}s.", ExtractFileName(fileName).c_str(), errCode, errMessage.c_str()); - if (errCode == REQUIRED_KEY_NOT_AVAILABLE || errCode == REQUIRED_KEY_REVOKED) { - std::string operationMsg = "Read Xml file when the screen is locked."; - const ReportParam reportParam = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_OPERAT_IS_LOCKED, errCode, operationMsg}; - PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::USE_WHEN_SCREEN_LOCKED); + if (ReportNonCorruptError("read failed", fileName, bundleName, errCode)) { return nullptr; } if (!RenameToBrokenFile(fileName)) { @@ -174,29 +517,32 @@ static xmlDoc *XmlReadFile(const std::string &fileName, const std::string &bundl isReport = true; } - if (RenameFromBackupFile(fileName, bundleName, isReport)) { + bool isExist = true; + if (RenameFromBackupFile(fileName, bundleName, isReport, isExist)) { int bakErrCode = 0; doc = ReadFile(fileName, bakErrCode); xmlErrorPtr xmlErr = xmlGetLastError(); std::string message = (xmlErr != nullptr) ? xmlErr->message : "null"; errMessage.append(" bak: errno is " + std::to_string(bakErrCode) + ", errMessage is " + message); } - if (!isMultiProcessing) { - if (isReport) { - const std::string operationMsg = "operation: failed to read XML format file, errMessage:" + errMessage; - ReportXmlFileCorrupted(fileName, bundleName, operationMsg, errCode); - } - } else { - ReportParam param = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_OPERAT_IS_CROSS_PROESS, errCode, "Cross-process operations exist during file reading." }; - PreferencesDfxManager::Report(param, EVENT_NAME_PREFERENCES_FAULT); + if (isMultiProcessing) { + ReportFaultParam param = { "read failed", bundleName, NORMAL_DB, ExtractFileName(fileName), + E_OPERAT_IS_CROSS_PROESS, "Cross-process operations." }; + PreferencesDfxManager::ReportFault(param); + return doc; + } + if (isReport) { + ReportFaultParam param = { "read failed", bundleName, NORMAL_DB, ExtractFileName(fileName), + E_ERROR, "read failed, " + errMessage}; + isExist ? ReportXmlFileCorrupted(fileName, bundleName, errMessage, errCode) : + PreferencesDfxManager::ReportFault(param); } return doc; } /* static */ bool PreferencesXmlUtils::ReadSettingXml(const std::string &fileName, const std::string &bundleName, - std::vector &settings) + std::unordered_map &conMap) { if (fileName.size() == 0) { LOG_ERROR("The length of the file name is 0."); @@ -220,7 +566,7 @@ bool PreferencesXmlUtils::ReadSettingXml(const std::string &fileName, const std: Element element; if (ParseNodeElement(cur, element)) { - settings.push_back(element); + ReadXmlElement(element, conMap); } else { success = false; LOG_ERROR("The error occurred during getting xml child elements."); @@ -233,6 +579,12 @@ bool PreferencesXmlUtils::ReadSettingXml(const std::string &fileName, const std: /* static */ bool ParseNodeElement(const xmlNode *node, Element &element) { + if (!xmlStrcmp(node->name, reinterpret_cast("string")) + || !xmlStrcmp(node->name, reinterpret_cast("uint8Array")) + || !xmlStrcmp(node->name, reinterpret_cast("object"))) { + return ParseStringNodeElement(node, element); + } + if (!xmlStrcmp(node->name, reinterpret_cast("int")) || !xmlStrcmp(node->name, reinterpret_cast("long")) || !xmlStrcmp(node->name, reinterpret_cast("bool")) @@ -242,12 +594,6 @@ bool ParseNodeElement(const xmlNode *node, Element &element) return ParsePrimitiveNodeElement(node, element); } - if (!xmlStrcmp(node->name, reinterpret_cast("string")) - || !xmlStrcmp(node->name, reinterpret_cast("uint8Array")) - || !xmlStrcmp(node->name, reinterpret_cast("object"))) { - return ParseStringNodeElement(node, element); - } - if (!xmlStrcmp(node->name, reinterpret_cast("boolArray")) || !xmlStrcmp(node->name, reinterpret_cast("stringArray")) || !xmlStrcmp(node->name, reinterpret_cast("doubleArray")) @@ -263,8 +609,8 @@ bool ParseNodeElement(const xmlNode *node, Element &element) /* static */ bool ParsePrimitiveNodeElement(const xmlNode *node, Element &element) { - xmlChar *key = xmlGetProp(node, reinterpret_cast("key")); - xmlChar *value = xmlGetProp(node, reinterpret_cast("value")); + xmlChar *key = xmlGetProp(node, reinterpret_cast(ATTR_KEY)); + xmlChar *value = xmlGetProp(node, reinterpret_cast(ATTR_VALUE)); bool success = false; if (value != nullptr) { @@ -290,7 +636,7 @@ bool ParsePrimitiveNodeElement(const xmlNode *node, Element &element) /* static */ bool ParseStringNodeElement(const xmlNode *node, Element &element) { - xmlChar *key = xmlGetProp(node, (const xmlChar *)"key"); + xmlChar *key = xmlGetProp(node, (const xmlChar *)ATTR_KEY); xmlChar *text = xmlNodeGetContent(node); bool success = false; @@ -317,7 +663,7 @@ bool ParseStringNodeElement(const xmlNode *node, Element &element) /* static */ bool ParseArrayNodeElement(const xmlNode *node, Element &element) { - xmlChar *key = xmlGetProp(node, (const xmlChar *)"key"); + xmlChar *key = xmlGetProp(node, (const xmlChar *)ATTR_KEY); const xmlNode *children = node->children; bool success = false; @@ -348,228 +694,146 @@ bool ParseArrayNodeElement(const xmlNode *node, Element &element) return success; } -static std::pair SaveFormatFileEnc(const std::string &fileName, xmlDoc *doc) +static void ReportSaveFileFault(const std::string fileName, const std::string &bundleName, + bool &isReport, bool isMultiProcessing) { - return {xmlSaveFormatFileEnc(fileName.c_str(), doc, "UTF-8", 1) > 0, errno}; + int errCode = errno; + bool isExist = false; + xmlErrorPtr xmlErr = xmlGetLastError(); + std::string errMessage = (xmlErr != nullptr) ? xmlErr->message : "null"; + LOG_ERROR("Failed to save file: %{public}s, errno is %{public}d, error is %{public}s.", + ExtractFileName(fileName).c_str(), errCode, errMessage.c_str()); + if (IsFileExist(fileName)) { + RenameToBrokenFile(fileName); + isReport = true; + } + RenameFromBackupFile(fileName, bundleName, isReport, isExist); + if (ReportNonCorruptError("write failed", fileName, bundleName, errCode)) { + return; + } + if (isMultiProcessing) { + ReportFaultParam param = { "write failed", bundleName, NORMAL_DB, ExtractFileName(fileName), + E_OPERAT_IS_CROSS_PROESS, "Cross-process operations." }; + PreferencesDfxManager::ReportFault(param); + return; + } + if (isReport) { + ReportFaultParam param = { "write failed", bundleName, NORMAL_DB, ExtractFileName(fileName), + E_ERROR, "write failed, " + errMessage}; + isExist ? ReportXmlFileCorrupted(fileName, bundleName, errMessage, errCode) : + PreferencesDfxManager::ReportFault(param); + } } -bool XmlSaveFormatFileEnc(const std::string &fileName, const std::string &bundleName, xmlDoc *doc) +static bool SaveXmlFile(const std::string &fileName, const std::string &bundleName, xmlBufferPtr buf) { - PreferencesFileLock fileLock(fileName); + bool isReport = false; bool isMultiProcessing = false; + PreferencesFileLock fileLock(fileName); fileLock.WriteLock(isMultiProcessing); - LOG_INFO("save xml file:%{public}s, muti processing status is %{public}d.", ExtractFileName(fileName).c_str(), - isMultiProcessing); + LOG_INFO("save xml file:%{public}s, process is %{public}d.", ExtractFileName(fileName).c_str(), isMultiProcessing); if (IsFileExist(fileName) && !RenameToBackupFile(fileName)) { return false; } - - auto [ret, errCode] = SaveFormatFileEnc(fileName, doc); - if (!ret) { - bool isReport = false; - xmlErrorPtr xmlErr = xmlGetLastError(); - std::string errMessage = (xmlErr != nullptr) ? xmlErr->message : "null"; - LOG_ERROR("failed to save XML format file: %{public}s, errno is %{public}d, error is %{public}s.", - ExtractFileName(fileName).c_str(), errCode, errMessage.c_str()); - if (errCode == REQUIRED_KEY_NOT_AVAILABLE || errCode == REQUIRED_KEY_REVOKED) { - std::string operationMsg = "Write Xml file when the screen is locked."; - const ReportParam reportParam = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_OPERAT_IS_LOCKED, errCode, operationMsg}; - PreferencesDfxManager::ReportAbnormalOperation(reportParam, ReportedFaultBitMap::USE_WHEN_SCREEN_LOCKED); - return false; - } - if (IsFileExist(fileName)) { - RenameToBrokenFile(fileName); - isReport = true; - } - RenameFromBackupFile(fileName, bundleName, isReport); - if (!isMultiProcessing) { - if (isReport) { - const std::string operationMsg = "operation: failed to save XML format file, errMessage:" + errMessage; - ReportXmlFileCorrupted(fileName, bundleName, operationMsg, errCode); - } - } else { - ReportParam param = { bundleName, NORMAL_DB, ExtractFileName(fileName), - E_OPERAT_IS_CROSS_PROESS, errCode, "Cross-process operations exist during file writing." - }; - PreferencesDfxManager::Report(param, EVENT_NAME_PREFERENCES_FAULT); - } + int fd = Open(fileName.c_str()); + if (fd == -1) { + LOG_ERROR("failed open xml file:%{public}s", ExtractFileName(fileName).c_str()); + ReportSaveFileFault(fileName, bundleName, isReport, isMultiProcessing); return false; } - - // make sure the file is written to disk. - if (!Fsync(fileName)) { - LOG_WARN("failed to write the file to the disk."); + if (Write(fd, buf->content, buf->use) < 0) { + LOG_ERROR("Failed to write file: %{public}s", ExtractFileName(fileName).c_str()); + ReportSaveFileFault(fileName, bundleName, isReport, isMultiProcessing); + Close(fd); + return false; } - + if (!Fsync(fd)) { + LOG_WARN("Failed to write the file to the disk."); + } + Close(fd); RemoveBackupFile(fileName); - PreferencesXmlUtils::LimitXmlPermission(fileName); - LOG_DEBUG("successfully saved the XML format file"); + return true; +} + +static bool WriteXmlElement(xmlTextWriterPtr writer, const std::string &key, const PreferencesValue &value) +{ + ValueVisitor visitor; + std::visit(visitor, value.value_); + const char* tag = visitor.typeTag.data(); + const char* keyPtr = key.c_str(); + if (visitor.typeTag == "string" || visitor.typeTag == "uint8Array" || visitor.typeTag == "object") { + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST tag) >= 0, "Start element failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST ATTR_KEY, BAD_CAST keyPtr) >= 0, "Write attr failed"); + XML_CHECK(xmlTextWriterWriteString(writer, BAD_CAST visitor.valueStr.c_str()) >= 0, "Write value failed"); + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End element failed"); + } else if (visitor.typeTag == "int" || visitor.typeTag == "long" ||visitor.typeTag == "float" || + visitor.typeTag == "bool" || visitor.typeTag == "double") { + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST tag) >= 0, "Start element failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST ATTR_KEY, BAD_CAST keyPtr) >= 0, "Write attr failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST ATTR_VALUE, BAD_CAST visitor.valueStr.c_str()) >= 0, + "Write attr failed"); + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End element failed"); + } else if (visitor.typeTag == "doubleArray" || visitor.typeTag == "stringArray" || + visitor.typeTag == "boolArray" || visitor.typeTag == "BigInt") { + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST tag) >= 0, "Start element failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST ATTR_KEY, BAD_CAST keyPtr) >= 0, "Write attr failed"); + for (auto &child : visitor.arrayValues) { + if (visitor.childrenTag == "string") { + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST visitor.childrenTag.data()) >= 0, + "Start element failed"); + XML_CHECK(xmlTextWriterWriteString(writer, BAD_CAST child.c_str()) >= 0, "Write value failed"); + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End element failed"); + } else { + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST visitor.childrenTag.data()) >= 0, + "Start element failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST ATTR_VALUE, BAD_CAST child.c_str()) >= 0, + "Write attr failed"); + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End element failed"); + } + } + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End element failed"); + } return true; } /* static */ bool PreferencesXmlUtils::WriteSettingXml(const std::string &fileName, const std::string &bundleName, - const std::vector &settings) + const std::unordered_map &writeToDiskMap) { - if (fileName.size() == 0) { + if (fileName.empty()) { LOG_ERROR("The length of the file name is 0."); return false; } - // define doc and root Node - auto doc = std::shared_ptr(xmlNewDoc(BAD_CAST "1.0"), [](xmlDoc *doc) { xmlFreeDoc(doc); }); - if (doc == nullptr) { - LOG_ERROR("Failed to initialize the xmlDoc."); + XmlBufferWrapper bufferWrapper(xmlBufferCreateSize(writeToDiskMap.size() * PRE_ALLOCATE_BUFFER_SIZE)); + if (!bufferWrapper.get()) { + LOG_ERROR("Failed to create XML buffer"); return false; } - xmlNode *rootNode = xmlNewNode(NULL, BAD_CAST "preferences"); - if (rootNode == nullptr) { - LOG_ERROR("The xmlDoc failed to initialize the root node."); + XmlWriterWrapper writerWrapper(xmlNewTextWriterMemory(bufferWrapper.get(), 0)); + if (!writerWrapper.get()) { + LOG_ERROR("Failed to create XML writer"); return false; } - xmlNewProp(rootNode, BAD_CAST "version", BAD_CAST "1.0"); - // set root node - xmlDocSetRootElement(doc.get(), rootNode); + xmlTextWriterPtr writer = writerWrapper.get(); + xmlTextWriterSetIndent(writer, 0); - // set children node - for (Element element : settings) { - xmlNode *node = CreateElementNode(element); - if (node == nullptr) { - LOG_ERROR("The xmlDoc failed to initialize the element node."); - return false; - } - if (xmlAddChild(rootNode, node) == nullptr) { - /* free node in case of error */ - LOG_ERROR("The xmlDoc failed to add the child node."); - xmlFreeNode(node); + XML_CHECK(xmlTextWriterStartDocument(writer, nullptr, "UTF-8", nullptr) >= 0, "Start document failed"); + XML_CHECK(xmlTextWriterStartElement(writer, BAD_CAST TAG_PREFERENCES) >= 0, "Start root element failed"); + XML_CHECK(xmlTextWriterWriteAttribute(writer, BAD_CAST TAG_VERSION, BAD_CAST VERSION_VALUE) >= 0, + "Write version failed"); + + for (const auto& [key, prefValue] : writeToDiskMap) { + if (!WriteXmlElement(writer, key, prefValue)) { return false; } } - /* 1: formatting spaces are added. */ - bool result = XmlSaveFormatFileEnc(fileName, bundleName, doc.get()); - return result; -} + XML_CHECK(xmlTextWriterEndElement(writer) >= 0, "End root failed"); + XML_CHECK(xmlTextWriterEndDocument(writer) >= 0, "End document failed"); -/* static */ -xmlNode *CreateElementNode(Element &element) -{ - if ((element.tag_.compare("int") == 0) || (element.tag_.compare("long") == 0) - || (element.tag_.compare("float") == 0) || (element.tag_.compare("bool") == 0) - || (element.tag_.compare("double") == 0)) { - return CreatePrimitiveNode(element); - } - - if (element.tag_.compare("string") == 0 || element.tag_.compare("uint8Array") == 0 - || element.tag_.compare("object") == 0) { - return CreateStringNode(element); - } - - if ((element.tag_.compare("doubleArray") == 0) || (element.tag_.compare("stringArray") == 0) - || (element.tag_.compare("boolArray") == 0) || (element.tag_.compare("BigInt") == 0)) { - return CreateArrayNode(element); - } - - LOG_ERROR("An unsupported element type was encountered in parsing = %{public}s.", element.tag_.c_str()); - return nullptr; -} - -/* static */ -xmlNode *CreatePrimitiveNode(Element &element) -{ - xmlNode *node = xmlNewNode(NULL, BAD_CAST element.tag_.c_str()); - if (node == nullptr) { - LOG_ERROR("The xmlDoc failed to initialize the primitive element node."); - return nullptr; - } - if (!element.key_.empty()) { - const char *key = element.key_.c_str(); - xmlNewProp(node, BAD_CAST "key", BAD_CAST key); - } - - const char *value = element.value_.c_str(); - xmlNewProp(node, BAD_CAST "value", BAD_CAST value); - return node; + return SaveXmlFile(fileName, bundleName, bufferWrapper.get()); } - -xmlNode *CreateStringNode(Element &element) -{ - xmlNode *node = xmlNewNode(NULL, BAD_CAST element.tag_.c_str()); - if (node == nullptr) { - LOG_ERROR("The xmlDoc failed to initialize the string element node."); - return nullptr; - } - - if (!element.key_.empty()) { - const char *key = element.key_.c_str(); - xmlNewProp(node, BAD_CAST "key", BAD_CAST key); - } - - const char *value = element.value_.c_str(); - xmlNodePtr text = xmlNewText(BAD_CAST value); - if (xmlAddChild(node, text) == nullptr) { - xmlFreeNode(text); - } - return node; -} - -xmlNode *CreateArrayNode(Element &element) -{ - xmlNode *node = xmlNewNode(NULL, BAD_CAST element.tag_.c_str()); - if (node == nullptr) { - LOG_ERROR("The xmlDoc failed to initialize the array element node."); - return nullptr; - } - - const char *key = element.key_.c_str(); - xmlNewProp(node, BAD_CAST "key", BAD_CAST key); - - if (element.children_.empty()) { - return node; - } - Element flag = element.children_[0]; - if ((flag.tag_.compare("bool") == 0) || (flag.tag_.compare("double") == 0) || - (flag.tag_.compare("uint64_t") == 0)) { - for (Element &child : element.children_) { - xmlNode *childNode = CreatePrimitiveNode(child); - if (childNode == nullptr) { - continue; - } - if (xmlAddChild(node, childNode) == nullptr) { - xmlFreeNode(childNode); - } - } - } else if (flag.tag_.compare("string") == 0) { - for (Element child : element.children_) { - xmlNode *childNode = CreateStringNode(child); - if (childNode == nullptr) { - continue; - } - if (xmlAddChild(node, childNode) == nullptr) { - xmlFreeNode(childNode); - } - } - } - return node; -} - -void PreferencesXmlUtils::LimitXmlPermission(const std::string &fileName) -{ - /* clear execute permission of owner, clear execute permission of group, clear all permission of group. */ - struct stat fileStat = { 0 }; - if (stat(fileName.c_str(), &fileStat) != 0) { - LOG_ERROR("Failed to obtain stat of file, errno:%{public}d.", errno); - return; - } - if ((fileStat.st_mode & (S_IXUSR | S_IXGRP | S_IRWXO)) != 0) { - int result = chmod(fileName.c_str(), fileStat.st_mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)); - if (result != 0) { - LOG_ERROR("Failed to chmod file, errno:%{public}d.", errno); - } - } -} - } // End of namespace NativePreferences } // End of namespace OHOS \ No newline at end of file diff --git a/preferences/frameworks/ndk/include/oh_preferences_impl.h b/preferences/frameworks/ndk/include/oh_preferences_impl.h index 6790dcd5..c035a953 100644 --- a/preferences/frameworks/ndk/include/oh_preferences_impl.h +++ b/preferences/frameworks/ndk/include/oh_preferences_impl.h @@ -87,7 +87,7 @@ struct OH_PreferencesOption { std::string GetFileName(); std::string GetBundleName(); std::string GetDataGroupId(); - Preferences_StorageType storageType = Preferences_StorageType::PREFERENCES_STORAGE_CLKV; + Preferences_StorageType storageType = Preferences_StorageType::PREFERENCES_STORAGE_GSKV; void SetStorageType(const Preferences_StorageType &type); Preferences_StorageType GetStorageType(); std::shared_mutex opMutex_; diff --git a/preferences/frameworks/ndk/src/oh_convertor.cpp b/preferences/frameworks/ndk/src/oh_convertor.cpp index d7004924..652dec73 100644 --- a/preferences/frameworks/ndk/src/oh_convertor.cpp +++ b/preferences/frameworks/ndk/src/oh_convertor.cpp @@ -43,8 +43,8 @@ int OHConvertor::NativeErrToNdk(int nativeCode) OHOS::NativePreferences::StorageType OHConvertor::NdkStorageTypeToNative(const Preferences_StorageType &type) { - if (type == Preferences_StorageType::PREFERENCES_STORAGE_CLKV) { - return OHOS::NativePreferences::StorageType::CLKV; + if (type == Preferences_StorageType::PREFERENCES_STORAGE_GSKV) { + return OHOS::NativePreferences::StorageType::GSKV; } return OHOS::NativePreferences::StorageType::XML; } diff --git a/preferences/frameworks/ndk/src/oh_preferences.cpp b/preferences/frameworks/ndk/src/oh_preferences.cpp index b713d1c5..5f618a74 100644 --- a/preferences/frameworks/ndk/src/oh_preferences.cpp +++ b/preferences/frameworks/ndk/src/oh_preferences.cpp @@ -86,7 +86,7 @@ OH_Preferences *OH_Preferences_Open(OH_PreferencesOption *option, int *errCode) Preferences_StorageType type = option->GetStorageType(); OHOS::NativePreferences::Options nativeOptions(filePath, option->GetBundleName(), - option->GetDataGroupId(), type == PREFERENCES_STORAGE_CLKV); + option->GetDataGroupId(), type == PREFERENCES_STORAGE_GSKV); int nativeErr = OHOS::NativePreferences::E_OK; std::shared_ptr innerPreferences= @@ -394,7 +394,7 @@ int OH_Preferences_UnregisterDataObserver(OH_Preferences *preference, void *cont int OH_Preferences_IsStorageTypeSupported(Preferences_StorageType type, bool *isSupported) { if (type < Preferences_StorageType::PREFERENCES_STORAGE_XML || - type > Preferences_StorageType::PREFERENCES_STORAGE_CLKV || isSupported == nullptr) { + type > Preferences_StorageType::PREFERENCES_STORAGE_GSKV || isSupported == nullptr) { LOG_ERROR("param check failed, type: %{public}d, isSupported is null: %{public}d", static_cast(type), isSupported == nullptr); return OH_Preferences_ErrCode::PREFERENCES_ERROR_INVALID_PARAM; diff --git a/preferences/frameworks/ndk/src/oh_preferences_option.cpp b/preferences/frameworks/ndk/src/oh_preferences_option.cpp index 5c58edd0..70595a5e 100644 --- a/preferences/frameworks/ndk/src/oh_preferences_option.cpp +++ b/preferences/frameworks/ndk/src/oh_preferences_option.cpp @@ -85,7 +85,7 @@ OH_PreferencesOption* OH_PreferencesOption_Create(void) } option->cid = PreferencesNdkStructId::PREFERENCES_OH_OPTION_CID; if (!OHOS::NativePreferences::PreferencesHelper::IsStorageTypeSupported( - OHConvertor::NdkStorageTypeToNative(Preferences_StorageType::PREFERENCES_STORAGE_CLKV))) { + OHConvertor::NdkStorageTypeToNative(Preferences_StorageType::PREFERENCES_STORAGE_GSKV))) { option->SetStorageType(Preferences_StorageType::PREFERENCES_STORAGE_XML); } return option; @@ -140,7 +140,7 @@ int OH_PreferencesOption_SetStorageType(OH_PreferencesOption *option, Preference return OH_Preferences_ErrCode::PREFERENCES_ERROR_INVALID_PARAM; } if (type < Preferences_StorageType::PREFERENCES_STORAGE_XML || - type > Preferences_StorageType::PREFERENCES_STORAGE_CLKV) { + type > Preferences_StorageType::PREFERENCES_STORAGE_GSKV) { LOG_ERROR("set option's storage type failed, type invalid: %{public}d", static_cast(type)); return OH_Preferences_ErrCode::PREFERENCES_ERROR_INVALID_PARAM; } diff --git a/preferences/interfaces/inner_api/include/preferences.h b/preferences/interfaces/inner_api/include/preferences.h index b5ba2453..fe7442de 100644 --- a/preferences/interfaces/inner_api/include/preferences.h +++ b/preferences/interfaces/inner_api/include/preferences.h @@ -16,7 +16,7 @@ #ifndef PREFERENCES_H #define PREFERENCES_H -#include +#include #include #include #include @@ -31,7 +31,7 @@ namespace NativePreferences { using RegisterMode = PreferencesObserver::RegisterMode; enum StorageType { XML = 0, - CLKV + GSKV }; struct Options { public: @@ -436,6 +436,18 @@ public: { return ""; } + + /** + * @brief Obtains all the keys and values of a preferences. + * + * This function is used to get all keys and values in an object. + * + * @return Returns a unordered_map, the key is string type and the value is PreferencesValue type. + */ + virtual std::unordered_map GetAllDatas() + { + return {}; + } }; } // End of namespace NativePreferences } // End of namespace OHOS diff --git a/preferences/interfaces/ndk/include/oh_preferences_option.h b/preferences/interfaces/ndk/include/oh_preferences_option.h index 9100f405..3a88d509 100644 --- a/preferences/interfaces/ndk/include/oh_preferences_option.h +++ b/preferences/interfaces/ndk/include/oh_preferences_option.h @@ -55,13 +55,13 @@ typedef struct OH_PreferencesOption OH_PreferencesOption; /** * @brief Enumerates the preferences storage types. * - * @since 16 + * @since 18 */ typedef enum Preferences_StorageType { /** XML storage*/ PREFERENCES_STORAGE_XML = 0, - /** CLKV storage */ - PREFERENCES_STORAGE_CLKV + /** GSKV storage */ + PREFERENCES_STORAGE_GSKV } Preferences_StorageType; /** diff --git a/preferences/test/js/unittest/preferences/src/BUILD.gn b/preferences/test/js/unittest/preferences/src/BUILD.gn index 45142c9f..57831ab3 100644 --- a/preferences/test/js/unittest/preferences/src/BUILD.gn +++ b/preferences/test/js/unittest/preferences/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "preferences/js" +module_output_path = "preferences/preferences/js" ohos_js_unittest("PreferencesJsTest") { module_out_path = module_output_path diff --git a/preferences/test/js/unittest/preferences/src/PreferencesSyncJsunit.test.js b/preferences/test/js/unittest/preferences/src/PreferencesSyncJsunit.test.js index 35bcbe6b..b48786e5 100644 --- a/preferences/test/js/unittest/preferences/src/PreferencesSyncJsunit.test.js +++ b/preferences/test/js/unittest/preferences/src/PreferencesSyncJsunit.test.js @@ -124,6 +124,59 @@ describe('preferencesSyncTest', function () { } }) + /** + * @tc.name getAll sync interface test + * @tc.number SUB_DDM_AppDataFWK_JSPreferences_Preferences_0133 + * @tc.desc getAll sync interface test + */ + it('testPreferencesGetAll0002', 0, function () { + const key1 = '0'; + const key2 = '1'; + const key3 = '1234567890'; + const key4 = '-1'; + const key5 = '-1234567890'; + const key6 = '0.0'; + const key7 = '1.0'; + const key8 = '3.14159'; + const key9 = '00123'; + const key10 = '0.00123'; + const key11 = '123456789012345678901234567890'; + const key12 = '-1234567890.12345678901234567890'; + const key13 = '1.23e-10'; + mPreferences.putSync(key1, KEY_TEST_INT_ELEMENT); + mPreferences.putSync(key2, KEY_TEST_INT_ELEMENT); + mPreferences.putSync(key3, KEY_TEST_INT_ELEMENT); + mPreferences.putSync(key4, KEY_TEST_INT_ELEMENT); + mPreferences.putSync(key5, KEY_TEST_INT_ELEMENT); + mPreferences.putSync(key6, KEY_TEST_STRING_ELEMENT); + mPreferences.putSync(key7, KEY_TEST_STRING_ELEMENT); + mPreferences.putSync(key8, KEY_TEST_STRING_ELEMENT); + mPreferences.putSync(key9, KEY_TEST_STRING_ELEMENT); + mPreferences.putSync(key10, KEY_TEST_STRING_ELEMENT); + mPreferences.putSync(key11, KEY_TEST_FLOAT_ELEMENT); + mPreferences.putSync(key12, KEY_TEST_FLOAT_ELEMENT); + mPreferences.putSync(key13, KEY_TEST_FLOAT_ELEMENT); + + try { + let obj = mPreferences.getAllSync(); + expect(KEY_TEST_INT_ELEMENT).assertEqual(obj[key1.toString()]); + expect(KEY_TEST_INT_ELEMENT).assertEqual(obj[key2.toString()]); + expect(KEY_TEST_INT_ELEMENT).assertEqual(obj[key3.toString()]); + expect(KEY_TEST_INT_ELEMENT).assertEqual(obj[key4.toString()]); + expect(KEY_TEST_INT_ELEMENT).assertEqual(obj[key5.toString()]); + expect(KEY_TEST_STRING_ELEMENT).assertEqual(obj[key6.toString()]); + expect(KEY_TEST_STRING_ELEMENT).assertEqual(obj[key7.toString()]); + expect(KEY_TEST_STRING_ELEMENT).assertEqual(obj[key8.toString()]); + expect(KEY_TEST_STRING_ELEMENT).assertEqual(obj[key9.toString()]); + expect(KEY_TEST_STRING_ELEMENT).assertEqual(obj[key10.toString()]); + expect(KEY_TEST_FLOAT_ELEMENT).assertEqual(obj[key11.toString()]); + expect(KEY_TEST_FLOAT_ELEMENT).assertEqual(obj[key12.toString()]); + expect(KEY_TEST_FLOAT_ELEMENT).assertEqual(obj[key13.toString()]); + } catch (err) { + expect(false).assertTrue(); + } + }) + /** * @tc.name clear sync interface test * @tc.number SUB_DDM_AppDataFWK_JSPreferences_Promise_0010 @@ -368,4 +421,21 @@ describe('preferencesSyncTest', function () { let per2 = mPreferences.getSync(KEY_TEST_STRING_ELEMENT, "defaultvalue"); expect('string').assertEqual(per2); }) -}) \ No newline at end of file + + /** + * @tc.name value is not utf-8 + * @tc.number SUB_DDM_AppDataFWK_JSPreferences_Preferences_0164 + * @tc.desc put String sync interface test + */ + it('testPreferenceflushSync0164', 0, async function () { + let pref = data_preferences.getPreferencesSync(context, {name: "test_preference_164"}); + pref.put('key_test_utf_8', "éöüÄÖÜ€‚Š‘’“"); + pref.flushSync(); + data_preferences.removePreferencesFromCacheSync(context, {name: "test_preference_164"}); + pref = null; + pref = data_preferences.getPreferencesSync(context, {name: "test_preference_164"}); + let value = pref.getSync('key_test_utf_8', 'defaultValue'); + expect('defaultValue').assertEqual(value); + await data_preferences.deletePreferences(context, {name: "test_preference_164"}); + }) +}) diff --git a/preferences/test/js/unittest/stage_unittest/preferences/src/BUILD.gn b/preferences/test/js/unittest/stage_unittest/preferences/src/BUILD.gn index b2b6cc77..32cba43a 100644 --- a/preferences/test/js/unittest/stage_unittest/preferences/src/BUILD.gn +++ b/preferences/test/js/unittest/stage_unittest/preferences/src/BUILD.gn @@ -13,8 +13,6 @@ import("//build/test.gni") -want_output_path = "preferences/jsStage" - ohos_js_stage_unittest("stagePreferencesJsTest") { hap_profile = "entry/src/main/module.json" deps = [ @@ -26,7 +24,7 @@ ohos_js_stage_unittest("stagePreferencesJsTest") { hap_name = "stagePreferencesJsTest" subsystem_name = "distributeddatamgr" part_name = "preferences" - module_out_path = want_output_path + module_out_path = "preferences/preferences/jsStage" } ohos_app_scope("stageprejstest_app_profile") { diff --git a/preferences/test/js/unittest/stage_unittest/preferences/src/entry/src/main/ets/test/StagePreferencesSynctest.ets b/preferences/test/js/unittest/stage_unittest/preferences/src/entry/src/main/ets/test/StagePreferencesSynctest.ets index 6c2c95d2..fcaf3215 100644 --- a/preferences/test/js/unittest/stage_unittest/preferences/src/entry/src/main/ets/test/StagePreferencesSynctest.ets +++ b/preferences/test/js/unittest/stage_unittest/preferences/src/entry/src/main/ets/test/StagePreferencesSynctest.ets @@ -211,17 +211,17 @@ export default function stagePreferencesSyncJsTest() { try { // normal storage type console.info(TAG + "enum value of xml: ${data_preferences.StorageType.XML}"); - console.info(TAG + "enum value of clkv: ${data_preferences.StorageType.CLKV}"); + console.info(TAG + "enum value of GSKV: ${data_preferences.StorageType.GSKV}"); expect(0).assertEqual(data_preferences.StorageType.XML); - expect(1).assertEqual(data_preferences.StorageType.CLKV); + expect(1).assertEqual(data_preferences.StorageType.GSKV); let isXmlSupported = data_preferences.isStorageTypeSupported(data_preferences.StorageType.XML); console.info(TAG + "isXmlSupported: ${isXmlSupported}"); expect(isXmlSupported).assertTrue() - let isClkvSupported = data_preferences.isStorageTypeSupported(data_preferences.StorageType.CLKV); - console.info(TAG + "isClkvSupported: ${isClkvSupported}"); - expect(isClkvSupported).assertFalse() + let isGskvSupported = data_preferences.isStorageTypeSupported(data_preferences.StorageType.GSKV); + console.info(TAG + "isGskvSupported: ${isGskvSupported}"); + expect(isGskvSupported).assertFalse() console.info("====>testPreferencesStorageType001 success: part1") } catch (err) { @@ -320,14 +320,14 @@ export default function stagePreferencesSyncJsTest() { * @tc.desc test StorageType */ it('testPreferencesStorageType005', 0, () => { - // open with xml type but clkv exists + // open with xml type but GSKV exists // it should return not supported - // firstly create clkv - let isEnhance = data_preferences.isStorageTypeSupported(data_preferences.StorageType.CLKV); + // firstly create GSKV + let isEnhance = data_preferences.isStorageTypeSupported(data_preferences.StorageType.GSKV); if (isEnhance) { const Options : data_preferences.Options = { name: "storage_005", - storageType: data_preferences.StorageType.CLKV + storageType: data_preferences.StorageType.GSKV }; let sp = data_preferences.getPreferencesSync(context, Options); expect(sp != null).assertTrue(); @@ -354,7 +354,7 @@ export default function stagePreferencesSyncJsTest() { * @tc.desc test StorageType */ it('testPreferencesStorageType006', 0, async () => { - // open with clkv type but xml exists + // open with GSKV type but xml exists // it should return not supported // firstly create xml const Options : data_preferences.Options = { @@ -367,10 +367,10 @@ export default function stagePreferencesSyncJsTest() { data_preferences.removePreferencesFromCacheSync(context, Options); expect(isFileExists(context.preferencesDir + '/' + Options.name)).assertTrue(); - // open with clkv + // open with GSKV const Options2 : data_preferences.Options = { name: "storage_006", - storageType: data_preferences.StorageType.CLKV + storageType: data_preferences.StorageType.GSKV } try { sp = data_preferences.getPreferencesSync(context, Options2); @@ -386,14 +386,14 @@ export default function stagePreferencesSyncJsTest() { * @tc.desc test StorageType */ it('testPreferencesStorageType007', 0, () => { - // open with clkv type and open again - // firstly create clkv - let isEnhance = data_preferences.isStorageTypeSupported(data_preferences.StorageType.CLKV); + // open with GSKV type and open again + // firstly create GSKV + let isEnhance = data_preferences.isStorageTypeSupported(data_preferences.StorageType.GSKV); if (isEnhance) { try { const Options : data_preferences.Options = { name: "storage_007", - storageType: data_preferences.StorageType.CLKV + storageType: data_preferences.StorageType.GSKV }; let sp = data_preferences.getPreferencesSync(context, Options); expect(sp != null).assertTrue(); diff --git a/preferences/test/js/unittest/storage/src/BUILD.gn b/preferences/test/js/unittest/storage/src/BUILD.gn index afc52c1d..f921ba4d 100644 --- a/preferences/test/js/unittest/storage/src/BUILD.gn +++ b/preferences/test/js/unittest/storage/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "preferences/js" +module_output_path = "preferences/preferences/js" ohos_js_unittest("StorageJsTest") { module_out_path = module_output_path diff --git a/preferences/test/js/unittest/system_storage/src/BUILD.gn b/preferences/test/js/unittest/system_storage/src/BUILD.gn index 23e68105..58c5a1e9 100644 --- a/preferences/test/js/unittest/system_storage/src/BUILD.gn +++ b/preferences/test/js/unittest/system_storage/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "preferences/js" +module_output_path = "preferences/preferences/js" ohos_js_unittest("SystemStorageJsTest") { module_out_path = module_output_path diff --git a/preferences/test/native/BUILD.gn b/preferences/test/native/BUILD.gn index 375a25b1..9731cc36 100644 --- a/preferences/test/native/BUILD.gn +++ b/preferences/test/native/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/preferences/preferences.gni") -module_output_path = "preferences/native_preferences" +module_output_path = "preferences/preferences/native_preferences" ############################################################################### config("module_private_config") { diff --git a/preferences/test/native/fuzztest/preferences_fuzzer/preferences_fuzzer.cpp b/preferences/test/native/fuzztest/preferences_fuzzer/preferences_fuzzer.cpp index 382025d9..fb74f6e9 100644 --- a/preferences/test/native/fuzztest/preferences_fuzzer/preferences_fuzzer.cpp +++ b/preferences/test/native/fuzztest/preferences_fuzzer/preferences_fuzzer.cpp @@ -49,6 +49,8 @@ void PreferencesFuzzTest::SetUpTestCase(void) void PreferencesFuzzTest::TearDownTestCase(void) { + Preferences_->Clear(); + PreferencesHelper::RemovePreferencesFromCache("/data/test/test"); } void PreferencesFuzzTest::SetUp(void) @@ -227,5 +229,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) OHOS::PutLongFuzz(data, size); OHOS::PutDoubleFuzz(data, size); OHOS::GetLongFuzz(data, size); + OHOS::PreferencesFuzzTest::TearDownTestCase(); return 0; } \ No newline at end of file diff --git a/preferences/test/native/unittest/base64_helper_test.cpp b/preferences/test/native/unittest/base64_helper_test.cpp index 884c0e75..f0c23ad8 100644 --- a/preferences/test/native/unittest/base64_helper_test.cpp +++ b/preferences/test/native/unittest/base64_helper_test.cpp @@ -52,7 +52,6 @@ void Base64HelperTest::TearDown(void) * @tc.name: Base64HelperTest_0001 * @tc.desc: normal testcase of base64 encode * @tc.type: FUNC - * @tc.require: SR000CU2BL * @tc.author: xiuhongju */ HWTEST_F(Base64HelperTest, Base64HelperTest_001, TestSize.Level1) diff --git a/preferences/test/native/unittest/preferences_file_test.cpp b/preferences/test/native/unittest/preferences_file_test.cpp index f3a607cf..cc3482d5 100644 --- a/preferences/test/native/unittest/preferences_file_test.cpp +++ b/preferences/test/native/unittest/preferences_file_test.cpp @@ -66,7 +66,6 @@ int PreferencesPutValue(std::shared_ptr pref, const std::string &in * @tc.name: NativePreferencesFileTest_001 * @tc.desc: normal testcase of backup file * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: liulinna */ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_001, TestSize.Level1) @@ -77,13 +76,9 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_001, TestSize.Level1) std::remove(file.c_str()); std::remove(backupFile.c_str()); - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = std::string("int"); - elem.value_ = std::to_string(10); - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(backupFile, "", settings); + std::unordered_map values; + values.insert({"intKey", 10}); + PreferencesXmlUtils::WriteSettingXml(backupFile, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -106,7 +101,6 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_001, TestSize.Level1) * @tc.name: NativePreferencesFileTest_002 * @tc.desc: normal testcase of file permission * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: liulinna */ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_002, TestSize.Level1) @@ -136,7 +130,6 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_002, TestSize.Level1) * @tc.name: NativePreferencesFileTest_003 * @tc.desc: test FlushSync one times and five times * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: liulinna */ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_003, TestSize.Level1) @@ -194,7 +187,6 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_003, TestSize.Level1) * @tc.name: NativePreferencesFileTest_004 * @tc.desc: test Flush one times and five times * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: liulinna */ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_004, TestSize.Level3) @@ -274,19 +266,10 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_005, TestSize.Level1) pref->PutInt("intKey", 2); OHOS::NativePreferences::Mkdir(path); - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = std::string("int"); - elem.value_ = std::to_string(10); - Element elem1; - elem1.key_ = "intKey1"; - elem1.tag_ = std::string("int"); - elem1.value_ = std::to_string(10); - settings.push_back(elem); - settings.push_back(elem1); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); - + std::unordered_map values; + values.insert({"intKey", 10}); + values.insert({"intKey1", 10}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); ret = pref->GetInt("intKey", 0); EXPECT_EQ(ret, 2); ret = pref->GetInt("intKey1", 0); @@ -318,32 +301,25 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_006, TestSize.Level1) pref->PutInt("intKey", 2); OHOS::NativePreferences::Mkdir(path); - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = std::string("int"); - elem.value_ = std::to_string(20); - Element elem1; - elem1.key_ = "intKey1"; - elem1.tag_ = std::string("int"); - elem1.value_ = std::to_string(20); - settings.push_back(elem); - settings.push_back(elem1); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + int value = 20; + values.insert({"intKey", value}); + values.insert({"intKey1", value}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); pref->FlushSync(); - std::vector settingsRes = {}; - bool res = PreferencesXmlUtils::ReadSettingXml(file, "", settingsRes); + std::unordered_map allDatas; + bool res = PreferencesXmlUtils::ReadSettingXml(file, "", allDatas); EXPECT_EQ(res, true); - EXPECT_EQ(settingsRes.empty(), false); - EXPECT_EQ(elem.key_, settingsRes[0].key_); - EXPECT_EQ(elem.tag_, settingsRes[0].tag_); - EXPECT_EQ(std::to_string(2), settingsRes[0].value_); + EXPECT_EQ(allDatas.empty(), false); + auto it = allDatas.find("intKey"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(2, int(it->second)); - EXPECT_EQ(elem1.key_, settingsRes[1].key_); - EXPECT_EQ(elem1.tag_, settingsRes[1].tag_); - EXPECT_EQ(elem1.value_, settingsRes[1].value_); + it = allDatas.find("intKey1"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(PreferencesValue(value) == it->second, true); pref = nullptr; ret = PreferencesHelper::DeletePreferences(file); @@ -367,18 +343,10 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_007, TestSize.Level1) EXPECT_EQ(errCode, E_OK); pref->PutInt("intKey", 7); - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = std::string("int"); - elem.value_ = std::to_string(70); - Element elem1; - elem1.key_ = "intKey1"; - elem1.tag_ = std::string("int"); - elem1.value_ = std::to_string(70); - settings.push_back(elem); - settings.push_back(elem1); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + values.insert({"intKey", 70}); + values.insert({"intKey1", 70}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); ret = pref->GetInt("intKey", 0); EXPECT_EQ(ret, 7); @@ -408,32 +376,25 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_008, TestSize.Level1) EXPECT_EQ(false, pref->HasKey("intKey1")); pref->PutInt("intKey", 8); - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = std::string("int"); - elem.value_ = std::to_string(80); - Element elem1; - elem1.key_ = "intKey1"; - elem1.tag_ = std::string("int"); - elem1.value_ = std::to_string(80); - settings.push_back(elem); - settings.push_back(elem1); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + int value = 80; + values.insert({"intKey", value}); + values.insert({"intKey1", value}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); pref->FlushSync(); - std::vector settingsRes = {}; - bool res = PreferencesXmlUtils::ReadSettingXml(file, "", settingsRes); + std::unordered_map allDatas; + bool res = PreferencesXmlUtils::ReadSettingXml(file, "", allDatas); EXPECT_EQ(res, true); - EXPECT_EQ(settingsRes.empty(), false); - EXPECT_EQ(elem.key_, settingsRes[0].key_); - EXPECT_EQ(elem.tag_, settingsRes[0].tag_); - EXPECT_EQ(std::to_string(8), settingsRes[0].value_); + EXPECT_EQ(allDatas.empty(), false); + auto it = allDatas.find("intKey"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(8, int(it->second)); - EXPECT_EQ(elem1.key_, settingsRes[1].key_); - EXPECT_EQ(elem1.tag_, settingsRes[1].tag_); - EXPECT_EQ(elem1.value_, settingsRes[1].value_); + it = allDatas.find("intKey1"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(PreferencesValue(value) == it->second, true); pref = nullptr; ret = PreferencesHelper::DeletePreferences(file); @@ -458,16 +419,16 @@ HWTEST_F(PreferencesFileTest, NativePreferencesFileTest_009, TestSize.Level1) pref->FlushSync(); - std::vector settingsRes = {}; - bool res = PreferencesXmlUtils::ReadSettingXml(file, "", settingsRes); + std::unordered_map allDatas; + bool res = PreferencesXmlUtils::ReadSettingXml(file, "", allDatas); EXPECT_EQ(res, true); - EXPECT_EQ(settingsRes.empty(), false); - EXPECT_EQ("intKey", settingsRes[0].key_); - EXPECT_EQ(std::string("int"), settingsRes[0].tag_); - EXPECT_EQ(std::to_string(9), settingsRes[0].value_); + EXPECT_EQ(allDatas.empty(), false); + auto it = allDatas.find("intKey"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(9, int(it->second)); pref = nullptr; ret = PreferencesHelper::DeletePreferences(file); EXPECT_EQ(ret, E_OK); } -} +} \ No newline at end of file diff --git a/preferences/test/native/unittest/preferences_helper_test.cpp b/preferences/test/native/unittest/preferences_helper_test.cpp index c1ee5e15..cbd51913 100644 --- a/preferences/test/native/unittest/preferences_helper_test.cpp +++ b/preferences/test/native/unittest/preferences_helper_test.cpp @@ -58,7 +58,6 @@ void PreferencesHelperTest::TearDown(void) * @tc.name: NativePreferencesHelperTest_001 * @tc.desc: normal testcase of DeletePreferences * @tc.type: FUNC - * @tc.require: SR000CU2BL * @tc.author: xiuhongju */ HWTEST_F(PreferencesHelperTest, NativePreferencesHelperTest_001, TestSize.Level1) diff --git a/preferences/test/native/unittest/preferences_storage_type_test.cpp b/preferences/test/native/unittest/preferences_storage_type_test.cpp new file mode 100644 index 00000000..b8cffde3 --- /dev/null +++ b/preferences/test/native/unittest/preferences_storage_type_test.cpp @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include "log_print.h" +#include "preferences.h" +#include "preferences_errno.h" +#include "preferences_file_operation.h" +#include "preferences_helper.h" +#include "preferences_observer.h" +#include "preferences_value.h" + +using namespace testing::ext; +using namespace OHOS::NativePreferences; + +namespace { + +static void CreateDirectoryRecursively(const std::string &path) +{ + std::string::size_type pos = path.find_last_of('/'); + if (pos == std::string::npos || path.front() != '/') { + printf("path can not be relative path.\n"); + } + std::string dir = path.substr(0, pos); + + std::string tempDirectory = dir; + std::vector directories; + + pos = tempDirectory.find('/'); + while (pos != std::string::npos) { + std::string directory = tempDirectory.substr(0, pos); + if (!directory.empty()) { + directories.push_back(directory); + } + tempDirectory = tempDirectory.substr(pos + 1); + pos = tempDirectory.find('/'); + } + directories.push_back(tempDirectory); + + std::string databaseDirectory; + for (const std::string& directory : directories) { + databaseDirectory = databaseDirectory + "/" + directory; + if (OHOS::NativePreferences::Access(databaseDirectory.c_str()) != F_OK) { + if (OHOS::NativePreferences::Mkdir(databaseDirectory)) { + printf("failed to mkdir, errno %d, %s \n", errno, databaseDirectory.c_str()); + return; + } + } + } +} + +static bool IsFileExist(const std::string &path) +{ + struct stat buffer; + return (stat(path.c_str(), &buffer) == 0); +} + +class PreferencesStorageTypeTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void PreferencesStorageTypeTest::SetUpTestCase(void) +{ + CreateDirectoryRecursively("/data/test/"); +} + +void PreferencesStorageTypeTest::TearDownTestCase(void) +{ +} + +void PreferencesStorageTypeTest::SetUp(void) +{ + CreateDirectoryRecursively("/data/test/"); +} + +void PreferencesStorageTypeTest::TearDown(void) +{ +} + +/** + * @tc.name: StorageTypeBaseTest000 + * @tc.desc: test IsStorageTypeSupported api + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeBaseTest000, TestSize.Level0) +{ + ASSERT_EQ(PreferencesHelper::IsStorageTypeSupported(StorageType::XML), true); + ASSERT_EQ(PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV), false); +} + +/** + * @tc.name: StorageTypeBaseTest002 + * @tc.desc: base test, use different storage type to get preferences and check file type + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeBaseTest002, TestSize.Level0) +{ + printf("test with xml\n"); + // default option constructor + int errCode = E_OK; + std::string filePath = "/data/test/XML002_1"; + std::shared_ptr pref = PreferencesHelper::GetPreferences(filePath, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + filePath = "/data/test/XML002_2"; + Options option = Options(filePath, "", "", false); + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + filePath = "/data/test/GSKV002"; + Options option2 = Options(filePath, "", "", true); + pref = PreferencesHelper::GetPreferences(option2, errCode); + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + printf("test with GSKV\n"); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } else { + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + } +} + +/** + * @tc.name: StorageTypeBaseTest004 + * @tc.desc: base test, test get again with different mode + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeBaseTest004, TestSize.Level0) +{ + printf("test with xml\n"); + // xml exist and open with GSKV mode + int errCode = E_OK; + std::string filePath = "/data/test/XML004"; + Options option = Options(filePath, "", "", false); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + option.isEnhance = true; + // in cache, not check type, just return + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + // not in cahce, invalid storage type + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + printf("test with GSKV\n"); + // GSKV exist and open with xml mode + filePath = "/data/test/GSKV004"; + option.filePath = filePath; + option.isEnhance = true; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + option.isEnhance = false; + // in cache, not check type, just return + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + // not in cahce, invalid storage type + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } +} + +/** + * @tc.name: StorageTypeCombineTest005 + * @tc.desc: combine test, test xml mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest005, TestSize.Level0) +{ + // new with xml mode in white list + printf("test with xml\n"); + int errCode = E_OK; + std::string filePath = "/data/test/White005"; + Options option = Options(filePath, "abcuttestabc", ""); // XML type + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(errCode, E_OK); + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + printf("test with GSKV\n"); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + } else { + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + } + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + // not white + filePath = "/data/test/Normal005"; + option.filePath = filePath; + option.bundleName = "abcabc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); +} + +/** + * @tc.name: StorageTypeCombineTest006 + * @tc.desc: combine test, test xml mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest006, TestSize.Level0) +{ + // new with xml exists in xml mode + // create xml firstly + int errCode = E_OK; + std::string filePath = "/data/test/XML006"; + Options option = Options(filePath, "", ""); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + + // in white list, when xml exist, open with xml mode, it should be xml even in enhance + option.isEnhance = false; + option.bundleName = "abcuttestabc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + printf("test with GSKV\n"); + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + } + + // not in white list, open with xml mode when xml exists + Options option2 = Options(filePath, "abc", "", false); + pref = PreferencesHelper::GetPreferences(option2, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); +} + +/** + * @tc.name: StorageTypeCombineTest007 + * @tc.desc: combine test, test xml mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest007, TestSize.Level0) +{ + // create fake GSKV firstly + int errCode = E_OK; + std::string filePath = "/data/test/XML007"; + Options option = Options(filePath + ".db", "", ""); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (!isEnhance) { + // open with white list in xml mode + // it should be xml + Options option1 = Options(filePath, "abcuttestabc", "", false); + pref = PreferencesHelper::GetPreferences(option1, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + } + + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath + ".db"), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + if (isEnhance) { + // create GSKV firstly + printf("test with GSKV\n"); + filePath = "/data/test/GSKV007"; + option.filePath = filePath; + option.isEnhance = true; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + + // open in xml mode with white list, it should be GSKV + option.bundleName = "abcuttestabc"; + option.isEnhance = false; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } +} + +/** + * @tc.name: StorageTypeCombineTest008 + * @tc.desc: combine test, test xml mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest008, TestSize.Level0) +{ + // create fake GSKV firstly + int errCode = E_OK; + printf("test with xml\n"); + std::string filePath = "/data/test/XML008"; + Options option1 = Options(filePath + ".db", "", ""); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option1, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + + // open with xml mode, not in white list + // GSKV exists, should return E_NOT_SUPPORTED + Options option = Options(filePath, "abc", "", false); + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath + ".db"), E_OK); + + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + // same case in enhance + printf("test with GSKV\n"); + filePath = "/data/test/GSKV008"; + option.filePath = filePath; + option.isEnhance = true; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + + option.isEnhance = false; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } +} + +/** + * @tc.name: StorageTypeCombineTest009 + * @tc.desc: combine test, test GSKV mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest009, TestSize.Level0) +{ + // new with GSKV mode + // white list + int errCode = E_OK; + std::string filePath = "/data/test/White009"; + Options option = Options(filePath, "abcuttestabc", "", true); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + printf("test with GSKV\n"); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + } else { + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(IsFileExist(filePath), true); + } + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + + // new with GSKV mode and not in white list + filePath = "/data/test/Normal009"; + option.filePath = filePath; + option.bundleName = "abc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + + if (isEnhance) { + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } else { + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + } +} + +/** + * @tc.name: StorageTypeCombineTest010 + * @tc.desc: combine test, test GSKV mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest010, TestSize.Level0) +{ + // xml exists when GSKV mode + // create xml firstly + int errCode = E_OK; + std::string filePath = "/data/test/Test010"; + Options option = Options(filePath, "", ""); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + + // open with white list, it should be xml even in enhance + option.bundleName = "abcuttestabc"; + option.isEnhance = true; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(IsFileExist(filePath), true); + ASSERT_EQ(IsFileExist(filePath + ".db"), false); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + + // when not in white list + option.bundleName = "abc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); +} + +/** + * @tc.name: StorageTypeCombineTest011 + * @tc.desc: combine test, test GSKV mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest011, TestSize.Level0) +{ + // GSKV exists, open with GSKV mode + // create fake GSKV firstly + int errCode = E_OK; + std::string filePath = "/data/test/Test011"; + std::string filePathFakeKV = "/data/test/Test011.db"; + Options option = Options(filePathFakeKV, "", ""); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(pref->PutString("key", "value"), E_OK); + ASSERT_EQ(pref->FlushSync(), E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePathFakeKV), E_OK); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(IsFileExist(filePathFakeKV), true); + + // open in white list when not enhance + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (!isEnhance) { + option.bundleName = "uttest"; + option.isEnhance = true; + option.filePath = filePath; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + // when not white list + option.bundleName = "abc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_NOT_SUPPORTED); + } + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath + ".db"), E_OK); +} + +/** + * @tc.name: StorageTypeCombineTest012 + * @tc.desc: combine test, test GSKV mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeCombineTest012, TestSize.Level0) +{ + // GSKV exists, open with GSKV mode + // it should be GSKV enven not in white list + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + // create GSKV firstly + int errCode = E_OK; + std::string filePath = "/data/test/Test012"; + Options option = Options(filePath, "", "", true); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + + option.bundleName = "uttest"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + + option.bundleName = "abc"; + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(IsFileExist(filePath), false); + ASSERT_EQ(IsFileExist(filePath + ".db"), true); + } +} + +/** + * @tc.name: StorageTypeApiTest013 + * @tc.desc: api test, test GSKV mode in different cases + * @tc.type: FUNC + * @tc.author: huangboxin + */ +HWTEST_F(PreferencesStorageTypeTest, StorageTypeApiTest013, TestSize.Level0) +{ + bool isEnhance = PreferencesHelper::IsStorageTypeSupported(StorageType::GSKV); + if (isEnhance) { + int errCode = E_OK; + std::string filePath = "/data/test/Test013"; + Options option = Options(filePath, "", "", true); + std::shared_ptr pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(pref->PutString("test013_key_" + std::to_string(i), "test013_value_" + std::to_string(i)), E_OK); + } + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + for (int i = 0; i < 100; i++) { + std::string expValue = "test013_value_" + std::to_string(i); + ASSERT_EQ(pref->GetString("test013_key_" + std::to_string(i), "def"), expValue); + } + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + + pref = PreferencesHelper::GetPreferences(option, errCode); + ASSERT_EQ(errCode, E_OK); + auto set = pref->GetAll(); + ASSERT_EQ(set.size(), 100); + ASSERT_EQ(pref->Clear(), E_OK); + auto set2 = pref->GetAll(); + ASSERT_EQ(set2.size(), 0); + ASSERT_EQ(PreferencesHelper::RemovePreferencesFromCache(filePath), E_OK); + ASSERT_EQ(PreferencesHelper::DeletePreferences(filePath), E_OK); + } +} + +} // namespace \ No newline at end of file diff --git a/preferences/test/native/unittest/preferences_test.cpp b/preferences/test/native/unittest/preferences_test.cpp index 2a9efb7f..b081b534 100644 --- a/preferences/test/native/unittest/preferences_test.cpp +++ b/preferences/test/native/unittest/preferences_test.cpp @@ -164,7 +164,6 @@ void PreferencesObserverCrossProcess::OnChange(const std::string &key) * @tc.name: NativePreferencesGroupIdTest_001 * @tc.desc: normal testcase of GetGroupId * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: lirui */ HWTEST_F(PreferencesTest, NativePreferencesGroupIdTest_001, TestSize.Level1) @@ -185,7 +184,6 @@ HWTEST_F(PreferencesTest, NativePreferencesGroupIdTest_001, TestSize.Level1) * @tc.name: NativePreferencesTest_001 * @tc.desc: normal testcase of FlushSync * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_001, TestSize.Level1) @@ -204,7 +202,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_001, TestSize.Level1) * @tc.name: NativePreferencesTest_002 * @tc.desc: normal testcase of HasKey * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_002, TestSize.Level1) @@ -228,7 +225,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_002, TestSize.Level1) * @tc.name: NativePreferencesTest_003 * @tc.desc: normal testcase of pref * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_003, TestSize.Level1) @@ -247,7 +243,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_003, TestSize.Level1) * @tc.name: NativePreferencesTest_004 * @tc.desc: normal testcase of GetBool * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_004, TestSize.Level1) @@ -273,7 +268,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_004, TestSize.Level1) * @tc.name: NativePreferencesTest_005 * @tc.desc: normal testcase of GetFloat * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_005, TestSize.Level1) @@ -299,7 +293,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_005, TestSize.Level1) * @tc.name: NativePreferencesTest_006 * @tc.desc: normal testcase of GetInt * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_006, TestSize.Level1) @@ -325,7 +318,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_006, TestSize.Level1) * @tc.name: NativePreferencesTest_007 * @tc.desc: normal testcase of GetLong * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_007, TestSize.Level1) @@ -351,7 +343,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_007, TestSize.Level1) * @tc.name: NativePreferencesTest_008 * @tc.desc: normal testcase of GetString * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_008, TestSize.Level1) @@ -377,7 +368,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_008, TestSize.Level1) * @tc.name: NativePreferencesTest_009 * @tc.desc: normal testcase of GetDefValue * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_009, TestSize.Level1) @@ -402,7 +392,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_009, TestSize.Level1) * @tc.name: NativePreferencesTest_010 * @tc.desc: normal testcase of PutBool * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_010, TestSize.Level1) @@ -430,7 +419,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_010, TestSize.Level1) * @tc.name: NativePreferencesTest_011 * @tc.desc: normal testcase of PutFloat * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_011, TestSize.Level1) @@ -458,7 +446,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_011, TestSize.Level1) * @tc.name: NativePreferencesTest_012 * @tc.desc: normal testcase of PutInt * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_012, TestSize.Level1) @@ -486,7 +473,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_012, TestSize.Level1) * @tc.name: NativePreferencesTest_013 * @tc.desc: normal testcase of PutLong * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_013, TestSize.Level1) @@ -514,7 +500,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_013, TestSize.Level1) * @tc.name: NativePreferencesTest_014 * @tc.desc: normal testcase of PutString * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_014, TestSize.Level1) @@ -542,7 +527,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_014, TestSize.Level1) * @tc.name: NativePreferencesTest_015 * @tc.desc: normal testcase of Delete * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_015, TestSize.Level1) @@ -557,13 +541,14 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_015, TestSize.Level1) pref->FlushSync(); ret = pref->GetString("test", "defaultValue"); EXPECT_EQ(ret, "defaultValue"); + int res = pref->Delete(""); + EXPECT_EQ(res, E_KEY_EMPTY); } /** * @tc.name: NativePreferencesTest_016 * @tc.desc: normal testcase of RegisterPreferencesObserver * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_016, TestSize.Level1) @@ -588,7 +573,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_016, TestSize.Level1) * @tc.name: NativePreferencesTest_017 * @tc.desc: normal testcase of UnRegisterPreferencesObserver * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_017, TestSize.Level1) @@ -613,7 +597,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_017, TestSize.Level1) * @tc.name: NativePreferencesTest_018 * @tc.desc: normal testcase of Clear * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, NativePreferencesTest_018, TestSize.Level1) @@ -1028,7 +1011,6 @@ HWTEST_F(PreferencesTest, NativePreferencesTest_033, TestSize.Level1) * @tc.name: OperatorTest_001 * @tc.desc: normal testcase of PreferencesValue Operator * @tc.type: FUNC - * @tc.require: AR000CU2BN * @tc.author: xiuhongju */ HWTEST_F(PreferencesTest, PreferencesValueTest_001, TestSize.Level1) diff --git a/preferences/test/native/unittest/preferences_xml_utils_test.cpp b/preferences/test/native/unittest/preferences_xml_utils_test.cpp index 9bb98f33..40753679 100644 --- a/preferences/test/native/unittest/preferences_xml_utils_test.cpp +++ b/preferences/test/native/unittest/preferences_xml_utils_test.cpp @@ -23,7 +23,7 @@ #include "preferences.h" #include "preferences_errno.h" #include "preferences_helper.h" -#include "preferences_impl.h" +#include "preferences_utils.h" using namespace testing::ext; using namespace OHOS::NativePreferences; @@ -60,15 +60,15 @@ void PreferencesXmlUtilsTest::TearDown(void) */ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_001, TestSize.Level1) { - std::vector settings = {}; - bool ret = PreferencesXmlUtils::ReadSettingXml("", "", settings); + std::unordered_map allDatas; + bool ret = PreferencesXmlUtils::ReadSettingXml("", "", allDatas); EXPECT_EQ(ret, false); std::string path = "/data/test/test_helper" + std::string(4096, 't'); - ret = PreferencesXmlUtils::ReadSettingXml(path, "", settings); + ret = PreferencesXmlUtils::ReadSettingXml(path, "", allDatas); EXPECT_EQ(ret, false); - ret = PreferencesXmlUtils::ReadSettingXml("data/test/test_helper", "", settings); + ret = PreferencesXmlUtils::ReadSettingXml("data/test/test_helper", "", allDatas); EXPECT_EQ(ret, false); } @@ -101,21 +101,17 @@ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_003, TestSize.Level1) { std::string file = "/data/test/test01"; - std::vector settings; - Element elem; - elem.key_ = "testKet"; - elem.tag_ = "int"; - elem.value_ = "999"; - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + values.insert({"testKey", 999}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); - std::vector settingsRes = {}; - bool ret = PreferencesXmlUtils::ReadSettingXml(file, "", settingsRes); + std::unordered_map allDatas; + bool ret = PreferencesXmlUtils::ReadSettingXml(file, "", allDatas); EXPECT_EQ(ret, true); - EXPECT_EQ(settingsRes.empty(), false); - EXPECT_EQ(elem.key_, settingsRes.front().key_); - EXPECT_EQ(elem.tag_, settingsRes.front().tag_); - EXPECT_EQ(elem.value_, settingsRes.front().value_); + EXPECT_EQ(allDatas.empty(), false); + auto it = allDatas.find("testKey"); + EXPECT_EQ(it != allDatas.end(), true); + EXPECT_EQ(999, int(it->second)); std::remove(file.c_str()); } @@ -127,23 +123,22 @@ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_003, TestSize.Level1) */ HWTEST_F(PreferencesXmlUtilsTest, UnnormalReadSettingXml_001, TestSize.Level1) { - std::vector settings = {}; - PreferencesXmlUtils::WriteSettingXml("", "", settings); - bool ret = PreferencesXmlUtils::ReadSettingXml("", "", settings); + std::unordered_map values; + PreferencesXmlUtils::WriteSettingXml("", "", values); + bool ret = PreferencesXmlUtils::ReadSettingXml("", "", values); EXPECT_EQ(ret, false); std::string path = "/data/test/test_helper" + std::string(4096, 't'); - ret = PreferencesXmlUtils::ReadSettingXml(path, "", settings); + ret = PreferencesXmlUtils::ReadSettingXml(path, "", values); EXPECT_EQ(ret, false); - ret = PreferencesXmlUtils::ReadSettingXml("data/test/test_helper", "", settings); + ret = PreferencesXmlUtils::ReadSettingXml("data/test/test_helper", "", values); EXPECT_EQ(ret, false); - Element elem; - settings.push_back(elem); + values.insert({}); path = "data/test/test_helper"; - PreferencesXmlUtils::WriteSettingXml(path, "", settings); - ret = PreferencesXmlUtils::ReadSettingXml(path, "", settings); + PreferencesXmlUtils::WriteSettingXml(path, "", values); + ret = PreferencesXmlUtils::ReadSettingXml(path, "", values); EXPECT_EQ(ret, false); } @@ -157,19 +152,15 @@ HWTEST_F(PreferencesXmlUtilsTest, StringNodeElementTest_001, TestSize.Level1) std::string file = "/data/test/test01"; std::remove(file.c_str()); - std::vector settings; - Element elem; - elem.key_ = "stringKey"; - elem.tag_ = std::string("string"); - elem.value_ = "test"; - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + values.insert({"stringKey", "test"}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); EXPECT_EQ(errCode, E_OK); std::string retString = pref->GetString("stringKey", ""); - EXPECT_EQ(retString, elem.value_); + EXPECT_EQ(retString, "test"); int ret = PreferencesHelper::DeletePreferences(file); EXPECT_EQ(ret, E_OK); @@ -184,24 +175,10 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_001, TestSize.Level1) { std::string file = "/data/test/test02"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "stringArrayKey"; - elem.tag_ = std::string("stringArray"); - elem.value_ = "testStringArray"; - - Element elemChild; - elemChild.key_ = "stringKey"; - elemChild.tag_ = std::string("string"); - - elemChild.value_ = "test_child1"; - elem.children_.push_back(elemChild); - elemChild.value_ = "test_child2"; - elem.children_.push_back(elemChild); - settings.push_back(elem); + std::unordered_map values; std::vector inputStringArray = { "test_child1", "test_child2" }; - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + values.insert({"stringArrayKey", inputStringArray}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -223,25 +200,10 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_002, TestSize.Level1) { std::string file = "/data/test/test03"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "doubleArrayKey"; - elem.tag_ = std::string("doubleArray"); - elem.value_ = std::to_string(10.0); - - Element elemChild; - elemChild.key_ = "doubleKey"; - elemChild.tag_ = std::string("double"); - - elemChild.value_ = std::to_string(1.0); - elem.children_.push_back(elemChild); - - elemChild.value_ = std::to_string(2.0); - elem.children_.push_back(elemChild); - settings.push_back(elem); + std::unordered_map values; std::vector inputDoubleArray = { 1.0, 2.0 }; - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + values.insert({"doubleArrayKey", inputDoubleArray}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -263,25 +225,10 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_003, TestSize.Level1) { std::string file = "/data/test/test04"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "boolArrayKey"; - elem.tag_ = std::string("boolArray"); - elem.value_ = std::to_string(false); - - Element elemChild; - elemChild.key_ = "boolKey"; - elemChild.tag_ = std::string("bool"); - - elemChild.value_ = "false"; - elem.children_.push_back(elemChild); - - elemChild.value_ = "true"; - elem.children_.push_back(elemChild); - settings.push_back(elem); + std::unordered_map values; std::vector inputBoolArray = { false, true }; - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + values.insert({"boolArrayKey", inputBoolArray}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -301,17 +248,12 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_003, TestSize.Level1) */ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_004, TestSize.Level1) { - std::string file = "/data/test/test05"; + std::string file = "/data/test/testttt05"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "boolArrayKey"; - elem.tag_ = std::string("boolArray"); - elem.value_ = std::to_string(false); - - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + std::vector value = {}; + values.insert({"boolArrayKey", value}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -333,17 +275,12 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_004, TestSize.Level1) */ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_005, TestSize.Level1) { - std::string file = "/data/test/test06"; + std::string file = "/data/test/testttt06"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "stringArrayKey"; - elem.tag_ = std::string("stringArray"); - elem.value_ = std::to_string(false); - - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + std::vector value = {}; + values.insert({"stringArrayKey", value}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -367,15 +304,10 @@ HWTEST_F(PreferencesXmlUtilsTest, ArrayNodeElementTest_006, TestSize.Level1) { std::string file = "/data/test/test07"; std::remove(file.c_str()); - std::vector settings; - - Element elem; - elem.key_ = "doubleArrayKey"; - elem.tag_ = std::string("doubleArray"); - elem.value_ = std::to_string(1); - - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(file, "", settings); + std::unordered_map values; + std::vector value = {}; + values.insert({"doubleArrayKey", value}); + PreferencesXmlUtils::WriteSettingXml(file, "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(file, errCode); @@ -402,14 +334,9 @@ HWTEST_F(PreferencesXmlUtilsTest, RenameToBrokenFileTest_001, TestSize.Level1) std::ofstream oss(fileName); oss << "corrupted"; - std::vector settings; - Element elem; - elem.key_ = "intKey"; - elem.tag_ = "int"; - elem.value_ = "2"; - - settings.push_back(elem); - PreferencesXmlUtils::WriteSettingXml(MakeFilePath(fileName, STR_BACKUP), "", settings); + std::unordered_map values; + values.insert({"intKey", 2}); + PreferencesXmlUtils::WriteSettingXml(MakeFilePath(fileName, STR_BACKUP), "", values); int errCode = E_OK; std::shared_ptr pref = PreferencesHelper::GetPreferences(fileName, errCode); @@ -437,8 +364,8 @@ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_004, TestSize.Level1) std::ofstream ossBak(MakeFilePath(fileName, STR_BACKUP)); ossBak << "corruptedBak"; - std::vector settings; - bool res = PreferencesXmlUtils::ReadSettingXml(fileName, "", settings); + std::unordered_map values; + bool res = PreferencesXmlUtils::ReadSettingXml(fileName, "", values); EXPECT_EQ(res, false); int ret = PreferencesHelper::DeletePreferences(fileName); @@ -453,17 +380,12 @@ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_004, TestSize.Level1) HWTEST_F(PreferencesXmlUtilsTest, WriteSettingXmlWhenFileIsNotExistTest_001, TestSize.Level1) { std::string fileName = "/data/test/test01"; - std::vector settings; - Element elem; - elem.key_ = "stringKey"; - elem.tag_ = "string"; - elem.value_ = ""; - - settings.push_back(elem); - bool result = PreferencesXmlUtils::WriteSettingXml("/data/test/preferences/test01", "", settings); + std::unordered_map values; + values.insert({"stringKey", ""}); + bool result = PreferencesXmlUtils::WriteSettingXml("/data/test/preferences/testttt01", "", values); EXPECT_EQ(result, false); - result = PreferencesXmlUtils::WriteSettingXml(fileName, "", settings); + result = PreferencesXmlUtils::WriteSettingXml(fileName, "", values); EXPECT_EQ(result, true); } @@ -480,17 +402,13 @@ HWTEST_F(PreferencesXmlUtilsTest, ReadSettingXmlTest_005, TestSize.Level1) std::ofstream oss(fileName); oss << "corrupted"; - std::vector settings; - Element elem; - elem.key_ = "stringKey"; - elem.tag_ = "string"; - elem.value_ = ""; - - settings.push_back(elem); - bool result = PreferencesXmlUtils::WriteSettingXml(MakeFilePath(fileName, STR_BACKUP), "", settings); + std::unordered_map values; + values.insert({"stringKey", ""}); + bool result = PreferencesXmlUtils::WriteSettingXml(MakeFilePath(fileName, STR_BACKUP), "", values); EXPECT_EQ(result, true); - bool res = PreferencesXmlUtils::ReadSettingXml(fileName, "", settings); + std::unordered_map allDatas; + bool res = PreferencesXmlUtils::ReadSettingXml(fileName, "", values); EXPECT_EQ(res, true); int ret = PreferencesHelper::DeletePreferences(fileName); diff --git a/preferences/test/ndk/BUILD.gn b/preferences/test/ndk/BUILD.gn index 0fe19ace..64542c7c 100644 --- a/preferences/test/ndk/BUILD.gn +++ b/preferences/test/ndk/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/preferences/preferences.gni") -module_output_path = "preferences/ndk_preferences" +module_output_path = "preferences/preferences/ndk_preferences" ############################################################################### diff --git a/preferences/test/ndk/unittest/preferences_ndk_storage_type_test.cpp b/preferences/test/ndk/unittest/preferences_ndk_storage_type_test.cpp index 0ea5c28f..cefa9893 100644 --- a/preferences/test/ndk/unittest/preferences_ndk_storage_type_test.cpp +++ b/preferences/test/ndk/unittest/preferences_ndk_storage_type_test.cpp @@ -87,7 +87,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeBaseTest_001, TestSize.Lev ASSERT_EQ(errCode, PREFERENCES_OK); ASSERT_EQ(isSupport, true); - errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_CLKV, &isSupport); + errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_GSKV, &isSupport); ASSERT_EQ(errCode, PREFERENCES_OK); ASSERT_EQ(isSupport, false); @@ -96,7 +96,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeBaseTest_001, TestSize.Lev ASSERT_EQ(errCode, PREFERENCES_ERROR_INVALID_PARAM); errCode = OH_Preferences_IsStorageTypeSupported( - static_cast(Preferences_StorageType::PREFERENCES_STORAGE_CLKV + 1), &isSupport); + static_cast(Preferences_StorageType::PREFERENCES_STORAGE_GSKV + 1), &isSupport); ASSERT_EQ(errCode, PREFERENCES_ERROR_INVALID_PARAM); } @@ -110,7 +110,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeBaseTest_001, TestSize.Lev HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeBaseTest_002, TestSize.Level1) { // without setting storage type - // it should be xml when not in enhance by default, or clkv in enhance + // it should be xml when not in enhance by default, or GSKV in enhance OH_PreferencesOption *option = OH_PreferencesOption_Create(); const char *fileName = "CStorageTypeTest002"; ASSERT_EQ(OH_PreferencesOption_SetFileName(option, fileName), PREFERENCES_OK); @@ -122,7 +122,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeBaseTest_002, TestSize.Lev EXPECT_EQ(OH_Preferences_Close(pref), PREFERENCES_OK); bool isEnhance = false; - errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_CLKV, &isEnhance); + errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_GSKV, &isEnhance); ASSERT_EQ(errCode, PREFERENCES_OK); if (isEnhance) { ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName) + ".db"), true); @@ -175,15 +175,15 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeXMLTest_003, TestSize.Leve /** * @tc.name: NDKStorageTypeXMLTest_003 - * @tc.desc: test storage type when clkv exists and open with xml + * @tc.desc: test storage type when GSKV exists and open with xml * @tc.type: FUNC * @tc.require: NA * @tc.author: huangboxin */ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeXMLTest_004, TestSize.Level1) { - // clkv exists, open in xml mode, should return invalid_args - // create fake clkv firstly + // GSKV exists, open in xml mode, should return invalid_args + // create fake GSKV firstly OH_PreferencesOption *option = OH_PreferencesOption_Create(); const char *fileName = "CStorageTypeTest004"; ASSERT_EQ(OH_PreferencesOption_SetFileName(option, "CStorageTypeTest004.db"), PREFERENCES_OK); @@ -207,25 +207,25 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeXMLTest_004, TestSize.Leve } /** - * @tc.name: NDKStorageTypeCLKVTest_005 - * @tc.desc: test storage type when new with clkv + * @tc.name: NDKStorageTypeGSKVTest_005 + * @tc.desc: test storage type when new with GSKV * @tc.type: FUNC * @tc.require: NA * @tc.author: huangboxin */ -HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_005, TestSize.Level1) +HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeGSKVTest_005, TestSize.Level1) { - // new with clkv mode + // new with GSKV mode OH_PreferencesOption *option = OH_PreferencesOption_Create(); const char *fileName = "CStorageTypeTest005"; ASSERT_EQ(OH_PreferencesOption_SetFileName(option, fileName), PREFERENCES_OK); bool isEnhance = false; - int errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_CLKV, &isEnhance); + int errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_GSKV, &isEnhance); ASSERT_EQ(errCode, PREFERENCES_OK); if (isEnhance) { - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); OH_Preferences *pref = OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_OK); @@ -234,7 +234,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_005, TestSize.Lev ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName) + ".db"), true); ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName)), false); } else { - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); (void)OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_ERROR_NOT_SUPPORTED); @@ -247,15 +247,15 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_005, TestSize.Lev } /** - * @tc.name: NDKStorageTypeCLKVTest_006 - * @tc.desc: test storage type when xml exists but open with clkv + * @tc.name: NDKStorageTypeGSKVTest_006 + * @tc.desc: test storage type when xml exists but open with GSKV * @tc.type: FUNC * @tc.require: NA * @tc.author: huangboxin */ -HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_006, TestSize.Level1) +HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeGSKVTest_006, TestSize.Level1) { - // xml exists but open with clkv + // xml exists but open with GSKV // create xml firstly OH_PreferencesOption *option = OH_PreferencesOption_Create(); const char *fileName = "CStorageTypeTest006"; @@ -270,17 +270,17 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_006, TestSize.Lev ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName) + ".db"), false); ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName)), true); - // open with clkv + // open with GSKV bool isEnhance = false; - errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_CLKV, &isEnhance); + errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_GSKV, &isEnhance); ASSERT_EQ(errCode, PREFERENCES_OK); if (isEnhance) { - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); pref = OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_ERROR_NOT_SUPPORTED); } else { - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); pref = OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_ERROR_NOT_SUPPORTED); @@ -292,24 +292,24 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_006, TestSize.Lev } /** - * @tc.name: NDKStorageTypeCLKVTest_007 - * @tc.desc: test storage type when clkv exists but open with clkv + * @tc.name: NDKStorageTypeGSKVTest_007 + * @tc.desc: test storage type when GSKV exists but open with GSKV * @tc.type: FUNC * @tc.require: NA * @tc.author: huangboxin */ -HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_007, TestSize.Level1) +HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeGSKVTest_007, TestSize.Level1) { - // clkv exists, open in clkv mode + // GSKV exists, open in GSKV mode bool isEnhance = false; - int errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_CLKV, &isEnhance); + int errCode = OH_Preferences_IsStorageTypeSupported(Preferences_StorageType::PREFERENCES_STORAGE_GSKV, &isEnhance); ASSERT_EQ(errCode, PREFERENCES_OK); OH_PreferencesOption *option = OH_PreferencesOption_Create(); if (isEnhance) { - // create clkv firstly + // create GSKV firstly const char *fileName = "CStorageTypeTest007"; ASSERT_EQ(OH_PreferencesOption_SetFileName(option, fileName), PREFERENCES_OK); - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); OH_Preferences *pref = OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_OK); @@ -323,7 +323,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_007, TestSize.Lev ASSERT_EQ(errCode, PREFERENCES_OK); ASSERT_EQ(OH_Preferences_Close(pref), PREFERENCES_OK); } else { - // create fake clkv firstly + // create fake GSKV firstly const char *fileName = "CStorageTypeTest007Fake.db"; ASSERT_EQ(OH_PreferencesOption_SetFileName(option, fileName), PREFERENCES_OK); // xml by default when not enhance @@ -335,7 +335,7 @@ HWTEST_F(PreferencesNdkStorageTypeTest, NDKStorageTypeCLKVTest_007, TestSize.Lev ASSERT_EQ(IsFileExist(TEST_PATH + std::string(fileName)), true); // open again - ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_CLKV), + ASSERT_EQ(OH_PreferencesOption_SetStorageType(option, Preferences_StorageType::PREFERENCES_STORAGE_GSKV), PREFERENCES_OK); pref = OH_Preferences_Open(option, &errCode); ASSERT_EQ(errCode, PREFERENCES_ERROR_NOT_SUPPORTED); diff --git a/relational_store/CMakeLists.txt b/relational_store/CMakeLists.txt index e8e4d52d..a31d9a62 100644 --- a/relational_store/CMakeLists.txt +++ b/relational_store/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.2) project(relational_store) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl -Wno-deprecated") diff --git a/relational_store/bundle.json b/relational_store/bundle.json index 654a5b8f..81a4db74 100644 --- a/relational_store/bundle.json +++ b/relational_store/bundle.json @@ -89,7 +89,7 @@ "//foundation/distributeddatamgr/relational_store/frameworks/js/napi/common:commontype_napi", "//foundation/distributeddatamgr/relational_store/frameworks/js/napi/sendablerelationalstore:sendablerelationalstore", "//foundation/distributeddatamgr/relational_store/frameworks/native/icu:build_module", - "//foundation/distributeddatamgr/relational_store/interfaces/inner_api/gdb:framework_graphstore", + "//foundation/distributeddatamgr/relational_store/interfaces/inner_api/gdb:native_graphstore", "//foundation/distributeddatamgr/relational_store/frameworks/js/napi/graphstore:graphstore" ], "inner_kits": [ @@ -178,6 +178,24 @@ ], "header_base": "//foundation/distributeddatamgr/relational_store/interfaces/inner_api/cloud_data/include" } + }, + { + "name": "//foundation/distributeddatamgr/relational_store/interfaces/inner_api/gdb:native_graphstore", + "header": { + "header_files": [ + "edge.h", + "gdb_errors.h", + "gdb_helper.h", + "gdb_store.h", + "gdb_store_config.h", + "gdb_transaction.h", + "path.h", + "path_segment.h", + "result.h", + "vertex.h" + ], + "header_base": "//foundation/distributeddatamgr/relational_store/interfaces/inner_api/gdb/include" + } } ], "test": [ diff --git a/relational_store/frameworks/cj/src/relational_store_ffi.cpp b/relational_store/frameworks/cj/src/relational_store_ffi.cpp index 25149ce4..f986a755 100644 --- a/relational_store/frameworks/cj/src/relational_store_ffi.cpp +++ b/relational_store/frameworks/cj/src/relational_store_ffi.cpp @@ -22,6 +22,8 @@ #include "napi_rdb_js_utils.h" #include "rdb_errno.h" #include "relational_store_utils.h" +#include "rdb_errno.h" +#include "js_utils.h" using namespace OHOS::FFI; @@ -368,7 +370,8 @@ int64_t FfiOHOSRelationalStoreQuery( } auto resultSet = nativeRdbStore->Query(*nativeRdbPredicates, columns, columnsSize); if (resultSet == nullptr) { - *errCode = RelationalStoreJsKit::E_INNER_ERROR; + // If the API version is greater than or equal to 20, throw E_ALREADY_CLOSED. + *errCode = (AppDataMgrJsKit::JSUtils::GetHapVersion() >= 20) ? NativeRdb::E_ALREADY_CLOSED : NativeRdb::E_ERROR; return -1; } else { *errCode = RelationalStoreJsKit::OK; diff --git a/relational_store/frameworks/cj/src/relational_store_impl_rdbstore.cpp b/relational_store/frameworks/cj/src/relational_store_impl_rdbstore.cpp index a331f5cb..44af5f03 100644 --- a/relational_store/frameworks/cj/src/relational_store_impl_rdbstore.cpp +++ b/relational_store/frameworks/cj/src/relational_store_impl_rdbstore.cpp @@ -31,6 +31,7 @@ #include "rdb_common.h" #include "native_log.h" #include "relational_store_impl_rdbstore.h" +#include "js_utils.h" #ifndef PATH_SPLIT #define PATH_SPLIT '/' @@ -758,7 +759,9 @@ namespace Relational { if (*errCode != NativeRdb::E_OK) { return; } - *errCode = NativeRdb::RdbHelper::DeleteRdbStore(rdbConfig.path); + // If the API version is greater than or equal to 20, close the connection. + bool isClose = (AppDataMgrJsKit::JSUtils::GetHapVersion() >= 20); + *errCode = NativeRdb::RdbHelper::DeleteRdbStore(rdbConfig.path, isClose); return; } @@ -779,7 +782,9 @@ namespace Relational { if (*errCode != NativeRdb::E_OK) { return; } - *errCode = NativeRdb::RdbHelper::DeleteRdbStore(rdbConfig.path); + // If the API version is greater than or equal to 20, close the connection. + bool isClose = (AppDataMgrJsKit::JSUtils::GetHapVersion() >= 20); + *errCode = NativeRdb::RdbHelper::DeleteRdbStore(rdbConfig.path, isClose); return; } } diff --git a/relational_store/frameworks/cj/src/relational_store_impl_resultsetproxy.cpp b/relational_store/frameworks/cj/src/relational_store_impl_resultsetproxy.cpp index 35882fc0..368d7937 100644 --- a/relational_store/frameworks/cj/src/relational_store_impl_resultsetproxy.cpp +++ b/relational_store/frameworks/cj/src/relational_store_impl_resultsetproxy.cpp @@ -18,6 +18,8 @@ #include "napi_rdb_error.h" #include "value_object.h" #include "native_log.h" +#include "js_utils.h" +#include "rdb_errno.h" namespace OHOS { namespace Relational { @@ -206,8 +208,12 @@ namespace Relational { int32_t ResultSetImpl::GetColumnIndex(char* columnName, int32_t* rtnCode) { - int32_t result; + int32_t result = -1; *rtnCode = resultSetValue->GetColumnIndex(columnName, result); + // If the API version is less than 13, directly return. + if (AppDataMgrJsKit::JSUtils::GetHapVersion() < 13 || (*rtnCode == NativeRdb::E_INVALID_ARGS)) { + *rtnCode = E_OK; + } return result; } diff --git a/relational_store/frameworks/js/napi/cloud_data/src/js_cloud_utils.cpp b/relational_store/frameworks/js/napi/cloud_data/src/js_cloud_utils.cpp index 6d169485..722e1b58 100644 --- a/relational_store/frameworks/js/napi/cloud_data/src/js_cloud_utils.cpp +++ b/relational_store/frameworks/js/napi/cloud_data/src/js_cloud_utils.cpp @@ -266,6 +266,7 @@ napi_value Convert2JSValue(napi_env env, const CloudSyncInfo &value) napi_set_named_property(env, jsValue, "startTime", startTime); napi_set_named_property(env, jsValue, "finishTime", finishTime); napi_set_named_property(env, jsValue, "code", Convert2JSValue(env, value.code)); + napi_set_named_property(env, jsValue, "syncStatus", Convert2JSValue(env, value.syncStatus)); return jsValue; } }; // namespace JSUtils diff --git a/relational_store/frameworks/js/napi/cloud_data/src/js_const_properties.cpp b/relational_store/frameworks/js/napi/cloud_data/src/js_const_properties.cpp index 4924b18b..5f8886e9 100644 --- a/relational_store/frameworks/js/napi/cloud_data/src/js_const_properties.cpp +++ b/relational_store/frameworks/js/napi/cloud_data/src/js_const_properties.cpp @@ -108,11 +108,22 @@ static napi_value ExportNetWorkStrategy(napi_env env) return netStrategy; } +static napi_value ExportSyncStatus(napi_env env) +{ + napi_value syncStatus = nullptr; + napi_create_object(env, &syncStatus); + SetNamedProperty(env, syncStatus, "RUNNING", SyncStatus::RUNNING); + SetNamedProperty(env, syncStatus, "FINISHED", SyncStatus::FINISHED); + napi_object_freeze(env, syncStatus); + return syncStatus; +} + napi_status InitConstProperties(napi_env env, napi_value exports) { const napi_property_descriptor properties[] = { DECLARE_NAPI_PROPERTY("Action", ExportAction(env)), DECLARE_NAPI_PROPERTY("ClearAction", ExportAction(env)), + DECLARE_NAPI_PROPERTY("SyncStatus", ExportSyncStatus(env)), DECLARE_NAPI_PROPERTY("DATA_CHANGE_EVENT_ID", AppDataMgrJsKit::JSUtils::Convert2JSValue(env, std::string(CloudData::DATA_CHANGE_EVENT_ID))), }; diff --git a/relational_store/frameworks/js/napi/graphstore/BUILD.gn b/relational_store/frameworks/js/napi/graphstore/BUILD.gn index 19a9f63a..e970ce11 100644 --- a/relational_store/frameworks/js/napi/graphstore/BUILD.gn +++ b/relational_store/frameworks/js/napi/graphstore/BUILD.gn @@ -22,14 +22,13 @@ ohos_shared_library("graphstore") { cfi_cross_dso = true debug = false } + cflags_cc = [ "-fvisibility=hidden" ] include_dirs = [ "${relational_store_common_path}/include", - "${relational_store_frameworks_path}/native/gdb/include", - "${relational_store_frameworks_path}/common/include", "${relational_store_frameworks_path}/js/napi/common/include", "${relational_store_frameworks_path}/js/napi/graphstore/include", - "${relational_store_innerapi_path}/gdb/include", - "${relational_store_innerapi_path}/rdb/include", + "${relational_store_native_path}/gdb/include", + "${relational_store_native_path}/rdb/include", ] sources = [ "${relational_store_frameworks_path}/js/napi/common/src/js_ability.cpp", @@ -45,7 +44,7 @@ ohos_shared_library("graphstore") { "${relational_store_frameworks_path}/js/napi/graphstore/src/napi_gdb_store_helper.cpp", "${relational_store_frameworks_path}/js/napi/graphstore/src/napi_gdb_transaction.cpp", ] - deps = [ "${relational_store_innerapi_path}/gdb:framework_graphstore" ] + deps = [ "${relational_store_innerapi_path}/gdb:native_graphstore" ] external_deps = [ "ability_runtime:abilitykit_native", "ability_runtime:extensionkit_native", diff --git a/relational_store/frameworks/js/napi/graphstore/include/napi_async_call.h b/relational_store/frameworks/js/napi/graphstore/include/napi_async_call.h index c2fdca11..160200e6 100644 --- a/relational_store/frameworks/js/napi/graphstore/include/napi_async_call.h +++ b/relational_store/frameworks/js/napi/graphstore/include/napi_async_call.h @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "logger.h" #include "napi/native_api.h" #include "napi/native_common.h" diff --git a/relational_store/frameworks/js/napi/graphstore/include/napi_gdb_transaction.h b/relational_store/frameworks/js/napi/graphstore/include/napi_gdb_transaction.h index 2d3e1759..98565623 100644 --- a/relational_store/frameworks/js/napi/graphstore/include/napi_gdb_transaction.h +++ b/relational_store/frameworks/js/napi/graphstore/include/napi_gdb_transaction.h @@ -20,12 +20,12 @@ #include #include +#include "gdb_transaction.h" #include "js_proxy.h" #include "js_uv_queue.h" #include "napi/native_api.h" #include "napi/native_common.h" #include "napi/native_node_api.h" -#include "transaction.h" namespace OHOS::GraphStoreJsKit { using Transaction = DistributedDataAip::Transaction; diff --git a/relational_store/frameworks/js/napi/graphstore/src/napi_gdb_error.cpp b/relational_store/frameworks/js/napi/graphstore/src/napi_gdb_error.cpp index 037b7341..042c9c64 100644 --- a/relational_store/frameworks/js/napi/graphstore/src/napi_gdb_error.cpp +++ b/relational_store/frameworks/js/napi/graphstore/src/napi_gdb_error.cpp @@ -17,7 +17,7 @@ #include -#include "aip_errors.h" +#include "gdb_errors.h" namespace OHOS::GraphStoreJsKit { using namespace DistributedDataAip; diff --git a/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_js_utils.h b/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_js_utils.h index 8b1d01a9..52f7b5ab 100644 --- a/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_js_utils.h +++ b/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_js_utils.h @@ -57,6 +57,7 @@ struct RdbConfig { bool vector = false; bool allowRebuild = false; bool isReadOnly = false; + bool persist = true; SecurityLevel securityLevel = SecurityLevel::LAST; Tokenizer tokenizer = Tokenizer::NONE_TOKENIZER; std::string dataGroupId; diff --git a/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_store.h b/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_store.h index b7ddc661..2d506fe5 100644 --- a/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_store.h +++ b/relational_store/frameworks/js/napi/relationalstore/include/napi_rdb_store.h @@ -64,8 +64,6 @@ private: static napi_value Attach(napi_env env, napi_callback_info info); static napi_value Detach(napi_env env, napi_callback_info info); static napi_value GetPath(napi_env env, napi_callback_info info); - static napi_value IsMemoryRdb(napi_env env, napi_callback_info info); - static napi_value IsReadOnly(napi_env env, napi_callback_info info); static napi_value BeginTransaction(napi_env env, napi_callback_info info); static napi_value BeginTrans(napi_env env, napi_callback_info info); static napi_value RollBack(napi_env env, napi_callback_info info); @@ -107,14 +105,10 @@ private: static constexpr int WAIT_TIME_LIMIT = 300; napi_value OnRemote(napi_env env, size_t argc, napi_value *argv); - napi_value OnLocal(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback); - napi_value RegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, - std::map>> &observers, napi_value callback); + napi_value RegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback); napi_value OffRemote(napi_env env, size_t argc, napi_value *argv); - napi_value OffLocal(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback); - napi_value UnRegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, - std::map>> &observers, napi_value callback); + napi_value UnRegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback); class SyncObserver : public DistributedRdb::DetailProgressObserver, public std::enable_shared_from_this { diff --git a/relational_store/frameworks/js/napi/relationalstore/mock/include/napi_rdb_store.h b/relational_store/frameworks/js/napi/relationalstore/mock/include/napi_rdb_store.h index a083f073..1122a89f 100644 --- a/relational_store/frameworks/js/napi/relationalstore/mock/include/napi_rdb_store.h +++ b/relational_store/frameworks/js/napi/relationalstore/mock/include/napi_rdb_store.h @@ -51,6 +51,7 @@ private: static napi_value Update(napi_env env, napi_callback_info info); static napi_value Insert(napi_env env, napi_callback_info info); static napi_value BatchInsert(napi_env env, napi_callback_info info); + static napi_value BatchInsertWithConflictResolution(napi_env env, napi_callback_info info); static napi_value Query(napi_env env, napi_callback_info info); static napi_value QuerySql(napi_env env, napi_callback_info info); static napi_value ExecuteSql(napi_env env, napi_callback_info info); @@ -60,9 +61,6 @@ private: static napi_value Attach(napi_env env, napi_callback_info info); static napi_value Detach(napi_env env, napi_callback_info info); static napi_value GetPath(napi_env env, napi_callback_info info); - static napi_value IsMemoryRdb(napi_env env, napi_callback_info info); - static napi_value IsHoldingConnection(napi_env env, napi_callback_info info); - static napi_value IsReadOnly(napi_env env, napi_callback_info info); static napi_value BeginTransaction(napi_env env, napi_callback_info info); static napi_value BeginTrans(napi_env env, napi_callback_info info); static napi_value RollBack(napi_env env, napi_callback_info info); diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_const_properties.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_const_properties.cpp index 0fd52437..c306b3e9 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_const_properties.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_const_properties.cpp @@ -24,12 +24,12 @@ using OHOS::DistributedRdb::SubscribeMode; using OHOS::DistributedRdb::SyncMode; - #endif using OHOS::DistributedRdb::DistributedTableType; using OHOS::DistributedRdb::ProgressCode; using OHOS::NativeRdb::ConflictResolution; using OHOS::NativeRdb::SecurityLevel; +using OHOS::DistributedRdb::ColumnType; #define SET_NAPI_PROPERTY(object, prop, value) \ napi_set_named_property((env), (object), (prop), AppDataMgrJsKit::JSUtils::Convert2JSValue((env), (value))) @@ -324,6 +324,27 @@ static napi_value ExportTokenizer(napi_env env) return tokenizerType; } +static napi_value ExportColumnType(napi_env env) +{ + napi_value columnType = nullptr; + napi_status status = napi_create_object(env, &columnType); + if (status != napi_ok) { + return nullptr; + } + + SET_NAPI_PROPERTY(columnType, "NULL", int32_t(ColumnType::TYPE_NULL)); + SET_NAPI_PROPERTY(columnType, "INTEGER", int32_t(ColumnType::TYPE_INTEGER)); + SET_NAPI_PROPERTY(columnType, "REAL", int32_t(ColumnType::TYPE_FLOAT)); + SET_NAPI_PROPERTY(columnType, "TEXT", int32_t(ColumnType::TYPE_STRING)); + SET_NAPI_PROPERTY(columnType, "BLOB", int32_t(ColumnType::TYPE_BLOB)); + SET_NAPI_PROPERTY(columnType, "ASSET", int32_t(ColumnType::TYPE_ASSET)); + SET_NAPI_PROPERTY(columnType, "ASSETS", int32_t(ColumnType::TYPE_ASSETS)); + SET_NAPI_PROPERTY(columnType, "FLOAT_VECTOR", int32_t(ColumnType::TYPE_FLOAT32_ARRAY)); + SET_NAPI_PROPERTY(columnType, "UNLIMITED_INT", int32_t(ColumnType::TYPE_BIGINT)); + napi_object_freeze(env, columnType); + return columnType; +} + napi_status InitConstProperties(napi_env env, napi_value exports) { const napi_property_descriptor properties[] = { @@ -347,6 +368,7 @@ napi_status InitConstProperties(napi_env env, napi_value exports) DECLARE_NAPI_PROPERTY("KdfAlgo", ExportKdfAlgo(env)), DECLARE_NAPI_PROPERTY("TransactionType", ExportTransactionType(env)), DECLARE_NAPI_PROPERTY("Tokenizer", ExportTokenizer(env)), + DECLARE_NAPI_PROPERTY("ColumnType", ExportColumnType(env)), }; size_t count = sizeof(properties) / sizeof(properties[0]); diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_error.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_error.cpp index c1a93da7..d6418cd4 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_error.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_error.cpp @@ -53,6 +53,7 @@ static constexpr JsErrorCode JS_ERROR_CODE_MSGS[] = { { NativeRdb::E_CONFIG_INVALID_CHANGE, 14800017, "Config changed." }, { NativeRdb::E_INVALID_SECRET_KEY, 14800020, "The secret key is corrupted or lost." }, { NativeRdb::E_SQLITE_IOERR_FULL, 14800028, "SQLite: Some kind of disk I/O error occurred." }, + { NativeRdb::E_INVALID_ARGS_NEW, 401, "Invalid args." }, { NativeRdb::E_NOT_SUPPORT, 801, "Capability not support." }, }; diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_js_utils.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_js_utils.cpp index 48498e2c..bdbf1f38 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_js_utils.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_js_utils.cpp @@ -102,6 +102,7 @@ int32_t Convert2Value(napi_env env, napi_value input, DistributedRdb::Distribute NAPI_CALL_RETURN_ERR(GetNamedProperty(env, input, "references", output.references, true), napi_invalid_arg); NAPI_CALL_RETURN_ERR( GetNamedProperty(env, input, "asyncDownloadAsset", output.asyncDownloadAsset, true), napi_invalid_arg); + NAPI_CALL_RETURN_ERR(GetNamedProperty(env, input, "enableCloud", output.enableCloud, true), napi_invalid_arg); return napi_ok; } @@ -393,6 +394,9 @@ int32_t Convert2Value(napi_env env, napi_value jsValue, RdbConfig &rdbConfig) status = GetNamedProperty(env, jsValue, "tokenizer", tokenizer, true); ASSERT(OK == status, "get tokenizer failed.", napi_invalid_arg); rdbConfig.tokenizer = static_cast(tokenizer); + + status = GetNamedProperty(env, jsValue, "persist", rdbConfig.persist, true); + ASSERT(OK == status, "get persist failed.", napi_invalid_arg); return napi_ok; } @@ -519,7 +523,8 @@ RdbStoreConfig GetRdbStoreConfig(const RdbConfig &rdbConfig, const ContextParam rdbStoreConfig.SetEncryptStatus(rdbConfig.isEncrypt); rdbStoreConfig.SetSearchable(rdbConfig.isSearchable); rdbStoreConfig.SetIsVector(rdbConfig.vector); - rdbConfig.vector ? rdbStoreConfig.SetDBType(DB_VECTOR) : rdbStoreConfig.SetDBType(DB_SQLITE); + rdbStoreConfig.SetDBType(rdbConfig.vector ? DB_VECTOR : DB_SQLITE); + rdbStoreConfig.SetStorageMode(rdbConfig.persist ? StorageMode::MODE_DISK : StorageMode::MODE_MEMORY); rdbStoreConfig.SetAutoClean(rdbConfig.isAutoClean); rdbStoreConfig.SetSecurityLevel(rdbConfig.securityLevel); rdbStoreConfig.SetDataGroupId(rdbConfig.dataGroupId); diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store.cpp index 0b2f730e..167b9bad 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store.cpp @@ -84,7 +84,7 @@ void RdbStoreProxy::UnregisterAll() return; } #if !defined(CROSS_PLATFORM) - for (int32_t mode = DistributedRdb::REMOTE; mode < DistributedRdb::LOCAL; mode++) { + for (int32_t mode = SubscribeMode::REMOTE; mode < SubscribeMode::LOCAL; mode++) { for (auto &obs : observers_[mode]) { if (obs == nullptr) { continue; @@ -92,25 +92,19 @@ void RdbStoreProxy::UnregisterAll() rdbStore->UnSubscribe({ static_cast(mode) }, obs.get()); } } + rdbStore->UnsubscribeObserver({ SubscribeMode::LOCAL_DETAIL }, nullptr); for (const auto &[event, observers] : localObservers_) { - for (const auto &obs : observers) { - if (obs == nullptr) { - continue; - } - rdbStore->UnSubscribe({ static_cast(DistributedRdb::LOCAL), event }, obs.get()); - } + rdbStore->UnSubscribe({ static_cast(DistributedRdb::LOCAL), event }, nullptr); } for (const auto &[event, observers] : localSharedObservers_) { - for (const auto &obs : observers) { - if (obs == nullptr) { - continue; - } - rdbStore->UnSubscribe({ static_cast(DistributedRdb::LOCAL_SHARED), event }, obs.get()); - } + rdbStore->UnSubscribe({ static_cast(DistributedRdb::LOCAL_SHARED), event }, nullptr); } for (const auto &obs : syncObservers_) { rdbStore->UnregisterAutoSyncCallback(obs); } + for (const auto &obs : statisticses_) { + DistributedRdb::SqlStatistic::Unsubscribe(obs); + } #endif } @@ -816,8 +810,8 @@ napi_value RdbStoreProxy::Query(napi_env env, napi_callback_info info) context->resultSet = context->rdbStore->Query(*(context->rdbPredicates), context->columns); #endif context->rdbStore = nullptr; - // If the API version is greater than or equal to 16, throw E_ALREADY_CLOSED. - return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 16) ? E_ALREADY_CLOSED : E_ERROR; + // If the API version is greater than or equal to 20, throw E_ALREADY_CLOSED. + return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 20) ? E_ALREADY_CLOSED : E_ERROR; }; auto output = [context](napi_env env, napi_value &result) { result = ResultSetProxy::NewInstance(env, std::move(context->resultSet)); @@ -887,8 +881,8 @@ napi_value RdbStoreProxy::QuerySql(napi_env env, napi_callback_info info) context->resultSet = context->rdbStore->QuerySql(context->sql, context->bindArgs); #endif context->rdbStore = nullptr; - // If the API version is greater than or equal to 16, throw E_ALREADY_CLOSED. - return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 16) ? E_ALREADY_CLOSED : E_ERROR; + // If the API version is greater than or equal to 20, throw E_ALREADY_CLOSED. + return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 20) ? E_ALREADY_CLOSED : E_ERROR; }; auto output = [context](napi_env env, napi_value &result) { result = ResultSetProxy::NewInstance(env, std::move(context->resultSet)); @@ -1278,8 +1272,8 @@ napi_value RdbStoreProxy::QueryByStep(napi_env env, napi_callback_info info) auto rdbStore = std::move(context->rdbStore); context->resultSet = context->isQuerySql ? rdbStore->QueryByStep(context->sql, context->bindArgs) : rdbStore->QueryByStep(*(context->rdbPredicates), context->columns); - // If the API version is greater than or equal to 16, throw E_ALREADY_CLOSED. - return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 16) ? E_ALREADY_CLOSED : E_ERROR; + // If the API version is greater than or equal to 20, throw E_ALREADY_CLOSED. + return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 20) ? E_ALREADY_CLOSED : E_ERROR; }; auto output = [context](napi_env env, napi_value &result) { result = ResultSetProxy::NewInstance(env, std::move(context->resultSet)); @@ -1662,9 +1656,10 @@ napi_value RdbStoreProxy::OnRemote(napi_env env, size_t argc, napi_value *argv) return nullptr; } -napi_value RdbStoreProxy::RegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, - std::map>> &observers, napi_value callback) +napi_value RdbStoreProxy::RegisteredObserver( + napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback) { + auto &observers = option.mode == SubscribeMode::LOCAL ? localObservers_ : localSharedObservers_; observers.try_emplace(option.event); auto &list = observers.find(option.event)->second; bool result = @@ -1683,14 +1678,6 @@ napi_value RdbStoreProxy::RegisteredObserver(napi_env env, const DistributedRdb: return nullptr; } -napi_value RdbStoreProxy::OnLocal(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback) -{ - if (option.mode == SubscribeMode::LOCAL) { - return RegisteredObserver(env, option, localObservers_, callback); - } - return RegisteredObserver(env, option, localSharedObservers_, callback); -} - napi_value RdbStoreProxy::OffRemote(napi_env env, size_t argc, napi_value *argv) { napi_valuetype type = napi_undefined; @@ -1734,9 +1721,10 @@ napi_value RdbStoreProxy::OffRemote(napi_env env, size_t argc, napi_value *argv) return nullptr; } -napi_value RdbStoreProxy::UnRegisteredObserver(napi_env env, const DistributedRdb::SubscribeOption &option, - std::map>> &observers, napi_value callback) +napi_value RdbStoreProxy::UnRegisteredObserver( + napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback) { + auto &observers = option.mode == SubscribeMode::LOCAL ? localObservers_ : localSharedObservers_; auto obs = observers.find(option.event); if (obs == observers.end()) { LOG_INFO("Observer not found, event: %{public}s", option.event.c_str()); @@ -1765,14 +1753,6 @@ napi_value RdbStoreProxy::UnRegisteredObserver(napi_env env, const DistributedRd return nullptr; } -napi_value RdbStoreProxy::OffLocal(napi_env env, const DistributedRdb::SubscribeOption &option, napi_value callback) -{ - if (option.mode == SubscribeMode::LOCAL) { - return UnRegisteredObserver(env, option, localObservers_, callback); - } - return UnRegisteredObserver(env, option, localSharedObservers_, callback); -} - napi_value RdbStoreProxy::OnEvent(napi_env env, napi_callback_info info) { size_t argc = 3; @@ -1806,9 +1786,9 @@ napi_value RdbStoreProxy::OnEvent(napi_env env, napi_callback_info info) RDB_NAPI_ASSERT(env, type == napi_function, std::make_shared("observer", "function")); SubscribeOption option; option.event = event; - valueBool ? option.mode = SubscribeMode::LOCAL_SHARED : option.mode = SubscribeMode::LOCAL; + option.mode = valueBool ? SubscribeMode::LOCAL_SHARED : SubscribeMode::LOCAL; // 'argv[2]' represents a callback function - return proxy->OnLocal(env, option, argv[2]); + return proxy->RegisteredObserver(env, option, argv[2]); } napi_value RdbStoreProxy::OffEvent(napi_env env, napi_callback_info info) @@ -1850,7 +1830,7 @@ napi_value RdbStoreProxy::OffEvent(napi_env env, napi_callback_info info) option.event = event; valueBool ? option.mode = SubscribeMode::LOCAL_SHARED : option.mode = SubscribeMode::LOCAL; // 'argv[2]' represents a callback function, 'argc == 3' represents determine if 'argc' is equal to '3' - return proxy->OffLocal(env, option, argc == 3 ? argv[2] : nullptr); + return proxy->UnRegisteredObserver(env, option, argc == 3 ? argv[2] : nullptr); } napi_value RdbStoreProxy::Notify(napi_env env, napi_callback_info info) @@ -2088,8 +2068,8 @@ napi_value RdbStoreProxy::QueryLockedRow(napi_env env, napi_callback_info info) context->rdbPredicates->EqualTo(AbsRdbPredicates::LOCK_STATUS, AbsRdbPredicates::LOCK_CHANGED)->EndWrap(); context->resultSet = context->rdbStore->QueryByStep(*(context->rdbPredicates), context->columns); context->rdbStore = nullptr; - // If the API version is greater than or equal to 16, throw E_ALREADY_CLOSED. - return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 16) ? E_ALREADY_CLOSED : E_ERROR; + // If the API version is greater than or equal to 20, throw E_ALREADY_CLOSED. + return (context->resultSet != nullptr) ? E_OK : (JSUtils::GetHapVersion() >= 20) ? E_ALREADY_CLOSED : E_ERROR; }; auto output = [context](napi_env env, napi_value &result) { result = ResultSetProxy::NewInstance(env, std::move(context->resultSet)); diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store_helper.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store_helper.cpp index 91e09773..83c547ad 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store_helper.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_rdb_store_helper.cpp @@ -71,7 +71,13 @@ napi_value GetRdbStore(napi_env env, napi_callback_info info) CHECK_RETURN_SET_E(context->config.cryptoParam.IsValid(), std::make_shared("Illegal CryptoParam.")); CHECK_RETURN_SET_E(context->config.tokenizer >= NONE_TOKENIZER && context->config.tokenizer < TOKENIZER_END, std::make_shared("Illegal tokenizer.")); - + CHECK_RETURN_SET_E(RdbHelper::IsSupportedTokenizer(context->config.tokenizer), + std::make_shared(NativeRdb::E_NOT_SUPPORT)); + if (!context->config.persist) { + CHECK_RETURN_SET_E(context->config.rootDir.empty(), + std::make_shared(NativeRdb::E_NOT_SUPPORT)); + return; + } auto [code, err] = GetRealPath(env, argv[0], context->config, context->param); if (!context->config.rootDir.empty()) { context->config.isReadOnly = true; @@ -128,16 +134,16 @@ napi_value DeleteRdbStore(napi_env env, napi_callback_info info) CHECK_RETURN_SET_E(OK == code, err); }; auto exec = [context]() -> int { - // If the API version is greater than or equal to 16, close the connection. + // If the API version is greater than or equal to 20, close the connection. RdbStoreConfig storeConfig = GetRdbStoreConfig(context->config, context->param); if (context->onlyPath) { storeConfig.SetDBType(DB_SQLITE); - int errCodeSqlite = RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 16); + int errCodeSqlite = RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 20); storeConfig.SetDBType(DB_VECTOR); - int errCodeVector = RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 16); + int errCodeVector = RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 20); return (errCodeSqlite == E_OK && errCodeVector == E_OK) ? E_OK : E_REMOVE_FILE; } - return RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 16); + return RdbHelper::DeleteRdbStore(storeConfig, JSUtils::GetHapVersion() >= 20); }; auto output = [context](napi_env env, napi_value &result) { napi_status status = napi_create_int64(env, OK, &result); diff --git a/relational_store/frameworks/js/napi/relationalstore/src/napi_result_set.cpp b/relational_store/frameworks/js/napi/relationalstore/src/napi_result_set.cpp index 36590ab4..2e464046 100644 --- a/relational_store/frameworks/js/napi/relationalstore/src/napi_result_set.cpp +++ b/relational_store/frameworks/js/napi/relationalstore/src/napi_result_set.cpp @@ -12,6 +12,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include +#include "js_native_api.h" +#include "js_native_api_types.h" #define LOG_TAG "ResultSetProxy" #include "napi_result_set.h" @@ -214,15 +218,54 @@ napi_value ResultSetProxy::GetColumnCount(napi_env env, napi_callback_info info) napi_value ResultSetProxy::GetColumnType(napi_env env, napi_callback_info info) { - DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); - int32_t columnIndex; - auto resultSet = ParseInt32FieldByName(env, info, columnIndex, "columnIndex"); - ColumnType columnType = ColumnType::TYPE_NULL; - if (resultSet != nullptr) { - resultSet->GetColumnType(columnIndex, columnType); - } - - return JSUtils::Convert2JSValue(env, int32_t(columnType)); + struct TypeContextBase : public ContextBase { + public: + int32_t columnIndex = 0; + std::string columnName; + ColumnType columnType = ColumnType::TYPE_NULL; + std::weak_ptr resultSet; + }; + std::shared_ptr context = std::make_shared(); + auto resultSet = GetInnerResultSet(env, info); + RDB_NAPI_ASSERT(env, resultSet != nullptr, std::make_shared(E_ALREADY_CLOSED)); + context->resultSet = resultSet; + auto input = [context](napi_env env, size_t argc, napi_value *argv, napi_value self) { + CHECK_RETURN_SET_E(argc > 0, std::make_shared("1")); + napi_valuetype type = napi_undefined; + napi_typeof(env, argv[0], &type); + if (type == napi_number) { + auto errCode = JSUtils::Convert2ValueExt(env, argv[0], context->columnIndex); + CHECK_RETURN_SET_E( + OK == errCode && context->columnIndex >= 0, std::make_shared("Invalid columnIndex")); + } else { + auto errCode = JSUtils::Convert2Value(env, argv[0], context->columnName); + CHECK_RETURN_SET_E(OK == errCode && !context->columnName.empty(), + std::make_shared("columnName", "a non empty string.")); + } + }; + auto exec = [context]() -> int { + auto result = context->resultSet.lock(); + if (result == nullptr) { + return E_ALREADY_CLOSED; + } + int errCode = E_OK; + if (!context->columnName.empty()) { + errCode = result->GetColumnIndex(context->columnName, context->columnIndex); + } + if (errCode == E_OK) { + errCode = result->GetColumnType(context->columnIndex, context->columnType); + } + if (errCode == E_INVALID_ARGS) { + return E_INVALID_ARGS_NEW; + } + return errCode; + }; + auto output = [context](napi_env env, napi_value &result) { + result = JSUtils::Convert2JSValue(env, static_cast(context->columnType)); + }; + context->SetAction(env, info, input, exec, output); + CHECK_RETURN_NULL(context->error == nullptr || context->error->GetCode() == OK); + return ASYNC_CALL(env, context); } napi_value ResultSetProxy::GetRowCount(napi_env env, napi_callback_info info) @@ -713,7 +756,8 @@ void ResultSetProxy::Init(napi_env env, napi_value exports) std::vector properties = { DECLARE_NAPI_FUNCTION("goToRow", GoToRow), DECLARE_NAPI_FUNCTION("getLong", GetLong), - DECLARE_NAPI_FUNCTION("getColumnType", GetColumnType), + DECLARE_NAPI_FUNCTION_WITH_DATA("getColumnType", GetColumnType, ASYNC), + DECLARE_NAPI_FUNCTION_WITH_DATA("getColumnTypeSync", GetColumnType, SYNC), DECLARE_NAPI_FUNCTION("goTo", GoTo), DECLARE_NAPI_FUNCTION("getColumnIndex", GetColumnIndex), DECLARE_NAPI_FUNCTION("getColumnName", GetColumnName), diff --git a/relational_store/frameworks/native/cloud_data/src/cloud_types_util.cpp b/relational_store/frameworks/native/cloud_data/src/cloud_types_util.cpp index 0b72056e..abf93c05 100644 --- a/relational_store/frameworks/native/cloud_data/src/cloud_types_util.cpp +++ b/relational_store/frameworks/native/cloud_data/src/cloud_types_util.cpp @@ -145,11 +145,11 @@ bool Marshalling(const CommonAsset &input, MessageParcel &data) template<> bool Marshalling(const CloudSyncInfo &input, MessageParcel &data) { - return Marshal(data, input.startTime, input.finishTime, input.code); + return Marshal(data, input.startTime, input.finishTime, input.code, input.syncStatus); } template<> bool Unmarshalling(CloudSyncInfo &output, MessageParcel &data) { - return Unmarshal(data, output.startTime, output.finishTime, output.code); + return Unmarshal(data, output.startTime, output.finishTime, output.code, output.syncStatus); } } // namespace OHOS::ITypesUtil \ No newline at end of file diff --git a/relational_store/frameworks/native/dfx/include/rdb_dfx_errno.h b/relational_store/frameworks/native/dfx/include/rdb_dfx_errno.h new file mode 100644 index 00000000..eec0d0ce --- /dev/null +++ b/relational_store/frameworks/native/dfx/include/rdb_dfx_errno.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DISTRIBUTEDDATAMGR_RDB_DFX_ERRNO_H +#define DISTRIBUTEDDATAMGR_RDB_DFX_ERRNO_H +#include +namespace OHOS { +namespace NativeRdb { + +/** +* @brief The base code of the exception dfx error code, base value is 0x1A28000. +*/ +constexpr int E_DFX_BASE = ErrCodeOffset(SUBSYS_DISTRIBUTEDDATAMNG, 2) + 0x8000; + +/** + * @brief Database db err message is not create. + */ +static constexpr int E_DFX_IS_NOT_CREATE = (E_DFX_BASE + 0x1); + +/** + * @brief Database db err message is delete. + */ +static constexpr int E_DFX_IS_DELETE = (E_DFX_BASE + 0x2); + +/** + * @brief Database db err message is rename. + */ +static constexpr int E_DFX_IS_RENAME = (E_DFX_BASE + 0x3); + +/** + * @brief Database db err message is not exist. + */ +static constexpr int E_DFX_IS_NOT_EXIST = (E_DFX_BASE + 0x4); + +/** + * @brief Only use for dfx, sqlite error log. + */ +static constexpr int E_DFX_SQLITE_LOG = (E_DFX_BASE + 0x5); + +/** + * @brief Only use for dfx, batch insert args size too big. + */ +static constexpr int E_DFX_BATCH_INSERT_ARGS_SIZE = (E_DFX_BASE + 0x6); + +/** + * @brief Only use for dfx, get journal mode fail. + */ +static constexpr int E_DFX_GET_JOURNAL_FAIL = (E_DFX_BASE + 0x7); + +/** + * @brief Only use for dfx, set journal mode fail. + */ +static constexpr int E_DFX_SET_JOURNAL_FAIL = (E_DFX_BASE + 0x8); + +/** + * @brief Only use for dfx, print dump info. + */ +static constexpr int E_DFX_DUMP_INFO = (E_DFX_BASE + 0x9); + +} // namespace NativeRdb +} // namespace OHOS + +#endif // DISTRIBUTEDDATAMGR_RDB_DFX_ERRNO_H \ No newline at end of file diff --git a/relational_store/frameworks/native/dfx/include/rdb_fault_hiview_reporter.h b/relational_store/frameworks/native/dfx/include/rdb_fault_hiview_reporter.h index fabf3609..96eb7713 100644 --- a/relational_store/frameworks/native/dfx/include/rdb_fault_hiview_reporter.h +++ b/relational_store/frameworks/native/dfx/include/rdb_fault_hiview_reporter.h @@ -26,8 +26,10 @@ #include "connection.h" #include "rdb_store_config.h" #include "rdb_types.h" +#include "rdb_dfx_errno.h" namespace OHOS::NativeRdb { using DebugInfo = OHOS::DistributedRdb::RdbDebugInfo; +using DfxInfo = OHOS::DistributedRdb::RdbDfxInfo; struct RdbCorruptedEvent { std::string bundleName; std::string moduleName; @@ -43,27 +45,12 @@ struct RdbCorruptedEvent { time_t errorOccurTime; std::string path; std::map debugInfos; + DfxInfo dfxInfo; }; -struct RdbFaultCounter { - uint8_t full{ 0 }; - uint8_t corrupt{ 0 }; - uint8_t perm{ 0 }; - uint8_t busy{ 0 }; - uint8_t noMem{ 0 }; - uint8_t ioErr{ 0 }; - uint8_t cantOpen{ 0 }; - uint8_t constraint{ 0 }; - uint8_t notDb{ 0 }; - uint8_t rootKeyFault{ 0 }; - uint8_t rootKeyNotLoad{ 0 }; - uint8_t workKeyFault{ 0 }; - uint8_t workkeyEencrypt{ 0 }; - uint8_t workKeyDcrypt{ 0 }; - uint8_t setEncrypt{ 0 }; - uint8_t setNewEncrypt{ 0 }; - uint8_t setServiceEncrypt{ 0 }; - uint8_t checkPoint{ 0 }; +struct RdbFaultCode { + int nativeCode; + uint8_t faultCounter; }; // Fault Type Define @@ -72,6 +59,9 @@ static constexpr const char *FT_CURD = "CURD_DB"; static constexpr const char *FT_EX_FILE = "EX_FILE"; static constexpr const char *FT_EX_HUKS = "EX_HUKS"; static constexpr const char *FT_CP = "CHECK_POINT"; +static constexpr const char *FT_SQLITE = "SQLITE"; + +static constexpr const char *BUNDLE_NAME_COMMON = "common"; class API_EXPORT RdbFaultEvent { public: @@ -117,8 +107,11 @@ public: class API_EXPORT RdbFaultHiViewReporter { public: - static RdbCorruptedEvent Create(const RdbStoreConfig &config, int32_t errCode, const std::string &appendix = ""); - static bool RegCollector(Connection::Collector collector); + using Collector = int32_t (*)(const RdbStoreConfig &config, std::map&, DistributedRdb::RdbDfxInfo&); + static RdbCorruptedEvent Create(const RdbStoreConfig &config, int32_t errCode, const std::string &appendix = "", + bool needSyncParaFromSrv = true); + static bool RegCollector(Collector collector); static void ReportCorrupted(const RdbCorruptedEvent &eventInfo); static void ReportCorruptedOnce(const RdbCorruptedEvent &eventInfo); static void ReportFault(const RdbFaultEvent &faultEvent); @@ -131,9 +124,10 @@ private: static void CreateCorruptedFlag(const std::string &dbPath); static void DeleteCorruptedFlag(const std::string &dbPath); static bool IsReportFault(const std::string &bundleName, int32_t errCode); - static uint8_t *GetFaultCounter(RdbFaultCounter &counter, int32_t errCode); - static Connection::Collector collector_; - static RdbFaultCounter faultCounter_; + static uint8_t *GetFaultCounter(int32_t errCode); + static Collector collector_; + static RdbFaultCode faultCounters_[]; + static bool memCorruptReportedFlg_; }; } // namespace OHOS::NativeRdb #endif // DISTRIBUTEDDATAMGR_RDB_FAULT_HIVIEW_REPORTER_H diff --git a/relational_store/frameworks/native/dfx/include/rdb_stat_reporter.h b/relational_store/frameworks/native/dfx/include/rdb_stat_reporter.h new file mode 100644 index 00000000..d719a65c --- /dev/null +++ b/relational_store/frameworks/native/dfx/include/rdb_stat_reporter.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RDB_STAT_REPORTER_H +#define RDB_STAT_REPORTER_H + +#include +#include + +#include "rdb_store_config.h" +#include "rdb_types.h" +namespace OHOS::NativeRdb { +enum StatType : int32_t { + RDB_PERF = 1, +}; + +enum SubType : int32_t { + INSERT = 1, + BATCHINSERT, + UPDATE, + DELETE, + EXECUTESQL, + EXECUTE, + BEGINTRANSACTION, + COMMIT, + ROLLBACK, +}; + +enum TimeType : int32_t { + TIME_LEVEL_NONE = 0, + TIME_LEVEL_FIRST, + TIME_LEVEL_SECOND, + TIME_LEVEL_THIRD, +}; + +class RdbStatReporter { +public: + using ReportFunc = std::function; + RdbStatReporter(StatType statType, SubType subType, const RdbStoreConfig &config, std::shared_ptr func); + ~RdbStatReporter(); + static TimeType GetTimeType(uint32_t costTime); +private: + std::chrono::steady_clock::time_point startTime_; + DistributedRdb::RdbStatEvent statEvent_; + std::shared_ptr reportFunc_ = nullptr; + static std::atomic reportTime_; +}; + +} // namespace OHOS::NativeRdb +#endif // RDB_STAT_REPORTER_H \ No newline at end of file diff --git a/relational_store/frameworks/native/dfx/src/rdb_fault_hiview_reporter.cpp b/relational_store/frameworks/native/dfx/src/rdb_fault_hiview_reporter.cpp index 7146b422..e561d799 100644 --- a/relational_store/frameworks/native/dfx/src/rdb_fault_hiview_reporter.cpp +++ b/relational_store/frameworks/native/dfx/src/rdb_fault_hiview_reporter.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -45,14 +44,48 @@ static constexpr const char *FAULT_EVENT = "DISTRIBUTED_DATA_RDB_FAULT"; static constexpr const char *DISTRIBUTED_DATAMGR = "DISTDATAMGR"; static constexpr const char *DB_CORRUPTED_POSTFIX = ".corruptedflg"; static constexpr int MAX_FAULT_TIMES = 1; -Connection::Collector RdbFaultHiViewReporter::collector_ = nullptr; -RdbFaultCounter RdbFaultHiViewReporter::faultCounter_ = { 0 }; +RdbFaultHiViewReporter::Collector RdbFaultHiViewReporter::collector_ = nullptr; + +RdbFaultCode RdbFaultHiViewReporter::faultCounters_[] = { + { E_CREATE_FOLDER_FAIL, 0}, + { E_SQLITE_FULL, 0 }, + { E_SQLITE_CORRUPT, 0 }, + { E_SQLITE_PERM, 0 }, + { E_SQLITE_BUSY, 0 }, + { E_SQLITE_NOMEM, 0 }, + { E_SQLITE_IOERR, 0 }, + { E_SQLITE_CANTOPEN, 0 }, + { E_SQLITE_CONSTRAINT, 0 }, + { E_SQLITE_NOT_DB, 0 }, + { E_ROOT_KEY_FAULT, 0 }, + { E_ROOT_KEY_NOT_LOAD, 0 }, + { E_WORK_KEY_FAIL, 0 }, + { E_WORK_KEY_ENCRYPT_FAIL, 0 }, + { E_WORK_KEY_DECRYPT_FAIL, 0 }, + { E_SET_ENCRYPT_FAIL, 0 }, + { E_SET_NEW_ENCRYPT_FAIL, 0 }, + { E_SET_SERVICE_ENCRYPT_FAIL, 0 }, + { E_CHECK_POINT_FAIL, 0 }, + { E_SQLITE_META_RECOVERED, 0 }, + { E_DFX_IS_NOT_CREATE, 0 }, + { E_DFX_IS_DELETE, 0 }, + { E_DFX_IS_RENAME, 0 }, + { E_DFX_IS_NOT_EXIST, 0 }, + { E_DFX_SQLITE_LOG, 0 }, + { E_DFX_BATCH_INSERT_ARGS_SIZE, 0 }, + { E_DFX_GET_JOURNAL_FAIL, 0 }, + { E_DFX_SET_JOURNAL_FAIL, 0 }, + { E_DFX_DUMP_INFO, 0 }, +}; + +bool RdbFaultHiViewReporter::memCorruptReportedFlg_ = false; void RdbFaultHiViewReporter::ReportCorruptedOnce(const RdbCorruptedEvent &eventInfo) { if (IsReportCorruptedFault(eventInfo.path)) { RdbCorruptedEvent eventInfoAppend = eventInfo; eventInfoAppend.appendix += SqliteUtils::FormatDebugInfo(eventInfoAppend.debugInfos, ""); + eventInfoAppend.appendix += SqliteUtils::FormatDfxInfo(eventInfo.dfxInfo); LOG_WARN("Corrupted %{public}s errCode:0x%{public}x [%{public}s]", SqliteUtils::Anonymous(eventInfoAppend.storeName).c_str(), eventInfoAppend.errorCode, eventInfoAppend.appendix.c_str()); @@ -103,7 +136,7 @@ void RdbFaultHiViewReporter::ReportCorrupted(const RdbCorruptedEvent &eventInfo) bool RdbFaultHiViewReporter::IsReportCorruptedFault(const std::string &dbPath) { - if (dbPath.empty()) { + if (dbPath.empty() || memCorruptReportedFlg_) { return false; } @@ -116,6 +149,7 @@ bool RdbFaultHiViewReporter::IsReportCorruptedFault(const std::string &dbPath) void RdbFaultHiViewReporter::CreateCorruptedFlag(const std::string &dbPath) { + memCorruptReportedFlg_ = true; if (dbPath.empty()) { return; } @@ -143,7 +177,7 @@ void RdbFaultHiViewReporter::DeleteCorruptedFlag(const std::string &dbPath) } RdbCorruptedEvent RdbFaultHiViewReporter::Create( - const RdbStoreConfig &config, int32_t errCode, const std::string &appendix) + const RdbStoreConfig &config, int32_t errCode, const std::string &appendix, bool needSyncParaFromSrv) { RdbCorruptedEvent eventInfo; eventInfo.bundleName = config.GetBundleName(); @@ -160,13 +194,16 @@ RdbCorruptedEvent RdbFaultHiViewReporter::Create( eventInfo.errorOccurTime = time(nullptr); eventInfo.debugInfos = Connection::Collect(config); SqliteGlobalConfig::GetDbPath(config, eventInfo.path); - if (collector_ != nullptr && !IsReportCorruptedFault(eventInfo.path)) { - Update(eventInfo.debugInfos, collector_(config)); + if (collector_ != nullptr && needSyncParaFromSrv) { + std::map serviceDebugInfos; + if (collector_(config, serviceDebugInfos, eventInfo.dfxInfo) == E_OK) { + Update(eventInfo.debugInfos, serviceDebugInfos); + } } return eventInfo; } -bool RdbFaultHiViewReporter::RegCollector(Connection::Collector collector) +bool RdbFaultHiViewReporter::RegCollector(Collector collector) { if (collector_ != nullptr) { return false; @@ -214,48 +251,16 @@ std::string RdbFaultHiViewReporter::GetBundleName(const std::string &bundleName, return SqliteUtils::Anonymous(storeName); } -uint8_t *RdbFaultHiViewReporter::GetFaultCounter(RdbFaultCounter &counter, int32_t errCode) +uint8_t *RdbFaultHiViewReporter::GetFaultCounter(int32_t errCode) { - switch (errCode) { - case E_SQLITE_FULL: - return &counter.full; - case E_SQLITE_CORRUPT: - return &counter.corrupt; - case E_SQLITE_PERM: - return &counter.perm; - case E_SQLITE_BUSY: - return &counter.busy; - case E_SQLITE_NOMEM: - return &counter.noMem; - case E_SQLITE_IOERR: - return &counter.ioErr; - case E_SQLITE_CANTOPEN: - return &counter.cantOpen; - case E_SQLITE_CONSTRAINT: - return &counter.constraint; - case E_SQLITE_NOT_DB: - return &counter.notDb; - case E_ROOT_KEY_FAULT: - return &counter.rootKeyFault; - case E_ROOT_KEY_NOT_LOAD: - return &counter.rootKeyNotLoad; - case E_WORK_KEY_FAIL: - return &counter.workKeyFault; - case E_WORK_KEY_ENCRYPT_FAIL: - return &counter.workkeyEencrypt; - case E_WORK_KEY_DECRYPT_FAIL: - return &counter.workKeyDcrypt; - case E_SET_ENCRYPT_FAIL: - return &counter.setEncrypt; - case E_SET_NEW_ENCRYPT_FAIL: - return &counter.setNewEncrypt; - case E_SET_SERVICE_ENCRYPT_FAIL: - return &counter.setServiceEncrypt; - case E_CHECK_POINT_FAIL: - return &counter.checkPoint; - default: - return nullptr; - }; + auto it = std::lower_bound(faultCounters_, faultCounters_ + sizeof(faultCounters_) / sizeof(RdbFaultCode), errCode, + [](const RdbFaultCode& faultCode, int32_t code) { + return faultCode.nativeCode < code; + }); + if (it != faultCounters_ + sizeof(faultCounters_) / sizeof(RdbFaultCode) && it->nativeCode == errCode) { + return &it->faultCounter; + } + return nullptr; } bool RdbFaultHiViewReporter::IsReportFault(const std::string &bundleName, int32_t errCode) @@ -263,11 +268,11 @@ bool RdbFaultHiViewReporter::IsReportFault(const std::string &bundleName, int32_ if (bundleName.empty()) { return false; } - uint8_t *counter = GetFaultCounter(faultCounter_, errCode); + uint8_t *counter = GetFaultCounter(errCode); if (counter == nullptr) { return false; } - if (*counter < UCHAR_MAX) { + if (*counter < UINT8_MAX) { (*counter)++; } return *counter <= MAX_FAULT_TIMES; @@ -300,7 +305,8 @@ void RdbFaultEvent::Report() const { .name = "FAULT_TIME", .t = HISYSEVENT_STRING, .v = { .s = occurTime.data() }, .arraySize = 0 }, { .name = "FAULT_TYPE", .t = HISYSEVENT_STRING, .v = { .s = faultType.data() }, .arraySize = 0 }, { .name = "BUNDLE_NAME", .t = HISYSEVENT_STRING, .v = { .s = bundleName.data() }, .arraySize = 0 }, - { .name = "ERROR_CODE", .t = HISYSEVENT_INT32, .v = { .ui32 = static_cast(errorCode_) }, .arraySize = 0 }, + { .name = "ERROR_CODE", .t = HISYSEVENT_INT32, .v = { .ui32 = static_cast(errorCode_) }, + .arraySize = 0 }, { .name = "APPENDIX", .t = HISYSEVENT_STRING, .v = { .s = appendInfo.data() }, .arraySize = 0 }, }; auto size = sizeof(params) / sizeof(params[0]); @@ -309,9 +315,8 @@ void RdbFaultEvent::Report() const RdbFaultDbFileEvent::RdbFaultDbFileEvent(const std::string &faultType, int32_t errorCode, const RdbStoreConfig &config, const std::string &custLog, bool printDbInfo) - : RdbFaultEvent(faultType, errorCode, "", custLog), config_(config), printDbInfo_(printDbInfo) + : RdbFaultEvent(faultType, errorCode, config.GetBundleName(), custLog), config_(config), printDbInfo_(printDbInfo) { - SetBundleName(RdbFaultHiViewReporter::GetBundleName(config_.GetBundleName(), config_.GetName())); } RdbEmptyBlobEvent::RdbEmptyBlobEvent(const std::string &bundleName) @@ -354,7 +359,8 @@ void RdbFaultDbFileEvent::Report() const { .name = "MODULE_NAME", .t = HISYSEVENT_STRING, .v = { .s = moduleName.data() }, .arraySize = 0 }, { .name = "STORE_NAME", .t = HISYSEVENT_STRING, .v = { .s = storeName.data() }, .arraySize = 0 }, { .name = "BUSINESS_TYPE", .t = HISYSEVENT_STRING, .v = { .s = businessType.data() }, .arraySize = 0 }, - { .name = "ERROR_CODE", .t = HISYSEVENT_INT32, .v = { .ui32 = static_cast(GetErrCode())}, .arraySize = 0 }, + { .name = "ERROR_CODE", .t = HISYSEVENT_INT32, .v = { .ui32 = static_cast(GetErrCode())}, + .arraySize = 0 }, { .name = "APPENDIX", .t = HISYSEVENT_STRING, .v = { .s = appendInfo.data() }, .arraySize = 0 }, }; auto size = sizeof(params) / sizeof(params[0]); @@ -366,7 +372,6 @@ std::string RdbFaultDbFileEvent::BuildConfigLog() const std::string errNoStr = std::to_string(static_cast(errno)); std::string dbPath; SqliteGlobalConfig::GetDbPath(config_, dbPath); - dbPath = SqliteUtils::Anonymous(dbPath); std::vector> logInfo; logInfo.emplace_back("S_L", std::to_string(static_cast(config_.GetSecurityLevel()))); logInfo.emplace_back("P_A", std::to_string(static_cast(config_.GetArea()))); @@ -387,16 +392,15 @@ std::string RdbFaultDbFileEvent::BuildConfigLog() const std::string RdbFaultDbFileEvent::BuildLogInfo() const { std::string appendInfo = GetLogInfo(); - std::string dbFileInfo = SqliteUtils::FormatDebugInfo( - RdbFaultHiViewReporter::Create(config_, GetErrCode()).debugInfos, ""); - if (GetErrCode() == E_SQLITE_NOT_DB) { std::string dbPath; SqliteGlobalConfig::GetDbPath(config_, dbPath); appendInfo += SqliteUtils::ReadFileHeader(dbPath); } if (printDbInfo_) { - appendInfo += ("\n" + BuildConfigLog() + "\n" + dbFileInfo); + RdbCorruptedEvent eventInfo = RdbFaultHiViewReporter::Create(config_, GetErrCode()); + appendInfo += ("\n" + BuildConfigLog() + "\n" + SqliteUtils::FormatDfxInfo(eventInfo.dfxInfo) + "\n" + + SqliteUtils::FormatDebugInfo(eventInfo.debugInfos, "")); } return appendInfo; } diff --git a/relational_store/frameworks/native/dfx/src/rdb_stat_reporter.cpp b/relational_store/frameworks/native/dfx/src/rdb_stat_reporter.cpp new file mode 100644 index 00000000..c56e3757 --- /dev/null +++ b/relational_store/frameworks/native/dfx/src/rdb_stat_reporter.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "RdbStatReporter" + +#include "rdb_stat_reporter.h" + +#include + +#include "logger.h" +#include "rdb_errno.h" +#include "rdb_time_utils.h" +#include "sqlite_global_config.h" +#include "sqlite_utils.h" +#include "task_executor.h" + +namespace OHOS::NativeRdb { +using namespace OHOS::Rdb; +constexpr int32_t WAIT_TIME = 600; +constexpr int32_t TIMEOUT_FIRST = 1500; +constexpr int32_t TIMEOUT_SECOND = 5000; +constexpr int32_t TIMEOUT_THIRD = 10000; +std::atomic RdbStatReporter::reportTime_ = + std::chrono::steady_clock::time_point(); + +RdbStatReporter::RdbStatReporter( + StatType statType, SubType subType, const RdbStoreConfig &config, std::shared_ptr func) +{ + if (func == nullptr) { + return; + } + startTime_ = std::chrono::steady_clock::now(); + statEvent_.statType = static_cast(statType); + statEvent_.bundleName = config.GetBundleName(); + statEvent_.storeName = SqliteUtils::Anonymous(config.GetName()); + statEvent_.subType = static_cast(subType); + reportFunc_ = std::move(func); +} + +RdbStatReporter::~RdbStatReporter() +{ + if (reportFunc_ == nullptr) { + return; + } + auto endTime = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(endTime - startTime_).count(); + auto loadedReportTime = reportTime_.load(); + auto reportInterval = std::chrono::duration_cast(endTime - loadedReportTime).count(); + statEvent_.costTime = GetTimeType(static_cast(duration)); + if (statEvent_.costTime > TIME_LEVEL_NONE && reportInterval >= WAIT_TIME) { + auto pool = TaskExecutor::GetInstance().GetExecutor(); + if (pool == nullptr) { + LOG_WARN("task pool err when RdbStatReporter"); + } + pool->Execute([report = std::move(reportFunc_), statEvent = std::move(statEvent_)]() { + (*report)(statEvent); + }); + reportTime_.store(std::chrono::steady_clock::now()); + } +} + +TimeType RdbStatReporter::GetTimeType(uint32_t costTime) +{ + if (costTime < TIMEOUT_FIRST) { + return TIME_LEVEL_NONE; + } else if (costTime < TIMEOUT_SECOND) { + return TIME_LEVEL_FIRST; + } else if (costTime < TIMEOUT_THIRD) { + return TIME_LEVEL_SECOND; + } + return TIME_LEVEL_THIRD; +} + +} // namespace OHOS::NativeRdb \ No newline at end of file diff --git a/relational_store/frameworks/native/gdb/adapter/include/grd_adapter.h b/relational_store/frameworks/native/gdb/adapter/include/grd_adapter.h index e6f8950b..a59ed59b 100644 --- a/relational_store/frameworks/native/gdb/adapter/include/grd_adapter.h +++ b/relational_store/frameworks/native/gdb/adapter/include/grd_adapter.h @@ -29,7 +29,6 @@ namespace OHOS::DistributedDataAip { class GrdAdapter { public: - static int TransErrno(int err); static ColumnType TransColType(int grdColType); static int Open(const char *dbPath, const char *configStr, uint32_t flags, GRD_DB **db); static int Repair(const char *dbPath, const char *configStr); @@ -56,6 +55,8 @@ public: static int Rekey(const char *dbFile, const char *configStr, const std::vector &encryptedKey); private: + static int TransErrno(int err); + static std::map GRD_ERRNO_MAP; }; diff --git a/relational_store/frameworks/native/gdb/adapter/include/grd_error.h b/relational_store/frameworks/native/gdb/adapter/include/grd_error.h deleted file mode 100644 index cb2d8e16..00000000 --- a/relational_store/frameworks/native/gdb/adapter/include/grd_error.h +++ /dev/null @@ -1,164 +0,0 @@ -/* -* Copyright (c) 2024 Huawei Device Co., Ltd. -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -#ifndef OHOS_DISTRIBUTED_DATA_OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRD_ERROR_H -#define OHOS_DISTRIBUTED_DATA_OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRD_ERROR_H - -#include "grd_type_export.h" - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus - -// Error category -#define GRD_OK 0 - -// Error category -#define GRD_NOT_SUPPORT (-1000) -#define GRD_OVER_LIMIT (-2000) -#define GRD_INVALID_ARGS (-3000) -#define GRD_SYSTEM_ERR (-4000) -#define GRD_FAILED_FILE_OPERATION (-5000) -#define GRD_INVALID_FILE_FORMAT (-6000) -#define GRD_INSUFFICIENT_SPACE (-7000) -#define GRD_INNER_ERR (-8000) -#define GRD_RESOURCE_BUSY (-9000) - -#define GRD_NO_DATA (-11000) -#define GRD_FAILED_MEMORY_ALLOCATE (-13000) -#define GRD_FAILED_MEMORY_RELEASE (-14000) -#define GRD_DATA_CONFLICT (-16000) -#define GRD_DATATYPE_MISMATCH (-17000) -#define GRD_NOT_AVAILABLE (-19000) -#define GRD_ACTIVE_TRANSACTION (-20000) -#define GRD_UNIQUE_VIOLATION (-21000) -#define GRD_DUPLICATE_TABLE (-22000) -#define GRD_UNDEFINED_TABLE (-23000) -#define GRD_INVALID_BIND_VALUE (-36000) -#define GRD_INVALID_FORMAT (-37000) -#define GRD_REBUILD_DATABASE (-38000) -#define GRD_TIME_OUT (-39000) -#define GRD_DB_INSTANCE_ABNORMAL (-40000) -#define GRD_DISK_SPACE_FULL (-41000) -#define GRD_CRC_CHECK_DISABLED (-42000) -#define GRD_PERMISSION_DENIED (-43000) -#define GRD_SYNC_PREREQUISITES_ABNORMAL (-44000) -#define GRD_DATA_CORRUPTED (-45000) -#define GRD_DB_BUSY (-46000) -#define GRD_CALC_MODE_SET_PERMISSION_DENIED (-47000) - -#define GRD_CIPHER_ERROR (-48000) -#define GRD_PASSWORD_NEED_REKEY (-48001) -#define GRD_PASSWORD_UNMATCHED (-48002) - -#define GRD_SCHEMA_CHANGED (-49000) -// not support -#define GRD_JSON_OPERATION_NOT_SUPPORT (-5001001) -#define GRD_MODEL_NOT_SUPPORT (-5001002) -#define GRD_FEATURE_NOT_SUPPORTED (-5001003) - -// Exceed limit -#define GRD_JSON_LEN_LIMIT (-5002001) -#define GRD_SUBSCRIPTION_EXCEEDED_LIMIT (-5002002) -#define GRD_SYNC_EXCEED_TASK_QUEUE_LIMIT (-5002003) -#define GRD_SHARED_OBJ_ENABLE_UNDO_EXCEED_LIMIT (-5002004) -#define GRD_TABLE_LIMIT_EXCEEDED (-5002005) - -// Invalid parameter -#define GRD_FIELD_TYPE_NOT_MATCH (-5003001) -#define GRD_LARGE_JSON_NEST (-5003002) -#define GRD_INVALID_JSON_TYPE (-5003003) -#define GRD_INVALID_CONFIG_VALUE (-5003004) -#define GRD_INVALID_OPERATOR (-5003005) -#define GRD_INVALID_PROJECTION_FIELD (-5003006) -#define GRD_INVALID_PROJECTION_VALUE (-5003007) -#define GRD_COLLECTION_NOT_EXIST (-5003008) -#define GRD_DB_NOT_EXIST (-5003009) -#define GRD_INVALID_VALUE (-5003010) -#define GRD_SHARED_OBJ_NOT_EXIST (-5003020) -#define GRD_SUBSCRIBE_NOT_EXIST (-5003021) -#define GRD_SHARED_OBJ_UNDO_MANAGER_NOT_EXIST (-5003022) -#define GRD_SHARED_OBJ_INVALID_UNDO (-5003023) -#define GRD_SHARED_OBJ_INVALID_REDO (-5003024) -#define GRD_NAME_TOO_LONG (-5003025) -#define GRD_INVALID_TABLE_DEFINITION (-5003026) -#define GRD_WRONG_STMT_OBJECT (-5003027) -#define GRD_SEMANTIC_ERROR (-5003028) -#define GRD_SYNTAX_ERROR (-5003029) - -// System err -#define GRD_JSON_LIB_HANDLE_FAILED (-5004001) -#define GRD_DIRECTORY_OPERATE_FAILED (-5004002) -#define GRD_FILE_OPERATE_FAILED (-5004003) -#define GRD_LOAD_THIRD_PARTY_LIBRARY_FAILED (-5004004) -#define GRD_THIRD_PARTY_FUNCTION_EXECUTE_FAILED (-5004005) -#define GRD_INSUFFICIENT_RESOURCES (-5004006) - -// resource busy -#define GRD_RESULTSET_BUSY (-5009001) - -// no data -#define GRD_RECORD_NOT_FOUND (-5011001) -#define GRD_FIELD_NOT_FOUND (-5011002) -#define GRD_ARRAY_INDEX_NOT_FOUND (-5011003) - -// data conflicted -#define GRD_KEY_CONFLICT (-5016001) -#define GRD_FIELD_TYPE_CONFLICT (-5016002) -#define GRD_SHARED_OBJ_CONFLICT (-5016003) -#define GRD_SUBSCRIBE_CONFLICT (-5016004) -#define GRD_EQUIP_ID_CONFLICT (-5016005) -#define GRD_SHARED_OBJ_ENABLE_UNDO_CONFLICT (-5016006) - -// data exception -#define GRD_DATA_EXCEPTION (-5017001) -#define GRD_FIELD_OVERFLOW (-5017002) -#define GRD_DIVISION_BY_ZERO (-5017003) - -// Cursor or ResultSet not available -#define GRD_RESULT_SET_NOT_AVAILABLE (-5019001) -#define GRD_SHARED_OBJ_UNDO_NOT_AVAILABLE (-5019002) -#define GRD_SHARED_OBJ_REDO_NOT_AVAILABLE (-5019003) - -// Transaction -#define GRD_TRANSACTION_ROLLBACK (-5020001) -#define GRD_NO_ACTIVE_TRANSACTION (-5020002) - -// violation -#define GRD_PRIMARY_KEY_VIOLATION (-5021001) -#define GRD_RESTRICT_VIOLATION (-5021002) -#define GRD_CONSTRAINT_CHECK_VIOLATION (-5021003) - -// duplicate -#define GRD_DUPLICATE_COLUMN (-5022001) -#define GRD_DUPLICATE_OBJECT (-5022002) - -// undefined -#define GRD_UNDEFINE_COLUMN (-5023001) -#define GRD_UNDEFINED_OBJECT (-5023002) - -// Invalid format -#define GRD_INVALID_JSON_FORMAT (-5037001) -#define GRD_INVALID_KEY_FORMAT (-5037002) -#define GRD_INVALID_COLLECTION_NAME (-5037003) -#define GRD_INVALID_EQUIP_ID (-5037004) - -// time out -#define GRD_REQUEST_TIME_OUT (-5039001) - -#ifdef __cplusplus -} -#endif // __cplusplus -#endif // OHOS_DISTRIBUTED_DATA_OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRD_ERROR_H \ No newline at end of file diff --git a/relational_store/frameworks/native/gdb/adapter/src/grd_adapter.cpp b/relational_store/frameworks/native/gdb/adapter/src/grd_adapter.cpp index 0d63f891..1b2129ce 100644 --- a/relational_store/frameworks/native/gdb/adapter/src/grd_adapter.cpp +++ b/relational_store/frameworks/native/gdb/adapter/src/grd_adapter.cpp @@ -19,7 +19,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "gdb_utils.h" #include "grd_adapter_manager.h" #include "grd_error.h" @@ -167,13 +167,11 @@ int GrdAdapter::Open(const char *dbPath, const char *configStr, uint32_t flags, return E_NOT_SUPPORT; } auto ret = g_adapterHolder.Open(dbPath, configStr, flags, db); - LOG_DEBUG("Open ret=%{public}d, *db=%{public}p", ret, *db); return TransErrno(ret); } int GrdAdapter::Close(GRD_DB *db, uint32_t flags) { - LOG_DEBUG("Close *db=%{public}p", db); if (g_adapterHolder.Close == nullptr) { g_adapterHolder = GetAdapterHolder(); } @@ -241,7 +239,6 @@ int GrdAdapter::Rekey(const char *dbFile, const char *configStr, const std::vect int32_t GrdAdapter::Prepare(GRD_DB *db, const char *str, uint32_t strLen, GRD_StmtT **stmt, const char **unusedStr) { - LOG_DEBUG("Prepare *db=%{public}p", db); if (g_adapterHolder.Prepare == nullptr) { g_adapterHolder = GetAdapterHolder(); } @@ -249,7 +246,6 @@ int32_t GrdAdapter::Prepare(GRD_DB *db, const char *str, uint32_t strLen, GRD_St return E_NOT_SUPPORT; } int32_t ret = g_adapterHolder.Prepare(db, str, strLen, stmt, unusedStr); - LOG_DEBUG("Prepare ret=%{public}d, stmt=%{public}p", ret, *stmt); return TransErrno(ret); } @@ -266,7 +262,6 @@ int32_t GrdAdapter::Reset(GRD_StmtT *stmt) int32_t GrdAdapter::Finalize(GRD_StmtT *stmt) { - LOG_DEBUG("Finalize *stmt=%{public}p", stmt); if (g_adapterHolder.Finalize == nullptr) { g_adapterHolder = GetAdapterHolder(); } @@ -280,7 +275,6 @@ int32_t GrdAdapter::Finalize(GRD_StmtT *stmt) int32_t GrdAdapter::Step(GRD_StmtT *stmt) { - LOG_DEBUG("Step *stmt=%{public}p", stmt); if (g_adapterHolder.Step == nullptr) { g_adapterHolder = GetAdapterHolder(); } diff --git a/relational_store/frameworks/native/gdb/include/db_store_impl.h b/relational_store/frameworks/native/gdb/include/db_store_impl.h index f754a07d..a50062fb 100644 --- a/relational_store/frameworks/native/gdb/include/db_store_impl.h +++ b/relational_store/frameworks/native/gdb/include/db_store_impl.h @@ -23,7 +23,7 @@ #include "connection_pool.h" #include "gdb_store.h" #include "gdb_store_config.h" -#include "transaction.h" +#include "gdb_transaction.h" namespace OHOS::DistributedDataAip { class DBStoreImpl final : public DBStore { diff --git a/relational_store/frameworks/native/gdb/include/gdb_utils.h b/relational_store/frameworks/native/gdb/include/gdb_utils.h index b339570b..055e91c3 100644 --- a/relational_store/frameworks/native/gdb/include/gdb_utils.h +++ b/relational_store/frameworks/native/gdb/include/gdb_utils.h @@ -17,8 +17,10 @@ #define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GDB_UTILS_H #include +#include "rdb_visibility.h" + namespace OHOS::DistributedDataAip { -class GdbUtils { +class API_EXPORT GdbUtils { public: static bool IsTransactionGql(const std::string &gql); static int CreateDirectory(const std::string &databaseDir); @@ -29,7 +31,8 @@ public: private: static constexpr int DIR_RWXRWS__X = 0771; static constexpr const char *GRD_OPEN_CONFIG_STR = - "\"pageSize\":4, \"crcCheckEnable\":0"; + R"("pageSize": 8, "crcCheckEnable": 0, "defaultIsolationLevel": 3, "redoFlushByTrx": 1, )" + R"("metaInfoBak": 1, "maxConnNum": 500, "bufferPoolSize": 10240)"; static std::string GetAnonymousName(const std::string& fileName); static std::string AnonyDigits(const std::string& fileName); }; diff --git a/relational_store/frameworks/native/gdb/include/graph_connection.h b/relational_store/frameworks/native/gdb/include/graph_connection.h index 58542bec..670a5a1d 100644 --- a/relational_store/frameworks/native/gdb/include/graph_connection.h +++ b/relational_store/frameworks/native/gdb/include/graph_connection.h @@ -44,6 +44,7 @@ private: int InnerOpen(const StoreConfig &config); int32_t ResetKey(const StoreConfig &config); + bool IsEncryptInvalidChanged(const StoreConfig &config); GRD_DB *dbHandle_ = nullptr; const StoreConfig config_; bool isWriter_ = false; diff --git a/relational_store/frameworks/native/gdb/include/transaction_impl.h b/relational_store/frameworks/native/gdb/include/transaction_impl.h index b191f09b..9673bfd2 100644 --- a/relational_store/frameworks/native/gdb/include/transaction_impl.h +++ b/relational_store/frameworks/native/gdb/include/transaction_impl.h @@ -14,16 +14,16 @@ */ #ifndef ARKDATA_INTELLIGENCE_PLATFORM_TRANSACTION_IMPL_H #define ARKDATA_INTELLIGENCE_PLATFORM_TRANSACTION_IMPL_H - + #include #include #include - + #include "connection.h" #include "gdb_store_config.h" +#include "gdb_transaction.h" #include "result.h" -#include "transaction.h" - + namespace OHOS::DistributedDataAip { class DBStore; class TransactionImpl : public Transaction { diff --git a/relational_store/frameworks/native/gdb/src/connection.cpp b/relational_store/frameworks/native/gdb/src/connection.cpp index 3c9ab193..52e6d8a4 100644 --- a/relational_store/frameworks/native/gdb/src/connection.cpp +++ b/relational_store/frameworks/native/gdb/src/connection.cpp @@ -14,7 +14,7 @@ */ #include "connection.h" -#include "aip_errors.h" +#include "gdb_errors.h" namespace OHOS::DistributedDataAip { static Connection::Creator g_creators[static_cast(DBType::DB_BUTT)] = { nullptr, nullptr }; diff --git a/relational_store/frameworks/native/gdb/src/connection_pool.cpp b/relational_store/frameworks/native/gdb/src/connection_pool.cpp index 6b36459a..24ea15cb 100644 --- a/relational_store/frameworks/native/gdb/src/connection_pool.cpp +++ b/relational_store/frameworks/native/gdb/src/connection_pool.cpp @@ -18,7 +18,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "gdb_utils.h" #include "logger.h" #include "rdb_store_config.h" diff --git a/relational_store/frameworks/native/gdb/src/db_helper.cpp b/relational_store/frameworks/native/gdb/src/db_helper.cpp index 38369dac..44465920 100644 --- a/relational_store/frameworks/native/gdb/src/db_helper.cpp +++ b/relational_store/frameworks/native/gdb/src/db_helper.cpp @@ -14,7 +14,7 @@ */ #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_store_manager.h" #include "db_trace.h" #include "gdb_helper.h" diff --git a/relational_store/frameworks/native/gdb/src/db_store_impl.cpp b/relational_store/frameworks/native/gdb/src/db_store_impl.cpp index 6817e29a..bb7808dc 100644 --- a/relational_store/frameworks/native/gdb/src/db_store_impl.cpp +++ b/relational_store/frameworks/native/gdb/src/db_store_impl.cpp @@ -17,12 +17,12 @@ #include -#include "aip_errors.h" #include "connection.h" #include "db_trace.h" -#include "logger.h" +#include "gdb_errors.h" +#include "gdb_transaction.h" #include "gdb_utils.h" -#include "transaction.h" +#include "logger.h" #include "transaction_impl.h" namespace OHOS::DistributedDataAip { diff --git a/relational_store/frameworks/native/gdb/src/db_store_manager.cpp b/relational_store/frameworks/native/gdb/src/db_store_manager.cpp index 65c1ca1d..aa232801 100644 --- a/relational_store/frameworks/native/gdb/src/db_store_manager.cpp +++ b/relational_store/frameworks/native/gdb/src/db_store_manager.cpp @@ -19,7 +19,7 @@ #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_store_impl.h" #include "gdb_utils.h" #include "logger.h" diff --git a/relational_store/frameworks/native/gdb/src/edge.cpp b/relational_store/frameworks/native/gdb/src/edge.cpp index 9b36e883..1b3fb582 100644 --- a/relational_store/frameworks/native/gdb/src/edge.cpp +++ b/relational_store/frameworks/native/gdb/src/edge.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbEdge" #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "full_result.h" #include "logger.h" diff --git a/relational_store/frameworks/native/gdb/src/full_result.cpp b/relational_store/frameworks/native/gdb/src/full_result.cpp index bde9228a..fdc8a026 100644 --- a/relational_store/frameworks/native/gdb/src/full_result.cpp +++ b/relational_store/frameworks/native/gdb/src/full_result.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbDataSet" #include "full_result.h" -#include "aip_errors.h" +#include "gdb_errors.h" #include "statement.h" #include "logger.h" @@ -54,7 +54,7 @@ std::pair> FullResult::GetR return { E_NO_DATA, res }; } - for (int i = 0; i < columnCount; i++) { + for (uint32_t i = 0; i < columnCount; i++) { auto [ret, key] = stmt->GetColumnName(i); if (ret != E_OK) { LOG_ERROR("GetKeys failed ret=%{public}d.", ret); diff --git a/relational_store/frameworks/native/gdb/src/gdb_utils.cpp b/relational_store/frameworks/native/gdb/src/gdb_utils.cpp index b23b250e..ee35ad14 100644 --- a/relational_store/frameworks/native/gdb/src/gdb_utils.cpp +++ b/relational_store/frameworks/native/gdb/src/gdb_utils.cpp @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" namespace OHOS::DistributedDataAip { constexpr int32_t CONTINUOUS_DIGITS_MINI_SIZE = 5; diff --git a/relational_store/frameworks/native/gdb/src/graph_connection.cpp b/relational_store/frameworks/native/gdb/src/graph_connection.cpp index e2181356..a0a0fcf3 100644 --- a/relational_store/frameworks/native/gdb/src/graph_connection.cpp +++ b/relational_store/frameworks/native/gdb/src/graph_connection.cpp @@ -20,7 +20,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "gdb_utils.h" #include "graph_statement.h" #include "logger.h" @@ -49,6 +49,10 @@ std::pair> GraphConnection::Create(const St conn = connection; break; } + if (errCode == E_GRD_INVALID_ARGS && connection->IsEncryptInvalidChanged(config)) { + errCode = E_CONFIG_INVALID_CHANGE; + break; + } } return result; } @@ -152,4 +156,15 @@ int32_t GraphConnection::ResetKey(const StoreConfig &config) config.ChangeEncryptKey(); return E_OK; } + +bool GraphConnection::IsEncryptInvalidChanged(const StoreConfig &config) +{ + if (config.GetFullPath().empty() || config.IsEncrypt()) { + LOG_WARN("Config has no path or config is encrypted, path: %{public}s, isEncrypt: %{public}d", + GdbUtils::Anonymous(config.GetFullPath()).c_str(), config.IsEncrypt()); + return false; + } + return NativeRdb::RdbSecurityManager::GetInstance().IsKeyFileExists(config.GetFullPath(), + NativeRdb::RdbSecurityManager::KeyFileType::PUB_KEY_FILE); +} } // namespace OHOS::DistributedDataAip diff --git a/relational_store/frameworks/native/gdb/src/graph_statement.cpp b/relational_store/frameworks/native/gdb/src/graph_statement.cpp index 9d96d0c4..d1d0b3ce 100644 --- a/relational_store/frameworks/native/gdb/src/graph_statement.cpp +++ b/relational_store/frameworks/native/gdb/src/graph_statement.cpp @@ -17,7 +17,7 @@ #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "connection.h" #include "full_result.h" #include "grd_error.h" @@ -33,13 +33,12 @@ GraphStatement::GraphStatement(GRD_DB *db, const std::string &gql, std::shared_p return; } - int32_t ret = GrdAdapter::Prepare(dbHandle_, gql_.c_str(), gql_.size(), &stmtHandle_, nullptr); - if (ret != GRD_OK) { - LOG_ERROR("GRD_GqlPrepare failed. ret=%{public}d", ret); + errCode = GrdAdapter::Prepare(dbHandle_, gql_.c_str(), gql_.size(), &stmtHandle_, nullptr); + if (errCode != E_OK) { + LOG_ERROR("GRD_GqlPrepare failed. ret=%{public}d", errCode); if (stmtHandle_ != nullptr) { GrdAdapter::Finalize(stmtHandle_); } - errCode = GrdAdapter::TransErrno(ret); } } @@ -55,10 +54,10 @@ int32_t GraphStatement::Prepare() } int32_t ret = GrdAdapter::Prepare(dbHandle_, gql_.c_str(), gql_.size(), &stmtHandle_, nullptr); - if (ret != GRD_OK) { + if (ret != E_OK) { LOG_ERROR("GRD_GqlPrepare failed. ret=%{public}d", ret); } - return GrdAdapter::TransErrno(ret); + return ret; } int32_t GraphStatement::Step() @@ -67,10 +66,10 @@ int32_t GraphStatement::Step() return E_STEP_CHECK_FAILED; } int32_t ret = GrdAdapter::Step(stmtHandle_); - if (ret != GRD_OK) { + if (ret != E_OK && ret != E_GRD_NO_DATA) { LOG_ERROR("GRD_GqlStep failed. ret=%{public}d", ret); } - return GrdAdapter::TransErrno(ret); + return ret; } int32_t GraphStatement::Finalize() @@ -79,9 +78,9 @@ int32_t GraphStatement::Finalize() return E_OK; } int32_t ret = GrdAdapter::Finalize(stmtHandle_); - if (ret != GRD_OK) { + if (ret != E_OK) { LOG_ERROR("GRD_GqlFinalize failed. ret=%{public}d", ret); - return GrdAdapter::TransErrno(ret); + return ret; } stmtHandle_ = nullptr; gql_ = ""; diff --git a/relational_store/frameworks/native/gdb/src/path.cpp b/relational_store/frameworks/native/gdb/src/path.cpp index ac9d388b..d390b319 100644 --- a/relational_store/frameworks/native/gdb/src/path.cpp +++ b/relational_store/frameworks/native/gdb/src/path.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbPath" #include "path.h" -#include "aip_errors.h" +#include "gdb_errors.h" #include "path_segment.h" #include "grd_error.h" #include "logger.h" diff --git a/relational_store/frameworks/native/gdb/src/path_segment.cpp b/relational_store/frameworks/native/gdb/src/path_segment.cpp index df9f7b74..a4c501eb 100644 --- a/relational_store/frameworks/native/gdb/src/path_segment.cpp +++ b/relational_store/frameworks/native/gdb/src/path_segment.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbPath" #include "path_segment.h" -#include "aip_errors.h" +#include "gdb_errors.h" #include "grd_error.h" #include "logger.h" diff --git a/relational_store/frameworks/native/gdb/src/store_config.cpp b/relational_store/frameworks/native/gdb/src/store_config.cpp index 9d54f2e8..cda3da4c 100644 --- a/relational_store/frameworks/native/gdb/src/store_config.cpp +++ b/relational_store/frameworks/native/gdb/src/store_config.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbStoreConfig" #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "gdb_store_config.h" #include "gdb_utils.h" #include "logger.h" @@ -23,8 +23,8 @@ namespace OHOS::DistributedDataAip { StoreConfig::StoreConfig( - std::string name, std::string path, DBType dbType, bool status, const std::vector &encryptKey) - : name_(std::move(name)), path_(std::move(path)), dbType_(dbType), isEncrypt_(status), encryptKey_(encryptKey) + std::string name, std::string path, DBType dbType, bool isEncrypt, const std::vector &encryptKey) + : name_(std::move(name)), path_(std::move(path)), dbType_(dbType), isEncrypt_(isEncrypt), encryptKey_(encryptKey) { } @@ -58,12 +58,6 @@ bool StoreConfig::IsEncrypt() const return isEncrypt_; } -std::string StoreConfig::GetJson() const -{ - return "{\"pageSize\":" + std::to_string(pageSize_) + - ", \"defaultIsolationLevel\":" + std::to_string(defaultIsolationLevel_) + "}"; -} - std::string StoreConfig::GetFullPath() const { return path_ + "/" + name_ + ".db"; diff --git a/relational_store/frameworks/native/gdb/src/trans_db.cpp b/relational_store/frameworks/native/gdb/src/trans_db.cpp index 57d9fd93..3c9a6804 100644 --- a/relational_store/frameworks/native/gdb/src/trans_db.cpp +++ b/relational_store/frameworks/native/gdb/src/trans_db.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "TransDB" #include "trans_db.h" -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_trace.h" #include "gdb_utils.h" #include "logger.h" diff --git a/relational_store/frameworks/native/gdb/src/transaction.cpp b/relational_store/frameworks/native/gdb/src/transaction.cpp index 775cfa47..de75d1b8 100644 --- a/relational_store/frameworks/native/gdb/src/transaction.cpp +++ b/relational_store/frameworks/native/gdb/src/transaction.cpp @@ -12,10 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "transaction.h" - -#include "aip_errors.h" - +#include "gdb_errors.h" +#include "gdb_transaction.h" + namespace OHOS::DistributedDataAip { std::pair> Transaction::Create(std::shared_ptr conn) { diff --git a/relational_store/frameworks/native/gdb/src/transaction_impl.cpp b/relational_store/frameworks/native/gdb/src/transaction_impl.cpp index 954f3171..a593a4ec 100644 --- a/relational_store/frameworks/native/gdb/src/transaction_impl.cpp +++ b/relational_store/frameworks/native/gdb/src/transaction_impl.cpp @@ -17,7 +17,7 @@ #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_trace.h" #include "logger.h" #include "trans_db.h" diff --git a/relational_store/frameworks/native/gdb/src/vertex.cpp b/relational_store/frameworks/native/gdb/src/vertex.cpp index bfb51f83..ffdb97d5 100644 --- a/relational_store/frameworks/native/gdb/src/vertex.cpp +++ b/relational_store/frameworks/native/gdb/src/vertex.cpp @@ -15,7 +15,7 @@ #define LOG_TAG "GdbElement" #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "full_result.h" #include "logger.h" diff --git a/relational_store/frameworks/native/icu/src/icu_collect.cpp b/relational_store/frameworks/native/icu/src/icu_collect.cpp index 1153ec08..b948c921 100644 --- a/relational_store/frameworks/native/icu/src/icu_collect.cpp +++ b/relational_store/frameworks/native/icu/src/icu_collect.cpp @@ -18,6 +18,7 @@ #include #include +#include "ohos/init_data.h" #include "logger.h" #include "rdb_errno.h" #include "rdb_visibility.h" @@ -63,22 +64,23 @@ void ICUCollect::LocalizedCollatorDestroy(void *collator) int32_t ICUCollect::Local(sqlite3 *dbHandle, const std::string &str) { - LOG_ERROR("bai:local success"); UErrorCode status = U_ZERO_ERROR; + SetHwIcuDirectory(); + UCollator *collator = ucol_open(str.c_str(), &status); if (U_FAILURE(status)) { - LOG_ERROR("Can not open collator."); + LOG_ERROR("Can not open collator, status:%{public}d.", status); return E_ERROR; } ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status); if (U_FAILURE(status)) { - LOG_ERROR("Set attribute of collator failed."); + LOG_ERROR("Set attribute of collator failed, status:%{public}d.", status); return E_ERROR; } int err = sqlite3_create_collation_v2(dbHandle, "LOCALES", SQLITE_UTF8, collator, ICUCollect::Collate8Compare, (void (*)(void *))ICUCollect::LocalizedCollatorDestroy); if (err != SQLITE_OK) { - LOG_ERROR("SCreate collator in sqlite3 failed."); + LOG_ERROR("SCreate collator in sqlite3 failed err:%{public}d.", err); return err; } return E_OK; diff --git a/relational_store/frameworks/native/rdb/include/connection.h b/relational_store/frameworks/native/rdb/include/connection.h index 2ae66395..ee06e2b1 100644 --- a/relational_store/frameworks/native/rdb/include/connection.h +++ b/relational_store/frameworks/native/rdb/include/connection.h @@ -25,6 +25,10 @@ #include "rdb_common.h" #include "rdb_types.h" #include "statement.h" + +namespace DistributedDB { +class StoreObserver; +} namespace OHOS::NativeRdb { class RdbStoreConfig; class Statement; @@ -52,8 +56,10 @@ public: int32_t SetId(int32_t id); int32_t GetId() const; + void SetIsRecyclable(bool recyclable); + bool IsRecyclable() const; virtual ~Connection() = default; - virtual int32_t OnInitialize() = 0; + virtual int32_t VerifyAndRegisterHook(const RdbStoreConfig &config) = 0; virtual std::pair CreateStatement(const std::string &sql, SConn conn) = 0; virtual int32_t GetDBType() const = 0; virtual bool IsWriter() const = 0; @@ -67,10 +73,8 @@ public: virtual int32_t GetMaxVariable() const = 0; virtual int32_t GetJournalMode() = 0; virtual int32_t ClearCache() = 0; - virtual int32_t Subscribe( - const std::string &event, const std::shared_ptr &observer) = 0; - virtual int32_t Unsubscribe( - const std::string &event, const std::shared_ptr &observer) = 0; + virtual int32_t Subscribe(const std::shared_ptr &observer) = 0; + virtual int32_t Unsubscribe(const std::shared_ptr &observer) = 0; virtual int32_t Backup(const std::string &databasePath, const std::vector &destEncryptKey, bool isAsync, SlaveStatus &slaveStatus) = 0; virtual int32_t Restore( @@ -79,6 +83,7 @@ public: private: int32_t id_ = 0; + bool isRecyclable_ = true; }; } // namespace OHOS::NativeRdb #endif // OHOS_DISTRIBUTED_DATA_RELATIONAL_STORE_FRAMEWORKS_NATIVE_RDB_INCLUDE_CONNECTION_H diff --git a/relational_store/frameworks/native/rdb/include/connection_pool.h b/relational_store/frameworks/native/rdb/include/connection_pool.h index 9e82a731..a85a7eb9 100644 --- a/relational_store/frameworks/native/rdb/include/connection_pool.h +++ b/relational_store/frameworks/native/rdb/include/connection_pool.h @@ -32,6 +32,7 @@ #include "rdb_common.h" #include "rdb_store_config.h" namespace OHOS { +class ExecutorPool; namespace NativeRdb { class ConnectionPool : public std::enable_shared_from_this { public: @@ -52,7 +53,7 @@ public: int32_t EnableWal(); int32_t Dump(bool isWriter, const char *header); - int RestartReaders(); + int RestartConns(); int ConfigLocale(const std::string &localeStr); int ChangeDbFileForRestore(const std::string &newPath, const std::string &backupPath, const std::vector &newKey, SlaveStatus &slaveStatus); @@ -77,6 +78,7 @@ private: int64_t GetUsingTime() const; bool IsWriter() const; int32_t Unused(int32_t count, bool timeout); + bool IsRecyclable(); }; struct Container { @@ -87,7 +89,6 @@ private: int max_ = 0; int total_ = 0; int count_ = 0; - int trans_ = 0; int32_t left_ = 0; int32_t right_ = 0; std::chrono::seconds timeout_; @@ -99,17 +100,20 @@ private: std::pair> Initialize( Creator creator, int32_t max, int32_t timeout, bool disable, bool acquire = false); int32_t ConfigLocale(const std::string &locale); - std::shared_ptr Acquire(std::chrono::milliseconds milliS); - std::list> AcquireAll(std::chrono::milliseconds milliS); + std::pair> Acquire(std::chrono::milliseconds milliS); + std::pair>> AcquireAll(std::chrono::milliseconds milliS); std::pair> Create(); + void InitMembers(Creator creator, int32_t max, int32_t timeout, bool disable); void Disable(); void Enable(); int32_t Release(std::shared_ptr node); - int32_t Drop(std::shared_ptr node); + int32_t ReleaseTrans(std::shared_ptr node); int32_t Clear(); bool IsFull(); + bool Empty(); int32_t Dump(const char *header, int32_t count); + int32_t ClearUnusedTrans(std::shared_ptr pool); private: int32_t ExtendNode(); @@ -123,16 +127,19 @@ private: void ReleaseNode(std::shared_ptr node, bool reuse = true); int RestoreMasterDb(const std::string &newPath, const std::string &backupPath, SlaveStatus &slaveStatus); bool CheckIntegrity(const std::string &dbPath); + void DelayClearTrans(); static constexpr uint32_t CHECK_POINT_INTERVAL = 5; // 5 min static constexpr int LIMITATION = 1024; static constexpr uint32_t ITER_V1 = 5000; static constexpr uint32_t ITERS_COUNT = 2; static constexpr uint32_t MAX_TRANS = 4; + static constexpr std::chrono::steady_clock::duration TRANS_CLEAR_INTERVAL = std::chrono::seconds(150); const RdbStoreConfig &config_; RdbStoreConfig attachConfig_; Container writers_; Container readers_; + Container trans_; int32_t maxReader_ = 0; std::stack transactionStack_; @@ -140,6 +147,7 @@ private: std::condition_variable transCondition_; std::mutex transMutex_; bool transactionUsed_; + bool isAttach_ = false; std::atomic isInTransaction_ = false; std::atomic transCount_ = 0; std::atomic failedTime_; diff --git a/relational_store/frameworks/native/rdb/include/grd_error.h b/relational_store/frameworks/native/rdb/include/grd_error.h index 436031e9..d67b36a7 100644 --- a/relational_store/frameworks/native/rdb/include/grd_error.h +++ b/relational_store/frameworks/native/rdb/include/grd_error.h @@ -40,7 +40,7 @@ extern "C" { #define GRD_FAILED_MEMORY_ALLOCATE (-13000) #define GRD_FAILED_MEMORY_RELEASE (-14000) #define GRD_DATA_CONFLICT (-16000) -#define GRD_DATA_MISMATCH (-17000) +#define GRD_DATATYPE_MISMATCH (-17000) #define GRD_NOT_AVAILABLE (-19000) #define GRD_ACTIVE_TRANSACTION (-20000) #define GRD_UNIQUE_VIOLATION (-21000) @@ -59,6 +59,11 @@ extern "C" { #define GRD_DB_BUSY (-46000) #define GRD_CALC_MODE_SET_PERMISSION_DENIED (-47000) +#define GRD_CIPHER_ERROR (-48000) +#define GRD_PASSWORD_NEED_REKEY (-48001) +#define GRD_PASSWORD_UNMATCHED (-48002) + +#define GRD_SCHEMA_CHANGED (-49000) // not support #define GRD_JSON_OPERATION_NOT_SUPPORT (-5001001) #define GRD_MODEL_NOT_SUPPORT (-5001002) @@ -66,6 +71,10 @@ extern "C" { // Exceed limit #define GRD_JSON_LEN_LIMIT (-5002001) +#define GRD_SUBSCRIPTION_EXCEEDED_LIMIT (-5002002) +#define GRD_SYNC_EXCEED_TASK_QUEUE_LIMIT (-5002003) +#define GRD_SHARED_OBJ_ENABLE_UNDO_EXCEED_LIMIT (-5002004) +#define GRD_TABLE_LIMIT_EXCEEDED (-5002005) // Invalid parameter #define GRD_FIELD_TYPE_NOT_MATCH (-5003001) @@ -75,7 +84,14 @@ extern "C" { #define GRD_INVALID_OPERATOR (-5003005) #define GRD_INVALID_PROJECTION_FIELD (-5003006) #define GRD_INVALID_PROJECTION_VALUE (-5003007) -#define GRD_ARRAY_INDEX_NOT_FOUND (-5003008) +#define GRD_COLLECTION_NOT_EXIST (-5003008) +#define GRD_DB_NOT_EXIST (-5003009) +#define GRD_INVALID_VALUE (-5003010) +#define GRD_SHARED_OBJ_NOT_EXIST (-5003020) +#define GRD_SUBSCRIBE_NOT_EXIST (-5003021) +#define GRD_SHARED_OBJ_UNDO_MANAGER_NOT_EXIST (-5003022) +#define GRD_SHARED_OBJ_INVALID_UNDO (-5003023) +#define GRD_SHARED_OBJ_INVALID_REDO (-5003024) #define GRD_NAME_TOO_LONG (-5003025) #define GRD_INVALID_TABLE_DEFINITION (-5003026) #define GRD_WRONG_STMT_OBJECT (-5003027) @@ -84,33 +100,63 @@ extern "C" { // System err #define GRD_JSON_LIB_HANDLE_FAILED (-5004001) +#define GRD_DIRECTORY_OPERATE_FAILED (-5004002) +#define GRD_FILE_OPERATE_FAILED (-5004003) +#define GRD_LOAD_THIRD_PARTY_LIBRARY_FAILED (-5004004) +#define GRD_THIRD_PARTY_FUNCTION_EXECUTE_FAILED (-5004005) +#define GRD_INSUFFICIENT_RESOURCES (-5004006) + +// resource busy +#define GRD_RESULTSET_BUSY (-5009001) // no data -#define GRD_COLLECTION_NOT_FOUND (-5011001) -#define GRD_RECORD_NOT_FOUND (-5011002) -#define GRD_FIELD_NOT_FOUND (-5011004) +#define GRD_RECORD_NOT_FOUND (-5011001) +#define GRD_FIELD_NOT_FOUND (-5011002) +#define GRD_ARRAY_INDEX_NOT_FOUND (-5011003) // data conflicted -#define GRD_COLLECTION_CONFLICT (-5016001) -#define GRD_KEY_CONFLICT (-5016002) -#define GRD_FIELD_TYPE_CONFLICT (-5016003) +#define GRD_KEY_CONFLICT (-5016001) +#define GRD_FIELD_TYPE_CONFLICT (-5016002) +#define GRD_SHARED_OBJ_CONFLICT (-5016003) +#define GRD_SUBSCRIBE_CONFLICT (-5016004) +#define GRD_EQUIP_ID_CONFLICT (-5016005) +#define GRD_SHARED_OBJ_ENABLE_UNDO_CONFLICT (-5016006) + +// data exception +#define GRD_DATA_EXCEPTION (-5017001) +#define GRD_FIELD_OVERFLOW (-5017002) +#define GRD_DIVISION_BY_ZERO (-5017003) // Cursor or ResultSet not available #define GRD_RESULT_SET_NOT_AVAILABLE (-5019001) +#define GRD_SHARED_OBJ_UNDO_NOT_AVAILABLE (-5019002) +#define GRD_SHARED_OBJ_REDO_NOT_AVAILABLE (-5019003) + +// Transaction +#define GRD_TRANSACTION_ROLLBACK (-5020001) +#define GRD_NO_ACTIVE_TRANSACTION (-5020002) // violation #define GRD_PRIMARY_KEY_VIOLATION (-5021001) #define GRD_RESTRICT_VIOLATION (-5021002) #define GRD_CONSTRAINT_CHECK_VIOLATION (-5021003) +// duplicate +#define GRD_DUPLICATE_COLUMN (-5022001) +#define GRD_DUPLICATE_OBJECT (-5022002) + +// undefined +#define GRD_UNDEFINE_COLUMN (-5023001) +#define GRD_UNDEFINED_OBJECT (-5023002) + // Invalid format #define GRD_INVALID_JSON_FORMAT (-5037001) #define GRD_INVALID_KEY_FORMAT (-5037002) #define GRD_INVALID_COLLECTION_NAME (-5037003) +#define GRD_INVALID_EQUIP_ID (-5037004) -#define GRD_CIPHER_ERROR (-48000) -#define GRD_PASSWORD_NEED_REKEY (-48001) -#define GRD_PASSWORD_UNMATCHED (-48002) +// time out +#define GRD_REQUEST_TIME_OUT (-5039001) #ifdef __cplusplus } diff --git a/relational_store/frameworks/native/rdb/include/rd_connection.h b/relational_store/frameworks/native/rdb/include/rd_connection.h index 55ce84e9..e9737849 100644 --- a/relational_store/frameworks/native/rdb/include/rd_connection.h +++ b/relational_store/frameworks/native/rdb/include/rd_connection.h @@ -37,7 +37,7 @@ public: static int32_t Delete(const RdbStoreConfig &config); RdConnection(const RdbStoreConfig &config, bool isWriteConnection); ~RdConnection(); - int32_t OnInitialize() override; + int32_t VerifyAndRegisterHook(const RdbStoreConfig &config) override; std::pair CreateStatement(const std::string &sql, SConn conn) override; int32_t GetDBType() const override; bool IsWriter() const override; @@ -51,10 +51,8 @@ public: int32_t GetMaxVariable() const override; int32_t GetJournalMode() override; int32_t ClearCache() override; - int32_t Subscribe( - const std::string &event, const std::shared_ptr &observer) override; - int32_t Unsubscribe( - const std::string &event, const std::shared_ptr &observer) override; + int32_t Subscribe(const std::shared_ptr &observer) override; + int32_t Unsubscribe(const std::shared_ptr &observer) override; int32_t Backup(const std::string &databasePath, const std::vector &destEncryptKey, bool isAsync, SlaveStatus &slaveStatus) override; int32_t Restore(const std::string &databasePath, const std::vector &destEncryptKey, diff --git a/relational_store/frameworks/native/rdb/include/rdb_service_proxy.h b/relational_store/frameworks/native/rdb/include/rdb_service_proxy.h index 54960482..244cf491 100644 --- a/relational_store/frameworks/native/rdb/include/rdb_service_proxy.h +++ b/relational_store/frameworks/native/rdb/include/rdb_service_proxy.h @@ -34,6 +34,11 @@ public: SubscribeOption subscribeOption{ SubscribeMode::REMOTE }; }; using Observers = ConcurrentMap>; + struct SyncObserverParam { + std::shared_ptr syncObserver = nullptr; + std::string bundleName; + }; + using SyncObservers = ConcurrentMap>; explicit RdbServiceProxy(const sptr &object); std::string ObtainDistributedTableName(const std::string &device, const std::string &table) override; @@ -70,6 +75,8 @@ public: int32_t AfterOpen(const RdbSyncerParam ¶m) override; + int32_t ReportStatistic(const RdbSyncerParam ¶m, const RdbStatEvent &statEvent) override; + int32_t Delete(const RdbSyncerParam ¶m) override; int32_t NotifyDataChange(const RdbSyncerParam ¶m, const RdbChangedData &clientChangedData, @@ -89,13 +96,16 @@ public: int32_t GetDebugInfo(const RdbSyncerParam ¶m, std::map &debugInfo) override; + int32_t GetDfxInfo(const RdbSyncerParam ¶m, DistributedRdb::RdbDfxInfo &dfxInfo) override; + int32_t VerifyPromiseInfo(const RdbSyncerParam ¶m) override; + SyncObservers ExportSyncObservers(); + void ImportSyncObservers(SyncObservers &SyncObservers); private: using ChangeInfo = RdbStoreObserver::ChangeInfo; using PrimaryFields = RdbStoreObserver::PrimaryFields; using SyncCallbacks = ConcurrentMap; - using SyncObservers = ConcurrentMap>>; std::pair DoSync( const RdbSyncerParam ¶m, const Option &option, const PredicatesMemo &predicates); diff --git a/relational_store/frameworks/native/rdb/include/rdb_store_impl.h b/relational_store/frameworks/native/rdb/include/rdb_store_impl.h index bdbb2a1f..7088ea05 100644 --- a/relational_store/frameworks/native/rdb/include/rdb_store_impl.h +++ b/relational_store/frameworks/native/rdb/include/rdb_store_impl.h @@ -42,6 +42,7 @@ class ExecutorPool; namespace OHOS::NativeRdb { class DelayNotify; +class RdbStoreLocalDbObserver; class RdbStoreLocalObserver { public: explicit RdbStoreLocalObserver(DistributedRdb::RdbStoreObserver *observer) : observer_(observer) {}; @@ -163,6 +164,7 @@ private: using RdbParam = DistributedRdb::RdbSyncerParam; using Options = DistributedRdb::RdbService::Option; using Memo = DistributedRdb::PredicatesMemo; + using ReportFunc = std::function; class CloudTables { public: int32_t AddTables(const std::vector &tables); @@ -175,22 +177,13 @@ private: std::set tables_; std::set changes_; }; - class SavePointers { - public: - SavePointers(std::shared_ptr pool, std::shared_ptr conn, - ConflictResolution resolution, int &errCode); - ~SavePointers(); - private: - std::shared_ptr pool_; - std::shared_ptr conn_; - ConflictResolution resolution_; - int &errCode_; - }; - class StoreSuspender; static void AfterOpen(const RdbParam ¶m, int32_t retry = 0); + static void RegisterDataChangeCallback( + std::shared_ptr delayNotifier, std::weak_ptr connPool, int retry); int InnerOpen(); - void InitSyncerParam(const RdbStoreConfig &config); + void InitReportFunc(const RdbParam ¶m); + void InitSyncerParam(const RdbStoreConfig &config, bool created); int ExecuteByTrxId(const std::string &sql, int64_t trxId, bool closeConnAfterExecute = false, const std::vector &bindArgs = {}); std::pair HandleDifferentSqlTypes( @@ -210,9 +203,7 @@ private: int32_t SubscribeLocalDetail(const SubscribeOption &option, const std::shared_ptr &observer); int SubscribeRemote(const SubscribeOption &option, RdbStoreObserver *observer); int UnSubscribeLocal(const SubscribeOption &option, RdbStoreObserver *observer); - int UnSubscribeLocalAll(const SubscribeOption &option); int UnSubscribeLocalShared(const SubscribeOption &option, RdbStoreObserver *observer); - int UnSubscribeLocalSharedAll(const SubscribeOption &option); int32_t UnsubscribeLocalDetail(const SubscribeOption &option, const std::shared_ptr &observer); int UnSubscribeRemote(const SubscribeOption &option, RdbStoreObserver *observer); int RegisterDataChangeCallback(); @@ -236,6 +227,9 @@ private: int HandleCloudSyncAfterSetDistributedTables( const std::vector &tables, const DistributedRdb::DistributedConfig &distributedConfig); std::pair> GetConn(bool isRead); + void HandleSchemaDDL(std::shared_ptr statement, + std::shared_ptr pool, const std::string &sql, int32_t &errCode); + void BatchInsertArgsDfx(int argsSize); static constexpr char SCHEME_RDB[] = "rdb://"; static constexpr uint32_t EXPANSION = 2; @@ -247,6 +241,7 @@ private: bool isOpen_ = false; bool isReadOnly_ = false; bool isMemoryRdb_; + std::atomic hasRegisterDataChange_ = false; uint32_t rebuild_ = RebuiltType::NONE; SlaveStatus slaveStatus_ = SlaveStatus::UNDEFINED; int64_t vSchema_ = 0; @@ -254,6 +249,8 @@ private: const RdbStoreConfig config_; // Can only be modified within the constructor DistributedRdb::RdbSyncerParam syncerParam_; + DistributedRdb::RdbStatEvent statEvent_; + std::shared_ptr reportFunc_ = nullptr; std::string path_; std::string name_; std::string fileType_; @@ -265,6 +262,7 @@ private: std::shared_ptr cloudInfo_ = std::make_shared(); std::map>> localObservers_; std::map>> localSharedObservers_; + std::list> localDetailObservers_; ConcurrentMap attachedInfo_; ConcurrentMap> trxConnMap_ = {}; std::list> transactions_; diff --git a/relational_store/frameworks/native/rdb/include/rdb_store_manager.h b/relational_store/frameworks/native/rdb/include/rdb_store_manager.h index 7c269320..2fecfd58 100644 --- a/relational_store/frameworks/native/rdb/include/rdb_store_manager.h +++ b/relational_store/frameworks/native/rdb/include/rdb_store_manager.h @@ -38,19 +38,23 @@ public: const RdbStoreConfig &config, int &errCode, int version, RdbOpenCallback &openCallback); void Clear(); bool Remove(const std::string &path, bool shouldClose); - bool Delete(const std::string &path, bool shouldClose); + bool Delete(const RdbStoreConfig &config, bool shouldClose); int SetSecurityLabel(const RdbStoreConfig &config); private: using Param = DistributedRdb::RdbSyncerParam; using Info = DistributedRdb::RdbDebugInfo; + using DebugInfos = std::map; + using DfxInfo = DistributedRdb::RdbDfxInfo; int ProcessOpenCallback(RdbStore &rdbStore, int version, RdbOpenCallback &openCallback); bool IsConfigInvalidChanged(const std::string &path, RdbStoreConfig &config); - bool IsPermitted(const DistributedRdb::RdbSyncerParam ¶m); int32_t GetParamFromService(DistributedRdb::RdbSyncerParam ¶m); + static bool IsPermitted(const DistributedRdb::RdbSyncerParam ¶m); + static int32_t CheckConfig(const RdbStoreConfig &config); static Param GetSyncParam(const RdbStoreConfig &config); - static std::map Collector(const RdbStoreConfig &config); - std::shared_ptr GetStoreFromCache(const RdbStoreConfig &config, const std::string &path); + static int32_t Collector(const RdbStoreConfig &config, DebugInfos &debugInfos, DfxInfo &dfxInfo); + std::shared_ptr GetStoreFromCache(const std::string &path); + std::pair> OpenStore(const RdbStoreConfig &config, const std::string &path); static constexpr uint32_t BUCKET_MAX_SIZE = 4; static const bool regCollector_; diff --git a/relational_store/frameworks/native/rdb/include/rdb_types_util.h b/relational_store/frameworks/native/rdb/include/rdb_types_util.h index fc3ce4e9..c5730d9f 100644 --- a/relational_store/frameworks/native/rdb/include/rdb_types_util.h +++ b/relational_store/frameworks/native/rdb/include/rdb_types_util.h @@ -45,7 +45,9 @@ using RdbChangedData = DistributedRdb::RdbChangedData; using RdbProperties = DistributedRdb::RdbChangeProperties; using Reference = DistributedRdb::Reference; using BigInt = NativeRdb::BigInteger; -using DebugInfo = DistributedRdb::RdbDebugInfo;; +using DebugInfo = DistributedRdb::RdbDebugInfo; +using StatReporter = DistributedRdb::RdbStatEvent; +using RdbDfxInfo = DistributedRdb::RdbDfxInfo; template<> API_EXPORT bool Marshalling(const SyncerParam &input, MessageParcel &data); template<> @@ -126,5 +128,13 @@ template<> API_EXPORT bool Marshalling(const DebugInfo &input, MessageParcel &data); template<> API_EXPORT bool Unmarshalling(DebugInfo &output, MessageParcel &data); +template<> +API_EXPORT bool Marshalling(const StatReporter &input, MessageParcel &data); +template<> +API_EXPORT bool Unmarshalling(StatReporter &output, MessageParcel &data); +template<> +API_EXPORT bool Marshalling(const RdbDfxInfo &input, MessageParcel &data); +template<> +API_EXPORT bool Unmarshalling(RdbDfxInfo &output, MessageParcel &data); } // namespace OHOS::ITypesUtil #endif // DISTRIBUTED_RDB_RDB_TYPES_UTIL_H diff --git a/relational_store/frameworks/native/rdb/include/shared_block_serializer_info.h b/relational_store/frameworks/native/rdb/include/shared_block_serializer_info.h index 651c9ab0..2d004cfa 100644 --- a/relational_store/frameworks/native/rdb/include/shared_block_serializer_info.h +++ b/relational_store/frameworks/native/rdb/include/shared_block_serializer_info.h @@ -43,6 +43,9 @@ public: if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_OK) { return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } sharedBlock_->FreeLastRow(); return SQLITE_FULL; } @@ -62,6 +65,11 @@ public: return astartPos; } + bool IsFull() const + { + return isFull; + } + private: AppDataFwk::SharedBlock *sharedBlock_; sqlite3_stmt *statement_ = nullptr; @@ -69,6 +77,7 @@ private: int atotalRows; int astartPos; int raddedRows; + bool isFull = false; }; struct Sqlite3SharedBlockMethods { diff --git a/relational_store/frameworks/native/rdb/include/sqlite_connection.h b/relational_store/frameworks/native/rdb/include/sqlite_connection.h index caef3af4..b9e120ea 100644 --- a/relational_store/frameworks/native/rdb/include/sqlite_connection.h +++ b/relational_store/frameworks/native/rdb/include/sqlite_connection.h @@ -24,7 +24,6 @@ #include #include "connection.h" -#include "rdb_local_db_observer.h" #include "rdb_store_config.h" #include "sqlite3sym.h" #include "sqlite_statement.h" @@ -48,7 +47,7 @@ public: static std::map Collect(const RdbStoreConfig &config); SqliteConnection(const RdbStoreConfig &config, bool isWriteConnection); ~SqliteConnection(); - int32_t OnInitialize() override; + int32_t VerifyAndRegisterHook(const RdbStoreConfig &config) override; int TryCheckPoint(bool timeout) override; int LimitWalSize() override; int ConfigLocale(const std::string &localeStr) override; @@ -62,10 +61,8 @@ public: int GetMaxVariable() const override; int32_t GetDBType() const override; int32_t ClearCache() override; - int32_t Subscribe( - const std::string &event, const std::shared_ptr &observer) override; - int32_t Unsubscribe( - const std::string &event, const std::shared_ptr &observer) override; + int32_t Subscribe(const std::shared_ptr &observer) override; + int32_t Unsubscribe(const std::shared_ptr &observer) override; int32_t Backup(const std::string &databasePath, const std::vector &destEncryptKey, bool isAsync, SlaveStatus &slaveStatus) override; int32_t Restore(const std::string &databasePath, const std::vector &destEncryptKey, @@ -96,15 +93,16 @@ private: int SetServiceKey(const RdbStoreConfig &config, int32_t errCode); int SetEncryptAgo(const RdbStoreConfig &config); int SetJournalMode(const RdbStoreConfig &config); - int SetJournalSizeLimit(const RdbStoreConfig &config); int SetAutoCheckpoint(const RdbStoreConfig &config); int SetWalFile(const RdbStoreConfig &config); int SetWalSyncMode(const std::string &syncMode); int SetTokenizer(const RdbStoreConfig &config); - void LimitPermission(const std::string &dbPath) const; + void LimitPermission(const RdbStoreConfig &config, const std::string &dbPath) const; - int SetPersistWal(); + int SetPersistWal(const RdbStoreConfig &config); int SetBusyTimeout(int timeout); + int SetCrcCheck(const RdbStoreConfig &config); + void SetDwrEnable(const RdbStoreConfig &config); int RegDefaultFunctions(sqlite3 *dbHandle); int SetCustomFunctions(const RdbStoreConfig &config); @@ -122,12 +120,15 @@ private: int SqliteNativeBackup(bool isRestore, SlaveStatus &curStatus); int VeritySlaveIntegrity(); bool IsDbVersionBelowSlave(); + int RegisterStoreObs(); + int RegisterClientObs(); + int RegisterHookIfNecessary(); static std::pair> InnerCreate( const RdbStoreConfig &config, bool isWrite); static int CopyDb(const RdbStoreConfig &config, const std::string &srcPath, const std::string &destPath); static constexpr SqliteConnection::Suffix FILE_SUFFIXES[] = { { "", "DB" }, { "-shm", "SHM" }, { "-wal", "WAL" }, - { "-journal", "JOURNAL" }, { "-slaveFailure", nullptr }, { "-syncInterrupt", nullptr }, - { ".corruptedflg", nullptr } }; + { "-dwr", "DWR" }, { "-journal", "JOURNAL" }, { "-slaveFailure", nullptr }, { "-syncInterrupt", nullptr }, + { ".corruptedflg", nullptr }, { "-compare", nullptr } }; static constexpr int CHECKPOINT_TIME = 500; static constexpr int DEFAULT_BUSY_TIMEOUT_MS = 2000; static constexpr int BACKUP_PAGES_PRE_STEP = 12800; // 1024 * 4 * 12800 == 50m @@ -138,25 +139,31 @@ private: static constexpr uint32_t DB_INDEX = 0; static constexpr uint32_t WAL_INDEX = 2; static constexpr uint32_t ITER_V1 = 5000; + static constexpr uint32_t SQLITE_CKSUMVFS_RESERVE_BYTES = 8; static const int32_t regCreator_; static const int32_t regRepairer_; static const int32_t regDeleter_; static const int32_t regCollector_; static const int32_t regRestorer_; + using EventHandle = int (SqliteConnection::*)(); + struct HandleInfo { + RegisterType Type; + EventHandle handle; + }; + static constexpr HandleInfo onEventHandlers_[RegisterType::OBSERVER_END] = { + { RegisterType::STORE_OBSERVER, &SqliteConnection::RegisterStoreObs }, + { RegisterType::CLIENT_OBSERVER, &SqliteConnection::RegisterClientObs }, + }; std::atomic backupId_; sqlite3 *dbHandle_; bool isWriter_; bool isReadOnly_; bool isConfigured_ = false; - bool hasClientObserver_ = false; JournalMode mode_ = JournalMode::MODE_WAL; int maxVariableNumber_; - std::mutex mutex_; - std::string filePath; std::shared_ptr slaveConnection_; std::map customScalarFunctions_; - std::map>> observers_; const RdbStoreConfig config_; }; } // namespace NativeRdb diff --git a/relational_store/frameworks/native/rdb/include/sqlite_errno.h b/relational_store/frameworks/native/rdb/include/sqlite_errno.h index 03aab636..8e643e53 100644 --- a/relational_store/frameworks/native/rdb/include/sqlite_errno.h +++ b/relational_store/frameworks/native/rdb/include/sqlite_errno.h @@ -22,6 +22,9 @@ #include "rdb_errno.h" +#ifndef SQLITE_META_RECOVERED +#define SQLITE_META_RECOVERED 66 +#endif namespace OHOS { namespace NativeRdb { const static std::map ERROR_CODE_MAPPINT_TABLE = { @@ -44,6 +47,7 @@ const static std::map ERROR_CODE_MAPPINT_TABLE = { { SQLITE_NOTADB, E_SQLITE_CORRUPT }, { SQLITE_DONE, E_NO_MORE_ROWS }, { SQLITE_ROW, E_OK }, + { SQLITE_META_RECOVERED, E_SQLITE_META_RECOVERED }, }; class SQLiteError { public: diff --git a/relational_store/frameworks/native/rdb/include/sqlite_global_config.h b/relational_store/frameworks/native/rdb/include/sqlite_global_config.h index 1caa28bf..21ec081e 100644 --- a/relational_store/frameworks/native/rdb/include/sqlite_global_config.h +++ b/relational_store/frameworks/native/rdb/include/sqlite_global_config.h @@ -29,6 +29,7 @@ public: static constexpr int SOFT_HEAP_LIMIT = 8 * 1024 * 1024; /* 8MB */ static constexpr int DB_PAGE_SIZE = 4096; /* default page size : 4k */ static constexpr int DB_JOURNAL_SIZE = 1024 * 1024; /* default file size : 1M */ + static constexpr int CLEAR_MEMORY_SIZE = 1024 * 1024; /* default file size : 1M */ static constexpr ssize_t DB_WAL_SIZE_LIMIT_MIN = 20 * 1024 * 1024; /* default wal file maximum size : 20M */ static constexpr ssize_t DB_WAL_WARNING_SIZE = 256 * 1024 * 1024; /* default wal file maximum size : 256M */ static constexpr ssize_t DB_WAL_DEFAULT_SIZE = 0x20000000; /* default wal file size 512M */ @@ -48,6 +49,8 @@ public: static constexpr char JOURNAL_MODE_WAL[] = "WAL"; static constexpr char DEFAULE_SYNC_MODE[] = "FULL"; static constexpr char MEMORY_DB_PATH[] = ":memory:"; + static constexpr char SHARED_MEMORY_DB_PATH_PREFIX[] = "file:"; + static constexpr char SHARED_MEMORY_DB_PATH_SUFFIX[] = "?mode=memory&cache=shared"; static constexpr char CODEC_HMAC_ALGO[] = "PRAGMA codec_hmac_algo=sha256"; static constexpr char CODEC_HMAC_ALGO_PREFIX[] = "PRAGMA codec_hmac_algo='"; static constexpr char CODEC_KDF_ALGO_PREFIX[] = "PRAGMA codec_kdf_algo='"; @@ -63,6 +66,7 @@ public: static constexpr char CIPHER_DEFAULT_ATTACH_KDF_ALGO_PREFIX[] = "PRAGMA cipher_default_attach_kdf_algo='"; static constexpr char CIPHER_DEFAULT_ATTACH_PAGE_SIZE_PREFIX[] = "PRAGMA cipher_default_attach_page_size="; static constexpr char CIPHER_DEFAULT_ATTACH_KDF_ITER_PREFIX[] = "PRAGMA cipher_default_attach_kdf_iter="; + static constexpr char PRAGMA_META_DOUBLE_WRITE[] = "PRAGMA meta_double_write=enabled"; }; class SqliteGlobalConfig { @@ -72,6 +76,7 @@ public: static void InitSqliteGlobalConfig(); static void Log(const void *data, int err, const char *msg); static std::string GetMemoryDbPath(); + static std::string GetSharedMemoryDbPath(const std::string &name); static int GetPageSize(); static std::string GetSyncMode(); static int GetJournalFileSize(); @@ -80,6 +85,9 @@ public: static int GetDbPath(const RdbStoreConfig &config, std::string &dbPath); static void Corruption(void *arg, const void *msg); static std::string GetLastCorruptionMsg(); + +private: + static void SqliteErrReport(int err, const char *msg); }; } // namespace NativeRdb diff --git a/relational_store/frameworks/native/rdb/include/sqlite_statement.h b/relational_store/frameworks/native/rdb/include/sqlite_statement.h index dfc353ba..e66eee14 100644 --- a/relational_store/frameworks/native/rdb/include/sqlite_statement.h +++ b/relational_store/frameworks/native/rdb/include/sqlite_statement.h @@ -22,6 +22,7 @@ #include "rdb_store_config.h" #include "share_block.h" #include "sqlite3sym.h" +#include "sqlite_utils.h" #include "statement.h" #include "value_object.h" @@ -90,6 +91,9 @@ private: ValueObject GetValueFromBlob(int32_t index, int32_t type) const; void ReadFile2Buffer(); void PrintInfoForDbError(int errCode, const std::string &sql); + void TableReport(const std::string &errMsg, const std::string &bundleName, ErrMsgState state); + void ColumnReport(const std::string &errMsg, const std::string &bundleName, ErrMsgState state); + void HandleErrMsg(const std::string &errMsg, const std::string &dbPath, const std::string &bundleName); static constexpr uint32_t BUFFER_LEN = 16; diff --git a/relational_store/frameworks/native/rdb/include/sqlite_utils.h b/relational_store/frameworks/native/rdb/include/sqlite_utils.h index 8c8ec18e..cb25c478 100644 --- a/relational_store/frameworks/native/rdb/include/sqlite_utils.h +++ b/relational_store/frameworks/native/rdb/include/sqlite_utils.h @@ -25,7 +25,14 @@ namespace OHOS { namespace NativeRdb { +struct ErrMsgState { + bool isCreated = false; + bool isDeleted = false; + bool isRenamed = false; +}; + using DebugInfo = OHOS::DistributedRdb::RdbDebugInfo; +using DfxInfo = OHOS::DistributedRdb::RdbDfxInfo; class SqliteUtils { public: static constexpr int STATEMENT_SELECT = 1; @@ -53,7 +60,7 @@ public: static bool IsSqlReadOnly(int sqlType); static bool IsSpecial(int sqlType); static const char *GetConflictClause(int conflictResolution); - static std::string StrToUpper(std::string s); + static std::string StrToUpper(const std::string &s); static void Replace(std::string &src, const std::string &rep, const std::string &dst); static bool DeleteFile(const std::string &filePath); static bool RenameFile(const std::string &srcFile, const std::string &destFile); @@ -72,11 +79,17 @@ public: static const char *EncryptAlgoDescription(int32_t encryptAlgo); static bool DeleteDirtyFiles(const std::string &backupFilePath); static std::pair Stat(const std::string &path); - static std::string FomatConfigChg(const std::string &path, const RdbStoreConfig &config, - const DistributedRdb::RdbSyncerParam &lastParam); + static void WriteSqlToFile(const std::string &comparePath, const std::string &sql); + static bool CleanFileContent(const std::string &filePath); + static std::string GetErrInfoFromMsg(const std::string &message, const std::string &errStr); + static ErrMsgState CompareTableFileContent(const std::string &dbPath, const std::string &bundleName, + const std::string &tableName); + static ErrMsgState CompareColumnFileContent(const std::string &dbPath, const std::string &bundleName, + const std::string &columnName); static std::string ReadFileHeader(const std::string &filePath); static std::string FormatDebugInfo(const std::map &debugs, const std::string &header); static std::string FormatDebugInfoBrief(const std::map &debugs, const std::string &header); + static std::string FormatDfxInfo(const DfxInfo &dfxInfo); private: struct SqlType { diff --git a/relational_store/frameworks/native/rdb/include/string_utils.h b/relational_store/frameworks/native/rdb/include/string_utils.h index 8012f97d..b231fed6 100644 --- a/relational_store/frameworks/native/rdb/include/string_utils.h +++ b/relational_store/frameworks/native/rdb/include/string_utils.h @@ -30,6 +30,7 @@ public: static std::vector Split(const std::string &str, const std::string &delim); static std::string ExtractFilePath(const std::string &fileFullName); static std::string ExtractFileName(const std::string &fileFullName); + static std::string TruncateAfterFirstParen(const std::string& str); static bool IsEmpty(std::string source) { return (source.empty()); diff --git a/relational_store/frameworks/native/rdb/include/transaction_impl.h b/relational_store/frameworks/native/rdb/include/transaction_impl.h index a3049314..fe133482 100644 --- a/relational_store/frameworks/native/rdb/include/transaction_impl.h +++ b/relational_store/frameworks/native/rdb/include/transaction_impl.h @@ -57,7 +57,7 @@ private: static std::string GetBeginSql(int32_t type); int32_t Begin(int32_t type); bool IsInTransaction(); - int32_t CloseInner(); + int32_t CloseInner(bool connRecycle = true); std::shared_ptr GetStore(); void AddResultSet(std::weak_ptr resultSet); diff --git a/relational_store/frameworks/native/rdb/mock/include/rdb_store_impl.h b/relational_store/frameworks/native/rdb/mock/include/rdb_store_impl.h index 8e5d9356..6ed5797e 100644 --- a/relational_store/frameworks/native/rdb/mock/include/rdb_store_impl.h +++ b/relational_store/frameworks/native/rdb/mock/include/rdb_store_impl.h @@ -89,6 +89,7 @@ public: private: using Stmt = std::shared_ptr; using RdbParam = DistributedRdb::RdbSyncerParam; + using ReportFunc = std::function; class CloudTables { public: int32_t AddTables(const std::vector &tables); @@ -101,19 +102,9 @@ private: std::set tables_; std::set changes_; }; - class StoreSuspender { - public: - StoreSuspender(RdbStoreImpl &rdbStore) : rdbStore_(rdbStore) - { - } - ~StoreSuspender() - { - } - private: - const RdbStoreImpl &rdbStore_; - }; int InnerOpen(); + void InitReportFunc(const RdbParam ¶m); void InitSyncerParam(const RdbStoreConfig &config, bool created); int ExecuteByTrxId(const std::string &sql, int64_t trxId, bool closeConnAfterExecute = false, const std::vector &bindArgs = {}); @@ -143,6 +134,9 @@ private: int HandleCloudSyncAfterSetDistributedTables( const std::vector &tables, const DistributedRdb::DistributedConfig &distributedConfig); std::pair> GetConn(bool isRead); + void HandleSchemaDDL(std::shared_ptr statement, + std::shared_ptr pool, const std::string &sql, int32_t &errCode); + void BatchInsertArgsDfx(int argsSize); static constexpr char SCHEME_RDB[] = "rdb://"; static constexpr uint32_t EXPANSION = 2; @@ -160,6 +154,8 @@ private: std::atomic newTrxId_ = 1; const RdbStoreConfig config_; DistributedRdb::RdbSyncerParam syncerParam_; + DistributedRdb::RdbStatEvent statEvent_; + std::shared_ptr reportFunc_ = nullptr; std::string path_; std::string name_; std::string fileType_; diff --git a/relational_store/frameworks/native/rdb/mock/src/rdb_fault_hiview_reporter.cpp b/relational_store/frameworks/native/rdb/mock/src/rdb_fault_hiview_reporter.cpp index bf72d265..2dc6a2b7 100644 --- a/relational_store/frameworks/native/rdb/mock/src/rdb_fault_hiview_reporter.cpp +++ b/relational_store/frameworks/native/rdb/mock/src/rdb_fault_hiview_reporter.cpp @@ -48,13 +48,17 @@ void RdbFaultHiViewReporter::DeleteCorruptedFlag(const std::string &dbPath) } RdbCorruptedEvent RdbFaultHiViewReporter::Create( - const RdbStoreConfig &config, int32_t errCode, const std::string &appendix) + const RdbStoreConfig &config, int32_t errCode, const std::string &appendix, bool needSyncParaFromSrv) { + (void)config; + (void)errCode; + (void)appendix; + (void)needSyncParaFromSrv; RdbCorruptedEvent eventInfo; return eventInfo; } -bool RdbFaultHiViewReporter::RegCollector(Connection::Collector collector) +bool RdbFaultHiViewReporter::RegCollector(Collector collector) { (void)collector; return true; diff --git a/relational_store/frameworks/native/rdb/src/connection.cpp b/relational_store/frameworks/native/rdb/src/connection.cpp index 713472d1..5588fa22 100644 --- a/relational_store/frameworks/native/rdb/src/connection.cpp +++ b/relational_store/frameworks/native/rdb/src/connection.cpp @@ -177,4 +177,14 @@ int Connection::GetId() const { return id_; } + +void Connection::SetIsRecyclable(bool isRecyclable) +{ + isRecyclable_ = isRecyclable; +} + +bool Connection::IsRecyclable() const +{ + return isRecyclable_; +} } // namespace OHOS::NativeRdb diff --git a/relational_store/frameworks/native/rdb/src/connection_pool.cpp b/relational_store/frameworks/native/rdb/src/connection_pool.cpp index 5d5c2cba..8eff9a1c 100644 --- a/relational_store/frameworks/native/rdb/src/connection_pool.cpp +++ b/relational_store/frameworks/native/rdb/src/connection_pool.cpp @@ -12,10 +12,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #define LOG_TAG "ConnectionPool" #include "connection_pool.h" #include + #include #include #include @@ -28,9 +30,9 @@ #include "rdb_errno.h" #include "rdb_fault_hiview_reporter.h" #include "rdb_sql_statistic.h" -#include "rdb_security_manager.h" #include "sqlite_global_config.h" #include "sqlite_utils.h" +#include "task_executor.h" namespace OHOS { namespace NativeRdb { @@ -53,15 +55,12 @@ std::shared_ptr ConnPool::Create(const RdbStoreConfig &config, int &er return nullptr; } std::shared_ptr conn; - auto originalIter = config.GetIter(); - std::tie(errCode, conn) = pool->Init(); - if (errCode == E_SQLITE_CORRUPT && originalIter == 0) { - config.SetIter(ITER_V1); + for (uint32_t retry = 0; retry < ITERS_COUNT; ++retry) { std::tie(errCode, conn) = pool->Init(); - // TOD: Report if successful - } - if (errCode == E_SQLITE_CORRUPT) { - config.SetIter(originalIter); + if (errCode != E_SQLITE_CORRUPT) { + break; + } + config.SetIter(ITER_V1); } std::string dbPath; (void)SqliteGlobalConfig::GetDbPath(config, dbPath); @@ -82,22 +81,27 @@ std::pair> ConnPool::HandleDataCorr std::pair> result; auto &[rebuiltType, pool] = result; -// RestoreReporter reporter(storeConfig, __FUNCTION__); int repairErrCode = Connection::Repair(storeConfig); - if (repairErrCode != E_OK && !storeConfig.GetAllowRebuild()) { - return result; - } if (repairErrCode == E_OK) { rebuiltType = RebuiltType::REPAIRED; -// reporter.SetRestoreType("Repair"); } else if (storeConfig.GetAllowRebuild()) { Connection::Delete(storeConfig); - RdbSecurityManager::GetInstance().DelAllKeyFiles(storeConfig.GetPath()); rebuiltType = RebuiltType::REBUILT; -// reporter.SetRestoreType("Rebuilt"); + } else if (storeConfig.IsEncrypt() && errCode == E_INVALID_SECRET_KEY) { + return result; + } else { + errCode = E_SQLITE_CORRUPT; + return result; } - pool = Create(storeConfig, errCode); + if (errCode != E_OK) { + LOG_WARN("failed, type %{public}d db %{public}s encrypt %{public}d error %{public}d errno %{public}d", + static_cast(rebuiltType), SqliteUtils::Anonymous(storeConfig.GetName()).c_str(), + storeConfig.IsEncrypt(), errCode, errno); + } else { + Reportor::ReportRestore(Reportor::Create(storeConfig, E_OK, "RestoreType:Rebuild", false), false); + } + return result; } @@ -106,6 +110,8 @@ ConnPool::ConnectionPool(const RdbStoreConfig &storeConfig) transactionUsed_(false) { attachConfig_.SetJournalMode(JournalMode::MODE_TRUNCATE); + trans_.right_ = Container::MIN_TRANS_ID; + trans_.left_ = trans_.right_; } std::pair> ConnPool::Init(bool isAttach, bool needWriter) @@ -121,18 +127,18 @@ std::pair> ConnPool::Init(bool isAttach, bo if ((config.GetRoleType() == OWNER || config.GetRoleType() == VISITOR_WRITE) && !config.IsReadOnly()) { // write connect count is 1 std::shared_ptr node; - std::tie(errCode, node) = writers_.Initialize( - [this, isAttach]() { - const RdbStoreConfig &config = isAttach ? attachConfig_ : config_; - return Connection::Create(config, true); - }, - 1, config.GetWriteTime(), true, needWriter); + auto create = [this, isAttach]() { + const RdbStoreConfig &config = isAttach ? attachConfig_ : config_; + return Connection::Create(config, true); + }; + std::tie(errCode, node) = writers_.Initialize(create, 1, config.GetWriteTime(), true, needWriter); conn = Convert2AutoConn(node); if (errCode != E_OK) { return result; } + trans_.InitMembers(create, MAX_TRANS, 0, false); } - + isAttach_ = isAttach; maxReader_ = GetMaxReaders(config); // max read connect count is 64 if (maxReader_ > 64) { @@ -173,16 +179,16 @@ std::shared_ptr ConnPool::Convert2AutoConn(std::shared_ptr if (conn == nullptr) { return nullptr; } + conn->VerifyAndRegisterHook(config_); if (isTrans) { transCount_++; } - return std::shared_ptr(conn.get(), [pool = weak_from_this(), node, isTrans](auto *) mutable { auto realPool = pool.lock(); if (realPool == nullptr) { return; } - realPool->ReleaseNode(node, !isTrans); + realPool->ReleaseNode(node, isTrans); if (isTrans) { realPool->transCount_--; } @@ -190,10 +196,27 @@ std::shared_ptr ConnPool::Convert2AutoConn(std::shared_ptr }); } +void ConnPool::DelayClearTrans() +{ + auto pool = TaskExecutor::GetInstance().GetExecutor(); + if (pool == nullptr) { + LOG_ERROR("pool is nullptr."); + return; + } + pool->Schedule(TRANS_CLEAR_INTERVAL, [pool = weak_from_this()]() { + auto realPool = pool.lock(); + if (realPool == nullptr) { + return; + } + realPool->trans_.ClearUnusedTrans(realPool); + }); +} + void ConnPool::CloseAllConnections() { writers_.Clear(); readers_.Clear(); + trans_.Clear(); } bool ConnPool::IsInTransaction() @@ -208,11 +231,15 @@ void ConnPool::SetInTransaction(bool isInTransaction) std::pair> ConnPool::CreateTransConn(bool limited) { - if (transCount_ >= MAX_TRANS && limited) { - writers_.Dump("NO TRANS", transCount_ + isInTransaction_); + if (transCount_.load() >= MAX_TRANS && limited) { + trans_.Dump("NO TRANS", transCount_ + isInTransaction_); + writers_.Dump("NO TRANS WRITE", transCount_ + isInTransaction_); return { E_DATABASE_BUSY, nullptr }; } - auto [errCode, node] = writers_.Create(); + if (trans_.Empty()) { + DelayClearTrans(); + } + auto [errCode, node] = trans_.Acquire(INVALID_TIME); return { errCode, Convert2AutoConn(node, true) }; } @@ -230,8 +257,8 @@ std::pair ConnPool::AcquireAll(int32_t time) auto &[writer, readers] = result; auto interval = duration_cast(seconds(time)); auto start = steady_clock::now(); - auto writerNodes = writers_.AcquireAll(interval); - if (writerNodes.empty()) { + auto [res, writerNodes] = writers_.AcquireAll(interval); + if (!res) { return {}; } writer = Convert2AutoConn(writerNodes.front()); @@ -246,8 +273,9 @@ std::pair ConnPool::AcquireAll(int32_t time) } readers_.Disable(); - auto nodes = readers_.AcquireAll(interval - usedTime); - if (nodes.empty()) { + std::list> nodes; + std::tie(res, nodes) = readers_.AcquireAll(interval - usedTime); + if (!res) { readers_.Enable(); return {}; } @@ -259,14 +287,25 @@ std::pair ConnPool::AcquireAll(int32_t time) } readers.push_back(conn); } + usedTime = duration_cast(steady_clock::now() - start); + if (usedTime >= interval) { + return {}; + } + trans_.Disable(); + std::list> trans; + std::tie(res, trans) = trans_.AcquireAll(interval - usedTime); + if (!res) { + trans_.Enable(); + return {}; + } return result; } std::shared_ptr ConnPool::Acquire(bool isReadOnly, std::chrono::milliseconds ms) { Container *container = (isReadOnly && maxReader_ != 0) ? &readers_ : &writers_; - auto node = container->Acquire(ms); - if (node == nullptr) { + auto [errCode, node] = container->Acquire(ms); + if (errCode != E_OK || node == nullptr) { const char *header = (isReadOnly && maxReader_ != 0) ? "readers_" : "writers_"; container->Dump(header, transCount_ + isInTransaction_); return nullptr; @@ -280,8 +319,8 @@ SharedConn ConnPool::AcquireRef(bool isReadOnly, std::chrono::milliseconds ms) if (maxReader_ != 0) { return Acquire(isReadOnly, ms); } - auto node = writers_.Acquire(ms); - if (node == nullptr) { + auto [errCode, node] = writers_.Acquire(ms); + if (errCode != E_OK || node == nullptr) { writers_.Dump("writers_", transCount_ + isInTransaction_); return nullptr; } @@ -296,7 +335,7 @@ SharedConn ConnPool::AcquireRef(bool isReadOnly, std::chrono::milliseconds ms) }); } -void ConnPool::ReleaseNode(std::shared_ptr node, bool reuse) +void ConnPool::ReleaseNode(std::shared_ptr node, bool isTrans) { if (node == nullptr) { return; @@ -306,10 +345,11 @@ void ConnPool::ReleaseNode(std::shared_ptr node, bool reuse) auto timeout = now > (failedTime_.load() + minutes(CHECK_POINT_INTERVAL)) || now < failedTime_.load() || failedTime_.load() == steady_clock::time_point(); auto transCount = transCount_ + isInTransaction_; - auto remainCount = reuse ? transCount : transCount - 1; + auto remainCount = isTrans ? transCount - 1 : transCount; auto errCode = node->Unused(remainCount, timeout); if (errCode == E_SQLITE_LOCKED || errCode == E_SQLITE_BUSY) { writers_.Dump("WAL writers_", transCount); + trans_.Dump("WAL trans_", transCount); readers_.Dump("WAL readers_", transCount); } @@ -317,11 +357,11 @@ void ConnPool::ReleaseNode(std::shared_ptr node, bool reuse) failedTime_ = errCode != E_OK ? now : steady_clock::time_point(); } - auto &container = node->IsWriter() ? writers_ : readers_; - if (reuse) { - container.Release(node); + if (isTrans) { + trans_.ReleaseTrans(node); } else { - container.Drop(node); + auto &container = node->IsWriter() ? writers_ : readers_; + container.Release(node); } } @@ -347,11 +387,23 @@ void ConnPool::ReleaseTransaction() transCondition_.notify_one(); } -int ConnPool::RestartReaders() +int ConnPool::RestartConns() { + const RdbStoreConfig &config = isAttach_ ? attachConfig_ : config_; readers_.Clear(); auto [errCode, node] = readers_.Initialize( - [this]() { return Connection::Create(config_, false); }, maxReader_, config_.GetReadTime(), maxReader_ == 0); + [this]() { + const RdbStoreConfig &config = isAttach_ ? attachConfig_ : config_; + return Connection::Create(config, false); + }, + maxReader_, config.GetReadTime(), maxReader_ == 0); + trans_.Clear(); + trans_.InitMembers( + [this]() { + const RdbStoreConfig &config = isAttach_ ? attachConfig_ : config_; + return Connection::Create(config, true); + }, + MAX_TRANS, 0, false); return errCode; } @@ -508,17 +560,23 @@ bool ConnPool::ConnNode::IsWriter() const return false; } +void ConnPool::Container::InitMembers(Creator creator, int32_t max, int32_t timeout, bool disable) +{ + std::unique_lock lock(mutex_); + disable_ = disable; + max_ = max; + creator_ = creator; + timeout_ = std::chrono::seconds(timeout); +} + std::pair> ConnPool::Container::Initialize( Creator creator, int32_t max, int32_t timeout, bool disable, bool acquire) { + InitMembers(creator, max, timeout, disable); std::shared_ptr connNode = nullptr; { std::unique_lock lock(mutex_); - disable_ = disable; - max_ = max; - creator_ = creator; - timeout_ = std::chrono::seconds(timeout); - for (int i = 0; i < max; ++i) { + for (int i = 0; i < max_; ++i) { auto errCode = ExtendNode(); if (errCode != E_OK) { nodes_.clear(); @@ -554,14 +612,15 @@ int32_t ConnPool::Container::ConfigLocale(const std::string &locale) return E_OK; } -std::shared_ptr ConnPool::Container::Acquire(std::chrono::milliseconds milliS) +std::pair> ConnPool::Container::Acquire(std::chrono::milliseconds milliS) { std::unique_lock lock(mutex_); auto interval = (milliS == INVALID_TIME) ? timeout_ : milliS; if (max_ == 0) { - return nullptr; + return {E_ERROR, nullptr}; } - auto waiter = [this]() -> bool { + int errCode = E_OK; + auto waiter = [this, &errCode]() -> bool { if (count_ > 0) { return true; } @@ -569,44 +628,22 @@ std::shared_ptr ConnPool::Container::Acquire(std::chrono::mi if (disable_) { return false; } - return ExtendNode() == E_OK; + errCode = ExtendNode(); + return errCode == E_OK; }; if (cond_.wait_for(lock, interval, waiter)) { if (nodes_.empty()) { LOG_ERROR("Nodes is empty.count %{public}d max %{public}d total %{public}d left %{public}d right%{public}d", count_, max_, total_, left_, right_); count_ = 0; - return nullptr; + return {E_ERROR, nullptr}; } auto node = nodes_.back(); nodes_.pop_back(); count_--; - return node; + return {E_OK, node}; } - return nullptr; -} - -std::pair> ConnPool::Container::Create() -{ - std::unique_lock lock(mutex_); - if (creator_ == nullptr) { - return { E_NOT_SUPPORT, nullptr }; - } - - auto [errCode, conn] = creator_(); - if (conn == nullptr) { - return { errCode, nullptr }; - } - - auto node = std::make_shared(conn); - if (node == nullptr) { - return { E_ERROR, nullptr }; - } - node->id_ = MIN_TRANS_ID + trans_; - conn->SetId(node->id_); - details_.push_back(node); - trans_++; - return { E_OK, node }; + return {errCode, nullptr}; } int32_t ConnPool::Container::ExtendNode() @@ -628,7 +665,8 @@ int32_t ConnPool::Container::ExtendNode() return E_OK; } -std::list> ConnPool::Container::AcquireAll(std::chrono::milliseconds milliS) +std::pair>> ConnPool::Container::AcquireAll( + std::chrono::milliseconds milliS) { std::list> nodes; int32_t count = 0; @@ -646,7 +684,7 @@ std::list> ConnPool::Container::AcquireAll(s count_ = count; nodes_ = std::move(nodes); nodes.clear(); - return nodes; + return {false, nodes}; } auto func = [](const std::list> &nodes) -> bool { for (auto &node : nodes) { @@ -667,7 +705,7 @@ std::list> ConnPool::Container::AcquireAll(s nodes_ = std::move(nodes); nodes.clear(); } - return nodes; + return {!failed, nodes}; } void ConnPool::Container::Disable() @@ -701,11 +739,20 @@ int32_t ConnPool::Container::Release(std::shared_ptr node) return E_OK; } -int32_t ConnectionPool::Container::Drop(std::shared_ptr node) +int32_t ConnectionPool::Container::ReleaseTrans(std::shared_ptr node) { { std::unique_lock lock(mutex_); - RelDetails(node); + if (node->id_ < left_ || node->id_ >= right_) { + return E_OK; + } + if (node->IsRecyclable()) { + nodes_.push_back(node); + count_++; + } else { + total_--; + RelDetails(node); + } } cond_.notify_one(); return E_OK; @@ -771,9 +818,16 @@ bool ConnPool::Container::IsFull() return total_ == count_; } +bool ConnPool::Container::Empty() +{ + std::unique_lock lock(mutex_); + return total_ == 0; +} + int32_t ConnPool::Container::Dump(const char *header, int32_t count) { std::string info; + std::string allInfo; std::vector> details; std::string title = "B_M_T_C[" + std::to_string(count) + "," + std::to_string(max_) + "," + std::to_string(total_) + "," + std::to_string(count_) + "]"; @@ -800,11 +854,42 @@ int32_t ConnPool::Container::Dump(const char *header, int32_t count) // 256 represent that limit to info length if (info.size() > 256) { LOG_WARN("%{public}s %{public}s:%{public}s", header, title.c_str(), info.c_str()); + allInfo.append(header).append(": ").append(title).append(std::move(info)).append("\n"); info.clear(); } } LOG_WARN("%{public}s %{public}s:%{public}s", header, title.c_str(), info.c_str()); + allInfo.append(header).append(": ").append(title).append(std::move(info)); + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_DUMP_INFO, BUNDLE_NAME_COMMON, allInfo)); return 0; } + +int32_t ConnectionPool::Container::ClearUnusedTrans(std::shared_ptr pool) +{ + std::unique_lock lock(mutex_); + int transCount = total_; + for (auto it = nodes_.begin(); it != nodes_.end();) { + auto unusedDuration = std::chrono::steady_clock::now() - (*it)->time_; + if (unusedDuration < TRANS_CLEAR_INTERVAL) { + it++; + continue; + } + RelDetails(*it); + it = nodes_.erase(it); + total_--; + count_--; + } + if (total_ != 0) { + pool->DelayClearTrans(); + } + LOG_INFO("%{public}d have been cleaned up, and there are %{public}d remaining to be cleaned up", + transCount - total_, total_); + return E_OK; +} + +bool ConnPool::ConnNode::IsRecyclable() +{ + return connect_->IsRecyclable(); +} } // namespace NativeRdb } // namespace OHOS diff --git a/relational_store/frameworks/native/rdb/src/rd_connection.cpp b/relational_store/frameworks/native/rdb/src/rd_connection.cpp index 79d64fb1..52c543e3 100644 --- a/relational_store/frameworks/native/rdb/src/rd_connection.cpp +++ b/relational_store/frameworks/native/rdb/src/rd_connection.cpp @@ -26,7 +26,6 @@ #include "rdb_security_manager.h" #include "sqlite_global_config.h" #include "sqlite_utils.h" - namespace OHOS { namespace NativeRdb { using namespace OHOS::Rdb; @@ -200,7 +199,7 @@ int RdConnection::InnerOpen(const RdbStoreConfig &config) return errCode; } -int32_t RdConnection::OnInitialize() +int32_t RdConnection::VerifyAndRegisterHook(const RdbStoreConfig &config) { return E_NOT_SUPPORT; } @@ -301,14 +300,12 @@ int32_t RdConnection::ClearCache() return E_NOT_SUPPORT; } -int32_t RdConnection::Subscribe( - const std::string &event, const std::shared_ptr &observer) +int32_t RdConnection::Subscribe(const std::shared_ptr &observer) { return E_NOT_SUPPORT; } -int32_t RdConnection::Unsubscribe( - const std::string &event, const std::shared_ptr &observer) +int32_t RdConnection::Unsubscribe(const std::shared_ptr &observer) { return E_NOT_SUPPORT; } diff --git a/relational_store/frameworks/native/rdb/src/rd_statement.cpp b/relational_store/frameworks/native/rdb/src/rd_statement.cpp index a390c946..be0b5b0e 100644 --- a/relational_store/frameworks/native/rdb/src/rd_statement.cpp +++ b/relational_store/frameworks/native/rdb/src/rd_statement.cpp @@ -142,7 +142,7 @@ int RdStatement::Prepare(GRD_DB *db, const std::string &newSql) stmtHandle_ = tmpStmt; columnCount_ = RdUtils::RdSqlColCnt(tmpStmt); readOnly_ = SqliteUtils::GetSqlStatementType(newSql) == SqliteUtils::STATEMENT_SELECT; - return PreGetColCount(); + return E_OK; } int RdStatement::Finalize() diff --git a/relational_store/frameworks/native/rdb/src/rd_utils.cpp b/relational_store/frameworks/native/rdb/src/rd_utils.cpp index 395e2911..db615d28 100644 --- a/relational_store/frameworks/native/rdb/src/rd_utils.cpp +++ b/relational_store/frameworks/native/rdb/src/rd_utils.cpp @@ -67,18 +67,91 @@ const GrdErrnoPair GRD_ERRNO_MAP[] = { { GRD_PASSWORD_NEED_REKEY, E_CHANGE_UNENCRYPTED_TO_ENCRYPTED }, { GRD_NAME_TOO_LONG, E_SQLITE_CONSTRAINT }, - { GRD_INVALID_TABLE_DEFINITION, E_SQLITE_SCHEMA }, + { GRD_INVALID_TABLE_DEFINITION, E_SQLITE_ERROR }, { GRD_SEMANTIC_ERROR, E_NOT_SUPPORT_THE_SQL }, { GRD_SYNTAX_ERROR, E_NOT_SUPPORT_THE_SQL }, - { GRD_DATA_MISMATCH, E_SQLITE_MISMATCH }, + { GRD_DATATYPE_MISMATCH, E_SQLITE_MISMATCH }, { GRD_WRONG_STMT_OBJECT, E_INVALID_OBJECT_TYPE }, - { GRD_DATA_CONFLICT, E_INVALID_CONFLICT_FLAG }, - + { GRD_DATA_CONFLICT, E_SQLITE_CONSTRAINT }, + + { GRD_ACTIVE_TRANSACTION, E_SQLITE_ERROR }, + { GRD_UNIQUE_VIOLATION, E_SQLITE_CONSTRAINT }, + { GRD_DUPLICATE_TABLE, E_SQLITE_ERROR }, + { GRD_UNDEFINED_TABLE, E_SQLITE_ERROR }, + { GRD_INVALID_BIND_VALUE, E_SQLITE_ERROR }, + { GRD_SCHEMA_CHANGED, E_SQLITE_ERROR }, + + { GRD_JSON_OPERATION_NOT_SUPPORT, E_NOT_SUPPORT_THE_SQL }, + { GRD_MODEL_NOT_SUPPORT, E_NOT_SUPPORT_THE_SQL }, + { GRD_FEATURE_NOT_SUPPORTED, E_NOT_SUPPORT_THE_SQL }, + + { GRD_JSON_LEN_LIMIT, E_SQLITE_TOOBIG }, + { GRD_SUBSCRIPTION_EXCEEDED_LIMIT, E_ERROR }, + { GRD_SYNC_EXCEED_TASK_QUEUE_LIMIT, E_ERROR }, + { GRD_SHARED_OBJ_ENABLE_UNDO_EXCEED_LIMIT, E_ERROR }, + { GRD_TABLE_LIMIT_EXCEEDED, E_SQLITE_CONSTRAINT }, + + { GRD_FIELD_TYPE_NOT_MATCH, E_SQLITE_MISMATCH }, + { GRD_LARGE_JSON_NEST, E_NOT_SUPPORT_THE_SQL }, + { GRD_INVALID_JSON_TYPE, E_SQLITE_ERROR }, + { GRD_INVALID_CONFIG_VALUE, E_CONFIG_INVALID_CHANGE }, + { GRD_INVALID_OPERATOR, E_SQLITE_ERROR }, + { GRD_INVALID_PROJECTION_FIELD, E_SQLITE_ERROR }, + { GRD_INVALID_PROJECTION_VALUE, E_SQLITE_ERROR }, + { GRD_COLLECTION_NOT_EXIST, E_ERROR }, + { GRD_DB_NOT_EXIST, E_ERROR }, + { GRD_INVALID_VALUE, E_SQLITE_ERROR }, + { GRD_SHARED_OBJ_NOT_EXIST, E_ERROR }, + { GRD_SUBSCRIBE_NOT_EXIST, E_ERROR }, + { GRD_SHARED_OBJ_UNDO_MANAGER_NOT_EXIST, E_ERROR }, + { GRD_SHARED_OBJ_INVALID_UNDO, E_ERROR }, + { GRD_SHARED_OBJ_INVALID_REDO, E_ERROR }, + + { GRD_JSON_LIB_HANDLE_FAILED, E_ERROR }, + { GRD_DIRECTORY_OPERATE_FAILED, E_SQLITE_IOERR_FULL }, + { GRD_FILE_OPERATE_FAILED, E_SQLITE_IOERR_FULL }, + { GRD_LOAD_THIRD_PARTY_LIBRARY_FAILED, E_ERROR }, + { GRD_THIRD_PARTY_FUNCTION_EXECUTE_FAILED, E_ERROR }, + { GRD_INSUFFICIENT_RESOURCES, E_ERROR }, + + { GRD_RESULTSET_BUSY, E_DATABASE_BUSY }, + { GRD_RECORD_NOT_FOUND, E_SQLITE_ERROR }, + { GRD_FIELD_NOT_FOUND, E_SQLITE_ERROR }, + { GRD_ARRAY_INDEX_NOT_FOUND, E_SQLITE_ERROR }, + { GRD_KEY_CONFLICT, E_SQLITE_CONSTRAINT }, + { GRD_FIELD_TYPE_CONFLICT, E_SQLITE_MISMATCH }, + { GRD_SHARED_OBJ_CONFLICT, E_SQLITE_CONSTRAINT }, + { GRD_SUBSCRIBE_CONFLICT, E_ERROR }, + { GRD_EQUIP_ID_CONFLICT, E_SQLITE_CONSTRAINT }, + { GRD_SHARED_OBJ_ENABLE_UNDO_CONFLICT, E_ERROR }, + + { GRD_DATA_EXCEPTION, E_SQLITE_CORRUPT }, + { GRD_FIELD_OVERFLOW, E_SQLITE_ERROR }, + { GRD_DIVISION_BY_ZERO, E_SQLITE_ERROR }, + { GRD_RESULT_SET_NOT_AVAILABLE, E_ERROR }, + { GRD_SHARED_OBJ_UNDO_NOT_AVAILABLE, E_ERROR }, + { GRD_SHARED_OBJ_REDO_NOT_AVAILABLE, E_ERROR }, + { GRD_TRANSACTION_ROLLBACK, E_SQLITE_ERROR }, + { GRD_NO_ACTIVE_TRANSACTION, E_SQLITE_ERROR }, + + { GRD_DUPLICATE_COLUMN, E_SQLITE_ERROR }, + { GRD_DUPLICATE_OBJECT, E_SQLITE_ERROR }, + { GRD_UNDEFINE_COLUMN, E_SQLITE_ERROR }, + { GRD_UNDEFINED_OBJECT, E_SQLITE_ERROR }, + { GRD_INVALID_JSON_FORMAT, E_ERROR }, + { GRD_INVALID_KEY_FORMAT, E_ERROR }, + { GRD_INVALID_COLLECTION_NAME, E_ERROR }, + { GRD_INVALID_EQUIP_ID, E_SQLITE_CONSTRAINT }, + { GRD_REQUEST_TIME_OUT, E_DATABASE_BUSY }, + + { GRD_SYNC_PREREQUISITES_ABNORMAL, E_ERROR }, + { GRD_CALC_MODE_SET_PERMISSION_DENIED, E_ERROR }, + { GRD_SYSTEM_ERR, E_ERROR }, { GRD_INNER_ERR, E_ERROR }, { GRD_FAILED_MEMORY_RELEASE, E_ERROR }, { GRD_NOT_AVAILABLE, E_ERROR }, { GRD_INVALID_FORMAT, E_ERROR }, - { GRD_TIME_OUT, E_ERROR }, + { GRD_TIME_OUT, E_DATABASE_BUSY }, { GRD_DB_INSTANCE_ABNORMAL, E_ERROR }, { GRD_CIPHER_ERROR, E_ERROR }, }; @@ -198,12 +271,12 @@ int RdUtils::RdSqlBindBlob(GRD_SqlStmt *stmt, uint32_t idx, const void *val, int LOG_ERROR("Invalid len %{public}d", len); return E_INVALID_ARGS; } - uint8_t *tmpVal = new uint8_t[len](); + uint8_t *tmpVal = new (std::nothrow)uint8_t[len](); if (tmpVal == nullptr) { return E_ERROR; } errno_t err = memcpy_s(tmpVal, len * sizeof(uint8_t), val, len * sizeof(uint8_t)); - if (err < 0) { + if (err != EOK) { delete[] tmpVal; LOG_ERROR("BindBlob failed due to memcpy %{public}d, len is %{public}d", err, len); return TransferGrdErrno(GRD_INNER_ERR); @@ -231,12 +304,12 @@ int RdUtils::RdSqlBindText(GRD_SqlStmt *stmt, uint32_t idx, const void *val, int LOG_ERROR("Invalid len %{public}d", len); return E_INVALID_ARGS; } - char *tmpVal = new char[len + 1](); + char *tmpVal = new (std::nothrow)char[len + 1](); if (tmpVal == nullptr) { return E_ERROR; } errno_t err = strcpy_s(tmpVal, len + 1, (const char *)val); - if (err < 0) { + if (err != EOK) { LOG_ERROR("BindText failed due to strycpy %{public}d, len is %{public}d", err, len + 1); delete[] tmpVal; return TransferGrdErrno(GRD_INNER_ERR); @@ -308,12 +381,12 @@ int RdUtils::RdSqlBindFloatVector(GRD_SqlStmt *stmt, uint32_t idx, float *val, u LOG_ERROR("Invalid dim %{public}d", dim); return E_INVALID_ARGS; } - float *tmpVal = new float[dim](); + float *tmpVal = new (std::nothrow)float[dim](); if (tmpVal == nullptr) { return E_ERROR; } errno_t err = memcpy_s(tmpVal, dim * sizeof(float), val, dim * sizeof(float)); - if (err < 0) { + if (err != EOK) { delete[] tmpVal; LOG_ERROR("BindFloat failed due to memcpy %{public}d, dim is %{public}d", err, dim); return TransferGrdErrno(GRD_INNER_ERR); @@ -458,8 +531,8 @@ const float *RdUtils::RdSqlColumnFloatVector(GRD_SqlStmt *stmt, uint32_t idx, ui void RdUtils::ClearAndZeroString(std::string &str) { - str.clear(); std::fill(str.begin(), str.end(), char(0)); + str.clear(); } const char *RdUtils::GetEncryptKey(const std::vector &encryptedKey, char outBuff[], size_t outBufSize) diff --git a/relational_store/frameworks/native/rdb/src/rdb_helper.cpp b/relational_store/frameworks/native/rdb/src/rdb_helper.cpp index 28178da5..67e4d1f9 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_helper.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_helper.cpp @@ -27,6 +27,7 @@ #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) #include "security_policy.h" #endif +#include "rdb_fault_hiview_reporter.h" namespace OHOS { namespace NativeRdb { @@ -38,7 +39,12 @@ std::shared_ptr RdbHelper::GetRdbStore( { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); SqliteGlobalConfig::InitSqliteGlobalConfig(); - return RdbStoreManager::GetInstance().GetRdbStore(config, errCode, version, openCallback); + auto rdb = RdbStoreManager::GetInstance().GetRdbStore(config, errCode, version, openCallback); + if (errCode != E_OK) { + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, errCode, config, "LOG:RdbHelper::GetRdbStore", true)); + } + + return rdb; } void RdbHelper::ClearCache() @@ -51,24 +57,35 @@ int RdbHelper::DeleteRdbStore(const std::string &dbFileName, bool shouldClose) { RdbStoreConfig config(dbFileName); config.SetDBType(DB_SQLITE); - int errCodeSqlite = DeleteRdbStore(config, shouldClose); + int errCode = DeleteRdbStore(config, shouldClose); + config.SetStorageMode(StorageMode::MODE_MEMORY); + errCode = DeleteRdbStore(config, shouldClose) == E_OK ? errCode : E_REMOVE_FILE; + + config.SetStorageMode(StorageMode::MODE_DISK); config.SetDBType(DB_VECTOR); - int errCodeVector = DeleteRdbStore(config, shouldClose); - return (errCodeSqlite == E_OK && errCodeVector == E_OK) ? E_OK : E_REMOVE_FILE; + errCode = DeleteRdbStore(config, shouldClose) == E_OK ? errCode : E_REMOVE_FILE; + return errCode; } int RdbHelper::DeleteRdbStore(const RdbStoreConfig &config, bool shouldClose) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); - auto dbFile = config.GetPath(); - if (dbFile.empty()) { + std::string dbFile; + auto errCode = SqliteGlobalConfig::GetDbPath(config, dbFile); + if (errCode != E_OK || dbFile.empty()) { return E_INVALID_FILE_PATH; } + if (config.IsMemoryRdb()) { + RdbStoreManager::GetInstance().Remove(dbFile, shouldClose); + LOG_INFO("Remove memory store, dbType:%{public}d, path %{public}s", config.GetDBType(), + SqliteUtils::Anonymous(dbFile).c_str()); + return E_OK; + } if (access(dbFile.c_str(), F_OK) == 0) { - RdbStoreManager::GetInstance().Delete(dbFile, shouldClose); + RdbStoreManager::GetInstance().Delete(config, shouldClose); } - Reportor::ReportRestore(Reportor::Create(config, E_OK, "RestoreType:Delete")); + Reportor::ReportRestore(Reportor::Create(config, E_OK, "RestoreType:Delete", false)); Connection::Delete(config); RdbSecurityManager::GetInstance().DelAllKeyFiles(dbFile); LOG_INFO("Delete rdb store, dbType:%{public}d, path %{public}s", config.GetDBType(), diff --git a/relational_store/frameworks/native/rdb/src/rdb_manager_impl.cpp b/relational_store/frameworks/native/rdb/src/rdb_manager_impl.cpp index 0f3f6982..ed6d1f80 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_manager_impl.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_manager_impl.cpp @@ -145,6 +145,7 @@ void RdbManagerImpl::OnRemoteDied() } auto proxy = std::static_pointer_cast(rdbService_); auto observers = proxy->ExportObservers(); + auto syncObservers = proxy->ExportSyncObservers(); ResetServiceHandle(); std::this_thread::sleep_for(std::chrono::seconds(WAIT_TIME)); @@ -157,6 +158,7 @@ void RdbManagerImpl::OnRemoteDied() return; } proxy->ImportObservers(observers); + proxy->ImportSyncObservers(syncObservers); } void RdbManagerImpl::ResetServiceHandle() diff --git a/relational_store/frameworks/native/rdb/src/rdb_security_manager.cpp b/relational_store/frameworks/native/rdb/src/rdb_security_manager.cpp index e90da835..68a0be70 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_security_manager.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_security_manager.cpp @@ -133,6 +133,9 @@ int32_t RdbSecurityManager::HksLoopUpdate(const struct HksBlob *handle, const st const struct HksBlob *inData, struct HksBlob *outData) { if (outData->size < inData->size * TIMES) { + HksAbort(handle, paramSet); + Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_FAIL, GetBundleNameByAlias(), + "HksLoopUpdate out size not enough")); return HKS_ERROR_INVALID_ARGUMENT; } @@ -177,7 +180,7 @@ int32_t RdbSecurityManager::HksEncryptThreeStage(const struct HksBlob *keyAlias, int32_t result = HksInit(keyAlias, paramSet, &handleBlob, nullptr); if (result != HKS_SUCCESS) { Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_ENCRYPT_FAIL, - GetBundleNameByAlias(), "HksInit ret=" + std::to_string(result))); + GetBundleNameByAlias(), "Encrypt HksInit ret=" + std::to_string(result))); LOG_ERROR("HksEncrypt failed with error %{public}d", result); return result; } @@ -192,11 +195,16 @@ int32_t RdbSecurityManager::HksDecryptThreeStage(const struct HksBlob *keyAlias, int32_t result = HksInit(keyAlias, paramSet, &handleBlob, nullptr); if (result != HKS_SUCCESS) { Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_DECRYPT_FAIL, GetBundleNameByAlias(), - "HksInit ret=" + std::to_string(result))); + "Decrypt HksInit ret=" + std::to_string(result))); LOG_ERROR("HksEncrypt failed with error %{public}d", result); return result; } - return HksLoopUpdate(&handleBlob, paramSet, cipherText, plainText); + result = HksLoopUpdate(&handleBlob, paramSet, cipherText, plainText); + if (result != HKS_SUCCESS) { + Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_DECRYPT_FAIL, GetBundleNameByAlias(), + "Decrypt HksLoopUpdate ret=" + std::to_string(result))); + } + return result; } RdbSecurityManager::RdbSecurityManager() @@ -328,7 +336,7 @@ std::vector RdbSecurityManager::EncryptWorkKey(std::vector &ke int32_t ret = HksInitParamSet(¶ms); if (ret != HKS_SUCCESS) { Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_ENCRYPT_FAIL, GetBundleNameByAlias(rootKeyAlias), - "HksInitParamSet ret=" + std::to_string(ret))); + "Encrypt HksInitParamSet ret=" + std::to_string(ret))); LOG_ERROR("HksInitParamSet() failed with error %{public}d", ret); return {}; } @@ -344,7 +352,7 @@ std::vector RdbSecurityManager::EncryptWorkKey(std::vector &ke ret = HksAddParams(params, hksParam, sizeof(hksParam) / sizeof(hksParam[0])); if (ret != HKS_SUCCESS) { Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_ENCRYPT_FAIL, GetBundleNameByAlias(rootKeyAlias), - "HksAddParams ret=" + std::to_string(ret))); + "Encrypt HksAddParams ret=" + std::to_string(ret))); LOG_ERROR("HksAddParams failed with error %{public}d", ret); HksFreeParamSet(¶ms); return {}; @@ -353,7 +361,7 @@ std::vector RdbSecurityManager::EncryptWorkKey(std::vector &ke ret = HksBuildParamSet(¶ms); if (ret != HKS_SUCCESS) { Reportor::ReportFault(RdbFaultEvent(FT_EX_HUKS, E_WORK_KEY_ENCRYPT_FAIL, GetBundleNameByAlias(rootKeyAlias), - "HksBuildParamSet ret=" + std::to_string(ret))); + "Encrypt HksBuildParamSet ret=" + std::to_string(ret))); LOG_ERROR("HksBuildParamSet failed with error %{public}d", ret); HksFreeParamSet(¶ms); return {}; @@ -704,15 +712,16 @@ bool RdbSecurityManager::IsKeyFileEmpty(const std::string &keyFile) int32_t RdbSecurityManager::RestoreKeyFile(const std::string &dbPath, const std::vector &key) { - if (key.empty()) { - LOG_ERROR("key is empty! Path:%{public}s.", SqliteUtils::Anonymous(dbPath).c_str()); - return E_ERROR; - } KeyFiles keyFiles(dbPath); keyFiles.Lock(); auto &keyFile = keyFiles.GetKeyFile(PUB_KEY_FILE); auto &reKeyFile = keyFiles.GetKeyFile(PUB_KEY_FILE_NEW_KEY); - if (!SaveSecretKeyToFile(reKeyFile, key) && !SaveSecretKeyToFile(keyFile, key)) { + { + std::lock_guard lock(mutex_); + SqliteUtils::DeleteFile(keyFile); + SqliteUtils::DeleteFile(reKeyFile); + } + if (!SaveSecretKeyToFile(keyFile, key)) { LOG_ERROR("failed, save key err:%{public}d, file:%{public}s.", errno, SqliteUtils::Anonymous(keyFile).c_str()); } keyFiles.Unlock(); diff --git a/relational_store/frameworks/native/rdb/src/rdb_service_proxy.cpp b/relational_store/frameworks/native/rdb/src/rdb_service_proxy.cpp index fdb7fc8a..dacd16bc 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_service_proxy.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_service_proxy.cpp @@ -311,6 +311,11 @@ RdbServiceProxy::Observers RdbServiceProxy::ExportObservers() return observers_; } +RdbServiceProxy::SyncObservers RdbServiceProxy::ExportSyncObservers() +{ + return syncObservers_; +} + void RdbServiceProxy::ImportObservers(Observers &observers) { observers.ForEach([this](const std::string &key, const std::list &value) { @@ -324,6 +329,19 @@ void RdbServiceProxy::ImportObservers(Observers &observers) }); } +void RdbServiceProxy::ImportSyncObservers(SyncObservers &syncObservers) +{ + syncObservers.ForEach([this](const std::string &key, const std::list &value) { + RdbSyncerParam syncerParam; + for (const auto ¶m : value) { + syncerParam.bundleName_ = param.bundleName; + syncerParam.storeName_ = key; + RegisterAutoSyncCallback(syncerParam, param.syncObserver); + } + return false; + }); +} + int32_t RdbServiceProxy::BeforeOpen(RdbSyncerParam ¶m) { MessageParcel reply; @@ -351,6 +369,18 @@ int32_t RdbServiceProxy::AfterOpen(const RdbSyncerParam ¶m) return status; } +int32_t RdbServiceProxy::ReportStatistic(const RdbSyncerParam ¶m, const RdbStatEvent &statEvent) +{ + MessageParcel reply; + int32_t status = + IPC_SEND(static_cast(RdbServiceCode::RDB_SERVICE_CMD_REPORT_STAT), reply, param, statEvent); + if (status != RDB_OK) { + LOG_ERROR("status:%{public}d, bundleName:%{public}s, storeName:%{public}s", status, param.bundleName_.c_str(), + SqliteUtils::Anonymous(param.storeName_).c_str()); + } + return status; +} + int32_t RdbServiceProxy::Delete(const RdbSyncerParam ¶m) { MessageParcel reply; @@ -398,14 +428,14 @@ int32_t RdbServiceProxy::RegisterAutoSyncCallback( auto name = RemoveSuffix(param.storeName_); syncObservers_.Compute(name, [this, ¶m, &status, observer](const auto &store, auto &observers) { for (const auto &element : observers) { - if (element.get() == observer.get()) { + if (element.syncObserver.get() == observer.get()) { LOG_ERROR("duplicate observer, storeName:%{public}s", SqliteUtils::Anonymous(store).c_str()); return true; } } status = DoRegister(param); if (status == RDB_OK) { - observers.push_back(observer); + observers.push_back({ observer, param.bundleName_ }); } return !observers.empty(); }); @@ -436,7 +466,7 @@ int32_t RdbServiceProxy::UnregisterAutoSyncCallback( auto name = RemoveSuffix(param.storeName_); syncObservers_.ComputeIfPresent(name, [this, ¶m, &status, observer](const auto &storeName, auto &observers) { for (auto it = observers.begin(); it != observers.end();) { - if (it->get() != observer.get()) { + if (it->syncObserver.get() != observer.get()) { ++it; continue; } @@ -497,9 +527,9 @@ void RdbServiceProxy::OnSyncComplete(const std::string &storeName, Details &&res syncObservers_.ComputeIfPresent(storeName, [&result](const auto &key, const auto &observers) { LOG_DEBUG("Sync complete, storeName%{public}s, result size:%{public}zu", SqliteUtils::Anonymous(key).c_str(), result.size()); - for (const auto &observer : observers) { - if (observer != nullptr) { - observer->ProgressNotification(result); + for (const auto &it : observers) { + if (it.syncObserver != nullptr) { + it.syncObserver->ProgressNotification(result); } } return true; @@ -616,6 +646,22 @@ int32_t RdbServiceProxy::GetDebugInfo(const RdbSyncerParam ¶m, std::map(RdbServiceCode::RDB_SERVICE_CMD_GET_DFX_INFO), reply, param); + if (status != RDB_OK) { + LOG_ERROR("fail, status:%{public}d, bundleName:%{public}s, storeName:%{public}s", status, + param.bundleName_.c_str(), SqliteUtils::Anonymous(param.storeName_).c_str()); + } + if (!ITypesUtil::Unmarshal(reply, dfxInfo)) { + LOG_ERROR("Unmarshal failed, bundleName:%{public}s, storeName:%{public}s", + param.bundleName_.c_str(), SqliteUtils::Anonymous(param.storeName_).c_str()); + status = RDB_ERROR; + } + return status; +} + int32_t RdbServiceProxy::VerifyPromiseInfo(const RdbSyncerParam ¶m) { MessageParcel reply; diff --git a/relational_store/frameworks/native/rdb/src/rdb_sql_utils.cpp b/relational_store/frameworks/native/rdb/src/rdb_sql_utils.cpp index a46970ed..798b23ce 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_sql_utils.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_sql_utils.cpp @@ -28,6 +28,8 @@ #include "rdb_platform.h" #include "sqlite_sql_builder.h" #include "sqlite_utils.h" +#include "rdb_fault_hiview_reporter.h" + namespace OHOS { using namespace Rdb; namespace NativeRdb { @@ -54,6 +56,8 @@ int RdbSqlUtils::CreateDirectory(const std::string &databaseDir) if (MkDir(databaseDirectory)) { LOG_ERROR("failed to mkdir errno[%{public}d] %{public}s", errno, SqliteUtils::Anonymous(databaseDirectory).c_str()); + RdbFaultHiViewReporter::ReportFault(RdbFaultEvent(FT_EX_FILE, E_CREATE_FOLDER_FAIL, BUNDLE_NAME_COMMON, + "failed to mkdir errno[ " + std::to_string(errno) + "]," + databaseDirectory)); return E_CREATE_FOLDER_FAIL; } // Set the default ACL attribute to the database root directory to ensure that files created by the server diff --git a/relational_store/frameworks/native/rdb/src/rdb_store_config.cpp b/relational_store/frameworks/native/rdb/src/rdb_store_config.cpp index 81c05797..c465419d 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_store_config.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_store_config.cpp @@ -22,6 +22,7 @@ #include "rdb_security_manager.h" #include "string_utils.h" #include "sqlite_global_config.h" +#include "sqlite_utils.h" #include "rdb_fault_hiview_reporter.h" namespace OHOS::NativeRdb { @@ -40,6 +41,7 @@ RdbStoreConfig::RdbStoreConfig(const std::string &name, StorageMode storageMode, walLimitSize_ = GlobalExpr::DB_WAL_DEFAULT_SIZE; checkpointSize_ = GlobalExpr::DB_WAL_WARNING_SIZE; startCheckpointSize_ = GlobalExpr::DB_WAL_SIZE_LIMIT_MIN; + clearMemorySize_ = GlobalExpr::CLEAR_MEMORY_SIZE; } RdbStoreConfig::~RdbStoreConfig() @@ -509,6 +511,11 @@ void RdbStoreConfig::SetWriteTime(int timeout) writeTimeout_ = std::max(MIN_TIMEOUT, std::min(MAX_TIMEOUT, timeout)); } +bool RdbStoreConfig::IsLocalOnly() const +{ + return localOnly_; +} + int RdbStoreConfig::GetReadTime() const { return readTimeout_; @@ -630,6 +637,30 @@ void RdbStoreConfig::SetWalLimitSize(ssize_t size) startCheckpointSize_ = (size >> 5) + (size >> 7); } +int32_t RdbStoreConfig::GetClearMemorySize() const +{ + return clearMemorySize_; +} + +void RdbStoreConfig::SetClearMemorySize(int32_t size) +{ + if (size < 0 || size > GlobalExpr::CLEAR_MEMORY_SIZE) { + LOG_WARN("size is inValid, size:%{public}d", size); + size = GlobalExpr::CLEAR_MEMORY_SIZE; + } + clearMemorySize_ = size; +} + +std::string RdbStoreConfig::GetCollatorLocales() const +{ + return loacles_; +} + +void RdbStoreConfig::SetCollatorLocales(const std::string &loacles) +{ + loacles_ = loacles; +} + ssize_t RdbStoreConfig::GetCheckpointSize() const { return checkpointSize_; @@ -640,6 +671,16 @@ ssize_t RdbStoreConfig::GetStartCheckpointSize() const return startCheckpointSize_; } +int32_t RdbStoreConfig::GetSubUser() const +{ + return subUser_; +} + +void RdbStoreConfig::SetSubUser(int32_t subUser) +{ + subUser_ = subUser; +} + void RdbStoreConfig::EnableRekey(bool enable) { autoRekey_ = enable; @@ -648,6 +689,9 @@ void RdbStoreConfig::EnableRekey(bool enable) void RdbStoreConfig::SetCryptoParam(RdbStoreConfig::CryptoParam cryptoParam) { cryptoParam_ = cryptoParam; + if (!(cryptoParam_.encryptKey_.empty())) { + localOnly_ = true; + } } RdbStoreConfig::CryptoParam RdbStoreConfig::GetCryptoParam() const @@ -695,6 +739,31 @@ bool RdbStoreConfig::CryptoParam::IsValid() const (cryptoPageSize & (cryptoPageSize - 1)) == 0; } +std::string RdbStoreConfig::ToString() const +{ + std::stringstream oss; + oss << " bundleName:" << bundleName_ << ","; + oss << " moduleName:" << moduleName_ << ","; + oss << " dataGroupId:" << dataGroupId_ << ","; + oss << " path:" << SqliteUtils::Anonymous(path_) << ","; + oss << " storageMode:" << static_cast(storageMode_) << ","; + oss << " journalMode:" << journalMode_ << ","; + oss << " syncMode:" << syncMode_ << ","; + oss << " databaseFileType:" << databaseFileType << ","; + oss << " isEncrypt:" << IsEncrypt() << ","; + oss << " isSearchable:" << IsSearchable() << ","; + oss << " readOnly_:" << readOnly_ << ","; + oss << " securityLevel:" << static_cast(securityLevel_) << ","; + oss << " journalSize:" << journalSize_ << ","; + oss << " pageSize:" << pageSize_ << ","; + oss << " dbType:" << dbType_ << ","; + oss << " customDir:" << SqliteUtils::Anonymous(customDir_) << ","; + oss << " haMode:" << haMode_ << ","; + oss << " pluginLibs size:" << pluginLibs_.size() << ","; + oss << " area:" << area_ << ","; + return oss.str(); +} + std::string RdbStoreConfig::FormatCfg(const RdbStoreConfig &first, const RdbStoreConfig &second) { std::stringstream oss; @@ -704,16 +773,33 @@ std::string RdbStoreConfig::FormatCfg(const RdbStoreConfig &first, const RdbStor oss << " syncMode:" << first.syncMode_ << "->" << second.syncMode_ << ","; oss << " databaseFileType:" << first.databaseFileType << "->" << second.databaseFileType << ","; oss << " isEncrypt:" << first.IsEncrypt() << "->" << second.IsEncrypt() << ","; + oss << " isSearchable:" << first.IsSearchable() << "->" << second.IsSearchable() << ","; oss << " readOnly_:" << first.readOnly_ << "->" << second.readOnly_ << ","; oss << " securityLevel:" << static_cast(first.securityLevel_) << "->" << static_cast(second.securityLevel_) << ","; oss << " journalSize:" << first.journalSize_ << "->" << second.journalSize_ << ","; oss << " pageSize:" << first.pageSize_ << "->" << second.pageSize_ << ","; oss << " dbType:" << first.dbType_ << "->" << second.dbType_ << ","; - oss << " customDir:" << first.customDir_ << "->" << second.customDir_ << ","; + oss << " customDir:" << SqliteUtils::Anonymous(first.customDir_) << "->" + << SqliteUtils::Anonymous(second.customDir_) << ","; oss << " haMode:" << first.haMode_ << "->" << second.haMode_ << ","; oss << " pluginLibs size:" << first.pluginLibs_.size() << "->" << second.pluginLibs_.size() << ","; oss << " area:" << first.area_ << "->" << second.area_ << ","; return oss.str(); } + +void RdbStoreConfig::SetRegisterInfo(RegisterType type, bool state) const +{ + registerInfo_.Set(type, state); +} + +bool RdbStoreConfig::GetRegisterInfo(RegisterType type) const +{ + return registerInfo_.Get(type); +} + +bool RdbStoreConfig::IsEqualRegisterInfo(const RdbStoreConfig& config) const +{ + return registerInfo_ == config.registerInfo_; +} } // namespace OHOS::NativeRdb diff --git a/relational_store/frameworks/native/rdb/src/rdb_store_impl.cpp b/relational_store/frameworks/native/rdb/src/rdb_store_impl.cpp index eaae4fc9..a2f5e0bf 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_store_impl.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_store_impl.cpp @@ -21,9 +21,11 @@ #include #include #include +#include #include #include #include +#include #include #include "cache_result_set.h" @@ -34,7 +36,9 @@ #include "rdb_common.h" #include "rdb_errno.h" #include "rdb_fault_hiview_reporter.h" +#include "rdb_local_db_observer.h" #include "rdb_radar_reporter.h" +#include "rdb_stat_reporter.h" #include "rdb_security_manager.h" #include "rdb_sql_statistic.h" #include "rdb_store.h" @@ -65,6 +69,7 @@ #else #define ISFILE(filePath) ((filePath.find("/") == std::string::npos)) #endif +#include "rdb_time_utils.h" namespace OHOS::NativeRdb { using namespace OHOS::Rdb; @@ -82,7 +87,7 @@ static constexpr const char *ROLLBACK_TRANSACTION_SQL = "rollback;"; static constexpr const char *BACKUP_RESTORE = "backup.restore"; constexpr int64_t TIME_OUT = 1500; -void RdbStoreImpl::InitSyncerParam(const RdbStoreConfig &config) +void RdbStoreImpl::InitSyncerParam(const RdbStoreConfig &config, bool created) { syncerParam_.bundleName_ = config.GetBundleName(); syncerParam_.hapName_ = config.GetModuleName(); @@ -101,17 +106,25 @@ void RdbStoreImpl::InitSyncerParam(const RdbStoreConfig &config) syncerParam_.uids_ = config.GetPromiseInfo().uids_; syncerParam_.user_ = config.GetPromiseInfo().user_; syncerParam_.permissionNames_ = config.GetPromiseInfo().permissionNames_; + syncerParam_.subUser_ = config.GetSubUser(); + syncerParam_.dfxInfo_.lastOpenTime_ = RdbTimeUtils::GetCurSysTimeWithMs(); + if (created) { + syncerParam_.infos_ = Connection::Collect(config); + } } int RdbStoreImpl::InnerOpen() { isOpen_ = true; #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) - if (isReadOnly_) { + if (isReadOnly_ || isMemoryRdb_ || config_.IsLocalOnly()) { return E_OK; } AfterOpen(syncerParam_); + if (config_.GetDBType() == DB_VECTOR || !config_.IsSearchable()) { + return E_OK; + } int errCode = RegisterDataChangeCallback(); if (errCode != E_OK) { LOG_ERROR("RegisterCallBackObserver is failed, err is %{public}d.", errCode); @@ -120,6 +133,26 @@ int RdbStoreImpl::InnerOpen() return E_OK; } +void RdbStoreImpl::InitReportFunc(const RdbParam ¶m) +{ +#if !defined(CROSS_PLATFORM) + reportFunc_ = std::make_shared([reportParam = param](const DistributedRdb::RdbStatEvent &event) { + auto [err, service] = RdbMgr::GetInstance().GetRdbService(reportParam); + if (err != E_OK || service == nullptr) { + LOG_ERROR("GetRdbService failed, err: %{public}d, storeName: %{public}s.", err, + SqliteUtils::Anonymous(reportParam.storeName_).c_str()); + return; + } + err = service->ReportStatistic(reportParam, event); + if (err != E_OK) { + LOG_ERROR("ReportStatistic failed, err: %{public}d, storeName: %{public}s.", err, + SqliteUtils::Anonymous(reportParam.storeName_).c_str()); + } + return; + }); +#endif +} + void RdbStoreImpl::Close() { { @@ -256,9 +289,9 @@ RdbStore::ModifyTime RdbStoreImpl::GetModifyTimeByRowId(const std::string &logTa int RdbStoreImpl::CleanDirtyData(const std::string &table, uint64_t cursor) { - if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR)) { - LOG_ERROR("Not support. table:%{public}s, isRead:%{public}d, dbType:%{public}d.", - SqliteUtils::Anonymous(table).c_str(), isReadOnly_, config_.GetDBType()); + if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR) || isMemoryRdb_) { + LOG_ERROR("Not support. table:%{public}s, isRead:%{public}d, dbType:%{public}d, isMemoryRdb:%{public}d.", + SqliteUtils::Anonymous(table).c_str(), isReadOnly_, config_.GetDBType(), isMemoryRdb_); return E_NOT_SUPPORT; } auto [errCode, conn] = GetConn(false); @@ -297,7 +330,7 @@ std::shared_ptr RdbStoreImpl::RemoteQuery( const std::string &device, const AbsRdbPredicates &predicates, const Fields &columns, int &errCode) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return nullptr; } std::vector selectionArgs = predicates.GetWhereArgs(); @@ -319,6 +352,10 @@ std::shared_ptr RdbStoreImpl::RemoteQuery( void RdbStoreImpl::NotifyDataChange() { + if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR) || !config_.GetRegisterInfo(RegisterType::CLIENT_OBSERVER)) { + return; + } + config_.SetRegisterInfo(RegisterType::CLIENT_OBSERVER, false); int errCode = RegisterDataChangeCallback(); if (errCode != E_OK) { LOG_ERROR("RegisterDataChangeCallback is failed, err is %{public}d.", errCode); @@ -333,7 +370,7 @@ int RdbStoreImpl::SetDistributedTables( const std::vector &tables, int32_t type, const DistributedRdb::DistributedConfig &distributedConfig) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); - if (config_.GetDBType() == DB_VECTOR || isReadOnly_) { + if (config_.GetDBType() == DB_VECTOR || isReadOnly_ || isMemoryRdb_) { return E_NOT_SUPPORT; } if (tables.empty()) { @@ -345,16 +382,16 @@ int RdbStoreImpl::SetDistributedTables( return errCode; } syncerParam_.asyncDownloadAsset_ = distributedConfig.asyncDownloadAsset; + syncerParam_.enableCloud_ = distributedConfig.enableCloud; int32_t errorCode = service->SetDistributedTables( syncerParam_, tables, distributedConfig.references, distributedConfig.isRebuild, type); - if (type == DistributedRdb::DISTRIBUTED_DEVICE) { - int SYNC_DATA_INDEX = 500; - Reportor::ReportCorrupted(Reportor::Create(config_, SYNC_DATA_INDEX, "RdbDeviceToDeviceDataSync")); - } if (errorCode != E_OK) { LOG_ERROR("Fail to set distributed tables, error=%{public}d.", errorCode); return errorCode; } + if (type == DistributedRdb::DISTRIBUTED_DEVICE && !config_.GetRegisterInfo(RegisterType::CLIENT_OBSERVER)) { + RegisterDataChangeCallback(); + } if (type != DistributedRdb::DISTRIBUTED_CLOUD) { return E_OK; } @@ -378,7 +415,7 @@ int RdbStoreImpl::HandleCloudSyncAfterSetDistributedTables( } { std::unique_lock lock(rwMutex_); - if (distributedConfig.autoSync) { + if (distributedConfig.enableCloud && distributedConfig.autoSync) { cloudInfo_->AddTables(tables); } else { cloudInfo_->RmvTables(tables); @@ -396,7 +433,7 @@ int RdbStoreImpl::HandleCloudSyncAfterSetDistributedTables( std::string RdbStoreImpl::ObtainDistributedTableName(const std::string &device, const std::string &table, int &errCode) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return ""; } std::string uuid; @@ -440,6 +477,9 @@ int RdbStoreImpl::Sync(const SyncOption &option, const std::vector int RdbStoreImpl::Sync(const SyncOption &option, const AbsRdbPredicates &predicate, const AsyncDetail &async) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); + if (isMemoryRdb_) { + return E_NOT_SUPPORT; + } DistributedRdb::RdbService::Option rdbOption; rdbOption.mode = option.mode; rdbOption.isAsync = !option.isBlock; @@ -525,18 +565,30 @@ int RdbStoreImpl::SubscribeLocalShared(const SubscribeOption &option, RdbStoreOb int32_t RdbStoreImpl::SubscribeLocalDetail( const SubscribeOption &option, const std::shared_ptr &observer) { + if (observer == nullptr) { + return E_OK; + } + std::lock_guard lock(mutex_); + for (auto it = localDetailObservers_.begin(); it != localDetailObservers_.end(); it++) { + if ((*it)->GetObserver() == observer) { + LOG_WARN("duplicate subscribe."); + return E_OK; + } + } + auto localStoreObserver = std::make_shared(observer); auto [errCode, conn] = GetConn(false); - if (errCode != E_OK) { - LOG_ERROR("Get connection failed, errCode:%{public}d.", errCode); + if (conn == nullptr) { return errCode; } - - errCode = conn->Subscribe(option.event, observer); + errCode = conn->Subscribe(localStoreObserver); if (errCode != E_OK) { LOG_ERROR("Subscribe local detail observer failed. db name:%{public}s errCode:%{public}." PRId32, SqliteUtils::Anonymous(config_.GetName()).c_str(), errCode); + return errCode; } - return errCode; + config_.SetRegisterInfo(RegisterType::STORE_OBSERVER, true); + localDetailObservers_.emplace_back(localStoreObserver); + return E_OK; } int RdbStoreImpl::SubscribeRemote(const SubscribeOption &option, RdbStoreObserver *observer) @@ -556,6 +608,9 @@ int RdbStoreImpl::Subscribe(const SubscribeOption &option, RdbStoreObserver *obs if (option.mode == SubscribeMode::LOCAL) { return SubscribeLocal(option, observer); } + if (isMemoryRdb_) { + return E_NOT_SUPPORT; + } if (option.mode == SubscribeMode::LOCAL_SHARED) { return SubscribeLocalShared(option, observer); } @@ -571,10 +626,14 @@ int RdbStoreImpl::UnSubscribeLocal(const SubscribeOption &option, RdbStoreObserv } auto &list = obs->second; - for (auto it = list.begin(); it != list.end(); it++) { - if ((*it)->getObserver() == observer) { + for (auto it = list.begin(); it != list.end();) { + if (observer == nullptr || (*it)->getObserver() == observer) { it = list.erase(it); - break; + if (observer != nullptr) { + break; + } + } else { + it++; } } @@ -584,18 +643,6 @@ int RdbStoreImpl::UnSubscribeLocal(const SubscribeOption &option, RdbStoreObserv return E_OK; } -int RdbStoreImpl::UnSubscribeLocalAll(const SubscribeOption &option) -{ - std::lock_guard lock(mutex_); - auto obs = localObservers_.find(option.event); - if (obs == localObservers_.end()) { - return E_OK; - } - - localObservers_.erase(option.event); - return E_OK; -} - int RdbStoreImpl::UnSubscribeLocalShared(const SubscribeOption &option, RdbStoreObserver *observer) { std::lock_guard lock(mutex_); @@ -611,15 +658,19 @@ int RdbStoreImpl::UnSubscribeLocalShared(const SubscribeOption &option, RdbStore } auto &list = obs->second; - for (auto it = list.begin(); it != list.end(); it++) { - if ((*it)->getObserver() == observer) { + for (auto it = list.begin(); it != list.end();) { + if (observer == nullptr || (*it)->getObserver() == observer) { int32_t err = client->UnregisterObserver(GetUri(option.event), *it); if (err != 0) { LOG_ERROR("UnSubscribeLocalShared failed."); return err; } - list.erase(it); - break; + it = list.erase(it); + if (observer != nullptr) { + break; + } + } else { + it++; } } if (list.empty()) { @@ -628,50 +679,31 @@ int RdbStoreImpl::UnSubscribeLocalShared(const SubscribeOption &option, RdbStore return E_OK; } -int RdbStoreImpl::UnSubscribeLocalSharedAll(const SubscribeOption &option) -{ - std::lock_guard lock(mutex_); - auto obs = localSharedObservers_.find(option.event); - if (obs == localSharedObservers_.end()) { - return E_OK; - } - - auto client = OHOS::AAFwk::DataObsMgrClient::GetInstance(); - if (client == nullptr) { - LOG_ERROR("Failed to get DataObsMgrClient."); - return E_GET_DATAOBSMGRCLIENT_FAIL; - } - - auto &list = obs->second; - auto it = list.begin(); - while (it != list.end()) { - int32_t err = client->UnregisterObserver(GetUri(option.event), *it); - if (err != 0) { - LOG_ERROR("UnSubscribe failed."); - return err; - } - it = list.erase(it); - } - - localSharedObservers_.erase(option.event); - return E_OK; -} - int32_t RdbStoreImpl::UnsubscribeLocalDetail( const SubscribeOption &option, const std::shared_ptr &observer) { auto [errCode, conn] = GetConn(false); - if (errCode != E_OK) { - LOG_ERROR("Get connection failed, errCode:%{public}d.", errCode); + if (conn == nullptr) { return errCode; } - - errCode = conn->Unsubscribe(option.event, observer); - if (errCode != E_OK) { - LOG_ERROR("Unsubscribe local detail observer failed. db name:%{public}s errCode:%{public}." PRId32, - SqliteUtils::Anonymous(config_.GetName()).c_str(), errCode); + std::lock_guard lock(mutex_); + for (auto it = localDetailObservers_.begin(); it != localDetailObservers_.end();) { + if (observer == nullptr || (*it)->GetObserver() == observer) { + int32_t err = conn->Unsubscribe(*it); + if (err != 0) { + LOG_ERROR("Unsubscribe local detail observer failed. db name:%{public}s errCode:%{public}." PRId32, + SqliteUtils::Anonymous(config_.GetName()).c_str(), errCode); + return err; + } + it = localDetailObservers_.erase(it); + if (observer != nullptr) { + break; + } + } else { + it++; + } } - return errCode; + return E_OK; } int RdbStoreImpl::UnSubscribeRemote(const SubscribeOption &option, RdbStoreObserver *observer) @@ -688,21 +720,21 @@ int RdbStoreImpl::UnSubscribe(const SubscribeOption &option, RdbStoreObserver *o if (config_.GetDBType() == DB_VECTOR) { return E_NOT_SUPPORT; } - if (option.mode == SubscribeMode::LOCAL && observer) { + if (option.mode == SubscribeMode::LOCAL) { return UnSubscribeLocal(option, observer); - } else if (option.mode == SubscribeMode::LOCAL && !observer) { - return UnSubscribeLocalAll(option); - } else if (option.mode == SubscribeMode::LOCAL_SHARED && observer) { + } + if (isMemoryRdb_) { + return E_NOT_SUPPORT; + } + if (option.mode == SubscribeMode::LOCAL_SHARED) { return UnSubscribeLocalShared(option, observer); - } else if (option.mode == SubscribeMode::LOCAL_SHARED && !observer) { - return UnSubscribeLocalSharedAll(option); } return UnSubscribeRemote(option, observer); } int RdbStoreImpl::SubscribeObserver(const SubscribeOption &option, const std::shared_ptr &observer) { - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return E_NOT_SUPPORT; } return SubscribeLocalDetail(option, observer); @@ -710,7 +742,7 @@ int RdbStoreImpl::SubscribeObserver(const SubscribeOption &option, const std::sh int RdbStoreImpl::UnsubscribeObserver(const SubscribeOption &option, const std::shared_ptr &observer) { - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return E_NOT_SUPPORT; } return UnsubscribeLocalDetail(option, observer); @@ -721,6 +753,20 @@ int RdbStoreImpl::Notify(const std::string &event) if (config_.GetDBType() == DB_VECTOR) { return E_NOT_SUPPORT; } + + { + std::lock_guard lock(mutex_); + auto obs = localObservers_.find(event); + if (obs != localObservers_.end()) { + auto &list = obs->second; + for (auto &it : list) { + it->OnChange(); + } + } + } + if (isMemoryRdb_) { + return E_OK; + } auto client = OHOS::AAFwk::DataObsMgrClient::GetInstance(); if (client == nullptr) { LOG_ERROR("Failed to get DataObsMgrClient."); @@ -730,20 +776,14 @@ int RdbStoreImpl::Notify(const std::string &event) if (err != 0) { LOG_ERROR("Notify failed."); } - - std::lock_guard lock(mutex_); - auto obs = localObservers_.find(event); - if (obs != localObservers_.end()) { - auto &list = obs->second; - for (auto &it : list) { - it->OnChange(); - } - } return E_OK; } int RdbStoreImpl::SetSearchable(bool isSearchable) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { + return E_NOT_SUPPORT; + } auto [errCode, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); if (errCode != E_OK || service == nullptr) { LOG_ERROR("GetRdbService is failed, err is %{public}d.", errCode); @@ -754,7 +794,7 @@ int RdbStoreImpl::SetSearchable(bool isSearchable) int RdbStoreImpl::RegisterAutoSyncCallback(std::shared_ptr observer) { - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return E_NOT_SUPPORT; } auto [errCode, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); @@ -766,7 +806,7 @@ int RdbStoreImpl::RegisterAutoSyncCallback(std::shared_ptr observer) { - if (config_.GetDBType() == DB_VECTOR) { + if (config_.GetDBType() == DB_VECTOR || isMemoryRdb_) { return E_NOT_SUPPORT; } auto [errCode, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); @@ -803,20 +843,43 @@ void RdbStoreImpl::InitDelayNotifier() int RdbStoreImpl::RegisterDataChangeCallback() { - if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR)) { - return E_NOT_SUPPORT; - } InitDelayNotifier(); - auto callBack = [delayNotifier = delayNotifier_](const DistributedRdb::RdbChangedData &rdbChangedData) { + auto connPool = GetPool(); + if (connPool == nullptr) { + return E_ALREADY_CLOSED; + } + + RegisterDataChangeCallback(delayNotifier_, connPool, 0); + config_.SetRegisterInfo(RegisterType::CLIENT_OBSERVER, true); + return E_OK; +} + +void RdbStoreImpl::RegisterDataChangeCallback( + std::shared_ptr delayNotifier, std::weak_ptr connPool, int retry) +{ + auto relConnPool = connPool.lock(); + if (relConnPool == nullptr) { + return; + } + auto conn = relConnPool->AcquireConnection(false); + if (conn == nullptr) { + relConnPool->Dump(true, "DATACHANGE"); + auto pool = TaskExecutor::GetInstance().GetExecutor(); + if (pool != nullptr && retry++ < MAX_RETRY_TIMES) { + pool->Schedule(std::chrono::seconds(1), + [delayNotifier, connPool, retry]() { RegisterDataChangeCallback(delayNotifier, connPool, retry); }); + } + return; + } + auto callBack = [delayNotifier](const DistributedRdb::RdbChangedData &rdbChangedData) { if (delayNotifier != nullptr) { delayNotifier->UpdateNotify(rdbChangedData); } }; - auto [errCode, conn] = GetConn(false); + auto errCode = conn->SubscribeTableChanges(callBack); if (errCode != E_OK) { - return errCode; + return; } - return conn->SubscribeTableChanges(callBack); } int RdbStoreImpl::GetHashKeyForLockRow(const AbsRdbPredicates &predicates, std::vector> &hashKeys) @@ -860,6 +923,9 @@ int RdbStoreImpl::GetHashKeyForLockRow(const AbsRdbPredicates &predicates, std:: int RdbStoreImpl::ModifyLockStatus(const AbsRdbPredicates &predicates, bool isLock) { + if (config_.IsVector() || isMemoryRdb_ || isReadOnly_) { + return E_NOT_SUPPORT; + } std::vector> hashKeys; int ret = GetHashKeyForLockRow(predicates, hashKeys); if (ret != E_OK) { @@ -887,6 +953,9 @@ int RdbStoreImpl::ModifyLockStatus(const AbsRdbPredicates &predicates, bool isLo std::pair RdbStoreImpl::LockCloudContainer() { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); + if (config_.IsVector() || isMemoryRdb_ || isReadOnly_) { + return { E_NOT_SUPPORT, 0 }; + } RdbRadar ret(Scene::SCENE_SYNC, __FUNCTION__, config_.GetBundleName()); auto [errCode, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); if (errCode == E_NOT_SUPPORT) { @@ -908,6 +977,9 @@ std::pair RdbStoreImpl::LockCloudContainer() int32_t RdbStoreImpl::UnlockCloudContainer() { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); + if (config_.IsVector() || isMemoryRdb_ || isReadOnly_) { + return E_NOT_SUPPORT; + } RdbRadar ret(Scene::SCENE_SYNC, __FUNCTION__, config_.GetBundleName()); auto [errCode, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); if (errCode == E_NOT_SUPPORT) { @@ -925,28 +997,6 @@ int32_t RdbStoreImpl::UnlockCloudContainer() } return errCode; } - -class RdbStoreImpl::StoreSuspender { -public: - StoreSuspender(RdbStoreImpl &rdbStore) : rdbStore_(rdbStore) - { - auto [err, service] = RdbMgr::GetInstance().GetRdbService(rdbStore_.syncerParam_); - if (service != nullptr) { - service->Disable(rdbStore_.syncerParam_); - } - service_ = service; - } - ~StoreSuspender() - { - if (service_ != nullptr) { - service_->Enable(rdbStore_.syncerParam_); - } - } - -private: - const RdbStoreImpl &rdbStore_; - std::shared_ptr service_; -}; #endif RdbStoreImpl::RdbStoreImpl(const RdbStoreConfig &config) @@ -965,30 +1015,43 @@ RdbStoreImpl::RdbStoreImpl(const RdbStoreConfig &config, int &errCode) SqliteGlobalConfig::GetDbPath(config_, path_); bool created = access(path_.c_str(), F_OK) != 0; connectionPool_ = ConnectionPool::Create(config_, errCode); - InitSyncerParam(config_); if (connectionPool_ == nullptr && (errCode == E_SQLITE_CORRUPT || errCode == E_INVALID_SECRET_KEY) && - !isReadOnly_ && config.GetRoleType() == OWNER) { + !isReadOnly_) { LOG_ERROR("database corrupt, errCode:0x%{public}x, %{public}s, %{public}s", errCode, SqliteUtils::Anonymous(name_).c_str(), SqliteUtils::FormatDebugInfoBrief(Connection::Collect(config_), "master").c_str()); - StoreSuspender suspender(*this); - // rebuild should use the original config - std::shared_ptr pool; - std::tie(rebuild_, pool) = ConnectionPool::HandleDataCorruption(config_, errCode); - if (pool != nullptr) { - connectionPool_ = pool; +#if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) + RdbParam param; + param.bundleName_ = config_.GetBundleName(); + param.storeName_ = config_.GetName(); + param.subUser_ = config_.GetSubUser(); + auto [err, service] = RdbMgr::GetInstance().GetRdbService(param); + if (service != nullptr) { + service->Disable(param); + } +#endif + config_.SetIter(0); + if (config_.IsEncrypt() && config_.GetAllowRebuild()) { + auto key = config_.GetEncryptKey(); + RdbSecurityManager::GetInstance().RestoreKeyFile(path_, key); + key.assign(key.size(), 0); } + std::tie(rebuild_, connectionPool_) = ConnectionPool::HandleDataCorruption(config_, errCode); created = true; +#if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) + if (service != nullptr) { + service->Enable(param); + } +#endif } if (connectionPool_ == nullptr || errCode != E_OK) { connectionPool_ = nullptr; - LOG_ERROR("Create connPool failed, err is %{public}d, rebuildType:%{public}d, path:%{public}s", errCode, - rebuild_, SqliteUtils::Anonymous(path_).c_str()); + LOG_ERROR("Create connPool failed, err is %{public}d, path:%{public}s", errCode, + SqliteUtils::Anonymous(path_).c_str()); return; } - if (created) { - syncerParam_.infos_ = Connection::Collect(config); - } + InitSyncerParam(config_, created); + InitReportFunc(syncerParam_); InnerOpen(); } @@ -1028,6 +1091,7 @@ std::pair RdbStoreImpl::Insert(const std::string &table, const Row if (conflictClause == nullptr) { return { E_INVALID_CONFLICT_FLAG, -1 }; } + RdbStatReporter reportStat(RDB_PERF, INSERT, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); std::string sql; sql.append("INSERT").append(conflictClause).append(" INTO ").append(table).append("("); @@ -1070,6 +1134,7 @@ std::pair RdbStoreImpl::BatchInsert(const std::string &table, cons return { E_OK, 0 }; } + RdbStatReporter reportStat(RDB_PERF, BATCHINSERT, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); auto pool = GetPool(); if (pool == nullptr) { @@ -1081,6 +1146,7 @@ std::pair RdbStoreImpl::BatchInsert(const std::string &table, cons } auto executeSqlArgs = SqliteSqlBuilder::GenerateSqls(table, rows, conn->GetMaxVariable()); + BatchInsertArgsDfx(static_cast(executeSqlArgs.size())); if (executeSqlArgs.empty()) { LOG_ERROR("empty, table=%{public}s, values:%{public}zu, max number:%{public}d.", table.c_str(), rows.RowSize(), conn->GetMaxVariable()); @@ -1091,8 +1157,7 @@ std::pair RdbStoreImpl::BatchInsert(const std::string &table, cons auto [errCode, statement] = GetStatement(sql, conn); if (statement == nullptr) { LOG_ERROR("statement is nullptr, errCode:0x%{public}x, args:%{public}zu, table:%{public}s, " - "app self can check the SQL", - errCode, bindArgs.size(), table.c_str()); + "app self can check the SQL", errCode, bindArgs.size(), table.c_str()); return { E_OK, -1 }; } for (const auto &args : bindArgs) { @@ -1113,6 +1178,14 @@ std::pair RdbStoreImpl::BatchInsert(const std::string &table, cons return { E_OK, int64_t(rows.RowSize()) }; } +void RdbStoreImpl::BatchInsertArgsDfx(int argsSize) +{ + if (argsSize > 1) { + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_BATCH_INSERT_ARGS_SIZE, config_.GetBundleName(), + "BatchInsert executeSqlArgs size[ " + std::to_string(argsSize) + "]")); + } +} + std::pair RdbStoreImpl::BatchInsertWithConflictResolution( const std::string &table, const ValuesBuckets &rows, Resolution resolution) { @@ -1135,33 +1208,13 @@ std::pair RdbStoreImpl::BatchInsertWithConflictResolution( } auto sqlArgs = SqliteSqlBuilder::GenerateSqls(table, rows, conn->GetMaxVariable(), resolution); - if (sqlArgs.empty()) { - LOG_ERROR("empty, table=%{public}s, values:%{public}zu, max number:%{public}d.", table.c_str(), rows.RowSize(), - conn->GetMaxVariable()); + // To ensure atomicity, execute SQL only once + if (sqlArgs.size() != 1 || sqlArgs.front().second.size() != 1) { + auto [fields, values] = rows.GetFieldsAndValues(); + LOG_ERROR("invalid args, table=%{public}s, rows:%{public}zu, fields:%{public}zu, max:%{public}d.", + table.c_str(), rows.RowSize(), fields != nullptr ? fields->size() : 0, conn->GetMaxVariable()); return { E_INVALID_ARGS, -1 }; } - - for (const auto &[sql, bindArgs] : sqlArgs) { - auto [errCode, statement] = GetStatement(sql, conn); - if (statement == nullptr) { - LOG_ERROR("statement is nullptr, errCode:0x%{public}x, args:%{public}zu, table:%{public}s, " - "app self can check the SQL", - errCode, bindArgs.size(), table.c_str()); - return { errCode, -1 }; - } - for (const auto &args : bindArgs) { - auto errCode = statement->Execute(args); - if (errCode == E_SQLITE_LOCKED || errCode == E_SQLITE_BUSY) { - pool->Dump(true, "BATCH"); - return { errCode, -1 }; - } - if (errCode != E_OK) { - LOG_ERROR("failed, errCode:%{public}d,args:%{public}zu,table:%{public}s,app self can check the SQL", - errCode, bindArgs.size(), table.c_str()); - return { E_OK, -1 }; - } - } - } auto &[sql, bindArgs] = sqlArgs.front(); auto [errCode, statement] = GetStatement(sql, conn); if (statement == nullptr) { @@ -1205,6 +1258,7 @@ std::pair RdbStoreImpl::Update( if (clause == nullptr) { return { E_INVALID_CONFLICT_FLAG, -1 }; } + RdbStatReporter reportStat(RDB_PERF, UPDATE, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); std::string sql; sql.append("UPDATE").append(clause).append(" ").append(table).append(" SET "); @@ -1249,6 +1303,7 @@ int RdbStoreImpl::Delete(int &deletedRows, const std::string &table, const std:: return E_EMPTY_TABLE_NAME; } + RdbStatReporter reportStat(RDB_PERF, DELETE, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); std::string sql; sql.append("DELETE FROM ").append(table); @@ -1312,6 +1367,42 @@ int RdbStoreImpl::Count(int64_t &outValue, const AbsRdbPredicates &predicates) return ExecuteAndGetLong(outValue, sql, predicates.GetBindArgs()); } +void WriteToCompareFile(const std::string &dbPath, const std::string &bundleName, const std::string &sql) +{ + auto poolTask = TaskExecutor::GetInstance().GetExecutor(); + if (poolTask != nullptr) { + poolTask->Execute([dbPath, bundleName, sql]() { + std::string comparePath = dbPath + "-compare"; + if (SqliteUtils::CleanFileContent(comparePath)) { + Reportor::ReportFault( + RdbFaultEvent(FT_CURD, E_DFX_IS_NOT_EXIST, bundleName, "compare file is deleted")); + } + SqliteUtils::WriteSqlToFile(comparePath, sql); + }); + } +} + +void RdbStoreImpl::HandleSchemaDDL(std::shared_ptr statement, + std::shared_ptr pool, const std::string &sql, int32_t &errCode) +{ + statement->Reset(); + statement->Prepare("PRAGMA schema_version"); + auto [err, version] = statement->ExecuteForValue(); + statement = nullptr; + if (vSchema_ < static_cast(version)) { + LOG_INFO("db:%{public}s exe DDL schema<%{public}" PRIi64 "->%{public}" PRIi64 + "> app self can check the SQL.", + SqliteUtils::Anonymous(name_).c_str(), vSchema_, static_cast(version)); + vSchema_ = version; + errCode = pool->RestartConns(); + if (!isMemoryRdb_) { + std::string dbPath = config_.GetPath(); + std::string bundleName = config_.GetBundleName(); + WriteToCompareFile(dbPath, bundleName, sql); + } + } +} + int RdbStoreImpl::ExecuteSql(const std::string &sql, const Values &args) { DISTRIBUTED_DATA_HITRACE(std::string(__FUNCTION__)); @@ -1322,6 +1413,7 @@ int RdbStoreImpl::ExecuteSql(const std::string &sql, const Values &args) if (ret != E_OK) { return ret; } + RdbStatReporter reportStat(RDB_PERF, EXECUTESQL, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); auto [errCode, statement] = BeginExecuteSql(sql); if (statement == nullptr) { @@ -1341,17 +1433,7 @@ int RdbStoreImpl::ExecuteSql(const std::string &sql, const Values &args) } int sqlType = SqliteUtils::GetSqlStatementType(sql); if (sqlType == SqliteUtils::STATEMENT_DDL) { - statement->Reset(); - statement->Prepare("PRAGMA schema_version"); - auto [err, version] = statement->ExecuteForValue(); - statement = nullptr; - if (vSchema_ < static_cast(version)) { - LOG_INFO("db:%{public}s exe DDL schema<%{public}" PRIi64 "->%{public}" PRIi64 - "> app self can check the SQL.", - SqliteUtils::Anonymous(name_).c_str(), vSchema_, static_cast(version)); - vSchema_ = version; - errCode = pool->RestartReaders(); - } + HandleSchemaDDL(statement, pool, sql, errCode); } statement = nullptr; if (errCode == E_OK && (sqlType == SqliteUtils::STATEMENT_UPDATE || sqlType == SqliteUtils::STATEMENT_INSERT)) { @@ -1367,6 +1449,7 @@ std::pair RdbStoreImpl::Execute(const std::string &sql, co return { E_NOT_SUPPORT, object }; } + RdbStatReporter reportStat(RDB_PERF, EXECUTE, config_, reportFunc_); SqlStatistic sqlStatistic("", SqlStatistic::Step::STEP_TOTAL); int sqlType = SqliteUtils::GetSqlStatementType(sql); if (!SqliteUtils::IsSupportSqlForExecute(sqlType)) { @@ -1439,16 +1522,7 @@ std::pair RdbStoreImpl::HandleDifferentSqlTypes( } if (sqlType == SqliteUtils::STATEMENT_DDL) { - statement->Reset(); - statement->Prepare("PRAGMA schema_version"); - auto [err, version] = statement->ExecuteForValue(); - if (vSchema_ < static_cast(version)) { - LOG_INFO("db:%{public}s exe DDL schema<%{public}" PRIi64 "->%{public}" PRIi64 - "> app self can check the SQL.", - SqliteUtils::Anonymous(name_).c_str(), vSchema_, static_cast(version)); - vSchema_ = version; - errCode = pool->RestartReaders(); - } + HandleSchemaDDL(statement, pool, sql, errCode); } return { errCode, object }; } @@ -1594,7 +1668,7 @@ int RdbStoreImpl::GetSlaveName(const std::string &path, std::string &backupFileP int RdbStoreImpl::Backup(const std::string &databasePath, const std::vector &encryptKey) { LOG_INFO("Backup db: %{public}s.", SqliteUtils::Anonymous(config_.GetName()).c_str()); - if (isReadOnly_) { + if (isReadOnly_ || isMemoryRdb_) { return E_NOT_SUPPORT; } std::string backupFilePath; @@ -1827,8 +1901,7 @@ int RdbStoreImpl::AttachInner(const RdbStoreConfig &config, const std::string &a return E_DATABASE_BUSY; } - if (config_.GetStorageMode() != StorageMode::MODE_MEMORY && - conn->GetJournalMode() == static_cast(JournalMode::MODE_WAL)) { + if (!isMemoryRdb_ && conn->GetJournalMode() == static_cast(JournalMode::MODE_WAL)) { // close first to prevent the connection from being put back. pool->CloseAllConnections(); conn = nullptr; @@ -1870,12 +1943,13 @@ int RdbStoreImpl::AttachInner(const RdbStoreConfig &config, const std::string &a std::pair RdbStoreImpl::Attach( const RdbStoreConfig &config, const std::string &attachName, int32_t waitTime) { - if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR) || config_.GetHaMode() != HAMode::SINGLE) { + if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR) || config_.GetHaMode() != HAMode::SINGLE || + (config.IsMemoryRdb() && config.IsEncrypt())) { return { E_NOT_SUPPORT, 0 }; } std::string dbPath; int err = SqliteGlobalConfig::GetDbPath(config, dbPath); - if (err != E_OK || access(dbPath.c_str(), F_OK) != E_OK) { + if (err != E_OK || (access(dbPath.c_str(), F_OK) != E_OK && !config.IsMemoryRdb())) { return { E_INVALID_FILE_PATH, 0 }; } @@ -1948,11 +2022,13 @@ std::pair RdbStoreImpl::Detach(const std::string &attachName, return { E_OK, attachedInfo_.Size() }; } statement = nullptr; - // close first to prevent the connection from being put back. - pool->CloseAllConnections(); - connection = nullptr; - readers.clear(); - errCode = pool->EnableWal(); + if (!isMemoryRdb_ && connection->GetJournalMode() == static_cast(JournalMode::MODE_WAL)) { + // close first to prevent the connection from being put back. + pool->CloseAllConnections(); + connection = nullptr; + readers.clear(); + errCode = pool->EnableWal(); + } return { errCode, 0 }; } @@ -2004,6 +2080,7 @@ int RdbStoreImpl::BeginTransaction() return E_NOT_SUPPORT; } // size + 1 means the number of transactions in process + RdbStatReporter reportStat(RDB_PERF, BEGINTRANSACTION, config_, reportFunc_); size_t transactionId = pool->GetTransactionStack().size() + 1; BaseTransaction transaction(pool->GetTransactionStack().size()); auto [errCode, statement] = GetStatement(transaction.GetTransactionStr()); @@ -2048,6 +2125,7 @@ std::pair RdbStoreImpl::BeginTrans() SqliteUtils::Anonymous(name_).c_str(), errCode); return { errCode, 0 }; } + tmpTrxId = newTrxId_.fetch_add(1); trxConnMap_.Insert(tmpTrxId, connection); errCode = ExecuteByTrxId(BEGIN_TRANSACTION_SQL, tmpTrxId); @@ -2071,6 +2149,7 @@ int RdbStoreImpl::RollBack() if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR)) { return E_NOT_SUPPORT; } + RdbStatReporter reportStat(RDB_PERF, ROLLBACK, config_, reportFunc_); size_t transactionId = pool->GetTransactionStack().size(); if (pool->GetTransactionStack().empty()) { @@ -2172,6 +2251,7 @@ int RdbStoreImpl::Commit() if (isReadOnly_ || (config_.GetDBType() == DB_VECTOR)) { return E_NOT_SUPPORT; } + RdbStatReporter reportStat(RDB_PERF, COMMIT, config_, reportFunc_); size_t transactionId = pool->GetTransactionStack().size(); if (pool->GetTransactionStack().empty()) { @@ -2366,7 +2446,7 @@ int RdbStoreImpl::GetDestPath(const std::string &backupPath, std::string &destPa int RdbStoreImpl::Restore(const std::string &backupPath, const std::vector &newKey) { LOG_INFO("Restore db: %{public}s.", SqliteUtils::Anonymous(config_.GetName()).c_str()); - if (isReadOnly_) { + if (isReadOnly_ || isMemoryRdb_) { return E_NOT_SUPPORT; } @@ -2379,33 +2459,38 @@ int RdbStoreImpl::Restore(const std::string &backupPath, const std::vectorChangeDbFileForRestore(path_, destPath, newKey, slaveStatus_); +#if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) + auto [err, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); + if (service != nullptr) { + service->Disable(syncerParam_); } +#endif + bool corrupt = Reportor::IsReportCorruptedFault(path_); + int errCode = pool->ChangeDbFileForRestore(path_, destPath, newKey, slaveStatus_); keyFiles.Unlock(); #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) SecurityPolicy::SetSecurityLabel(config_); - auto [code, service] = RdbMgr::GetInstance().GetRdbService(syncerParam_); - if (service != nullptr && errCode == E_OK) { - auto syncerParam = syncerParam_; - syncerParam.infos_ = Connection::Collect(config_); - service->AfterOpen(syncerParam); - NotifyDataChange(); + if (service != nullptr) { + service->Enable(syncerParam_); + if (errCode == E_OK) { + auto syncerParam = syncerParam_; + syncerParam.infos_ = Connection::Collect(config_); + service->AfterOpen(syncerParam); + NotifyDataChange(); + } } #endif if (errCode == E_OK) { ExchangeSlaverToMaster(); -// Reportor::ReportRestore(Reportor::Create(config_, E_OK), corrupt); + Reportor::ReportRestore(Reportor::Create(config_, E_OK, "ErrorType::RdbStoreImpl::Restore", false), corrupt); rebuild_ = RebuiltType::NONE; } DoCloudSync(""); @@ -2607,31 +2692,4 @@ std::set RdbStoreImpl::CloudTables::Steal() } return result; } - -RdbStoreImpl::SavePointers::SavePointers(std::shared_ptr pool, std::shared_ptr conn, - ConflictResolution resolution, int &errCode) - : pool_(pool), conn_(conn), resolution_(resolution), errCode_(errCode) -{ - std::lock_guard lockGuard(pool->GetTransactionStackMutex()); - size_t transactionId = pool->GetTransactionStack().size() + 1; - BaseTransaction transaction(pool->GetTransactionStack().size()); - auto [code, statement] = conn->CreateStatement(transaction.GetTransactionStr(), conn); - if (statement != nullptr) { - errCode = code; - return; - } - errCode = statement->Execute(); -} - -RdbStoreImpl::SavePointers::~SavePointers() -{ - if (errCode_ == E_OK) { - // commit() - } - if (errCode_ != E_SQLITE_CONSTRAINT || (resolution_ == ConflictResolution::ON_CONFLICT_ABORT || - resolution_ == ConflictResolution::ON_CONFLICT_REPLACE || - resolution_ == ConflictResolution::ON_CONFLICT_NONE)) { - // rollback(); - } -} } // namespace OHOS::NativeRdb diff --git a/relational_store/frameworks/native/rdb/src/rdb_store_manager.cpp b/relational_store/frameworks/native/rdb/src/rdb_store_manager.cpp index f828e708..17b2c0e6 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_store_manager.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_store_manager.cpp @@ -58,7 +58,7 @@ RdbStoreManager::RdbStoreManager() : configCache_(BUCKET_MAX_SIZE) { } -std::shared_ptr RdbStoreManager::GetStoreFromCache(const RdbStoreConfig &config, const std::string &path) +std::shared_ptr RdbStoreManager::GetStoreFromCache(const std::string &path) { auto it = storeCache_.find(path); if (it == storeCache_.end()) { @@ -69,50 +69,45 @@ std::shared_ptr RdbStoreManager::GetStoreFromCache(const RdbStoreC storeCache_.erase(it); return nullptr; } - auto oldConfig = rdbStore->GetConfig(); - if (oldConfig != config) { - storeCache_.erase(it); - LOG_WARN("app[%{public}s:%{public}s] path[%{public}s] cfg[%{public}s] debug[%{public}s]", - config.GetBundleName().c_str(), config.GetModuleName().c_str(), SqliteUtils::Anonymous(path).c_str(), - RdbStoreConfig::FormatCfg(oldConfig, config).c_str(), - SqliteUtils::FormatDebugInfoBrief(Connection::Collect(config), SqliteUtils::Anonymous(config.GetName())) - .c_str()); - return nullptr; - } return rdbStore; } std::shared_ptr RdbStoreManager::GetRdbStore( const RdbStoreConfig &config, int &errCode, int version, RdbOpenCallback &openCallback) { - // TOD this lock should only work on storeCache_, add one more lock for connectionPool - errCode = E_OK; - std::lock_guard lock(mutex_); - if (config.GetRoleType() == VISITOR_WRITE && !IsPermitted(GetSyncParam(config))) { - errCode = E_ERROR; + errCode = CheckConfig(config); + if (errCode != E_OK) { return nullptr; } - RdbStoreConfig modifyConfig = config; - auto path = config.GetRoleType() != OWNER ? config.GetVisitorDir() : config.GetPath(); - if (modifyConfig.GetRoleType() == OWNER && IsConfigInvalidChanged(path, modifyConfig)) { - errCode = E_CONFIG_INVALID_CHANGE; + std::string path; + errCode = SqliteGlobalConfig::GetDbPath(config, path); + if (errCode != E_OK) { return nullptr; } - std::shared_ptr rdbStore = GetStoreFromCache(modifyConfig, path); - if (rdbStore != nullptr) { + // TOD this lock should only work on storeCache_, add one more lock for connectionPool + std::lock_guard lock(mutex_); + std::shared_ptr rdbStore = GetStoreFromCache(path); + if (rdbStore != nullptr && rdbStore->GetConfig() == config) { return rdbStore; } - rdbStore = std::make_shared(modifyConfig, errCode); - if (errCode != E_OK && modifyConfig != config) { - LOG_WARN("Failed to GetRdbStore using modifyConfig. path:%{public}s, rc=%{public}d", - SqliteUtils::Anonymous(path).c_str(), errCode); - rdbStore = std::make_shared(config, errCode); // retry with input config + if (rdbStore != nullptr) { + auto log = RdbStoreConfig::FormatCfg(rdbStore->GetConfig(), config); + LOG_WARN("Diff config! app[%{public}s:%{public}s] path[%{public}s] cfg[%{public}s]", + config.GetBundleName().c_str(), config.GetModuleName().c_str(), SqliteUtils::Anonymous(path).c_str(), + log.c_str()); + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, E_CONFIG_INVALID_CHANGE, config, log)); + if (rdbStore->GetConfig().IsMemoryRdb() || config.IsMemoryRdb()) { + errCode = E_CONFIG_INVALID_CHANGE; + return nullptr; + } + storeCache_.erase(path); + rdbStore = nullptr; } - if (errCode != E_OK) { - LOG_ERROR("GetRdbStore failed. path:%{public}s, rc=%{public}d", SqliteUtils::Anonymous(path).c_str(), errCode); + std::tie(errCode, rdbStore) = OpenStore(config, path); + if (errCode != E_OK || rdbStore == nullptr) { + LOG_ERROR("OpenStore failed. path:%{public}s, rc=%{public}d", SqliteUtils::Anonymous(path).c_str(), errCode); return nullptr; } - if (rdbStore->GetConfig().GetRoleType() == OWNER && !rdbStore->GetConfig().IsReadOnly()) { errCode = SetSecurityLabel(rdbStore->GetConfig()); if (errCode != E_OK) { @@ -121,19 +116,23 @@ std::shared_ptr RdbStoreManager::GetRdbStore( (void)rdbStore->ExchangeSlaverToMaster(); errCode = ProcessOpenCallback(*rdbStore, version, openCallback); if (errCode != E_OK) { - LOG_ERROR("fail, storeName:%{public}s path:%{public}s ProcessOpenCallback errCode:%{public}d", - SqliteUtils::Anonymous(rdbStore->GetConfig().GetName()).c_str(), + LOG_ERROR("fail, path:%{public}s ProcessOpenCallback errCode:%{public}d", SqliteUtils::Anonymous(rdbStore->GetConfig().GetPath()).c_str(), errCode); return nullptr; } } - configCache_.Set(path, GetSyncParam(rdbStore->GetConfig())); + if (!rdbStore->GetConfig().IsMemoryRdb()) { + configCache_.Set(path, GetSyncParam(rdbStore->GetConfig())); + } storeCache_.insert_or_assign(std::move(path), rdbStore); return rdbStore; } bool RdbStoreManager::IsConfigInvalidChanged(const std::string &path, RdbStoreConfig &config) { + if (config.IsMemoryRdb()) { + return false; + } if (config.GetBundleName().empty()) { LOG_WARN("Config has no bundleName, path: %{public}s", SqliteUtils::Anonymous(path).c_str()); return false; @@ -152,13 +151,16 @@ bool RdbStoreManager::IsConfigInvalidChanged(const std::string &path, RdbStoreCo return false; } if (config.GetSecurityLevel() != SecurityLevel::LAST && - static_cast(config.GetSecurityLevel()) > lastParam.level_) { + static_cast(config.GetSecurityLevel()) < lastParam.level_) { LOG_WARN("Illegal change, storePath %{public}s, securityLevel: %{public}d -> %{public}d", SqliteUtils::Anonymous(path).c_str(), lastParam.level_, static_cast(config.GetSecurityLevel())); } - // Reset the first effective attribute - config.SetEncryptStatus(lastParam.isEncrypt_); + if (lastParam.isEncrypt_ != config.IsEncrypt()) { + LOG_WARN("Reset encrypt, storePath %{public}s, input:%{public}d original:%{public}d", + SqliteUtils::Anonymous(path).c_str(), config.IsEncrypt(), lastParam.isEncrypt_); + config.SetEncryptStatus(lastParam.isEncrypt_); + } return false; } @@ -180,6 +182,7 @@ DistributedRdb::RdbSyncerParam RdbStoreManager::GetSyncParam(const RdbStoreConfi syncerParam.uids_ = config.GetPromiseInfo().uids_; syncerParam.user_ = config.GetPromiseInfo().user_; syncerParam.permissionNames_ = config.GetPromiseInfo().permissionNames_; + syncerParam.subUser_ = config.GetSubUser(); return syncerParam; } @@ -287,13 +290,16 @@ int RdbStoreManager::ProcessOpenCallback(RdbStore &rdbStore, int version, RdbOpe return openCallback.OnOpen(rdbStore); } -bool RdbStoreManager::Delete(const std::string &path, bool shouldClose) +bool RdbStoreManager::Delete(const RdbStoreConfig &config, bool shouldClose) { + auto path = config.GetPath(); #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) auto tokens = StringUtils::Split(path, "/"); if (!tokens.empty()) { DistributedRdb::RdbSyncerParam param; + param.bundleName_ = config.GetBundleName(); param.storeName_ = *tokens.rbegin(); + param.subUser_ = config.GetSubUser(); auto [err, service] = DistributedRdb::RdbManagerImpl::GetInstance().GetRdbService(param); if (err != E_OK || service == nullptr) { LOG_DEBUG("GetRdbService failed, err is %{public}d.", err); @@ -312,30 +318,76 @@ bool RdbStoreManager::Delete(const std::string &path, bool shouldClose) int RdbStoreManager::SetSecurityLabel(const RdbStoreConfig &config) { + if (config.IsMemoryRdb()) { + return E_OK; + } #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) return SecurityPolicy::SetSecurityLabel(config); #endif return E_OK; } -std::map RdbStoreManager::Collector(const RdbStoreConfig &config) +int32_t RdbStoreManager::Collector(const RdbStoreConfig &config, DebugInfos &debugInfos, DfxInfo &dfxInfo) { - std::map debugInfos; #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) Param param = GetSyncParam(config); auto [err, service] = DistributedRdb::RdbManagerImpl::GetInstance().GetRdbService(param); if (err != E_OK || service == nullptr) { LOG_DEBUG("GetRdbService failed, err is %{public}d.", err); - return std::map(); + return E_ERROR; } + err = service->GetDebugInfo(param, debugInfos); if (err != E_OK) { LOG_ERROR("GetDebugInfo failed, storeName:%{public}s, err = %{public}d", SqliteUtils::Anonymous(param.storeName_).c_str(), err); - return std::map(); + return E_ERROR; + } + + err = service->GetDfxInfo(param, dfxInfo); + if (err != E_OK) { + LOG_ERROR("GetDfxInfo failed, storeName:%{public}s, err = %{public}d", + SqliteUtils::Anonymous(param.storeName_).c_str(), err); + return E_ERROR; } + #endif - return debugInfos; + return E_OK; +} + +int32_t RdbStoreManager::CheckConfig(const RdbStoreConfig &config) +{ + if (config.GetRoleType() == VISITOR_WRITE && !IsPermitted(GetSyncParam(config))) { + return E_NOT_SUPPORT; + } + if (config.IsMemoryRdb()) { + if (config.IsEncrypt() || config.IsVector() || config.GetRoleType() != OWNER || + config.GetHaMode() != HAMode::SINGLE || config.IsSearchable() || config.IsReadOnly() || + !config.GetDataGroupId().empty() || !config.GetCustomDir().empty()) { + LOG_ERROR("not support!config:%{public}s", config.ToString().c_str()); + return E_NOT_SUPPORT; + } + } + return E_OK; +} + +std::pair> RdbStoreManager::OpenStore( + const RdbStoreConfig &config, const std::string &path) +{ + RdbStoreConfig modifyConfig = config; + if (modifyConfig.GetRoleType() == OWNER && IsConfigInvalidChanged(path, modifyConfig)) { + return { E_CONFIG_INVALID_CHANGE, nullptr }; + } + + std::pair> result = { E_ERROR, nullptr }; + auto &[errCode, store] = result; + store = std::make_shared(modifyConfig, errCode); + if (errCode != E_OK && modifyConfig.IsEncrypt() != config.IsEncrypt()) { + LOG_WARN("Failed to OpenStore using modifyConfig. path:%{public}s, rc=%{public}d", + SqliteUtils::Anonymous(path).c_str(), errCode); + store = std::make_shared(config, errCode); // retry with input config + } + return result; } } // namespace NativeRdb } // namespace OHOS diff --git a/relational_store/frameworks/native/rdb/src/rdb_time_utils.cpp b/relational_store/frameworks/native/rdb/src/rdb_time_utils.cpp index d04319f1..88361b90 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_time_utils.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_time_utils.cpp @@ -34,10 +34,12 @@ std::string RdbTimeUtils::GetCurSysTimeWithMs() std::string RdbTimeUtils::GetTimeWithMs(time_t sec, int64_t nsec) { - std::tm *t = std::localtime(&sec); std::stringstream oss; - oss << std::put_time(t, "%Y-%m-%d %H:%M:%S") << '.' << std::setfill('0') << std::setw(MILLISECONDS_LEN) - << (nsec / NANO_TO_MILLI) % MILLI_PRE_SEC; + char buffer[MAX_TIME_BUF_LEN] = { 0 }; + std::tm local_time; + localtime_r(&sec, &local_time); + std::strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &local_time); + oss << buffer << '.' << std::setfill('0') << std::setw(MILLISECONDS_LEN) << (nsec / NANO_TO_MILLI) % MILLI_PRE_SEC; return oss.str(); } diff --git a/relational_store/frameworks/native/rdb/src/rdb_types_util.cpp b/relational_store/frameworks/native/rdb/src/rdb_types_util.cpp index e6953d6b..57bd4243 100644 --- a/relational_store/frameworks/native/rdb/src/rdb_types_util.cpp +++ b/relational_store/frameworks/native/rdb/src/rdb_types_util.cpp @@ -21,7 +21,7 @@ bool Marshalling(const SyncerParam &input, MessageParcel &data) return ITypesUtil::Marshal(data, input.bundleName_, input.hapName_, input.storeName_, input.area_, input.level_, input.type_, input.isEncrypt_, input.password_, input.customDir_, input.isAutoClean_, input.isSearchable_, input.haMode_, input.infos_, input.tokenIds_, input.uids_, input.user_, input.permissionNames_, - input.asyncDownloadAsset_); + input.asyncDownloadAsset_, input.enableCloud_, input.subUser_, input.dfxInfo_); } template<> bool Unmarshalling(SyncerParam &output, MessageParcel &data) @@ -29,7 +29,7 @@ bool Unmarshalling(SyncerParam &output, MessageParcel &data) return ITypesUtil::Unmarshal(data, output.bundleName_, output.hapName_, output.storeName_, output.area_, output.level_, output.type_, output.isEncrypt_, output.password_, output.customDir_, output.isAutoClean_, output.isSearchable_, output.haMode_, output.infos_, output.tokenIds_, output.uids_, output.user_, - output.permissionNames_, output.asyncDownloadAsset_); + output.permissionNames_, output.asyncDownloadAsset_, output.enableCloud_, output.subUser_, output.dfxInfo_); } template<> @@ -243,4 +243,27 @@ bool Unmarshalling(DebugInfo &output, MessageParcel &data) { return Unmarshal(data, output.inode_, output.mode_, output.uid_, output.gid_); } + +template<> +bool Marshalling(const StatReporter &input, MessageParcel &data) +{ + return Marshal(data, input.statType, input.bundleName, input.storeName, input.subType, input.costTime); +} +template<> +bool Unmarshalling(StatReporter &output, MessageParcel &data) +{ + return Unmarshal(data, output.statType, output.bundleName, output.storeName, output.subType, output.costTime); +} + +template<> +bool Marshalling(const RdbDfxInfo &input, MessageParcel &data) +{ + return Marshal(data, input.lastOpenTime_, input.curUserId_); +} + +template<> +bool Unmarshalling(RdbDfxInfo &output, MessageParcel &data) +{ + return Unmarshal(data, output.lastOpenTime_, output.curUserId_); +} } // namespace OHOS::ITypesUtil \ No newline at end of file diff --git a/relational_store/frameworks/native/rdb/src/share_block.cpp b/relational_store/frameworks/native/rdb/src/share_block.cpp index bd83211a..cfc30293 100644 --- a/relational_store/frameworks/native/rdb/src/share_block.cpp +++ b/relational_store/frameworks/native/rdb/src/share_block.cpp @@ -22,6 +22,8 @@ #include "logger.h" #include "shared_block_serializer_info.h" #include "sqlite_errno.h" +#include "sqlite_utils.h" +#include "string_utils.h" #include "value_object.h" namespace OHOS { @@ -131,6 +133,7 @@ int FillSharedBlockOpt(SharedBlockInfo *info, sqlite3_stmt *stmt) info->totalRows = serializer.GetTotalRows(); info->startPos = serializer.GetStartPos(); info->addedRows = serializer.GetAddedRows(); + info->isFull = serializer.IsFull(); if (errCode == SQLITE_DONE || errCode == SQLITE_ROW) { errCode = SQLITE_OK; @@ -303,8 +306,7 @@ FillOneRowResult FillOneRowOfBlob( auto action = &AppDataFwk::SharedBlock::PutBlob; auto *declType = sqlite3_column_decltype(statement, pos); if (declType != nullptr) { - std::string type(declType); - std::transform(type.begin(), type.end(), type.begin(), [](auto ch) { return std::toupper(ch); }); + auto type = StringUtils::TruncateAfterFirstParen(SqliteUtils::StrToUpper(declType)); action = (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutAsset : (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutAssets : (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutFloats @@ -342,7 +344,7 @@ bool ResetStatement(SharedBlockInfo *info, sqlite3_stmt *stmt) LOG_ERROR("startPos %{public}d > actual rows %{public}d", info->startPos, info->totalRows); } - if ((info->totalRows > 0 && info->addedRows == 0) && info->sharedBlock != nullptr) { + if ((info->isFull && info->totalRows > 0 && info->addedRows == 0) && info->sharedBlock != nullptr) { LOG_WARN("over 2MB[%{public}d, %{public}d]", info->totalRows, info->addedRows); return false; } diff --git a/relational_store/frameworks/native/rdb/src/shared_block_serializer_info.cpp b/relational_store/frameworks/native/rdb/src/shared_block_serializer_info.cpp index 19af4942..d2b54da1 100644 --- a/relational_store/frameworks/native/rdb/src/shared_block_serializer_info.cpp +++ b/relational_store/frameworks/native/rdb/src/shared_block_serializer_info.cpp @@ -16,6 +16,8 @@ #include "shared_block_serializer_info.h" #include "logger.h" +#include "sqlite_utils.h" +#include "string_utils.h" #include "value_object.h" namespace OHOS { @@ -39,6 +41,9 @@ int SharedBlockSerializerInfo::AddRow(int addedRows) raddedRows = addedRows + 1; return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } return SQLITE_FULL; } @@ -56,6 +61,7 @@ int SharedBlockSerializerInfo::Reset(int startPos) } astartPos = startPos; raddedRows = 0; + isFull = false; return SQLITE_OK; } @@ -72,6 +78,9 @@ int SharedBlockSerializerInfo::PutLong(int row, int column, sqlite3_int64 value) if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_OK) { return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } sharedBlock_->FreeLastRow(); return SQLITE_FULL; } @@ -82,6 +91,9 @@ int SharedBlockSerializerInfo::PutDouble(int row, int column, double value) if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_OK) { return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } sharedBlock_->FreeLastRow(); return SQLITE_FULL; } @@ -91,8 +103,7 @@ int SharedBlockSerializerInfo::PutBlob(int row, int column, const void *blob, in auto action = &AppDataFwk::SharedBlock::PutBlob; auto *declType = sqlite3_column_decltype(statement_, column); if (declType != nullptr) { - std::string type(declType); - std::transform(type.begin(), type.end(), type.begin(), [](auto ch) { return std::toupper(ch); }); + auto type = StringUtils::TruncateAfterFirstParen(SqliteUtils::StrToUpper(declType)); action = (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutAsset : (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutAssets : (type == ValueObject::DeclType()) ? &AppDataFwk::SharedBlock::PutFloats @@ -104,6 +115,9 @@ int SharedBlockSerializerInfo::PutBlob(int row, int column, const void *blob, in if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_OK) { return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } sharedBlock_->FreeLastRow(); return SQLITE_FULL; } @@ -114,6 +128,9 @@ int SharedBlockSerializerInfo::PutNull(int row, int column) if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_OK) { return SQLITE_OK; } + if (status == AppDataFwk::SharedBlock::SHARED_BLOCK_NO_MEMORY) { + isFull = true; + } sharedBlock_->FreeLastRow(); LOG_ERROR("Failed allocating space for a null in column %{public}d, error=%{public}d", column, status); return SQLITE_FULL; diff --git a/relational_store/frameworks/native/rdb/src/sqlite_connection.cpp b/relational_store/frameworks/native/rdb/src/sqlite_connection.cpp index a1565b34..55673379 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_connection.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_connection.cpp @@ -30,13 +30,14 @@ #include "raw_data_parser.h" #include "rdb_errno.h" #include "rdb_fault_hiview_reporter.h" +#include "rdb_local_db_observer.h" #include "rdb_security_manager.h" #include "rdb_sql_statistic.h" #include "rdb_store_config.h" #include "relational_store_client.h" #include "sqlite3.h" -#include "sqlite_errno.h" #include "sqlite_default_function.h" +#include "sqlite_errno.h" #include "sqlite_global_config.h" #include "sqlite_utils.h" #include "value_object.h" @@ -63,6 +64,7 @@ constexpr uint32_t SqliteConnection::NO_ITER; constexpr uint32_t SqliteConnection::DB_INDEX; constexpr uint32_t SqliteConnection::WAL_INDEX; constexpr uint32_t SqliteConnection::ITER_V1; +constexpr uint32_t SqliteConnection::SQLITE_CKSUMVFS_RESERVE_BYTES; __attribute__((used)) const int32_t SqliteConnection::regCreator_ = Connection::RegisterCreator(DB_SQLITE, SqliteConnection::Create); __attribute__((used)) @@ -132,7 +134,7 @@ std::map SqliteConnection::Collect(const RdbStore } SqliteConnection::SqliteConnection(const RdbStoreConfig &config, bool isWriteConnection) - : dbHandle_(nullptr), isWriter_(isWriteConnection), isReadOnly_(false), maxVariableNumber_(0), filePath(""), + : dbHandle_(nullptr), isWriter_(isWriteConnection), isReadOnly_(false), maxVariableNumber_(0), config_(config) { backupId_ = TaskExecutor::INVALID_TASK_ID; @@ -243,6 +245,9 @@ int SqliteConnection::InnerOpen(const RdbStoreConfig &config) isReadOnly_ = !isWriter_ || config.IsReadOnly(); int openFileFlags = config.IsReadOnly() ? (SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX) : (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + if (config.IsMemoryRdb()) { + openFileFlags |= SQLITE_OPEN_URI; + } errCode = OpenDatabase(dbPath, openFileFlags); if (errCode != E_OK) { Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, errCode, config, "", true)); @@ -273,8 +278,6 @@ int SqliteConnection::InnerOpen(const RdbStoreConfig &config) } } } - - filePath = dbPath; return E_OK; } @@ -357,20 +360,31 @@ int SqliteConnection::SetCustomScalarFunction(const std::string &functionName, i int SqliteConnection::Configure(const RdbStoreConfig &config, std::string &dbPath) { - if (config.GetStorageMode() == StorageMode::MODE_MEMORY || config.GetRoleType() == VISITOR) { + // there is a read-only dependency + if (!config.GetCollatorLocales().empty()) { + ConfigLocale(config.GetCollatorLocales()); + } + + if (config.GetRoleType() == VISITOR) { return E_OK; } - auto errCode = RegDefaultFunctions(dbHandle_); + auto errCode = SetCrcCheck(config); + if (errCode != E_OK) { + return errCode; + } + + errCode = RegDefaultFunctions(dbHandle_); if (errCode != E_OK) { return errCode; } SetBusyTimeout(DEFAULT_BUSY_TIMEOUT_MS); - LimitPermission(dbPath); + LimitPermission(config, dbPath); - errCode = SetPersistWal(); + SetDwrEnable(config); + errCode = SetPersistWal(config); if (errCode != E_OK) { return errCode; } @@ -393,11 +407,6 @@ int SqliteConnection::Configure(const RdbStoreConfig &config, std::string &dbPat // set the user version to the wal file; SetWalFile(config); - errCode = SetJournalSizeLimit(config); - if (errCode != E_OK) { - return errCode; - } - errCode = SetAutoCheckpoint(config); if (errCode != E_OK) { return errCode; @@ -407,7 +416,7 @@ int SqliteConnection::Configure(const RdbStoreConfig &config, std::string &dbPat if (errCode != E_OK) { return errCode; } - + RegisterHookIfNecessary(); return LoadExtension(config, dbHandle_); } @@ -420,13 +429,6 @@ SqliteConnection::~SqliteConnection() } } if (dbHandle_ != nullptr) { - if (hasClientObserver_) { - UnRegisterClientObserver(dbHandle_); - } - if (isWriter_) { - UnregisterStoreObserver(dbHandle_); - } - int errCode = sqlite3_close_v2(dbHandle_); if (errCode != SQLITE_OK) { LOG_ERROR("could not close database err = %{public}d, errno = %{public}d", errCode, errno); @@ -434,15 +436,58 @@ SqliteConnection::~SqliteConnection() } } -int32_t SqliteConnection::OnInitialize() +int32_t SqliteConnection::VerifyAndRegisterHook(const RdbStoreConfig &config) +{ + if (!isWriter_ || config_.IsEqualRegisterInfo(config)) { + return E_OK; + } + for (auto &eventInfo : onEventHandlers_) { + if (config.GetRegisterInfo(eventInfo.Type) && !config_.GetRegisterInfo(eventInfo.Type)) { + config_.SetRegisterInfo(eventInfo.Type, true); + (this->*(eventInfo.handle))(); + } + } + return E_OK; +} + +int32_t SqliteConnection::RegisterHookIfNecessary() +{ + if (!isWriter_) { + return E_OK; + } + for (auto &eventInfo : onEventHandlers_) { + if (config_.GetRegisterInfo(eventInfo.Type)) { + (this->*(eventInfo.handle))(); + } + } + return E_OK; +} + +int SqliteConnection::RegisterStoreObs() +{ + RegisterDbHook(dbHandle_); + auto status = CreateDataChangeTempTrigger(dbHandle_); + if (status != E_OK) { + LOG_ERROR("CreateDataChangeTempTrigger failed. status %{public}d", status); + return status; + } + return E_OK; +} + +int SqliteConnection::RegisterClientObs() { - return 0; + RegisterDbHook(dbHandle_); + return E_OK; } std::pair> SqliteConnection::CreateStatement( const std::string &sql, std::shared_ptr conn) { std::shared_ptr statement = std::make_shared(); + // When memory is not cleared, quick_check reads memory pages and detects damage but does not report it + if (sql == INTEGRITIES[1] && dbHandle_ != nullptr && mode_ == JournalMode::MODE_WAL) { + sqlite3_db_release_memory(dbHandle_); + } statement->config_ = &config_; int errCode = statement->Prepare(dbHandle_, sql); if (errCode != E_OK) { @@ -451,6 +496,9 @@ std::pair> SqliteConnection::CreateStatement( statement->conn_ = conn; if (slaveConnection_ && IsWriter()) { auto slaveStmt = std::make_shared(); + if (sql == INTEGRITIES[1] && dbHandle_ != nullptr && mode_ == JournalMode::MODE_WAL) { + sqlite3_db_release_memory(dbHandle_); + } slaveStmt->config_ = &slaveConnection_->config_; errCode = slaveStmt->Prepare(slaveConnection_->dbHandle_, sql); if (errCode != E_OK) { @@ -474,7 +522,7 @@ int SqliteConnection::SubscribeTableChanges(const Connection::Notifier ¬ifier if (!isWriter_ || notifier == nullptr) { return E_OK; } - hasClientObserver_ = true; + int32_t status = RegisterClientObserver(dbHandle_, [notifier](const ClientChangedData &clientData) { DistributedRdb::RdbChangedData rdbChangedData; for (auto &[key, val] : clientData.tableData) { @@ -488,6 +536,8 @@ int SqliteConnection::SubscribeTableChanges(const Connection::Notifier ¬ifier if (status != E_OK) { LOG_ERROR("RegisterClientObserver error, status:%{public}d", status); } + RegisterDbHook(dbHandle_); + config_.SetRegisterInfo(RegisterType::CLIENT_OBSERVER, true); return status; #endif return E_OK; @@ -608,27 +658,74 @@ int SqliteConnection::ReSetKey(const RdbStoreConfig &config) return E_OK; } +int SqliteConnection::SetCrcCheck(const RdbStoreConfig &config) +{ + if (config.IsEncrypt() || config.IsMemoryRdb()) { + return E_OK; + } + int n = -1; + auto errCode = SQLiteError::ErrNo(sqlite3_file_control(dbHandle_, 0, SQLITE_FCNTL_RESERVE_BYTES, &n)); + if (errCode != E_OK) { + LOG_ERROR("failed to set sqlite reserved bytes(SQLITE_FCNTL_RESERVE_BYTES), errCode=%{public}d", errCode); + return errCode; + } + if (n == 0) { + n = SQLITE_CKSUMVFS_RESERVE_BYTES; + (void)sqlite3_file_control(dbHandle_, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); + } + return E_OK; +} + +void SqliteConnection::SetDwrEnable(const RdbStoreConfig &config) +{ + if (config.IsEncrypt() || config.IsMemoryRdb()) { + return; + } + auto errCode = ExecuteSql(GlobalExpr::PRAGMA_META_DOUBLE_WRITE); + if (errCode == E_SQLITE_META_RECOVERED) { + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, errCode, config, "", true)); + } else if (errCode != E_OK) { + LOG_ERROR("meta double failed %{public}d", errCode); + } +} + int SqliteConnection::SetEncrypt(const RdbStoreConfig &config) { if (!config.IsEncrypt()) { return E_OK; } + if (config.IsMemoryRdb()) { + return E_NOT_SUPPORT; + } std::vector key = config.GetEncryptKey(); std::vector newKey = config.GetNewEncryptKey(); auto errCode = SetEncryptKey(key, config); key.assign(key.size(), 0); if (errCode != E_OK) { + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, E_SET_ENCRYPT_FAIL, config, "LOG:SetEncryptKey errcode=" + + std::to_string(errCode) + ",iter=" + std::to_string(config.GetIter()), true)); if (!newKey.empty()) { LOG_INFO("use new key, iter=%{public}d err=%{public}d errno=%{public}d name=%{public}s", config.GetIter(), errCode, errno, SqliteUtils::Anonymous(config.GetName()).c_str()); errCode = SetEncryptKey(newKey, config); + if (errCode != E_OK) { + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, E_SET_NEW_ENCRYPT_FAIL, config, + "LOG:new key SetEncryptKey errcode= "+ std::to_string(errCode) + + ",iter=" + std::to_string(config.GetIter()), true)); + } } newKey.assign(newKey.size(), 0); if (errCode != E_OK) { errCode = SetServiceKey(config, errCode); LOG_ERROR("fail, iter=%{public}d err=%{public}d errno=%{public}d name=%{public}s", config.GetIter(), errCode, errno, SqliteUtils::Anonymous(config.GetName()).c_str()); + if (errCode != E_OK) { + bool sameKey = (key == config.GetEncryptKey()) || (newKey == config.GetEncryptKey()); + Reportor::ReportFault(RdbFaultDbFileEvent(FT_OPEN, E_SET_SERVICE_ENCRYPT_FAIL, config, + "LOG:service key SetEncryptKey errcode=" + std::to_string(errCode) + + ",iter=" + std::to_string(config.GetIter()) + ",samekey=" + std::to_string(sameKey), true)); + } return errCode; } config.ChangeEncryptKey(); @@ -669,8 +766,11 @@ int SqliteConnection::SetEncryptKey(const std::vector &key, const RdbSt return errCode; } -int SqliteConnection::SetPersistWal() +int SqliteConnection::SetPersistWal(const RdbStoreConfig &config) { + if (config.IsMemoryRdb()) { + return E_OK; + } int opcode = 1; int errCode = sqlite3_file_control(dbHandle_, "main", SQLITE_FCNTL_PERSIST_WAL, &opcode); if (errCode != SQLITE_OK) { @@ -713,13 +813,15 @@ int SqliteConnection::RegDefaultFunctions(sqlite3 *dbHandle) int SqliteConnection::SetJournalMode(const RdbStoreConfig &config) { - if (isReadOnly_) { + if (isReadOnly_ || config.IsMemoryRdb()) { return E_OK; } auto [errCode, object] = ExecuteForValue("PRAGMA journal_mode"); if (errCode != E_OK) { LOG_ERROR("SetJournalMode fail to get journal mode : %{public}d, errno %{public}d", errCode, errno); + Reportor::ReportFault(RdbFaultEvent(FT_OPEN, E_DFX_GET_JOURNAL_FAIL, config_.GetBundleName(), + "PRAGMA journal_mode get fail: " + std::to_string(errCode) + "," + std::to_string(errno))); // errno: 28 No space left on device return (errCode == E_SQLITE_IOERR && sqlite3_system_errno(dbHandle_) == 28) ? E_SQLITE_IOERR_FULL : errCode; } @@ -733,6 +835,9 @@ int SqliteConnection::SetJournalMode(const RdbStoreConfig &config) auto [errorCode, journalMode] = ExecuteForValue("PRAGMA journal_mode=" + config.GetJournalMode()); if (errorCode != E_OK) { LOG_ERROR("SqliteConnection SetJournalMode: fail to set journal mode err=%{public}d", errorCode); + Reportor::ReportFault(RdbFaultEvent(FT_OPEN, E_DFX_SET_JOURNAL_FAIL, config_.GetBundleName(), + "PRAGMA journal_mode set fail: " + std::to_string(errCode) + "," + std::to_string(errno) + "," + + config.GetJournalMode())); return errorCode; } @@ -751,33 +856,9 @@ int SqliteConnection::SetJournalMode(const RdbStoreConfig &config) return errCode; } -int SqliteConnection::SetJournalSizeLimit(const RdbStoreConfig &config) -{ - if (isReadOnly_ || config.GetJournalSize() == GlobalExpr::DB_JOURNAL_SIZE) { - return E_OK; - } - - int targetValue = SqliteGlobalConfig::GetJournalFileSize(); - auto [errCode, currentValue] = ExecuteForValue("PRAGMA journal_size_limit"); - if (errCode != E_OK) { - LOG_ERROR("SqliteConnection SetJournalSizeLimit fail to get journal_size_limit : %{public}d", errCode); - return errCode; - } - - if (static_cast(currentValue) == targetValue) { - return E_OK; - } - - std::tie(errCode, currentValue) = ExecuteForValue("PRAGMA journal_size_limit=" + std::to_string(targetValue)); - if (errCode != E_OK) { - LOG_ERROR("SqliteConnection SetJournalSizeLimit fail to set journal_size_limit : %{public}d", errCode); - } - return errCode; -} - int SqliteConnection::SetAutoCheckpoint(const RdbStoreConfig &config) { - if (isReadOnly_ || !config.IsAutoCheck()) { + if (isReadOnly_ || config.IsMemoryRdb()) { return E_OK; } @@ -879,7 +960,12 @@ std::pair SqliteConnection::ExecuteForValue( int SqliteConnection::ClearCache() { if (dbHandle_ != nullptr && mode_ == JournalMode::MODE_WAL) { - sqlite3_db_release_memory(dbHandle_); + int usedBytes = 0; + int nEnyry = 0; + int errCode = sqlite3_db_status(dbHandle_, SQLITE_DBSTATUS_CACHE_USED, &usedBytes, &nEnyry, 0); + if (errCode == SQLITE_OK && usedBytes > config_.GetClearMemorySize()) { + sqlite3_db_release_memory(dbHandle_); + } } if (slaveConnection_) { int errCode = slaveConnection_->ClearCache(); @@ -890,8 +976,11 @@ int SqliteConnection::ClearCache() return E_OK; } -void SqliteConnection::LimitPermission(const std::string &dbPath) const +void SqliteConnection::LimitPermission(const RdbStoreConfig &config, const std::string &dbPath) const { + if (config.IsMemoryRdb()) { + return; + } struct stat st = { 0 }; if (stat(dbPath.c_str(), &st) == 0) { if ((st.st_mode & (S_IXUSR | S_IXGRP | S_IRWXO)) != 0) { @@ -993,88 +1082,29 @@ int SqliteConnection::LimitWalSize() return E_OK; } -int32_t SqliteConnection::Subscribe(const std::string &event, const std::shared_ptr &observer) +int32_t SqliteConnection::Subscribe(const std::shared_ptr &observer) { if (!isWriter_ || observer == nullptr) { return E_OK; } - std::lock_guard lock(mutex_); - observers_.try_emplace(event); - auto &list = observers_.find(event)->second; - for (auto it = list.begin(); it != list.end(); it++) { - if ((*it)->GetObserver() == observer) { - LOG_ERROR("duplicate subscribe."); - return E_OK; - } - } - auto localStoreObserver = std::make_shared(observer); - int32_t errCode = RegisterStoreObserver(dbHandle_, localStoreObserver); + int32_t errCode = RegisterStoreObserver(dbHandle_, observer); if (errCode != E_OK) { - LOG_ERROR("subscribe failed."); return errCode; } - observers_[event].push_back(std::move(localStoreObserver)); - return E_OK; -} - -int32_t SqliteConnection::Unsubscribe(const std::string &event, const std::shared_ptr &observer) -{ - if (!isWriter_) { - return E_OK; - } - if (observer) { - return UnsubscribeLocalDetail(event, observer); - } - return UnsubscribeLocalDetailAll(event); -} - -int32_t SqliteConnection::UnsubscribeLocalDetail( - const std::string &event, const std::shared_ptr &observer) -{ - std::lock_guard lock(mutex_); - auto observers = observers_.find(event); - if (observers == observers_.end()) { - return E_OK; - } - - auto &list = observers->second; - for (auto it = list.begin(); it != list.end(); it++) { - if ((*it)->GetObserver() == observer) { - int32_t err = UnregisterStoreObserver(dbHandle_, *it); - if (err != 0) { - LOG_ERROR("unsubscribeLocalShared failed."); - return err; - } - list.erase(it); - break; - } - } - if (list.empty()) { - observers_.erase(event); - } + RegisterDbHook(dbHandle_); + config_.SetRegisterInfo(RegisterType::STORE_OBSERVER, true); return E_OK; } -int32_t SqliteConnection::UnsubscribeLocalDetailAll(const std::string &event) +int32_t SqliteConnection::Unsubscribe(const std::shared_ptr &observer) { - std::lock_guard lock(mutex_); - auto observers = observers_.find(event); - if (observers == observers_.end()) { + if (!isWriter_ || observer == nullptr) { return E_OK; } - - auto &list = observers->second; - auto it = list.begin(); - while (it != list.end()) { - int32_t err = UnregisterStoreObserver(dbHandle_, *it); - if (err != 0) { - LOG_ERROR("unsubscribe failed."); - return err; - } - it = list.erase(it); + int32_t errCode = UnregisterStoreObserver(dbHandle_, observer); + if (errCode != 0) { + return errCode; } - - observers_.erase(event); return E_OK; } @@ -1183,6 +1213,7 @@ int SqliteConnection::SetServiceKey(const RdbStoreConfig &config, int32_t errCod param.isSearchable_ = config.IsSearchable(); param.haMode_ = config.GetHaMode(); param.password_ = {}; + param.subUser_ = config.GetSubUser(); std::vector> keys; #if !defined(WINDOWS_PLATFORM) && !defined(MAC_PLATFORM) && !defined(ANDROID_PLATFORM) && !defined(IOS_PLATFORM) auto [svcErr, service] = DistributedRdb::RdbManagerImpl::GetInstance().GetRdbService(param); diff --git a/relational_store/frameworks/native/rdb/src/sqlite_global_config.cpp b/relational_store/frameworks/native/rdb/src/sqlite_global_config.cpp index 665a049c..d75a1236 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_global_config.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_global_config.cpp @@ -23,11 +23,13 @@ #include #include #include +#include #include "logger.h" #include "rdb_errno.h" #include "sqlite3sym.h" #include "sqlite_utils.h" +#include "rdb_fault_hiview_reporter.h" namespace OHOS { namespace NativeRdb { @@ -55,6 +57,8 @@ SqliteGlobalConfig::SqliteGlobalConfig() sqlite3_soft_heap_limit(GlobalExpr::SOFT_HEAP_LIMIT); sqlite3_initialize(); + + sqlite3_register_cksumvfs(0); } SqliteGlobalConfig::~SqliteGlobalConfig() @@ -84,6 +88,19 @@ void SqliteGlobalConfig::Log(const void *data, int err, const char *msg) LOG_WARN("WARNING(%{public}d) %{public}s ", err, SqliteUtils::Anonymous(msg).c_str()); } else { LOG_ERROR("Error(%{public}d) errno is:%{public}d %{public}s.", err, errno, SqliteUtils::Anonymous(msg).c_str()); + SqliteErrReport(err, msg); + } +} + +void SqliteGlobalConfig::SqliteErrReport(int err, const char *msg) +{ + int lowErr = (err & 0xFF); + if (lowErr == SQLITE_NOMEM || lowErr == SQLITE_INTERRUPT || lowErr == SQLITE_FULL || lowErr == SQLITE_SCHEMA || + lowErr == SQLITE_NOLFS || lowErr == SQLITE_AUTH || lowErr == SQLITE_BUSY || lowErr == SQLITE_LOCKED || + lowErr == SQLITE_IOERR || lowErr == SQLITE_CANTOPEN) { + std::string log(msg == nullptr ? "" : SqliteUtils::Anonymous(msg).c_str()); + log.append(",errcode=").append(std::to_string(err)).append(",errno=").append(std::to_string(errno)); + RdbFaultHiViewReporter::ReportFault(RdbFaultEvent(FT_SQLITE, E_DFX_SQLITE_LOG, BUNDLE_NAME_COMMON, log)); } } @@ -92,6 +109,15 @@ std::string SqliteGlobalConfig::GetMemoryDbPath() return GlobalExpr::MEMORY_DB_PATH; } +std::string SqliteGlobalConfig::GetSharedMemoryDbPath(const std::string &name) +{ + static const std::regex pattern(R"(^[\w\-\.]+$)"); + if (!name.empty() && !std::regex_match(name, pattern)) { + return ""; + } + return GlobalExpr::SHARED_MEMORY_DB_PATH_PREFIX + name + GlobalExpr::SHARED_MEMORY_DB_PATH_SUFFIX; +} + int SqliteGlobalConfig::GetPageSize() { return GlobalExpr::DB_PAGE_SIZE; @@ -119,6 +145,15 @@ std::string SqliteGlobalConfig::GetDefaultJournalMode() int SqliteGlobalConfig::GetDbPath(const RdbStoreConfig &config, std::string &dbPath) { + if (config.GetStorageMode() == StorageMode::MODE_MEMORY) { + if (config.GetRoleType() != OWNER) { + LOG_ERROR("not support MODE_MEMORY, storeName:%{public}s, role:%{public}d", + SqliteUtils::Anonymous(config.GetName()).c_str(), config.GetRoleType()); + return E_NOT_SUPPORT; + } + dbPath = SqliteGlobalConfig::GetSharedMemoryDbPath(config.GetName()); + return dbPath.empty() ? E_INVALID_FILE_PATH : E_OK; + } std::string path; if (config.GetRoleType() == OWNER) { path = config.GetPath(); @@ -129,20 +164,12 @@ int SqliteGlobalConfig::GetDbPath(const RdbStoreConfig &config, std::string &dbP SqliteUtils::Anonymous(config.GetName()).c_str(), config.GetRoleType()); return E_NOT_SUPPORT; } - - if (config.GetStorageMode() == StorageMode::MODE_MEMORY) { - if (config.GetRoleType() != OWNER) { - LOG_ERROR("not support MODE_MEMORY, storeName:%{public}s, role:%{public}d", - SqliteUtils::Anonymous(config.GetName()).c_str(), config.GetRoleType()); - return E_NOT_SUPPORT; - } - dbPath = SqliteGlobalConfig::GetMemoryDbPath(); - } else if (path.empty() || (path.front() != '/' && path.at(1) != ':')) { - LOG_ERROR("SqliteConnection GetDbPath input empty database path."); + if (path.empty() || path.front() != '/') { + LOG_ERROR("invalid path, bundleName:%{public}s, role:%{public}d, %{public}s", + config.GetBundleName().c_str(), config.GetRoleType(), SqliteUtils::Anonymous(path).c_str()); return E_INVALID_FILE_PATH; - } else { - dbPath = path; } + dbPath = std::move(path); return E_OK; } diff --git a/relational_store/frameworks/native/rdb/src/sqlite_shared_result_set.cpp b/relational_store/frameworks/native/rdb/src/sqlite_shared_result_set.cpp index 682e818b..14a523fb 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_shared_result_set.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_shared_result_set.cpp @@ -166,7 +166,14 @@ int SqliteSharedResultSet::OnGo(int oldPosition, int newPosition) if ((uint32_t)newPosition < sharedBlock->GetStartPos() || (uint32_t)newPosition >= sharedBlock->GetLastPos() || oldPosition == rowCount_) { - return FillBlock(newPosition); + auto errCode = FillBlock(newPosition); + if (errCode == E_NO_MORE_ROWS && rowCount_ != Statement::INVALID_COUNT) { + rowCount_ = sharedBlock->GetLastPos() > INT_MAX ? Statement::INVALID_COUNT + : static_cast(sharedBlock->GetLastPos()); + rowPos_ = rowPos_ > rowCount_ ? rowCount_ : rowPos_; + errCode = E_ROW_OUT_RANGE; + } + return errCode; } return E_OK; } @@ -200,7 +207,7 @@ int SqliteSharedResultSet::FillBlock(int requiredPos) ", lastPos_= %{public}" PRIu32 ", blockPos_= %{public}" PRIu32 ".", rowCount_, requiredPos, block->GetStartPos(), block->GetLastPos(), block->GetBlockPos()); } - return E_OK; + return (uint32_t)requiredPos >= block->GetLastPos() ? E_NO_MORE_ROWS : E_OK; } void SqliteSharedResultSet::SetBlock(AppDataFwk::SharedBlock *block) diff --git a/relational_store/frameworks/native/rdb/src/sqlite_sql_builder.cpp b/relational_store/frameworks/native/rdb/src/sqlite_sql_builder.cpp index d7eed08b..2653f870 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_sql_builder.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_sql_builder.cpp @@ -305,12 +305,7 @@ SqliteSqlBuilder::BatchRefSqls SqliteSqlBuilder::GenerateSqls( auto columnSize = fields->size(); auto rowSize = buckets.RowSize(); std::vector> args(columnSize * rowSize, nullRef_); - auto conflictClause = SqliteUtils::GetConflictClause(static_cast(resolution)); - if (conflictClause == nullptr) { - return BatchRefSqls(); - } - std::string sql; - sql.append("INSERT").append(conflictClause).append(" INTO ").append(table).append("("); + std::string sql = "INSERT" + g_onConflictClause[static_cast(resolution)] + " INTO " + table + " ("; size_t columnIndex = 0; for (auto &field : *fields) { for (size_t row = 0; row < rowSize; ++row) { diff --git a/relational_store/frameworks/native/rdb/src/sqlite_statement.cpp b/relational_store/frameworks/native/rdb/src/sqlite_statement.cpp index f2d1f371..5ba7a10f 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_statement.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_statement.cpp @@ -37,7 +37,7 @@ #include "sqlite_errno.h" #include "sqlite_global_config.h" #include "sqlite_utils.h" - +#include "string_utils.h" namespace OHOS { namespace NativeRdb { using namespace OHOS::Rdb; @@ -46,6 +46,12 @@ using SqlStatistic = DistributedRdb::SqlStatistic; using Reportor = RdbFaultHiViewReporter; // Setting Data Precision constexpr SqliteStatement::Action SqliteStatement::ACTIONS[ValueObject::TYPE_MAX]; +static constexpr int ERR_MSG_SIZE = 2; +static constexpr const char *ERR_MSG[] = { + "no such table:", + "no such column:", + "has no column named" +}; SqliteStatement::SqliteStatement() : readOnly_(false), columnCount_(0), numParameters_(0), stmt_(nullptr), sql_("") { seqId_ = SqlStatistic::GenerateId(); @@ -60,6 +66,61 @@ SqliteStatement::~SqliteStatement() config_ = nullptr; } +void SqliteStatement::TableReport(const std::string &errMsg, const std::string &bundleName, ErrMsgState state) +{ + std::string custLog; + if (!state.isCreated) { + custLog = "table is not created " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_NOT_CREATE, bundleName, custLog)); + } else if (state.isDeleted) { + custLog = "table is deleted " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_DELETE, bundleName, custLog)); + } else if (state.isRenamed) { + custLog = "table is renamed " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_RENAME, bundleName, custLog)); + } else { + custLog = errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_NOT_EXIST, bundleName, custLog)); + } +} + +void SqliteStatement::ColumnReport(const std::string &errMsg, const std::string &bundleName, ErrMsgState state) +{ + std::string custLog; + if (!state.isCreated) { + custLog = "column is not created " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_NOT_CREATE, bundleName, custLog)); + } else if (state.isDeleted) { + custLog = "column is deleted " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_DELETE, bundleName, custLog)); + } else if (state.isRenamed) { + custLog = "column is renamed " + errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_RENAME, bundleName, custLog)); + } else { + custLog = errMsg; + Reportor::ReportFault(RdbFaultEvent(FT_CURD, E_DFX_IS_NOT_EXIST, bundleName, custLog)); + } +} + +void SqliteStatement::HandleErrMsg(const std::string &errMsg, const std::string &dbPath, const std::string &bundleName) +{ + for (auto err: ERR_MSG) { + if (errMsg.find(err) == std::string::npos) { + continue; + } + if (err == ERR_MSG[0]) { + std::string tableName = SqliteUtils::GetErrInfoFromMsg(errMsg, err); + ErrMsgState state = SqliteUtils::CompareTableFileContent(dbPath, bundleName, tableName); + TableReport(errMsg, bundleName, state); + } + if (err == ERR_MSG[1] || err == ERR_MSG[ERR_MSG_SIZE]) { + std::string columnName = SqliteUtils::GetErrInfoFromMsg(errMsg, err); + ErrMsgState state = SqliteUtils::CompareColumnFileContent(dbPath, bundleName, columnName); + ColumnReport(errMsg, bundleName, state); + } + } +} + int SqliteStatement::Prepare(sqlite3 *dbHandle, const std::string &newSql) { if (sql_.compare(newSql) == 0) { @@ -70,6 +131,10 @@ int SqliteStatement::Prepare(sqlite3 *dbHandle, const std::string &newSql) SqlStatistic sqlStatistic(newSql, SqlStatistic::Step::STEP_PREPARE, seqId_); int errCode = sqlite3_prepare_v2(dbHandle, newSql.c_str(), newSql.length(), &stmt, nullptr); if (errCode != SQLITE_OK) { + std::string errMsg(sqlite3_errmsg(dbHandle)); + if (errMsg.size() != 0) { + HandleErrMsg(errMsg, config_->GetPath(), config_->GetBundleName()); + } if (stmt != nullptr) { sqlite3_finalize(stmt); } @@ -80,11 +145,11 @@ int SqliteStatement::Prepare(sqlite3 *dbHandle, const std::string &newSql) if (config_ != nullptr && (errCode == SQLITE_CORRUPT || (errCode == SQLITE_NOTADB && config_->GetIter() != 0))) { Reportor::ReportCorruptedOnce(Reportor::Create(*config_, ret, - (errCode == SQLITE_CORRUPT ? SqliteGlobalConfig::GetLastCorruptionMsg() : ""))); + (errCode == SQLITE_CORRUPT ? SqliteGlobalConfig::GetLastCorruptionMsg() : "SqliteStatement::Prepare"))); } if (config_ != nullptr) { Reportor::ReportFault(RdbFaultDbFileEvent(FT_CURD, - (errCode == SQLITE_NOTADB ? E_SQLITE_NOT_DB : ret), *config_, "", true)); + (errCode == SQLITE_NOTADB ? E_SQLITE_NOT_DB : ret), *config_, "sqlite3_prepare_v2", true)); } PrintInfoForDbError(ret, newSql); return ret; @@ -129,7 +194,7 @@ void SqliteStatement::ReadFile2Buffer() FILE *file = fopen(fileName.c_str(), "r"); if (file == nullptr) { LOG_ERROR( - "open db file failed: %{public}s, errno is %{public}d", SqliteUtils::Anonymous(fileName).c_str(), errno); + "Open db file failed: %{public}s, errno is %{public}d", SqliteUtils::Anonymous(fileName).c_str(), errno); return; } size_t readSize = fread(buffer, sizeof(uint64_t), BUFFER_LEN, file); @@ -292,10 +357,10 @@ int SqliteStatement::InnerStep() int ret = SQLiteError::ErrNo(errCode); if (config_ != nullptr && (errCode == SQLITE_CORRUPT || (errCode == SQLITE_NOTADB && config_->GetIter() != 0))) { Reportor::ReportCorruptedOnce(Reportor::Create(*config_, ret, - (errCode == SQLITE_CORRUPT ? SqliteGlobalConfig::GetLastCorruptionMsg() : ""))); + (errCode == SQLITE_CORRUPT ? SqliteGlobalConfig::GetLastCorruptionMsg() : "SqliteStatement::InnerStep"))); } if (config_ != nullptr && ret != E_OK && !config_->GetBundleName().empty()) { - Reportor::ReportFault(RdbFaultDbFileEvent(FT_CURD, ret, *config_, "", true)); + Reportor::ReportFault(RdbFaultDbFileEvent(FT_CURD, ret, *config_, "sqlite3_step", true)); } PrintInfoForDbError(ret, sql_); return ret; @@ -359,12 +424,13 @@ int32_t SqliteStatement::Execute(const std::vectorIsWriter() && !ReadOnly()) { return E_EXECUTE_WRITE_IN_READ_CONNECTION; } - auto errCode = conn_->LimitWalSize(); + auto errCode = E_OK; + int sqlType = SqliteUtils::GetSqlStatementType(sql_); + if (sqlType != SqliteUtils::STATEMENT_COMMIT && sqlType != SqliteUtils::STATEMENT_ROLLBACK) { + errCode = conn_->LimitWalSize(); + } if (errCode != E_OK) { - int sqlType = SqliteUtils::GetSqlStatementType(sql_); - if (sqlType != SqliteUtils::STATEMENT_COMMIT && sqlType != SqliteUtils::STATEMENT_ROLLBACK) { - return errCode; - } + return errCode; } } @@ -477,7 +543,7 @@ std::pair SqliteStatement::GetColumnType(int index) const return { E_ERROR, int32_t(ColumnType::TYPE_NULL) }; } - auto declType = SqliteUtils::StrToUpper(std::string(decl)); + auto declType = StringUtils::TruncateAfterFirstParen(SqliteUtils::StrToUpper(decl)); if (declType == ValueObject::DeclType()) { types_[index] = int32_t(ColumnType::TYPE_ASSET); } else if (declType == ValueObject::DeclType()) { diff --git a/relational_store/frameworks/native/rdb/src/sqlite_utils.cpp b/relational_store/frameworks/native/rdb/src/sqlite_utils.cpp index 0f6bcc03..9a5fb81f 100644 --- a/relational_store/frameworks/native/rdb/src/sqlite_utils.cpp +++ b/relational_store/frameworks/native/rdb/src/sqlite_utils.cpp @@ -15,24 +15,29 @@ #define LOG_TAG "SqliteUtils" #include "sqlite_utils.h" +#include +#include #include #include #include #include +#include +#include #include #include #include #include -#include -#include +#include #include +#include +#include #include "logger.h" #include "rdb_errno.h" #include "rdb_store_config.h" -#include "rdb_time_utils.h" #include "string_utils.h" +#include "rdb_time_utils.h" namespace OHOS { namespace NativeRdb { @@ -46,6 +51,8 @@ constexpr int32_t AREA_MINI_SIZE = 4; constexpr int32_t AREA_OFFSET_SIZE = 5; constexpr int32_t PRE_OFFSET_SIZE = 1; constexpr int32_t DISPLAY_BYTE = 2; +constexpr int32_t PREFIX_LENGTH = 3; +constexpr int32_t FILE_MAX_SIZE = 20 * 1024; constexpr SqliteUtils::SqlType SqliteUtils::SQL_TYPE_MAP[]; constexpr const char *SqliteUtils::ON_CONFLICT_CLAUSE[]; @@ -65,7 +72,9 @@ int SqliteUtils::GetSqlStatementType(const std::string &sql) /* analyze the sql type through first 3 characters */ std::string prefixSql = StrToUpper(sql.substr(pos, 3)); SqlType type = { prefixSql.c_str(), STATEMENT_OTHER }; - auto comp = [](const SqlType &first, const SqlType &second) { return strcmp(first.sql, second.sql) < 0; }; + auto comp = [](const SqlType &first, const SqlType &second) { + return strcmp(first.sql, second.sql) < 0; + }; auto it = std::lower_bound(SQL_TYPE_MAP, SQL_TYPE_MAP + TYPE_SIZE, type, comp); if (it < SQL_TYPE_MAP + TYPE_SIZE && !comp(type, *it)) { return it->type; @@ -73,10 +82,11 @@ int SqliteUtils::GetSqlStatementType(const std::string &sql) return STATEMENT_OTHER; } -std::string SqliteUtils::StrToUpper(std::string s) +std::string SqliteUtils::StrToUpper(const std::string &s) { - std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::toupper(c); }); - return s; + std::string str = s; + std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) { return std::toupper(c); }); + return str; } void SqliteUtils::Replace(std::string &src, const std::string &rep, const std::string &dst) @@ -390,22 +400,11 @@ std::pair SqliteUtils::Stat(const std::st return std::pair{ E_OK, info }; } -std::string SqliteUtils::FomatConfigChg(const std::string &path, const RdbStoreConfig &config, - const DistributedRdb::RdbSyncerParam &lastParam) -{ - std::stringstream ss; - ss << "CFG_CHG:Store config invalid change, storePath=" << SqliteUtils::Anonymous(path).c_str() - << ",securityLevel:" << lastParam.level_ << "->" << static_cast(config.GetSecurityLevel()) - << ",area:" << lastParam.area_ << "->" << config.GetArea() - << ",isEncrypt:" << lastParam.isEncrypt_ << "->" << config.IsEncrypt(); - return ss.str(); -} - std::string SqliteUtils::ReadFileHeader(const std::string &filePath) { constexpr int MAX_SIZE = 98; std::ifstream file(filePath, std::ios::binary); - uint8_t data[MAX_SIZE] = { 0 }; + uint8_t data[MAX_SIZE] = {0}; if (file.is_open()) { file.read(reinterpret_cast(data), MAX_SIZE); file.close(); @@ -433,6 +432,126 @@ std::string SqliteUtils::GetFileStatInfo(const DebugInfo &debugInfo) return oss.str(); } +bool SqliteUtils::CleanFileContent(const std::string &filePath) +{ + struct stat fileStat; + if (stat(filePath.c_str(), &fileStat) != 0) { + return false; + } + if (fileStat.st_size < FILE_MAX_SIZE) { + return false; + } + return DeleteFile(filePath); +} + +void SqliteUtils::WriteSqlToFile(const std::string &comparePath, const std::string &sql) +{ + int fd = open(comparePath.c_str(), O_RDWR | O_CREAT, 0660); + if (fd == -1) { + LOG_ERROR("open file failed errno %{public}d %{public}s", errno, Anonymous(comparePath).c_str()); + return ; + } + if (flock(fd, LOCK_EX) == -1) { + LOG_ERROR("Failed to lock file errno %{public}d %{public}s", errno, Anonymous(comparePath).c_str()); + close(fd); + return ; + } + std::ofstream outFile(comparePath, std::ios::app); + if (!outFile) { + flock(fd, LOCK_UN); + close(fd); + return ; + } + + outFile << sql << "\n"; + outFile.close(); + if (flock(fd, LOCK_UN) == -1) { + LOG_ERROR("Failed to unlock file errno %{public}d %{public}s", errno, Anonymous(comparePath).c_str()); + } + close(fd); +} + +std::string SqliteUtils::GetErrInfoFromMsg(const std::string &message, const std::string &errStr) +{ + size_t startPos = message.find(errStr); + std::string result; + if (startPos != std::string::npos) { + startPos += errStr.length(); + size_t endPos = message.length(); + result = message.substr(startPos, endPos - startPos); + } + return result; +} + +ErrMsgState SqliteUtils::CompareTableFileContent( + const std::string &dbPath, const std::string &bundleName, const std::string &tableName) +{ + ErrMsgState state; + std::string compareFilePath = dbPath + "-compare"; + std::ifstream file(compareFilePath.c_str()); + if (!file.is_open()) { + LOG_ERROR("compare File open failed errno %{public}d %{public}s", errno, Anonymous(compareFilePath).c_str()); + return state; + } + + std::string line; + while (getline(file, line)) { + std::string target = line; + if (target.find(tableName) == std::string::npos) { + continue; + } + std::transform(target.begin(), target.end(), target.begin(), ::toupper); + if (target.substr(0, PREFIX_LENGTH) == "CRE") { + state.isCreated = true; + state.isDeleted = false; + state.isRenamed = false; + } else if (target.substr(0, PREFIX_LENGTH) == "DRO") { + state.isDeleted = true; + state.isRenamed = false; + } else if (target.substr(0, PREFIX_LENGTH) == "ALT" && target.find("RENAME") != std::string::npos) { + state.isDeleted = false; + state.isRenamed = true; + } + } + file.close(); + return state; +} + +ErrMsgState SqliteUtils::CompareColumnFileContent( + const std::string &dbPath, const std::string &bundleName, const std::string &columnName) +{ + ErrMsgState state; + std::string compareFilePath = dbPath + "-compare"; + std::ifstream file(compareFilePath.c_str()); + if (!file.is_open()) { + LOG_ERROR("compare File open failed errno %{public}d %{public}s", errno, Anonymous(compareFilePath).c_str()); + return state; + } + + std::string line; + while (getline(file, line)) { + std::string target = line; + if (target.find(columnName) == std::string::npos) { + continue; + } + std::transform(target.begin(), target.end(), target.begin(), ::toupper); + if (target.substr(0, PREFIX_LENGTH) == "CRE" || ( + target.substr(0, PREFIX_LENGTH) == "ALT" && target.find("ADD") != std::string::npos)) { + state.isCreated = true; + state.isDeleted = false; + state.isRenamed = false; + } else if (target.substr(0, PREFIX_LENGTH) == "ALT" && target.find("DROP") != std::string::npos) { + state.isDeleted = true; + state.isRenamed = false; + } else if (target.substr(0, PREFIX_LENGTH) == "ALT" && target.find("RENAME") != std::string::npos) { + state.isDeleted = false; + state.isRenamed = true; + } + } + file.close(); + return state; +} + std::string SqliteUtils::FormatDebugInfo(const std::map &debugs, const std::string &header) { if (debugs.empty()) { @@ -445,7 +564,8 @@ std::string SqliteUtils::FormatDebugInfo(const std::map return appendix; } -std::string SqliteUtils::FormatDebugInfoBrief(const std::map &debugs, const std::string &header) +std::string SqliteUtils::FormatDebugInfoBrief(const std::map &debugs, + const std::string &header) { if (debugs.empty()) { return ""; @@ -457,5 +577,11 @@ std::string SqliteUtils::FormatDebugInfoBrief(const std::map TransDB::BatchInsert(const std::string &table, const Ref LOG_ERROR("empty,table=%{public}s,rows:%{public}zu,max:%{public}d.", table.c_str(), rows.RowSize(), maxArgs_); return { E_INVALID_ARGS, -1 }; } - if (batchInfo.size() != 1 || batchInfo.front().second.size() != 1) { - auto [fields, values] = rows.GetFieldsAndValues(); - LOG_WARN("too much args, table=%{public}s, rows:%{public}zu, fields:%{public}zu, max:%{public}d.", - table.c_str(), rows.RowSize(), fields != nullptr ? fields->size() : 0, maxArgs_); - } - int64_t changes = 0; + for (const auto &[sql, batchArgs] : batchInfo) { auto [errCode, statement] = GetStatement(sql); if (statement == nullptr) { - return { errCode, changes }; + return { errCode, -1 }; } for (const auto &args : batchArgs) { errCode = statement->Execute(args); - if (errCode == E_SQLITE_CONSTRAINT || errCode == E_OK) { - changes += statement->Changes(); - } if (errCode == E_OK) { continue; } LOG_ERROR("failed(0x%{public}x) db:%{public}s table:%{public}s args:%{public}zu", errCode, SqliteUtils::Anonymous(name_).c_str(), table.c_str(), args.size()); - return { errCode, changes }; + return { errCode, -1 }; } } - return { E_OK, changes }; + return { E_OK, int64_t(rows.RowSize()) }; } std::pair TransDB::BatchInsertWithConflictResolution( @@ -181,10 +173,10 @@ std::pair TransDB::Update( } errCode = statement->Execute(totalArgs); - if (errCode == E_SQLITE_CONSTRAINT || errCode == E_OK) { - return { errCode, int32_t(statement->Changes()) }; + if (errCode != E_OK) { + return { errCode, 0 }; } - return { errCode, 0 }; + return { errCode, int32_t(statement->Changes()) }; } int TransDB::Delete(int &deletedRows, const std::string &table, const std::string &whereClause, const Values &args) diff --git a/relational_store/frameworks/native/rdb/src/transaction_impl.cpp b/relational_store/frameworks/native/rdb/src/transaction_impl.cpp index 4b260c00..0f0cc72d 100644 --- a/relational_store/frameworks/native/rdb/src/transaction_impl.cpp +++ b/relational_store/frameworks/native/rdb/src/transaction_impl.cpp @@ -35,7 +35,13 @@ TransactionImpl::TransactionImpl(std::shared_ptr connection, const s TransactionImpl::~TransactionImpl() { - CloseInner(); + // If the user does not commit the transaction, the next time using this connection to create the transaction will + // fail. Here, we attempt to roll back during the transaction object decomposition to prevent this situation + // from happening. + if (connection_ == nullptr) { + return; + } + Rollback(); } std::pair> TransactionImpl::Create( @@ -47,7 +53,7 @@ std::pair> TransactionImpl::Create( } auto errorCode = trans->Begin(type); if (errorCode != E_OK) { - LOG_ERROR("transaction begin failed, errorCode=%{public}d", errorCode); + LOG_ERROR("Transaction begin failed, errorCode=%{public}d", errorCode); return { errorCode, nullptr }; } return { E_OK, trans }; @@ -110,16 +116,17 @@ int32_t TransactionImpl::Commit() auto [errorCode, statement] = connection_->CreateStatement(COMMIT_SQL, connection_); if (errorCode != E_OK) { LOG_ERROR("create statement failed, errorCode=%{public}d", errorCode); - CloseInner(); + CloseInner(false); return errorCode; } errorCode = statement->Execute(); - CloseInner(); if (errorCode != E_OK) { + CloseInner(false); LOG_ERROR("statement execute failed, errorCode=%{public}d", errorCode); return errorCode; } + CloseInner(); return E_OK; } @@ -134,23 +141,27 @@ int32_t TransactionImpl::Rollback() auto [errorCode, statement] = connection_->CreateStatement(ROLLBACK_SQL, connection_); if (errorCode != E_OK) { LOG_ERROR("create statement failed, errorCode=%{public}d", errorCode); - CloseInner(); + CloseInner(false); return errorCode; } errorCode = statement->Execute(); - CloseInner(); if (errorCode != E_OK) { + CloseInner(false); LOG_ERROR("statement execute failed, errorCode=%{public}d", errorCode); return errorCode; } + CloseInner(); return E_OK; } -int32_t TransactionImpl::CloseInner() +int32_t TransactionImpl::CloseInner(bool connRecycle) { std::lock_guard lock(mutex_); store_ = nullptr; + if (connection_ != nullptr) { + connection_->SetIsRecyclable(connRecycle); + } connection_ = nullptr; for (auto &resultSet : resultSets_) { auto sp = resultSet.lock(); @@ -179,12 +190,7 @@ std::pair TransactionImpl::Insert(const std::string &table, const LOG_ERROR("transaction already close"); return { E_ALREADY_CLOSED, -1 }; } - auto [errCode, rows] = store->Insert(table, row, resolution); - if (resolution == Resolution::ON_CONFLICT_ROLLBACK && errCode == E_SQLITE_CONSTRAINT && !IsInTransaction()) { - LOG_WARN("transaction already rollback, start close!"); - CloseInner(); - } - return { errCode, rows }; + return store->Insert(table, row, resolution); } std::pair TransactionImpl::BatchInsert(const std::string &table, const Rows &rows) @@ -217,12 +223,7 @@ std::pair TransactionImpl::BatchInsertWithConflictResolution( LOG_ERROR("transaction already close"); return { E_ALREADY_CLOSED, -1 }; } - auto [errCode, changes] = store->BatchInsertWithConflictResolution(table, rows, resolution); - if (resolution == Resolution::ON_CONFLICT_ROLLBACK && errCode == E_SQLITE_CONSTRAINT && !IsInTransaction()) { - LOG_WARN("transaction already rollback, start close!"); - CloseInner(); - } - return { errCode, changes }; + return store->BatchInsertWithConflictResolution(table, rows, resolution); } std::pair TransactionImpl::Update( @@ -233,12 +234,7 @@ std::pair TransactionImpl::Update( LOG_ERROR("transaction already close"); return { E_ALREADY_CLOSED, -1 }; } - auto [errCode, rows] = store->Update(table, row, where, args, resolution); - if (resolution == Resolution::ON_CONFLICT_ROLLBACK && errCode == E_SQLITE_CONSTRAINT && !IsInTransaction()) { - LOG_WARN("transaction already rollback, start close!"); - CloseInner(); - } - return { errCode, rows }; + return store->Update(table, row, where, args, resolution); } std::pair TransactionImpl::Update( @@ -303,8 +299,8 @@ std::shared_ptr TransactionImpl::QueryByStep(const std::string &sql, return resultSet; } -std::shared_ptr TransactionImpl::QueryByStep(const AbsRdbPredicates &predicates, - const Fields &columns, bool preCount) +std::shared_ptr TransactionImpl::QueryByStep( + const AbsRdbPredicates &predicates, const Fields &columns, bool preCount) { auto store = GetStore(); if (store == nullptr) { diff --git a/relational_store/frameworks/native/rdb_data_share_adapter/src/rdb_utils.cpp b/relational_store/frameworks/native/rdb_data_share_adapter/src/rdb_utils.cpp index adec2f5a..4ec32ea8 100644 --- a/relational_store/frameworks/native/rdb_data_share_adapter/src/rdb_utils.cpp +++ b/relational_store/frameworks/native/rdb_data_share_adapter/src/rdb_utils.cpp @@ -80,12 +80,22 @@ void RdbUtils::NoSupport(const OperationItem &item, RdbPredicates &query) void RdbUtils::EqualTo(const OperationItem &item, RdbPredicates &predicates) { - predicates.EqualTo(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.EqualTo(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::NotEqualTo(const OperationItem &item, RdbPredicates &predicates) { - predicates.NotEqualTo(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.NotEqualTo(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::GreaterThan(const OperationItem &item, RdbPredicates &predicates) @@ -95,22 +105,37 @@ void RdbUtils::GreaterThan(const OperationItem &item, RdbPredicates &predicates) LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); return; } - predicates.GreaterThan(item.GetSingle(0), ToValueObject(item.singleParams[1])); + predicates.GreaterThan(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::LessThan(const OperationItem &item, RdbPredicates &predicates) { - predicates.LessThan(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.LessThan(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::GreaterThanOrEqualTo(const OperationItem &item, RdbPredicates &predicates) { - predicates.GreaterThanOrEqualTo(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.GreaterThanOrEqualTo(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::LessThanOrEqualTo(const OperationItem &item, RdbPredicates &predicates) { - predicates.LessThanOrEqualTo(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.LessThanOrEqualTo(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::And(const DataShare::OperationItem &item, RdbPredicates &predicates) @@ -125,52 +150,99 @@ void RdbUtils::Or(const DataShare::OperationItem &item, RdbPredicates &predicate void RdbUtils::IsNull(const OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.IsNull(item.GetSingle(0)); } void RdbUtils::IsNotNull(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.IsNotNull(item.GetSingle(0)); } void RdbUtils::In(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1 || item.multiParams.size() < 1) { + LOG_ERROR( + "SingleParams size is %{public}zu, MultiParams size is %{public}zu", + item.singleParams.size(), item.multiParams.size()); + return; + } predicates.In(item.GetSingle(0), MutliValue(item.multiParams[0])); } void RdbUtils::NotIn(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1 || item.multiParams.size() < 1) { + LOG_ERROR( + "SingleParams size is %{public}zu, MultiParams size is %{public}zu", + item.singleParams.size(), item.multiParams.size()); + return; + } predicates.NotIn(item.GetSingle(0), MutliValue(item.multiParams[0])); } void RdbUtils::Like(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.Like(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.Like(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::NotLike(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.NotLike(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.NotLike(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::OrderByAsc(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.OrderByAsc(item.GetSingle(0)); } void RdbUtils::OrderByDesc(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.OrderByDesc(item.GetSingle(0)); } void RdbUtils::Limit(const DataShare::OperationItem &item, RdbPredicates &predicates) { + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.Limit(item.GetSingle(0)); predicates.Offset(item.GetSingle(1)); } void RdbUtils::Offset(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.Offset(item.GetSingle(0)); } @@ -186,12 +258,22 @@ void RdbUtils::EndWrap(const DataShare::OperationItem &item, RdbPredicates &pred void RdbUtils::BeginsWith(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.BeginsWith(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.BeginsWith(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::EndsWith(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.EndsWith(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.EndsWith(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::Distinct(const DataShare::OperationItem &item, RdbPredicates &predicates) @@ -201,63 +283,116 @@ void RdbUtils::Distinct(const DataShare::OperationItem &item, RdbPredicates &pre void RdbUtils::GroupBy(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.multiParams.size() < 1) { + LOG_ERROR("MultiParams is missing elements, size is %{public}zu", item.multiParams.size()); + return; + } predicates.GroupBy(MutliValue(item.multiParams[0])); } void RdbUtils::IndexedBy(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.IndexedBy(item.GetSingle(0)); } void RdbUtils::Contains(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.Contains(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.Contains(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::NotContains(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.NotContains(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.NotContains(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::Glob(const DataShare::OperationItem &item, RdbPredicates &predicates) { - predicates.Glob(item.GetSingle(0), ToValueObject(item.singleParams[1])); + // 2 is the number of argument item.singleParams + if (item.singleParams.size() < 2) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } + predicates.Glob(item.GetSingle(0), ToValueObject(item.GetSingle(1))); } void RdbUtils::Between(const DataShare::OperationItem &item, RdbPredicates &predicates) { + // 3 is the number of argument item.singleParams + if (item.singleParams.size() < 3) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } // singleParams[2] is another param - predicates.Between(item.GetSingle(0), ToValueObject(item.singleParams[1]), ToValueObject(item.singleParams[2])); + predicates.Between(item.GetSingle(0), ToValueObject(item.GetSingle(1)), ToValueObject(item.GetSingle(2))); } void RdbUtils::NotBetween(const DataShare::OperationItem &item, RdbPredicates &predicates) { + // 3 is the number of argument item.singleParams + if (item.singleParams.size() < 3) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } // singleParams[2] is another param - predicates.NotBetween(item.GetSingle(0), ToValueObject(item.singleParams[1]), ToValueObject(item.singleParams[2])); + predicates.NotBetween(item.GetSingle(0), ToValueObject(item.GetSingle(1)), ToValueObject(item.GetSingle(2))); } void RdbUtils::CrossJoin(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.CrossJoin(item.GetSingle(0)); } void RdbUtils::InnerJoin(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.InnerJoin(item.GetSingle(0)); } void RdbUtils::LeftOuterJoin(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.singleParams.size() < 1) { + LOG_ERROR("SingleParams is missing elements, size is %{public}zu", item.singleParams.size()); + return; + } predicates.LeftOuterJoin(item.GetSingle(0)); } void RdbUtils::Using(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.multiParams.size() < 1) { + LOG_ERROR("MultiParams is missing elements, size is %{public}zu", item.multiParams.size()); + return; + } predicates.Using(MutliValue(item.multiParams[0])); } void RdbUtils::On(const DataShare::OperationItem &item, RdbPredicates &predicates) { + if (item.multiParams.size() < 1) { + LOG_ERROR("MultiParams is missing elements, size is %{public}zu", item.multiParams.size()); + return; + } predicates.On(MutliValue(item.multiParams[0])); } diff --git a/relational_store/interfaces/inner_api/cloud_data/include/cloud_types.h b/relational_store/interfaces/inner_api/cloud_data/include/cloud_types.h index ee58798c..c695bb25 100644 --- a/relational_store/interfaces/inner_api/cloud_data/include/cloud_types.h +++ b/relational_store/interfaces/inner_api/cloud_data/include/cloud_types.h @@ -63,28 +63,27 @@ struct StatisticInfo { }; struct QueryKey { + int32_t user; std::string accountId; std::string bundleName; std::string storeId; bool operator<(const QueryKey &queryKey) const { - if (accountId < queryKey.accountId) { - return true; - } else if (accountId == queryKey.accountId) { - if (bundleName < queryKey.bundleName) { - return true; - } else if (bundleName == queryKey.bundleName) { - return storeId < queryKey.storeId; - } - } - return false; + return std::tie(accountId, user, bundleName, storeId) < + std::tie(queryKey.accountId, queryKey.user, queryKey.bundleName, queryKey.storeId); } }; +enum SyncStatus : int32_t { + RUNNING, + FINISHED +}; + struct CloudSyncInfo { int64_t startTime = 0; int64_t finishTime = 0; int32_t code = -1; + int32_t syncStatus = SyncStatus::RUNNING; }; using StatisticInfos = std::vector; diff --git a/relational_store/interfaces/inner_api/gdb/BUILD.gn b/relational_store/interfaces/inner_api/gdb/BUILD.gn index ec43b43e..c593c4a0 100644 --- a/relational_store/interfaces/inner_api/gdb/BUILD.gn +++ b/relational_store/interfaces/inner_api/gdb/BUILD.gn @@ -14,12 +14,16 @@ import("//build/ohos.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -config("common_public_config") { +config("native_gdb_public_config") { visibility = [ ":*" ] + include_dirs = [ + "${relational_store_innerapi_path}/gdb/include", + "${relational_store_innerapi_path}/rdb/include", + ] } -ohos_shared_library("framework_graphstore") { - configs = [ ":common_public_config" ] +ohos_shared_library("native_graphstore") { + public_configs = [ ":native_gdb_public_config" ] sanitize = { boundary_sanitize = true ubsan = true @@ -27,13 +31,12 @@ ohos_shared_library("framework_graphstore") { cfi_cross_dso = true debug = false } + cflags_cc = [ "-fvisibility=hidden" ] include_dirs = [ "${relational_store_common_path}/include", "${relational_store_native_path}/gdb/adapter/include", "${relational_store_native_path}/gdb/include", "${relational_store_native_path}/rdb/include", - "${relational_store_innerapi_path}/gdb/include", - "${relational_store_innerapi_path}/rdb/include", ] sources = [ "${relational_store_native_path}/gdb/adapter/src/grd_adapter.cpp", diff --git a/relational_store/frameworks/native/gdb/include/edge.h b/relational_store/interfaces/inner_api/gdb/include/edge.h similarity index 65% rename from relational_store/frameworks/native/gdb/include/edge.h rename to relational_store/interfaces/inner_api/gdb/include/edge.h index d504063d..7414be40 100644 --- a/relational_store/frameworks/native/gdb/include/edge.h +++ b/relational_store/interfaces/inner_api/gdb/include/edge.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_EDGE_H -#define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_EDGE_H +#ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_EDGE_H +#define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_EDGE_H #include #include @@ -24,20 +24,21 @@ #include #include "nlohmann/json.hpp" +#include "rdb_visibility.h" #include "vertex.h" namespace OHOS::DistributedDataAip { class Edge : public Vertex { public: - Edge(); - Edge(std::string id, std::string label, std::string sourceId, std::string targetId); - Edge(const std::shared_ptr &element, std::string sourceId, std::string targetId); + API_EXPORT Edge(); + API_EXPORT Edge(std::string id, std::string label, std::string sourceId, std::string targetId); + API_EXPORT Edge(const std::shared_ptr &element, std::string sourceId, std::string targetId); static std::shared_ptr Parse(const nlohmann::json &json, int32_t &errCode); - std::string GetSourceId() const; - void SetSourceId(std::string sourceId); + API_EXPORT std::string GetSourceId() const; + API_EXPORT void SetSourceId(std::string sourceId); - std::string GetTargetId() const; - void SetTargetId(std::string targetId); + API_EXPORT std::string GetTargetId() const; + API_EXPORT void SetTargetId(std::string targetId); static constexpr const char *SOURCEID = "start"; static constexpr const char *TARGETID = "end"; @@ -49,4 +50,4 @@ private: }; } -#endif //OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_EDGE_H +#endif //OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_EDGE_H diff --git a/relational_store/frameworks/native/gdb/include/aip_errors.h b/relational_store/interfaces/inner_api/gdb/include/gdb_errors.h similarity index 95% rename from relational_store/frameworks/native/gdb/include/aip_errors.h rename to relational_store/interfaces/inner_api/gdb/include/gdb_errors.h index 030b5271..7ad53617 100644 --- a/relational_store/frameworks/native/gdb/include/aip_errors.h +++ b/relational_store/interfaces/inner_api/gdb/include/gdb_errors.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_GDB_AIP_ERRORS_H -#define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_AIP_ERRORS_H +#ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GDB_ERRORS_H +#define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GDB_ERRORS_H #include "errors.h" @@ -99,4 +99,4 @@ constexpr int E_GRD_NOT_SUPPORT = (E_BASE + 0x32); } // namespace DistributedDataAip } // namespace OHOS -#endif // OHOS_DISTRIBUTED_DATA_NATIVE_GDB_AIP_ERRORS_H \ No newline at end of file +#endif // OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GDB_ERRORS_H \ No newline at end of file diff --git a/relational_store/interfaces/inner_api/gdb/include/gdb_helper.h b/relational_store/interfaces/inner_api/gdb/include/gdb_helper.h index 8f0d71b7..f3f3e7f1 100644 --- a/relational_store/interfaces/inner_api/gdb/include/gdb_helper.h +++ b/relational_store/interfaces/inner_api/gdb/include/gdb_helper.h @@ -22,7 +22,7 @@ #include "rdb_visibility.h" namespace OHOS::DistributedDataAip { -class API_EXPORT GDBHelper { +class GDBHelper { public: API_EXPORT static std::shared_ptr GetDBStore(const StoreConfig &config, int &errCode); API_EXPORT static int DeleteDBStore(const StoreConfig &config); diff --git a/relational_store/interfaces/inner_api/gdb/include/gdb_store.h b/relational_store/interfaces/inner_api/gdb/include/gdb_store.h index 4d3b3c8f..fc251b50 100644 --- a/relational_store/interfaces/inner_api/gdb/include/gdb_store.h +++ b/relational_store/interfaces/inner_api/gdb/include/gdb_store.h @@ -15,9 +15,9 @@ #ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_DB_STORE_H #define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_DB_STORE_H +#include "gdb_transaction.h" #include "rdb_visibility.h" #include "result.h" -#include "transaction.h" namespace OHOS::DistributedDataAip { class API_EXPORT DBStore { diff --git a/relational_store/interfaces/inner_api/gdb/include/gdb_store_config.h b/relational_store/interfaces/inner_api/gdb/include/gdb_store_config.h index 362f8e0b..89962ccf 100644 --- a/relational_store/interfaces/inner_api/gdb/include/gdb_store_config.h +++ b/relational_store/interfaces/inner_api/gdb/include/gdb_store_config.h @@ -48,7 +48,7 @@ enum class DBType : int { DB_BUTT }; -class API_EXPORT StoreConfig { +class StoreConfig { public: API_EXPORT StoreConfig() = default; API_EXPORT ~StoreConfig(); @@ -60,7 +60,6 @@ public: API_EXPORT void SetEncryptStatus(bool status); API_EXPORT void SetSecurityLevel(int32_t securityLevel); API_EXPORT bool IsEncrypt() const; - API_EXPORT std::string GetJson() const; API_EXPORT std::string GetFullPath() const; API_EXPORT std::string GetPath() const; API_EXPORT std::string GetName() const; @@ -89,25 +88,12 @@ private: bool isEncrypt_ = false; mutable std::vector encryptKey_{}; mutable std::vector newEncryptKey_{}; - int32_t pageSize_ = 4; - int32_t defaultIsolationLevel_ = 3; // serialization mutable int32_t iter_ = 0; int32_t writeTimeout_ = 2; // seconds int32_t readTimeout_ = 1; // seconds int32_t readConSize_ = 4; int32_t securityLevel_ = SecurityLevel::S1; - [[maybe_unused]] int32_t redoFlushByTrx_ = 0; - [[maybe_unused]] int32_t redoPubBufSize_ = 1024; - [[maybe_unused]] int32_t maxConnNum_ = 100; - [[maybe_unused]] int32_t bufferPoolSize_ = 1024; - [[maybe_unused]] int32_t crcCheckEnable_ = 1; - std::string bufferPoolPolicy_ = "BUF_PRIORITY_TABLE"; - - [[maybe_unused]] int32_t sharedModeEnable_ = 0; - [[maybe_unused]] int32_t MetaInfoBak_ = 0; - [[maybe_unused]] int32_t dbFileSize_ = 128 * 1024 * 1024; - static constexpr int MAX_TIMEOUT = 300; // seconds static constexpr int MIN_TIMEOUT = 1; // seconds void ClearEncryptKey(); diff --git a/relational_store/interfaces/inner_api/gdb/include/transaction.h b/relational_store/interfaces/inner_api/gdb/include/gdb_transaction.h similarity index 76% rename from relational_store/interfaces/inner_api/gdb/include/transaction.h rename to relational_store/interfaces/inner_api/gdb/include/gdb_transaction.h index 01a67753..137ff740 100644 --- a/relational_store/interfaces/inner_api/gdb/include/transaction.h +++ b/relational_store/interfaces/inner_api/gdb/include/gdb_transaction.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef ARKDATA_INTELLIGENCE_PLATFORM_TRANSACTION_H -#define ARKDATA_INTELLIGENCE_PLATFORM_TRANSACTION_H +#ifndef ARKDATA_INTELLIGENCE_PLATFORM_GDB_TRANSACTION_H +#define ARKDATA_INTELLIGENCE_PLATFORM_GDB_TRANSACTION_H #include #include @@ -33,10 +33,10 @@ public: static std::pair> Create(std::shared_ptr conn); static int32_t RegisterCreator(Creator creator); - virtual ~Transaction() = default; + API_EXPORT virtual ~Transaction() = default; - virtual int32_t Commit() = 0; - virtual int32_t Rollback() = 0; + API_EXPORT virtual int32_t Commit() = 0; + API_EXPORT virtual int32_t Rollback() = 0; virtual int32_t Close() = 0; /** @@ -44,14 +44,14 @@ public: * * @param gql Indicates the GQL statement to execute. */ - virtual std::pair> Query(const std::string &gql) = 0; + API_EXPORT virtual std::pair> Query(const std::string &gql) = 0; /** * @brief Executes an GQL statement. * * @param gql Indicates the GQL statement to execute. */ - virtual std::pair> Execute(const std::string &gql) = 0; + API_EXPORT virtual std::pair> Execute(const std::string &gql) = 0; private: static inline Creator creator_; diff --git a/relational_store/frameworks/native/gdb/include/path.h b/relational_store/interfaces/inner_api/gdb/include/path.h similarity index 63% rename from relational_store/frameworks/native/gdb/include/path.h rename to relational_store/interfaces/inner_api/gdb/include/path.h index 0f5a67a2..d7095e2d 100644 --- a/relational_store/frameworks/native/gdb/include/path.h +++ b/relational_store/interfaces/inner_api/gdb/include/path.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_H -#define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_H +#ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_H +#define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_H #include #include #include @@ -25,25 +25,26 @@ #include "edge.h" #include "nlohmann/json.hpp" #include "path_segment.h" +#include "rdb_visibility.h" #include "vertex.h" namespace OHOS::DistributedDataAip { class Path { public: - Path(); - Path(std::shared_ptr start, std::shared_ptr end); - Path(std::shared_ptr start, std::shared_ptr end, uint32_t pathLen, + API_EXPORT Path(); + API_EXPORT Path(std::shared_ptr start, std::shared_ptr end); + API_EXPORT Path(std::shared_ptr start, std::shared_ptr end, uint32_t pathLen, std::vector> segments); static std::shared_ptr Parse(const nlohmann::json &json, int32_t &errCode); - uint32_t GetPathLength() const; - void SetPathLength(uint32_t pathLen); - std::shared_ptr GetStart() const; - void SetStart(std::shared_ptr start); - std::shared_ptr GetEnd() const; - void SetEnd(std::shared_ptr end); - const std::vector> &GetSegments() const; + API_EXPORT uint32_t GetPathLength() const; + API_EXPORT void SetPathLength(uint32_t pathLen); + API_EXPORT std::shared_ptr GetStart() const; + API_EXPORT void SetStart(std::shared_ptr start); + API_EXPORT std::shared_ptr GetEnd() const; + API_EXPORT void SetEnd(std::shared_ptr end); + API_EXPORT const std::vector> &GetSegments() const; static constexpr const char *PATHLEN = "length"; static constexpr const char *START = "start"; @@ -58,4 +59,4 @@ private: }; } // namespace OHOS::DistributedDataAip -#endif //OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_H +#endif //OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_H diff --git a/relational_store/frameworks/native/gdb/include/path_segment.h b/relational_store/interfaces/inner_api/gdb/include/path_segment.h similarity index 63% rename from relational_store/frameworks/native/gdb/include/path_segment.h rename to relational_store/interfaces/inner_api/gdb/include/path_segment.h index b927b605..31e00a00 100644 --- a/relational_store/frameworks/native/gdb/include/path_segment.h +++ b/relational_store/interfaces/inner_api/gdb/include/path_segment.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_SEGMENT_H -#define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_SEGMENT_H +#ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_SEGMENT_H +#define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_SEGMENT_H #include #include #include @@ -23,24 +23,25 @@ #include #include "edge.h" +#include "rdb_visibility.h" #include "vertex.h" namespace OHOS::DistributedDataAip { class PathSegment { public: - PathSegment(); - PathSegment(std::shared_ptr sourceVertex, std::shared_ptr targetVertex, + API_EXPORT PathSegment(); + API_EXPORT PathSegment(std::shared_ptr sourceVertex, std::shared_ptr targetVertex, std::shared_ptr edge); static std::shared_ptr Parse(const nlohmann::json &json, int32_t &errCode); - std::shared_ptr GetSourceVertex() const; - void SetSourceVertex(std::shared_ptr vertex); + API_EXPORT std::shared_ptr GetSourceVertex() const; + API_EXPORT void SetSourceVertex(std::shared_ptr vertex); - std::shared_ptr GetEdge() const; - void SetEdge(std::shared_ptr edge); + API_EXPORT std::shared_ptr GetEdge() const; + API_EXPORT void SetEdge(std::shared_ptr edge); - std::shared_ptr GetTargetVertex() const; - void SetTargetVertex(std::shared_ptr vertex); + API_EXPORT std::shared_ptr GetTargetVertex() const; + API_EXPORT void SetTargetVertex(std::shared_ptr vertex); static constexpr const char *SOURCE_VERTEX = "start"; static constexpr const char *TARGET_VERTEX = "end"; @@ -52,4 +53,4 @@ private: std::shared_ptr targetVertex_; }; } // namespace OHOS::DistributedDataAip -#endif //OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_PATH_SEGMENT_H +#endif //OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_PATH_SEGMENT_H diff --git a/relational_store/interfaces/inner_api/gdb/include/result.h b/relational_store/interfaces/inner_api/gdb/include/result.h index 4873c278..b4b17ed8 100644 --- a/relational_store/interfaces/inner_api/gdb/include/result.h +++ b/relational_store/interfaces/inner_api/gdb/include/result.h @@ -32,7 +32,7 @@ namespace OHOS::DistributedDataAip { using GraphValue = std::variant, std::shared_ptr, std::shared_ptr, std::shared_ptr, std::nullptr_t>; -class API_EXPORT Result { +class Result { public: API_EXPORT virtual std::vector> GetAllData() const = 0; }; diff --git a/relational_store/frameworks/native/gdb/include/vertex.h b/relational_store/interfaces/inner_api/gdb/include/vertex.h similarity index 61% rename from relational_store/frameworks/native/gdb/include/vertex.h rename to relational_store/interfaces/inner_api/gdb/include/vertex.h index 9e997240..f4f28882 100644 --- a/relational_store/frameworks/native/gdb/include/vertex.h +++ b/relational_store/interfaces/inner_api/gdb/include/vertex.h @@ -13,8 +13,8 @@ * limitations under the License. */ -#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_VERTEX_H -#define OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_VERTEX_H +#ifndef OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_VERTEX_H +#define OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_VERTEX_H #include #include #include @@ -23,25 +23,26 @@ #include #include "nlohmann/json.hpp" +#include "rdb_visibility.h" namespace OHOS::DistributedDataAip { using PropType = std::variant; class Vertex { public: - Vertex(); - Vertex(std::string id, std::string label); - Vertex(std::string id, std::string label, const std::unordered_map &properties); + API_EXPORT Vertex(); + API_EXPORT Vertex(std::string id, std::string label); + API_EXPORT Vertex(std::string id, std::string label, const std::unordered_map &properties); static std::shared_ptr Parse(const nlohmann::json &json, int32_t &errCode); - std::string GetId() const; - void SetId(std::string id); + API_EXPORT std::string GetId() const; + API_EXPORT void SetId(std::string id); - const std::string &GetLabel() const; - const std::vector &GetLabels() const; - void SetLabel(const std::string &label); + API_EXPORT const std::string &GetLabel() const; + API_EXPORT const std::vector &GetLabels() const; + API_EXPORT void SetLabel(const std::string &label); - const std::unordered_map &GetProperties() const; - void SetProperty(const std::string &key, PropType value); + API_EXPORT const std::unordered_map &GetProperties() const; + API_EXPORT void SetProperty(const std::string &key, PropType value); static constexpr const char *ID = "identity"; static constexpr const char *LABEL = "label"; @@ -54,4 +55,4 @@ protected: std::unordered_map properties_; }; } // namespace OHOS::DistributedDataAip -#endif //OHOS_DISTRIBUTED_DATA_NATIVE_GDB_GRAPH_VERTEX_H +#endif //OHOS_DISTRIBUTED_DATA_INTERFACE_GDB_GRAPH_VERTEX_H diff --git a/relational_store/interfaces/inner_api/rdb/BUILD.gn b/relational_store/interfaces/inner_api/rdb/BUILD.gn index 13e7d145..3888a55d 100644 --- a/relational_store/interfaces/inner_api/rdb/BUILD.gn +++ b/relational_store/interfaces/inner_api/rdb/BUILD.gn @@ -14,6 +14,7 @@ import("//build/ohos.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") base_sources = [ + "${relational_store_native_path}/dfx/src/rdb_stat_reporter.cpp", "${relational_store_native_path}/rdb/src/abs_predicates.cpp", "${relational_store_native_path}/rdb/src/abs_rdb_predicates.cpp", "${relational_store_native_path}/rdb/src/abs_result_set.cpp", @@ -427,7 +428,6 @@ if (is_ohos && !build_ohos_sdk) { config("native_rdb_config") { visibility = [ ":*" ] - cflags = [ "-Wno-c99-designator" ] include_dirs = [ "${relational_store_mock_path}/frameworks/native/rdb", "${distributedfile_path}/mod_securitylabel", diff --git a/relational_store/interfaces/inner_api/rdb/include/cache_result_set.h b/relational_store/interfaces/inner_api/rdb/include/cache_result_set.h index 1812ac1b..219637c3 100644 --- a/relational_store/interfaces/inner_api/rdb/include/cache_result_set.h +++ b/relational_store/interfaces/inner_api/rdb/include/cache_result_set.h @@ -26,6 +26,13 @@ #include "value_object.h" #include "values_bucket.h" +#define RDB_UTILS_PUSH_WARNING _Pragma("GCC diagnostic push") +#define RDB_UTILS_POP_WARNING _Pragma("GCC diagnostic pop") +#define RDB_UTILS_DISABLE_WARNING_INTERNAL2(warningName) #warningName +#define RDB_UTILS_DISABLE_WARNING(warningName) \ + _Pragma( \ + RDB_UTILS_DISABLE_WARNING_INTERNAL2(GCC diagnostic ignored warningName)) + namespace OHOS { namespace NativeRdb { /** @@ -310,6 +317,8 @@ public: int GetSize(int columnIndex, size_t &size) override; private: +RDB_UTILS_PUSH_WARNING +RDB_UTILS_DISABLE_WARNING("-Wc99-designator") static constexpr ColumnType COLUMNTYPES[ValueObject::TYPE_MAX] = { [ValueObject::TYPE_NULL] = ColumnType::TYPE_NULL, [ValueObject::TYPE_INT] = ColumnType::TYPE_INTEGER, @@ -320,6 +329,7 @@ private: [ValueObject::TYPE_ASSET] = ColumnType::TYPE_BLOB, [ValueObject::TYPE_ASSETS] = ColumnType::TYPE_BLOB, }; +RDB_UTILS_POP_WARNING int32_t row_; mutable std::shared_mutex rwMutex_; int32_t maxRow_; diff --git a/relational_store/interfaces/inner_api/rdb/include/distributeddata_relational_store_ipc_interface_code.h b/relational_store/interfaces/inner_api/rdb/include/distributeddata_relational_store_ipc_interface_code.h index fe1a1040..7d313863 100644 --- a/relational_store/interfaces/inner_api/rdb/include/distributeddata_relational_store_ipc_interface_code.h +++ b/relational_store/interfaces/inner_api/rdb/include/distributeddata_relational_store_ipc_interface_code.h @@ -90,6 +90,8 @@ enum class RdbServiceInterfaceCode { RDB_SERVICE_CMD_UNLOCK_CLOUD_CONTAINER, RDB_SERVICE_CMD_GET_DEBUG_INFO, RDB_SERVICE_CMD_VERIFY_PROMISE_INFO, + RDB_SERVICE_CMD_REPORT_STAT, + RDB_SERVICE_CMD_GET_DFX_INFO, RDB_SERVICE_CMD_MAX }; } // namespace RelationalStore diff --git a/relational_store/interfaces/inner_api/rdb/include/rdb_errno.h b/relational_store/interfaces/inner_api/rdb/include/rdb_errno.h index a7a220bf..04beefee 100644 --- a/relational_store/interfaces/inner_api/rdb/include/rdb_errno.h +++ b/relational_store/interfaces/inner_api/rdb/include/rdb_errno.h @@ -469,6 +469,16 @@ static constexpr int E_SET_SERVICE_ENCRYPT_FAIL = (E_BASE + 0x55); * @brief Database WAL file check point failed. */ static constexpr int E_CHECK_POINT_FAIL = (E_BASE + 0x56); + +/** + * @brief Database db meta recovered success. + */ +static constexpr int E_SQLITE_META_RECOVERED = (E_BASE + 0x57); + +/** +* @brief The error code for common invalid args. +*/ +static constexpr int E_INVALID_ARGS_NEW = (E_BASE + 0x58); } // namespace NativeRdb } // namespace OHOS diff --git a/relational_store/interfaces/inner_api/rdb/include/rdb_service.h b/relational_store/interfaces/inner_api/rdb/include/rdb_service.h index 74986a9a..0ff79324 100644 --- a/relational_store/interfaces/inner_api/rdb/include/rdb_service.h +++ b/relational_store/interfaces/inner_api/rdb/include/rdb_service.h @@ -90,7 +90,11 @@ public: virtual int32_t GetDebugInfo(const RdbSyncerParam ¶m, std::map &debugInfo) = 0; + virtual int32_t GetDfxInfo(const RdbSyncerParam ¶m, DistributedRdb::RdbDfxInfo &dfxInfo) = 0; + virtual int32_t VerifyPromiseInfo(const RdbSyncerParam ¶m) = 0; + + virtual int32_t ReportStatistic(const RdbSyncerParam ¶m, const RdbStatEvent &statEvent) = 0; }; } // namespace DistributedRdb } // namespace OHOS diff --git a/relational_store/interfaces/inner_api/rdb/include/rdb_store_config.h b/relational_store/interfaces/inner_api/rdb/include/rdb_store_config.h index 5c69c3a8..89323006 100644 --- a/relational_store/interfaces/inner_api/rdb/include/rdb_store_config.h +++ b/relational_store/interfaces/inner_api/rdb/include/rdb_store_config.h @@ -16,6 +16,9 @@ #ifndef NATIVE_RDB_RDB_STORE_CONFIG_H #define NATIVE_RDB_RDB_STORE_CONFIG_H +#include +#include +#include #include #include @@ -198,6 +201,44 @@ enum Tokenizer : int32_t { TOKENIZER_END }; +enum RegisterType : uint8_t { STORE_OBSERVER = 0, CLIENT_OBSERVER, OBSERVER_END }; + +struct RegisterInfo { +public: + RegisterInfo() + { + info_ = 0; + } + + RegisterInfo(const RegisterInfo &info) + { + info_ = info.info_; + } + + bool Get(RegisterType type) + { + uint8_t bit = type % sizeof(uint8_t); + std::lock_guard LockGuard(mutex_); + return (1 << bit) & info_; + } + + void Set(RegisterType type, bool state) + { + uint8_t bit = type % sizeof(uint8_t); + std::lock_guard LockGuard(mutex_); + info_ |= 1 << bit; + } + + bool operator==(const RegisterInfo& info) + { + std::lock_guard LockGuard(mutex_); + return info_ == info.info_; + } +private: + uint8_t info_; + std::mutex mutex_; +}; + /** * @brief Use DistributedType replace OHOS::DistributedRdb::RdbDistributedType. */ @@ -571,6 +612,11 @@ public: */ bool GetAutoClean() const; + /** + * @brief Obtains the cryptoParam field in this {@code StoreConfig} object. + */ + bool IsLocalOnly() const; + /** * @brief Set the isVector field in this {@code StoreConfig} object. */ @@ -617,22 +663,20 @@ public: return false; } } + if (storageMode_ != config.storageMode_ || journalMode_ != config.journalMode_ || + syncMode_ != config.syncMode_ || databaseFileType != config.databaseFileType || + journalSize_ != config.journalSize_ || pageSize_ != config.pageSize_ || dbType_ != config.dbType_ || + customDir_ != config.customDir_ || pluginLibs_ != config.pluginLibs_ || haMode_ != config.haMode_) { + return false; + } - return path_ == config.path_ && storageMode_ == config.storageMode_ && journalMode_ == config.journalMode_ && - syncMode_ == config.syncMode_ && databaseFileType == config.databaseFileType && - IsEncrypt() == config.IsEncrypt() && securityLevel_ == config.securityLevel_ && - journalSize_ == config.journalSize_ && pageSize_ == config.pageSize_ && dbType_ == config.dbType_ && - customDir_ == config.customDir_ && pluginLibs_ == config.pluginLibs_ && haMode_ == config.haMode_; + if (storageMode_ == StorageMode::MODE_MEMORY) { + return name_ == config.name_; + } else { + return path_ == config.path_ && securityLevel_ == config.securityLevel_; + } } - - /** - * @brief Overload the does not equal the number operator. - */ - bool operator!=(const RdbStoreConfig &config) const - { - return !(*this == config); - } - + /** * @brief Checks whether the database isSearchable necessary. */ @@ -698,10 +742,22 @@ public: void SetWalLimitSize(ssize_t size); + int32_t GetClearMemorySize() const; + + void SetClearMemorySize(int32_t size); + + std::string GetCollatorLocales() const; + + void SetCollatorLocales(const std::string &loacles); + int32_t GetHaMode() const; void SetHaMode(int32_t haMode); + int32_t GetSubUser() const; + + void SetSubUser(int32_t subUser); + void SetScalarFunctions(const std::map functions); void SetCryptoParam(CryptoParam cryptoParam); @@ -716,8 +772,15 @@ public: void SetNcandidates(int ncandidates); + std::string ToString() const; + static std::string FormatCfg(const RdbStoreConfig &first, const RdbStoreConfig &second); + void SetRegisterInfo(RegisterType type, bool state) const; + + bool GetRegisterInfo(RegisterType type) const; + + bool IsEqualRegisterInfo(const RdbStoreConfig& config) const; private: void ClearEncryptKey(); int32_t GenerateEncryptedKey() const; @@ -730,6 +793,7 @@ private: bool isAutoClean_ = true; bool isVector_ = false; bool autoRekey_ = false; + bool localOnly_ = false; int32_t journalSize_; int32_t pageSize_; int32_t readConSize_ = 4; @@ -754,6 +818,8 @@ private: ssize_t walLimitSize_; ssize_t checkpointSize_; ssize_t startCheckpointSize_; + int32_t clearMemorySize_; + std::string loacles_; // distributed rdb std::string bundleName_; std::string moduleName_; @@ -768,6 +834,8 @@ private: static constexpr int MAX_TIMEOUT = 300; // seconds static constexpr int MIN_TIMEOUT = 1; // seconds bool allowRebuilt_ = false; + int32_t subUser_ = 0; + mutable RegisterInfo registerInfo_; }; } // namespace OHOS::NativeRdb #endif diff --git a/relational_store/interfaces/inner_api/rdb/include/rdb_types.h b/relational_store/interfaces/inner_api/rdb/include/rdb_types.h index c0e784c2..2f2d68f5 100644 --- a/relational_store/interfaces/inner_api/rdb/include/rdb_types.h +++ b/relational_store/interfaces/inner_api/rdb/include/rdb_types.h @@ -53,6 +53,11 @@ struct RdbDebugInfo { uint32_t gid_ = 0; }; +struct RdbDfxInfo { + std::string lastOpenTime_; + int curUserId_; +}; + struct RdbSyncerParam { std::string bundleName_; std::string hapName_; @@ -73,6 +78,9 @@ struct RdbSyncerParam { std::string user_; std::vector permissionNames_ = {}; bool asyncDownloadAsset_ = false; + bool enableCloud_ = true; + int32_t subUser_ = 0; + RdbDfxInfo dfxInfo_; ~RdbSyncerParam() { password_.assign(password_.size(), 0); @@ -104,6 +112,37 @@ enum DistributedTableType { DISTRIBUTED_SEARCH }; +struct RdbStatEvent { + uint32_t statType = 0; + std::string bundleName = ""; + std::string storeName = ""; + uint32_t subType = 0; + uint32_t costTime = 0; + + bool operator<(const RdbStatEvent &other) const + { + if (statType != other.statType) { + return statType < other.statType; + } + if (bundleName.size() != other.bundleName.size()) { + return bundleName.size() < other.bundleName.size(); + } + if (bundleName != other.bundleName) { + return bundleName < other.bundleName; + } + if (storeName.size() != other.storeName.size()) { + return storeName.size() < other.storeName.size(); + } + if (storeName != other.storeName) { + return storeName < other.storeName; + } + if (subType != other.subType) { + return subType < other.subType; + } + return costTime < other.costTime; + } +}; + struct Reference { std::string sourceTable; std::string targetTable; @@ -115,6 +154,7 @@ struct DistributedConfig { std::vector references = {}; bool isRebuild = false; bool asyncDownloadAsset = false; + bool enableCloud = true; }; enum Progress { @@ -248,6 +288,32 @@ struct SubscribeOption { std::string event; }; +/** + * @brief Indicates the column type. + * + * Value returned by getColumnType(int) + */ +enum class ColumnType { + /** Indicates the column type is NULL.*/ + TYPE_NULL = 0, + /** Indicates the column type is INTEGER.*/ + TYPE_INTEGER, + /** Indicates the column type is FLOAT.*/ + TYPE_FLOAT, + /** Indicates the column type is STRING.*/ + TYPE_STRING, + /** Indicates the column type is BLOB.*/ + TYPE_BLOB, + /** Indicates the column type is ASSET.*/ + TYPE_ASSET, + /** Indicates the column type is ASSETS.*/ + TYPE_ASSETS, + /** Indicates the column type is Float32.*/ + TYPE_FLOAT32_ARRAY, + /** Indicates the column type is BigInt.*/ + TYPE_BIGINT +}; + struct Origin { enum OriginType : int32_t { ORIGIN_LOCAL, @@ -324,4 +390,4 @@ public: virtual void OnStatistic(const SqlExecutionInfo &info) = 0; }; } // namespace OHOS::DistributedRdb -#endif +#endif \ No newline at end of file diff --git a/relational_store/interfaces/inner_api/rdb/include/remote_result_set.h b/relational_store/interfaces/inner_api/rdb/include/remote_result_set.h index da3268f0..e168b0f6 100644 --- a/relational_store/interfaces/inner_api/rdb/include/remote_result_set.h +++ b/relational_store/interfaces/inner_api/rdb/include/remote_result_set.h @@ -20,36 +20,11 @@ #include #include "rdb_visibility.h" +#include "rdb_types.h" namespace OHOS { namespace NativeRdb { - -/** - * @brief Indicates the column type. - * - * Value returned by getColumnType(int) - */ -enum class ColumnType { - /** Indicates the column type is NULL.*/ - TYPE_NULL = 0, - /** Indicates the column type is INTEGER.*/ - TYPE_INTEGER, - /** Indicates the column type is FLOAT.*/ - TYPE_FLOAT, - /** Indicates the column type is STRING.*/ - TYPE_STRING, - /** Indicates the column type is BLOB.*/ - TYPE_BLOB, - /** Indicates the column type is ASSET.*/ - TYPE_ASSET, - /** Indicates the column type is ASSETS.*/ - TYPE_ASSETS, - /** Indicates the column type is Float32.*/ - TYPE_FLOAT32_ARRAY, - /** Indicates the column type is BigInt.*/ - TYPE_BIGINT -}; - +using ColumnType = DistributedRdb::ColumnType; /** * The RemoteResultSet class of RDB. * Provides methods for accessing a database result set generated by remote query the database. diff --git a/relational_store/interfaces/inner_api/rdb/include/transaction.h b/relational_store/interfaces/inner_api/rdb/include/transaction.h index a6ad7368..04563e9b 100644 --- a/relational_store/interfaces/inner_api/rdb/include/transaction.h +++ b/relational_store/interfaces/inner_api/rdb/include/transaction.h @@ -111,6 +111,7 @@ public: * @param rows Indicates the rows of data {@link RefRows} to be inserted into the table. */ virtual std::pair BatchInsert(const std::string &table, const RefRows &rows) = 0; + /** * @brief Inserts a batch of data into the target table. * diff --git a/relational_store/interfaces/inner_api/rdb/mock/include/rdb_store_config.h b/relational_store/interfaces/inner_api/rdb/mock/include/rdb_store_config.h index 68f155cb..a239f28f 100644 --- a/relational_store/interfaces/inner_api/rdb/mock/include/rdb_store_config.h +++ b/relational_store/interfaces/inner_api/rdb/mock/include/rdb_store_config.h @@ -18,9 +18,9 @@ #include #include +#include #include #include - namespace OHOS::NativeRdb { enum class IntegrityCheck { NONE, @@ -111,6 +111,43 @@ enum Tokenizer : int32_t { TOKENIZER_END }; +enum RegisterType : uint8_t { STORE_OBSERVER = 0, CLIENT_OBSERVER, OBSERVER_END }; + +struct RegisterInfo { +public: + RegisterInfo() + { + info_ = 0; + } + + RegisterInfo(const RegisterInfo &info) + { + info_ = info.info_; + } + + bool Get(RegisterType type) + { + uint8_t bit = type % sizeof(uint8_t); + std::lock_guard LockGuard(mutex_); + return (1 << bit) & info_; + } + + void Set(RegisterType type, bool state) + { + uint8_t bit = type % sizeof(uint8_t); + std::lock_guard LockGuard(mutex_); + info_ |= 1 << bit; + } + + bool operator==(const RegisterInfo& info) + { + std::lock_guard LockGuard(mutex_); + return info_ == info.info_; + } +private: + uint8_t info_; + std::mutex mutex_; +}; using ScalarFunction = std::function &)>; @@ -160,6 +197,7 @@ public: bool IsReadOnly() const; bool IsMemoryRdb() const; + bool IsLocalOnly() const; std::string GetDatabaseFileType() const; SecurityLevel GetSecurityLevel() const; void SetEncryptStatus(const bool status); @@ -222,25 +260,29 @@ public: std::string GetVisitorDir() const; bool operator==(const RdbStoreConfig &config) const { - if (this->customScalarFunctions.size() != config.customScalarFunctions.size()) { + if (customScalarFunctions_.size() != config.customScalarFunctions_.size()) { return false; } - auto iter1 = this->customScalarFunctions.begin(); - auto iter2 = config.customScalarFunctions.begin(); - for (; iter1 != this->customScalarFunctions.end(); ++iter1, ++iter2) { + auto iter1 = customScalarFunctions_.begin(); + auto iter2 = config.customScalarFunctions_.begin(); + for (; iter1 != customScalarFunctions_.end(); ++iter1, ++iter2) { if (iter1->first != iter2->first) { return false; } } + if (storageMode_ != config.storageMode_ || journalMode_ != config.journalMode_ || + syncMode_ != config.syncMode_ || databaseFileType != config.databaseFileType || + journalSize_ != config.journalSize_ || pageSize_ != config.pageSize_ || dbType_ != config.dbType_ || + customDir_ != config.customDir_ || pluginLibs_ != config.pluginLibs_ || haMode_ != config.haMode_) { + return false; + } - return this->path_ == config.path_ && this->storageMode_ == config.storageMode_ && - this->journalMode_ == config.journalMode_ && this->syncMode_ == config.syncMode_ && - this->databaseFileType == config.databaseFileType && IsEncrypt() == config.IsEncrypt() && - this->securityLevel_ == config.securityLevel_ && this->journalSize_ == config.journalSize_ && - this->pageSize_ == config.pageSize_ && this->readConSize_ == config.readConSize_ && - this->customDir_ == config.customDir_ && this->allowRebuilt_ == config.allowRebuilt_ && - this->pluginLibs_ == config.pluginLibs_ && this->haMode_ == config.haMode_; + if (storageMode_ == StorageMode::MODE_MEMORY) { + return name_ == config.name_; + } else { + return path_ == config.path_ && securityLevel_ == config.securityLevel_; + } } bool IsSearchable() const; @@ -268,6 +310,12 @@ public: ssize_t GetStartCheckpointSize() const; ssize_t GetWalLimitSize() const; void SetWalLimitSize(ssize_t size); + int32_t GetClearMemorySize() const; + void SetClearMemorySize(int32_t size); + std::string GetCollatorLocales() const; + void SetCollatorLocales(const std::string &loacles); + int32_t GetSubUser() const; + void SetSubUser(int32_t subUser); void SetHaMode(int32_t haMode); void SetScalarFunctions(const std::map functions); void SetCryptoParam(CryptoParam cryptoParam); @@ -276,7 +324,11 @@ public: void EnableRekey(bool enable); int GetNcandidates() const; void SetNcandidates(int ncandidates); - static std::string Format(const RdbStoreConfig &cacheConfig, const RdbStoreConfig &incomingConfig); + std::string ToString() const; + static std::string FormatCfg(const RdbStoreConfig &first, const RdbStoreConfig &second); + void SetRegisterInfo(RegisterType type, bool state) const; + bool GetRegisterInfo(RegisterType type) const; + bool IsEqualRegisterInfo(const RdbStoreConfig& config) const; private: void ClearEncryptKey(); @@ -290,6 +342,7 @@ private: bool isAutoClean_ = true; bool isVector_ = false; bool autoRekey_ = false; + bool localOnly_ = false; int32_t journalSize_; int32_t pageSize_; int32_t readConSize_ = 4; @@ -314,6 +367,8 @@ private: ssize_t walLimitSize_; ssize_t checkpointSize_; ssize_t startCheckpointSize_; + int32_t clearMemorySize_; + std::string loacles_; // distributed rdb std::string bundleName_; std::string moduleName_; @@ -321,13 +376,15 @@ private: std::string dataGroupId_; std::string customDir_; mutable std::vector newEncryptKey_{}; - std::map customScalarFunctions; + std::map customScalarFunctions_; std::vector pluginLibs_{}; int ncandidates_ = 128; static constexpr int MAX_TIMEOUT = 300; // seconds static constexpr int MIN_TIMEOUT = 1; // seconds bool allowRebuilt_ = false; + int32_t subUser_ = 0; + mutable RegisterInfo registerInfo_; }; } // namespace OHOS::NativeRdb #endif \ No newline at end of file diff --git a/relational_store/interfaces/inner_api/rdb_data_share_adapter/BUILD.gn b/relational_store/interfaces/inner_api/rdb_data_share_adapter/BUILD.gn index 403da4e0..13a54411 100644 --- a/relational_store/interfaces/inner_api/rdb_data_share_adapter/BUILD.gn +++ b/relational_store/interfaces/inner_api/rdb_data_share_adapter/BUILD.gn @@ -15,6 +15,8 @@ import("//foundation/distributeddatamgr/relational_store/relational_store.gni") config("rdb_data_share_adapter_config") { visibility = [ ":*" ] + + cflags = [ "-Wno-c99-designator" ] if (!is_mingw && !is_mac) { cflags_cc = [ "-fvisibility=hidden" ] } diff --git a/relational_store/interfaces/inner_api/rdb_data_share_adapter/include/rdb_utils.h b/relational_store/interfaces/inner_api/rdb_data_share_adapter/include/rdb_utils.h index 9c736280..00c37410 100644 --- a/relational_store/interfaces/inner_api/rdb_data_share_adapter/include/rdb_utils.h +++ b/relational_store/interfaces/inner_api/rdb_data_share_adapter/include/rdb_utils.h @@ -30,6 +30,7 @@ #include "result_set_bridge.h" #include "value_object.h" #include "values_bucket.h" +#include "cache_result_set.h" namespace OHOS { namespace RdbDataShareAdapter { @@ -137,6 +138,8 @@ private: ~RdbUtils(); static OHOS::NativeRdb::ValueObject ToValueObject(const DataSharePredicatesObject &predicatesObject); using OperateHandler = void (*)(const OperationItem &, RdbPredicates &); +RDB_UTILS_PUSH_WARNING +RDB_UTILS_DISABLE_WARNING("-Wc99-designator") static constexpr OperateHandler HANDLERS[DataShare::LAST_TYPE] = { [DataShare::INVALID_OPERATION] = &RdbUtils::NoSupport, [DataShare::EQUAL_TO] = &RdbUtils::EqualTo, @@ -176,6 +179,7 @@ private: [DataShare::USING] = &RdbUtils::Using, [DataShare::ON] = &RdbUtils::On, }; +RDB_UTILS_POP_WARNING }; } // namespace RdbDataShareAdapter } // namespace OHOS diff --git a/relational_store/interfaces/ndk/include/oh_rdb_types.h b/relational_store/interfaces/ndk/include/oh_rdb_types.h index 5080ae64..db356bc6 100644 --- a/relational_store/interfaces/ndk/include/oh_rdb_types.h +++ b/relational_store/interfaces/ndk/include/oh_rdb_types.h @@ -80,4 +80,3 @@ typedef enum Rdb_ConflictResolution { }; #endif #endif -/** @} */ \ No newline at end of file diff --git a/relational_store/interfaces/ndk/include/relational_store.h b/relational_store/interfaces/ndk/include/relational_store.h index 6eb5d353..54699112 100644 --- a/relational_store/interfaces/ndk/include/relational_store.h +++ b/relational_store/interfaces/ndk/include/relational_store.h @@ -355,6 +355,19 @@ int OH_Rdb_IsTokenizerSupported(Rdb_Tokenizer tokenizer, bool *isSupported); */ int OH_Rdb_SetTokenizer(OH_Rdb_ConfigV2 *config, Rdb_Tokenizer tokenizer); +/** + * @brief Set property isPersistent into config + * + * @param config Represents a pointer to {@link OH_Rdb_ConfigV2} instance. + * Indicates the configuration of the database related to this RDB store. + * @param isPersistent Indicates whether the database need persistence. + * @return Returns the status code of the execution. Successful execution returns RDB_OK, + * {@link RDB_OK} - success. + * {@link RDB_E_INVALID_ARGS} - The error code for common invalid args. + * @since 18 + */ +int OH_Rdb_SetPersistent(OH_Rdb_ConfigV2 *config, bool isPersistent); + /** * @brief Get support db type list * @param typeCount The output parameter, which is used to recieve the length of the support db type array. diff --git a/relational_store/interfaces/ndk/include/relational_store_error_code.h b/relational_store/interfaces/ndk/include/relational_store_error_code.h index ce09a438..ff98a52b 100644 --- a/relational_store/interfaces/ndk/include/relational_store_error_code.h +++ b/relational_store/interfaces/ndk/include/relational_store_error_code.h @@ -418,7 +418,7 @@ typedef enum OH_Rdb_ErrCode { RDB_E_TYPE_MISMATCH = (E_BASE + 64), /** - * @brief Data value type is null. + * @brief SQLite: Abort due to constraint violation. * * @since 16 */ diff --git a/relational_store/interfaces/ndk/src/oh_rdb_transaction.cpp b/relational_store/interfaces/ndk/src/oh_rdb_transaction.cpp index 9e383763..943967e4 100644 --- a/relational_store/interfaces/ndk/src/oh_rdb_transaction.cpp +++ b/relational_store/interfaces/ndk/src/oh_rdb_transaction.cpp @@ -118,7 +118,7 @@ int OH_RdbTrans_Insert(OH_Rdb_Transaction *trans, const char *table, const OH_VB auto [errCode, id] = trans->trans_->Insert(table, valuesBucket->Get()); *rowId = id; if (errCode != E_OK) { - LOG_ERROR("insert fail, errCode=%{public}d id=%{public}" PRId64 "" PRId64, errCode, id); + LOG_ERROR("insert fail, errCode=%{public}d id=%{public}" PRId64, errCode, id); } return ConvertorErrorCode::GetInterfaceCode(errCode); } @@ -142,7 +142,7 @@ int OH_RdbTrans_BatchInsert(OH_Rdb_Transaction *trans, const char *table, const trans->trans_->BatchInsertWithConflictResolution(table, datas, Utils::ConvertConflictResolution(resolution)); *changes = count; if (errCode != E_OK) { - LOG_ERROR("batch insert fail, errCode=%{public}d count=%{public}" PRId64 "" PRId64, errCode, count); + LOG_ERROR("batch insert fail, errCode=%{public}d count=%{public}" PRId64, errCode, count); } return ConvertorErrorCode::GetInterfaceCode(errCode); } diff --git a/relational_store/interfaces/ndk/src/relational_store.cpp b/relational_store/interfaces/ndk/src/relational_store.cpp index ae02e1ca..c66b7efa 100644 --- a/relational_store/interfaces/ndk/src/relational_store.cpp +++ b/relational_store/interfaces/ndk/src/relational_store.cpp @@ -54,6 +54,7 @@ struct OH_Rdb_ConfigV2 { std::string bundleName = ""; std::string moduleName = ""; bool isEncrypt = false; + bool persist = true; int securityLevel = 0; int area = 0; int dbType = RDB_SQLITE; @@ -208,6 +209,17 @@ int OH_Rdb_SetTokenizer(OH_Rdb_ConfigV2 *config, Rdb_Tokenizer tokenizer) return OH_Rdb_ErrCode::RDB_OK; } +int OH_Rdb_SetPersistent(OH_Rdb_ConfigV2 *config, bool isPersistent) +{ + if (config == nullptr || (config->magicNum != RDB_CONFIG_V2_MAGIC_CODE)) { + LOG_ERROR("config is null %{public}d or magicNum not valid %{public}d. isPersistent %{public}d", + (config == nullptr), (config == nullptr ? 0 : config->magicNum), isPersistent); + return OH_Rdb_ErrCode::RDB_E_INVALID_ARGS; + } + config->persist = isPersistent; + return OH_Rdb_ErrCode::RDB_OK; +} + const int *OH_Rdb_GetSupportedDbType(int *numType) { if (numType == nullptr) { @@ -360,19 +372,20 @@ static OHOS::NativeRdb::Tokenizer ConvertTokenizer2Native(Rdb_Tokenizer token) static OHOS::NativeRdb::RdbStoreConfig GetRdbStoreConfig(const OH_Rdb_ConfigV2 *config, int *errCode) { - if (config->magicNum != RDB_CONFIG_V2_MAGIC_CODE || - (OHOS::NativeRdb::SecurityLevel(config->securityLevel) < OHOS::NativeRdb::SecurityLevel::S1 || - OHOS::NativeRdb::SecurityLevel(config->securityLevel) >= OHOS::NativeRdb::SecurityLevel::LAST) || + if (config->magicNum != RDB_CONFIG_V2_MAGIC_CODE || (config->persist && + ((OHOS::NativeRdb::SecurityLevel(config->securityLevel) < OHOS::NativeRdb::SecurityLevel::S1 || + OHOS::NativeRdb::SecurityLevel(config->securityLevel) >= OHOS::NativeRdb::SecurityLevel::LAST))) || (config->area < RDB_SECURITY_AREA_EL1 || config->area > RDB_SECURITY_AREA_EL5) || (config->dbType < RDB_SQLITE || config->dbType > RDB_CAYLEY) || (config->token < RDB_NONE_TOKENIZER || config->token > RDB_CUSTOM_TOKENIZER)) { *errCode = OH_Rdb_ErrCode::RDB_E_INVALID_ARGS; LOG_ERROR("Config magic number is not valid %{public}x or securityLevel %{public}d area %{public}d" - "dbType %{public}d token %{public}d ret %{public}d", - config->magicNum, config->securityLevel, config->area, config->dbType, config->token, *errCode); + "dbType %{public}d token %{public}d ret %{public}d persist %{public}d", + config->magicNum, config->securityLevel, config->area, config->dbType, config->token, *errCode, + config->persist); return OHOS::NativeRdb::RdbStoreConfig(""); } - std::string realPath = + std::string realPath = !config->persist ? config->dataBaseDir: OHOS::NativeRdb::RdbSqlUtils::GetDefaultDatabasePath(config->dataBaseDir, config->storeName, *errCode); if (*errCode != 0) { *errCode = ConvertorErrorCode::NativeToNdk(*errCode); @@ -380,6 +393,9 @@ static OHOS::NativeRdb::RdbStoreConfig GetRdbStoreConfig(const OH_Rdb_ConfigV2 * return OHOS::NativeRdb::RdbStoreConfig(""); } OHOS::NativeRdb::RdbStoreConfig rdbStoreConfig(realPath); + if (!config->storeName.empty()) { + rdbStoreConfig.SetName(config->storeName); + } rdbStoreConfig.SetSecurityLevel(OHOS::NativeRdb::SecurityLevel(config->securityLevel)); rdbStoreConfig.SetEncryptStatus(config->isEncrypt); rdbStoreConfig.SetArea(config->area - 1); @@ -387,6 +403,8 @@ static OHOS::NativeRdb::RdbStoreConfig GetRdbStoreConfig(const OH_Rdb_ConfigV2 * rdbStoreConfig.SetBundleName(config->bundleName); rdbStoreConfig.SetName(config->storeName); rdbStoreConfig.SetTokenizer(ConvertTokenizer2Native(static_cast(config->token))); + rdbStoreConfig.SetStorageMode( + config->persist ? OHOS::NativeRdb::StorageMode::MODE_DISK : OHOS::NativeRdb::StorageMode::MODE_MEMORY); return rdbStoreConfig; } @@ -524,7 +542,7 @@ int OH_Rdb_BatchInsert(OH_Rdb_Store *store, const char *table, datas, Utils::ConvertConflictResolution(resolution)); *changes = count; if (errCode != OHOS::NativeRdb::E_OK) { - LOG_ERROR("batch insert fail, errCode=%{public}d count=%{public}" PRId64 "" PRId64, errCode, count); + LOG_ERROR("batch insert fail, errCode=%{public}d count=%{public}" PRId64, errCode, count); } return ConvertorErrorCode::GetInterfaceCode(errCode); } diff --git a/relational_store/rdbmock/frameworks/native/rdb/file_ex.h b/relational_store/rdbmock/frameworks/native/rdb/file_ex.h index 197fbf31..d2102fcb 100644 --- a/relational_store/rdbmock/frameworks/native/rdb/file_ex.h +++ b/relational_store/rdbmock/frameworks/native/rdb/file_ex.h @@ -28,19 +28,4 @@ static bool LoadBufferFromFile(const std::string &filePath, std::vector &c { return true; } - -#ifndef LOCK_EX -#define LOCK_EX 0 -#endif - -#ifndef LOCK_UN -#define LOCK_UN 1 -#endif -#ifdef __cplusplus -extern "C" { -#endif -#define flock(a, b) 0 -#ifdef __cplusplus -} -#endif #endif /* MOCK_UTILS_BASE_FILE_EX_H */ diff --git a/relational_store/rdbmock/frameworks/native/rdb/hks_api.h b/relational_store/rdbmock/frameworks/native/rdb/hks_api.h index 9437f90f..2a2caf1e 100644 --- a/relational_store/rdbmock/frameworks/native/rdb/hks_api.h +++ b/relational_store/rdbmock/frameworks/native/rdb/hks_api.h @@ -64,6 +64,17 @@ static int32_t HksFinish(const struct HksBlob *handle, const struct HksParamSet return HKS_SUCCESS; } +/** + * @brief Abort operation + * @param handle operation handle + * @param paramSet required parameter set + * @return error code, see hks_type.h + */ +static int32_t HksAbort(const struct HksBlob *handle, const struct HksParamSet *paramSet) +{ + return HKS_SUCCESS; +} + /** * @brief Generate key * @param keyAlias key alias diff --git a/relational_store/rdbmock/frameworks/native/rdb/relational_store_client.h b/relational_store/rdbmock/frameworks/native/rdb/relational_store_client.h index 7a083da3..cc94d325 100644 --- a/relational_store/rdbmock/frameworks/native/rdb/relational_store_client.h +++ b/relational_store/rdbmock/frameworks/native/rdb/relational_store_client.h @@ -51,4 +51,10 @@ DistributedDB::DBStatus UnLock( const std::string &tableName, const std::vector> &hashKey, sqlite3 *db); DistributedDB::DBStatus DropLogicDeletedData(sqlite3 *db, const std::string &tableName, uint64_t cursor); +void RegisterDbHook(sqlite3 *db); + +void UnregisterDbHook(sqlite3 *db); + +DistributedDB::DBStatus CreateDataChangeTempTrigger(sqlite3 *db); + #endif //RELATIONAL_STORE_RELATIONAL_STORE_CLIENT_H diff --git a/datamgr_service/services/distributeddataservice/framework/include/utils/time_statistic.h b/relational_store/rdbmock/frameworks/native/rdb/sys/file.h similarity index 68% rename from datamgr_service/services/distributeddataservice/framework/include/utils/time_statistic.h rename to relational_store/rdbmock/frameworks/native/rdb/sys/file.h index 1380fcfc..525ff1e4 100644 --- a/datamgr_service/services/distributeddataservice/framework/include/utils/time_statistic.h +++ b/relational_store/rdbmock/frameworks/native/rdb/sys/file.h @@ -12,15 +12,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef DISTRIBUTEDDATAMGR_DATAMGR_TIME_STATISTIC_H -#define DISTRIBUTEDDATAMGR_DATAMGR_TIME_STATISTIC_H -namespace OHOS { -namespace DistributedData { -class TimeStatistic { +#ifndef MOCK_SYSFILE_H +#define MOCK_SYSFILE_H -}; +#ifndef LOCK_EX +#define LOCK_EX 0 +#endif -} // namespace DistributedData -} // namespace OHOS -#endif //DISTRIBUTEDDATAMGR_DATAMGR_TIME_STATISTIC_H +#ifndef LOCK_UN +#define LOCK_UN 1 +#endif +#ifdef __cplusplus +extern "C" { +#endif +#define flock(fd, op) 0 +#ifdef __cplusplus +} +#endif +#endif /* MOCK_SYSFILE_H */ \ No newline at end of file diff --git a/relational_store/test/js/clouddata/unittest/src/BUILD.gn b/relational_store/test/js/clouddata/unittest/src/BUILD.gn index e54658a3..d99abd34 100644 --- a/relational_store/test/js/clouddata/unittest/src/BUILD.gn +++ b/relational_store/test/js/clouddata/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/relationalstore" +module_output_path = "relational_store/relational_store/relationalstore" ohos_js_unittest("CloudConfigJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/dataability/unittest/src/BUILD.gn b/relational_store/test/js/dataability/unittest/src/BUILD.gn index 1844e8cb..bf3a14a1 100644 --- a/relational_store/test/js/dataability/unittest/src/BUILD.gn +++ b/relational_store/test/js/dataability/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "dataability/dataability" +module_output_path = "relational_store/relational_store/dataability" ohos_js_unittest("DataAbJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/gdb/performance/src/BUILD.gn b/relational_store/test/js/gdb/performance/src/BUILD.gn index 2baf59ad..f5bc4ca2 100644 --- a/relational_store/test/js/gdb/performance/src/BUILD.gn +++ b/relational_store/test/js/gdb/performance/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/gdb_perf" +module_output_path = "relational_store/relational_store/gdb_perf" ohos_js_unittest("GdbPerfJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/gdb/unittest/src/BUILD.gn b/relational_store/test/js/gdb/unittest/src/BUILD.gn index 04e1cf5c..5ae4e323 100644 --- a/relational_store/test/js/gdb/unittest/src/BUILD.gn +++ b/relational_store/test/js/gdb/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/gdb" +module_output_path = "relational_store/relational_store/gdb" ohos_js_unittest("GdbJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/rdb/performance/src/BUILD.gn b/relational_store/test/js/rdb/performance/src/BUILD.gn index e7b7cdd2..3a81aa31 100644 --- a/relational_store/test/js/rdb/performance/src/BUILD.gn +++ b/relational_store/test/js/rdb/performance/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/performance" +module_output_path = "relational_store/relational_store/performance" ohos_js_unittest("RdbPerfJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/rdb/unittest/src/BUILD.gn b/relational_store/test/js/rdb/unittest/src/BUILD.gn index 138ba606..fffc5b3f 100644 --- a/relational_store/test/js/rdb/unittest/src/BUILD.gn +++ b/relational_store/test/js/rdb/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/rdb" +module_output_path = "relational_store/relational_store/rdb" ohos_js_unittest("RdbJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/rdb/unittest/src/RdbstorePredicatesJsunit.test.js b/relational_store/test/js/rdb/unittest/src/RdbstorePredicatesJsunit.test.js index f0d99b05..1e4bc962 100644 --- a/relational_store/test/js/rdb/unittest/src/RdbstorePredicatesJsunit.test.js +++ b/relational_store/test/js/rdb/unittest/src/RdbstorePredicatesJsunit.test.js @@ -298,7 +298,6 @@ describe('rdbPredicatesTest', function () { console.log(TAG + "************* testEqualTo0009 end *************"); }) - /** * @tc.name predicates notEqualTo normal test * @tc.number SUB_DDM_AppDataFWK_JSRDB_Predicates_0020 diff --git a/relational_store/test/js/relationalstore/performance/src/BUILD.gn b/relational_store/test/js/relationalstore/performance/src/BUILD.gn index d0814abe..a33549b1 100644 --- a/relational_store/test/js/relationalstore/performance/src/BUILD.gn +++ b/relational_store/test/js/relationalstore/performance/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/performance" +module_output_path = "relational_store/relational_store/performance" ohos_js_unittest("RelationalStorePerfJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/relationalstore/performance/src/RdbStorePromisePerf.js b/relational_store/test/js/relationalstore/performance/src/RdbStorePromisePerf.js index b03729bb..a0bcdf21 100644 --- a/relational_store/test/js/relationalstore/performance/src/RdbStorePromisePerf.js +++ b/relational_store/test/js/relationalstore/performance/src/RdbStorePromisePerf.js @@ -33,6 +33,7 @@ const BASE_COUNT = 1000; // loop times const BASE_LINE_TABLE = 1800; // callback tablet base line const BASE_LINE_PHONE = 3000; // callback phone base line const BASE_LINE = (deviceInfo.deviceType == "tablet") ? BASE_LINE_TABLE : BASE_LINE_PHONE; +const BASE_LINE_TRANS = 1500; describe('rdbStorePromisePerf', function () { beforeAll(async function () { @@ -84,4 +85,21 @@ describe('rdbStorePromisePerf', function () { console.info(TAG + "*************Unit Test End*************"); done(); }) + + it('SUB_DDM_PERF_RDB_createTransaction_Promise_001', 0, async function (done) { + console.info(TAG + "SUB_DDM_PERF_RDB_createTransaction_Promise_001 start"); + let totalTime = 0; + for (let i = 0; i < BASE_COUNT; i++) { + let startTime = new Date().getTime(); + let trans = await rdbStore?.createTransaction(); + let endTime = new Date().getTime(); + totalTime += endTime - startTime; + await trans?.commit(); + } + let averageTime = (totalTime * 1000) / BASE_COUNT; + console.info(TAG + " the SUB_DDM_PERF_RDB_createTransaction_Promise_001 average time is: " + averageTime + " μs"); + expect(averageTime < BASE_LINE_TRANS).assertTrue(); + console.info(TAG + "SUB_DDM_PERF_RDB_createTransaction_Promise_001 end"); + done(); + }); }) diff --git a/relational_store/test/js/relationalstore/performance/src/SceneGetValuesBucketPerf.js b/relational_store/test/js/relationalstore/performance/src/SceneGetValuesBucketPerf.js index 2113c85c..aec55042 100644 --- a/relational_store/test/js/relationalstore/performance/src/SceneGetValuesBucketPerf.js +++ b/relational_store/test/js/relationalstore/performance/src/SceneGetValuesBucketPerf.js @@ -163,7 +163,7 @@ describe('SceneGetValuesBucketPerf', function () { } for (let i = 0; i < 20; i++) { - switch (resultSet.getColumnType(indexes[i])) { + switch (resultSet.getColumnTypeSync(indexes[i])) { case 0: // TYPE_NULL values[FIELDS[i]] = null; break; diff --git a/relational_store/test/js/relationalstore/unittest/src/BUILD.gn b/relational_store/test/js/relationalstore/unittest/src/BUILD.gn index 59e02208..5c1e39bb 100644 --- a/relational_store/test/js/relationalstore/unittest/src/BUILD.gn +++ b/relational_store/test/js/relationalstore/unittest/src/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") -module_output_path = "relational_store/relationalstore" +module_output_path = "relational_store/relational_store/relationalstore" ohos_js_unittest("RelationalStoreJsTest") { module_out_path = module_output_path diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreAttachJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreAttachJsunit.test.js index 93439183..54b1af5b 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreAttachJsunit.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreAttachJsunit.test.js @@ -51,6 +51,12 @@ const STORE_CONFIG3 = { securityLevel: data_relationalStore.SecurityLevel.S1, } +const STORE_CONFIG_MEM = { + name: "memStore.db", + persist: false, + securityLevel: data_relationalStore.SecurityLevel.S1, +} + describe('ActsRdbStoreAttachTest', function () { beforeAll(async function () { console.info(TAG + 'beforeAll'); @@ -454,5 +460,91 @@ describe('ActsRdbStoreAttachTest', function () { console.log(TAG + "************* testRdbStoreAttach00014 end *************"); }) + /** + * @tc.name the attach function + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_attach_0015 + * @tc.desc attach memDb + */ + it('testRdbStoreAttach00015', 0, async function () { + console.log(TAG + "************* testRdbStoreAttach00015 start *************"); + let store = await data_relationalStore.getRdbStore(context, STORE_CONFIG); + let memStore = await data_relationalStore.getRdbStore(context, STORE_CONFIG_MEM); + await memStore.executeSql(CREATE_TABLE_TEST); + await attachBatchInsert(memStore, "test"); + try { + await store.attach(context, STORE_CONFIG_MEM, "attachMemDB"); + console.log("testRdbStoreAttach00015 attach success"); + let resultSet = await store.querySql("select * from attachMemDB.test"); + expect(resultSet.rowCount).assertEqual(10); + resultSet.close(); + } catch (e) { + console.log("testRdbStoreAttach00015 : failed, err: code=" + e.code + " message=" + e.message); + expect().assertFail(); + } finally { + store.close(); + memStore.close(); + } + console.log(TAG + "************* testRdbStoreAttach00015 end *************"); + }) + + /** + * @tc.name the attach function + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_attach_0016 + * @tc.desc memDb attach memDb + */ + it('testRdbStoreAttach00016', 0, async function () { + console.log(TAG + "************* testRdbStoreAttach00016 start *************"); + const STORE_CONFIG_MAIN = { + name: "mainMemStore.db", + persist: false, + securityLevel: data_relationalStore.SecurityLevel.S1, + } + let store = await data_relationalStore.getRdbStore(context, STORE_CONFIG_MAIN); + let memStore = await data_relationalStore.getRdbStore(context, STORE_CONFIG_MEM); + await memStore.executeSql(CREATE_TABLE_TEST); + await attachBatchInsert(memStore, "test"); + try { + await store.attach(context, STORE_CONFIG_MEM, "attachMemDB"); + console.log("testRdbStoreAttach00016 attach success"); + let resultSet = await store.querySql("select * from attachMemDB.test"); + expect(resultSet.rowCount).assertEqual(10); + resultSet.close(); + } catch (e) { + console.log("testRdbStoreAttach00016 : failed, err: code=" + e.code + " message=" + e.message); + expect().assertFail(); + } finally { + store.close(); + memStore.close(); + } + console.log(TAG + "************* testRdbStoreAttach00016 end *************"); + }) + + /** + * @tc.name the attach function + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_attach_0017 + * @tc.desc memDb attach walDb + */ + it('testRdbStoreAttach00017', 0, async function () { + console.log(TAG + "************* testRdbStoreAttach00017 start *************"); + let store = await data_relationalStore.getRdbStore(context, STORE_CONFIG); + await store.executeSql(CREATE_TABLE_TEST); + await attachBatchInsert(store, "test"); + let memStore = await data_relationalStore.getRdbStore(context, STORE_CONFIG_MEM); + try { + await memStore.attach(context, STORE_CONFIG, "attachWalDB"); + console.log("testRdbStoreAttach00017 attach success"); + let resultSet = memStore.querySqlSync("select * from attachWalDB.test"); + expect(resultSet.rowCount).assertEqual(10); + resultSet.close(); + } catch (e) { + console.log("testRdbStoreAttach00017 : failed, err: code=" + e.code + " message=" + e.message); + expect().assertFail(); + } finally { + store.close(); + memStore.close(); + } + console.log(TAG + "************* testRdbStoreAttach00017 end *************"); + }) + console.log(TAG + "*************Unit Test End*************"); }) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreCloud.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreCloud.test.js index 1a9ebb64..6bb706a2 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreCloud.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreCloud.test.js @@ -485,10 +485,99 @@ describe('rdbStoreCloud', function () { expect().assertFail(); done() }); + } catch (err) { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect(err.code == 401).assertTrue(); // Parameter error must be catched here. + done() + } + }) + + /** + * @tc.name set distributeded table cloud with promise + * @tc.number SUB_DDM_AppDataFWK_JSRDB_CLOUD_0017 + * @tc.desc set distributed table cloud with 'enableCloud' is true + */ + it('testRdbStoreCloud0017', 0, async function (done) { + console.log(TAG + "************* testRdbStoreCloud0017 start *************"); + try { + let config = { + autoSync: false, + enableCloud: true + } + await rdbStore.setDistributedTables(['employee'], + relationalStore.DistributedType.DISTRIBUTED_CLOUD, config).then(() => { + console.log(TAG + "set employee to be distributed cloud table success"); + expect(true).assertTrue(); + console.log(TAG + "************* testRdbStoreCloud0017 end *************"); + done(); + }).catch((err) => { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect().assertFail(); + done() + }); + } catch (err) { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect().assertFail(); + done() + } + }) + + /** + * @tc.name set distributeded table cloud with promise + * @tc.number SUB_DDM_AppDataFWK_JSRDB_CLOUD_0018 + * @tc.desc set distributed table cloud with 'enableCloud' is false + */ + it('testRdbStoreCloud0018', 0, async function (done) { + console.log(TAG + "************* testRdbStoreCloud0018 start *************"); + try { + let config = { + autoSync: false, + enableCloud: false + } + await rdbStore.setDistributedTables(['employee'], + relationalStore.DistributedType.DISTRIBUTED_CLOUD, config).then(() => { + console.log(TAG + "set employee to be distributed cloud table success"); + expect(true).assertTrue(); + console.log(TAG + "************* testRdbStoreCloud0018 end *************"); + done(); + }).catch((err) => { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect().assertFail(); + done() + }); + } catch (err) { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect().assertFail(); + done() + } + }) + + /** + * @tc.name set distributeded table cloud with promise + * @tc.number SUB_DDM_AppDataFWK_JSRDB_CLOUD_0019 + * @tc.desc set distributed table cloud with 'enableCloud' is 20 + */ + it('testRdbStoreCloud0019', 0, async function (done) { + console.log(TAG + "************* testRdbStoreCloud0019 start *************"); + try { + let config = { + autoSync: false, + enableCloud: 20 + } + await rdbStore.setDistributedTables(['employee'], + relationalStore.DistributedType.DISTRIBUTED_CLOUD, config).then(() => { + console.log(TAG + "set employee to be distributed cloud table success"); + expect().assertFail(); + console.log(TAG + "************* testRdbStoreCloud0019 end *************"); + done(); + }).catch((err) => { + console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); + expect().assertFail(); + done() + }); } catch (err) { console.log(TAG + `set employee to be distributed table failed, errcode:${JSON.stringify(err)}.`); expect(err.code == 401).assertTrue(); - expect(true).assertTrue(); // Parameter error must be catched here. done() } }) diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreRdExecuteJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreRdExecuteJsunit.test.js index b4c4570f..c303f9b3 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreRdExecuteJsunit.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreRdExecuteJsunit.test.js @@ -436,7 +436,7 @@ describe('rdbStoreRdExecuteTest', function () { await store?.execute(insert); } catch (e) { console.error(TAG + `testVectorSubSelectFailed0004 failed. err: ${JSON.stringify(e)}`); - expect(e.code).assertEqual(14800000); + expect(e.code).assertEqual(14800021); } console.log(TAG + "************* testVectorSubSelectFailed0004 end *************"); }); @@ -496,7 +496,7 @@ describe('rdbStoreRdExecuteTest', function () { await store?.execute(deleteSql); } catch (e) { console.error(TAG + `testVectorSubSelectFailed0005 failed. err: ${JSON.stringify(e)}`); - expect(e.code).assertEqual(14800000); + expect(e.code).assertEqual(14800021); } console.log(TAG + "************* testVectorSubSelectFailed0005 end *************"); }); diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetRow.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetRow.test.js index ccba5086..e687339c 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetRow.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetRow.test.js @@ -286,54 +286,6 @@ describe('rdbStoreResultSetGetRowTest', function () { console.log(TAG + "************* rdbStoreResultSetGetRowTest0009 end *************"); }) - /** - * @tc.name rdb store resultSet getRow test - * @tc.number rdbStoreResultSetGetRowTest0010 - * @tc.desc Insert a string greater than number.MAX_SAFE_INTEGER is also obtained as a string by getRow - */ - it('rdbStoreResultSetGetRowTest0010', 0, async () => { - console.info(TAG + "************* rdbStoreResultSetGetRowTest0010 start *************"); - let rowId = 0; - let valueNum = Number.MAX_SAFE_INTEGER.toString() + 0; - let valueBucket = { - data2: valueNum - }; - rowId = await rdbStore.insert("test", valueBucket); - expect(1).assertEqual(rowId); - let predicates = new data_relationalStore.RdbPredicates("test"); - let resultSet = await rdbStore.query(predicates); - resultSet.goToFirstRow(); - let res = resultSet.getRow(); - resultSet.close(); - console.info(TAG + 'valueNum ' + valueNum + ' getRow data2:' + res.data2 + ' type:' + typeof res.data2); - expect(valueNum).assertEqual(res.data2); - console.info(TAG + "************* rdbStoreResultSetGetRowTest0010 end *************"); - }); - - /** - * @tc.name rdb store resultSet getRow test - * @tc.number rdbStoreResultSetGetRowTest0011 - * @tc.desc Insert as a string equal to number.MAX_SAFE_INTEGER, getRow obtains the number - */ - it('rdbStoreResultSetGetRowTest0011', 0, async () => { - console.info(TAG + "************* rdbStoreResultSetGetRowTest0011 start *************"); - let rowId = 0; - let valueNum = Number.MAX_SAFE_INTEGER.toString(); - let valueBucket = { - data2: valueNum - }; - rowId = await rdbStore.insert("test", valueBucket); - expect(1).assertEqual(rowId); - let predicates = new data_relationalStore.RdbPredicates("test"); - let resultSet = await rdbStore.query(predicates); - resultSet.goToFirstRow(); - let res = resultSet.getRow(); - resultSet.close(); - console.info(TAG + 'valueNum ' + valueNum + ' getRow data2:' + res.data2 + ' type:' + typeof res.data2); - expect(Number.MAX_SAFE_INTEGER).assertEqual(res.data2); - console.info(TAG + "************* rdbStoreResultSetGetRowTest0011 end *************"); - }); - /** * @tc.name rdb store resultSet getRow test * @tc.number rdbStoreResultSetGetRows0001 diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetValue.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetValue.test.js index 84885534..5f56638e 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetValue.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetGetValue.test.js @@ -350,63 +350,5 @@ describe('rdbStoreResultSetGetValueTest', function () { console.log(TAG + "************* rdbStoreResultSetGetValueTest_0010 end *************"); }) - /** - * @tc.name rdb store resultSet getValue test - * @tc.number rdbStoreResultSetGetValueTest_0011 - * @tc.desc Insert a string greater than number.MAX_SAFE_INTEGER is also obtained as a string by getValue - */ - it('rdbStoreResultSetGetValueTest_0011', 0, async function () { - console.log(TAG + "************* rdbStoreResultSetGetValueTest_0011 start *************"); - let valueNum = Number.MAX_SAFE_INTEGER.toString() + 0; - let valueBucket = { - age: valueNum - }; - try { - let rowId = await rdbStore.insert("test", valueBucket); - expect(1).assertEqual(rowId); - } catch (err) { - console.error(`### insert failed, code:${err.code}, message:${err.message}`) - } - - try { - let resultSet = await rdbStore.querySql('SELECT * FROM test') - expect(true).assertEqual(resultSet.goToFirstRow()); - expect(valueNum).assertEqual(resultSet.getValue(resultSet.getColumnIndex('age'))); - resultSet.close(); - } catch (err) { - console.error(`### query failed, code:${err.code}, message:${err.message}`) - } - console.log(TAG + "************* rdbStoreResultSetGetValueTest_0011 end *************"); - }) - - /** - * @tc.name rdb store resultSet getValue test - * @tc.number rdbStoreResultSetGetValueTest_0012 - * @tc.desc Insert as a string equal to number.MAX_SAFE_INTEGER, getValue obtains the number - */ - it('rdbStoreResultSetGetValueTest_0012', 0, async function () { - console.log(TAG + "************* rdbStoreResultSetGetValueTest_0012 start *************"); - let valueNum = Number.MAX_SAFE_INTEGER.toString(); - let valueBucket = { - age: valueNum - }; - try { - let rowId = await rdbStore.insert("test", valueBucket); - expect(1).assertEqual(rowId); - } catch (err) { - console.error(`### insert failed, code:${err.code}, message:${err.message}`) - } - - try { - let resultSet = await rdbStore.querySql('SELECT * FROM test') - expect(true).assertEqual(resultSet.goToFirstRow()); - expect(Number.MAX_SAFE_INTEGER).assertEqual(resultSet.getValue(resultSet.getColumnIndex('age'))); - resultSet.close(); - } catch (err) { - console.error(`### query failed, code:${err.code}, message:${err.message}`) - } - console.log(TAG + "************* rdbStoreResultSetGetValueTest_0012 end *************"); - }) - console.log(TAG + "*************Unit Test End*************"); }) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetJsunit.test.js index c6447d7a..9607f421 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetJsunit.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetJsunit.test.js @@ -20,7 +20,8 @@ var context = ability_featureAbility.getContext() const TAG = "[RELATIONAL_STORE_JSKITS_TEST]" const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + - "data1 text," + "data2 long, " + "data3 double," + "data4 blob)"; + "data1 text," + "data2 long, " + "data3 double," + "data4 blob," + "data5 ASSET," + "data6 ASSETS," + + "data7 floatvector(128)," + "data8 UNLIMITED INT" + ")"; const STORE_CONFIG = { name: "Resultset.db", @@ -685,7 +686,7 @@ describe('rdbResultSetTest', function () { console.log(TAG + "************* testColumnCount0001 start *************"); let predicates = await new data_relationalStore.RdbPredicates("test") let resultSet = await rdbStore.query(predicates) - expect(5).assertEqual(resultSet.columnCount); + expect(9).assertEqual(resultSet.columnCount); resultSet.close(); resultSet = null; done(); @@ -2555,5 +2556,628 @@ describe('rdbResultSetTest', function () { done(); console.log(TAG + "************* testQuerySharingResource005 end *************"); }) + + /** + * @tc.name testgetColumnType001 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0245 + * @tc.desc testgetColumnType001 test + */ + it('testgetColumnType001', 0, async () => { + console.log(TAG + "************* testgetColumnType001 start *************"); + try { + const asset = { + name: "name4", + uri: "uri4", + createTime: "createTime4", + modifyTime: "modifyTime4", + size: "size4", + path: "path4", + } + const assets = [asset]; + const valueBucket = { + "data1": "hello world", + "data2": 3, + "data3": 10.5, + "data4": new Uint8Array([1, 2, 3]), + "data5":asset, + "data6":assets, + "data7":new Float32Array([1.5, 2.5]), + "data8":100n + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType001 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("id")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data1")); + expect(type).assertEqual(relationalStore.ColumnType.TEXT); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data2")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data3")); + expect(type).assertEqual(relationalStore.ColumnType.REAL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data4")); + expect(type).assertEqual(relationalStore.ColumnType.BLOB); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data5")); + expect(type).assertEqual(relationalStore.ColumnType.ASSET); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data6")); + expect(type).assertEqual(relationalStore.ColumnType.ASSETS); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data7")); + expect(type).assertEqual(relationalStore.ColumnType.FLOAT_VECTOR); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data8")); + expect(type).assertEqual(relationalStore.ColumnType.UNLIMITED_INT); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType001 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnType001 end *************"); + }); + + /** + * @tc.name testgetColumnType002 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0246 + * @tc.desc testgetColumnType002 test + */ + it('testgetColumnType002', 0, async () => { + console.log(TAG + "************* testgetColumnType002 start *************"); + try { + const valueBucket = { + "data1": null, + "data2": null, + "data3": null, + "data4": null, + "data5": null, + "data6": null, + "data7": null, + "data8": null + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType002 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("id")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data1")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data2")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data3")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data4")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data5")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data6")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data7")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType(resultSet?.getColumnIndex("data8")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType002 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnType002 end *************"); + }); + + /** + * @tc.name testgetColumnType003 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0247 + * @tc.desc testgetColumnType003 test + */ + it('testgetColumnType003', 0, async () => { + console.log(TAG + "************* testgetColumnType003 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType003 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.close(); + await resultSet?.getColumnType(resultSet?.getColumnIndex("id")); + console.error(TAG + "testgetColumnType003 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect('14800014').assertEqual(err.code); + console.error(TAG + "testgetColumnType003 getColumnType failed. " + err); + } + console.log(TAG + "************* testgetColumnType003 end *************"); + }); + + /** + * @tc.name testgetColumnType004 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0248 + * @tc.desc testgetColumnType004 test + */ + it('testgetColumnType004', 0, async () => { + console.log(TAG + "************* testgetColumnType004 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType004 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + await resultSet?.getColumnType(0); + console.error(TAG + "testgetColumnType004 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect(14800012).assertEqual(err.code); + resultSet.close(); + console.error(TAG + "testgetColumnType004 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnType004 end *************"); + }); + + /** + * @tc.name testgetColumnType005 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0248 + * @tc.desc testgetColumnType005 test + */ + it('testgetColumnType005', 0, async () => { + console.log(TAG + "************* testgetColumnType005 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType005 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + await resultSet?.getColumnType(columnCount); + console.error(TAG + "testgetColumnType005 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect(14800013).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnType005 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnType005 end *************"); + }); + + /** + * @tc.name testgetColumnType006 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0249 + * @tc.desc testgetColumnType006 test + */ + it('testgetColumnType006', 0, async () => { + console.log(TAG + "************* testgetColumnType006 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType006 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + await resultSet?.getColumnType(-1); + console.error(TAG + "testgetColumnType006 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect('401').assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnType006 getColumnType failed. " + err); + } + console.log(TAG + "************* testgetColumnType006 end *************"); + }); + + /** + * @tc.name testgetColumnType007 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0250 + * @tc.desc getColumnType test + */ + it('testgetColumnType007', 0, async () => { + console.log(TAG + "************* testgetColumnType007 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType007 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = await resultSet?.getColumnType(Number.MAX_SAFE_INTEGER); + console.error(TAG + "testgetColumnType007 failed type " + type); + expect(true).assertFail(); + } catch(err) { + console.error(TAG + "testgetColumnType007 success " + err); + expect('401').assertEqual(err.code); + resultSet?.close(); + } + console.log(TAG + "************* testgetColumnType007 end *************"); + }); + + /** + * @tc.name testgetColumnType008 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0251 + * @tc.desc getColumnType test + */ + it('testgetColumnType008', 0, async () => { + console.log(TAG + "************* testgetColumnType008 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType008 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = await resultSet?.getColumnType(Number.MAX_VALUE); + console.error(TAG + "testgetColumnType008 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType008 failed " + err); + } + console.log(TAG + "************* testgetColumnType008 end *************"); + }); + + /** + * @tc.name testgetColumnType009 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0252 + * @tc.desc getColumnType test + */ + it('testgetColumnType009', 0, async () => { + console.log(TAG + "************* testgetColumnType009 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType009 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = await resultSet?.getColumnType(Number.MIN_VALUE); + console.error(TAG + "testgetColumnType009 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType009 failed " + err); + } + console.log(TAG + "************* testgetColumnType009 end *************"); + }); + + /** + * @tc.name testgetColumnType010 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0253 + * @tc.desc getColumnType test + */ + it('testgetColumnType010', 0, async () => { + console.log(TAG + "************* testgetColumnType010 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType010 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = await resultSet?.getColumnType(Number.MIN_SAFE_INTEGER); + console.error(TAG + "testgetColumnType010 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType010 failed " + err); + } + console.log(TAG + "************* testgetColumnType010 end *************"); + }); + + /** + * @tc.name testgetColumnType011 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0254 + * @tc.desc getColumnType test + */ + it('testgetColumnType011', 0, async () => { + console.log(TAG + "************* testgetColumnType011 start *************"); + try { + const asset = { + name: "name4", + uri: "uri4", + createTime: "createTime4", + modifyTime: "modifyTime4", + size: "size4", + path: "path4", + } + const assets = [asset]; + const valueBucket = { + "data1": "hello world", + "data2": 3, + "data3": 100000, + "data4": new Uint8Array([1, 2, 3]), + "data5":asset, + "data6":assets, + "data7":new Float32Array([1.5, 2.5]), + "data8":100n + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType011 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = await resultSet?.getColumnType("id"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType("data1"); + expect(type).assertEqual(relationalStore.ColumnType.TEXT); + type = await resultSet?.getColumnType("data2"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType("data3"); + expect(type).assertEqual(relationalStore.ColumnType.REAL); + type = await resultSet?.getColumnType("data4"); + expect(type).assertEqual(relationalStore.ColumnType.BLOB); + type = await resultSet?.getColumnType("data5"); + expect(type).assertEqual(relationalStore.ColumnType.ASSET); + type = await resultSet?.getColumnType("data6"); + expect(type).assertEqual(relationalStore.ColumnType.ASSETS); + type = await resultSet?.getColumnType("data7"); + expect(type).assertEqual(relationalStore.ColumnType.FLOAT_VECTOR); + type = await resultSet?.getColumnType("data8"); + resultSet?.close(); + expect(type).assertEqual(relationalStore.ColumnType.UNLIMITED_INT); + } catch(err) { + console.error(TAG + "testgetColumnType011 querySql failed " + err); + } + console.log(TAG + "************* testgetColumnType011 end *************"); + }); + + /** + * @tc.name testgetColumnType012 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0255 + * @tc.desc getColumnType test + */ + it('testgetColumnType012', 0, async () => { + console.log(TAG + "************* testgetColumnType012 start *************"); + try { + const valueBucket = { + "data1": null, + "data2": null, + "data3": null, + "data4": null, + "data5": null, + "data6": null, + "data7": null, + "data8": null + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType012 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = await resultSet?.getColumnType("id"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = await resultSet?.getColumnType("data1"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data2"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data3"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data4"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data5"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data6"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data7"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = await resultSet?.getColumnType("data8"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnType012 failed " + err); + } + console.log(TAG + "************* testgetColumnType012 end *************"); + }); + + /** + * @tc.name testgetColumnType013 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0256 + * @tc.desc getColumnType test + */ + it('testgetColumnType013', 0, async () => { + console.log(TAG + "************* testgetColumnType013 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType013 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + await resultSet?.getColumnType(""); + console.error(TAG + "testgetColumnType013 failed "); + expect(true).assertFail(); + + } catch(err) { + expect('401').assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnType013 success " + err); + } + console.log(TAG + "************* testgetColumnType013 end *************"); + }); + + /** + * @tc.name testgetColumnType014 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0257 + * @tc.desc getColumnType test + */ + it('testgetColumnType014', 0, async () => { + console.log(TAG + "************* testgetColumnType014 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType014 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + await resultSet?.getColumnType("columnName"); + console.error(TAG + "testgetColumnType014 failed "); + expect(true).assertFail(); + + } catch(err) { + expect(401).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnType014 success " + err); + } + console.log(TAG + "************* testgetColumnType014 end *************"); + }); + + /** + * @tc.name testgetColumnType015 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0258 + * @tc.desc getColumnType test + */ + it('testgetColumnType015', 0, async () => { + console.log(TAG + "************* testgetColumnType015 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType015 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.close(); + await resultSet?.getColumnType("id"); + console.error(TAG + "testgetColumnType015 failed "); + expect(true).assertFail(); + } catch(err) { + expect('14800014').assertEqual(err.code); + console.error(TAG + "testgetColumnType015 success " + err); + } + console.log(TAG + "************* testgetColumnType015 end *************"); + }); + + /** + * @tc.name testgetColumnType016 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0259 + * @tc.desc getColumnType test + */ + it('testgetColumnType016', 0, async () => { + console.log(TAG + "************* testgetColumnType016 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType016 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + await resultSet?.getColumnType("id"); + console.error(TAG + "testgetColumnType016 failed "); + expect(true).assertFail(); + } catch(err) { + expect(14800012).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnType016 success " + err); + } + console.log(TAG + "************* testgetColumnType016 end *************"); + }); + + /** + * @tc.name testgetColumnType017 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0260 + * @tc.desc GetColumnType and close concurrent testing + */ + it('testgetColumnType017', 0, async () => { + console.log(TAG + "************* testgetColumnType017 start *************"); + try { + let predicates = new relationalStore.RdbPredicates("test"); + await rdbStore?.delete(predicates); + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnType017 insert failed " + err); + } + try { + for (let i = 0; i < 500; ++i) { + let resultSet = await rdbStore?.querySql("SELECT data1 FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.getColumnType(0); + resultSet?.close(); + } + } catch(err) { + expect('14800014').assertEqual(err.code); + console.error(TAG + "testgetColumnType017 success " + err); + } + console.log(TAG + "************* testgetColumnType017 end *************"); + }); console.log(TAG + "*************Unit Test End*************"); }) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetSyncJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetSyncJsunit.test.js index 33e202c7..e7909f4a 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetSyncJsunit.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreResultSetSyncJsunit.test.js @@ -16,11 +16,11 @@ import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from import data_relationalStore from '@ohos.data.relationalStore' import ability_featureAbility from '@ohos.ability.featureAbility' -var context = ability_featureAbility.getContext() - -const TAG = "[RELATIONAL_STORE_JSKITS_TEST]" +var context = ability_featureAbility.getContext(); +const TAG = "[RELATIONAL_STORE_JSKITS_RESULTSET_TEST]" const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + - "data1 text," + "data2 long, " + "data3 double," + "data4 blob)"; + "data1 text," + "data2 long, " + "data3 double," + "data4 blob," + "data5 ASSET," + "data6 ASSETS," + + "data7 floatvector(128)," + "data8 UNLIMITED INT" + ")"; const STORE_CONFIG = { name: "Resultset.db", @@ -685,7 +685,7 @@ describe('rdbResultSetSyncTest', function () { console.log(TAG + "************* testSyncColumnCount0001 start *************"); let predicates = await new data_relationalStore.RdbPredicates("test") let resultSet = rdbStore.querySync(predicates) - expect(5).assertEqual(resultSet.columnCount); + expect(9).assertEqual(resultSet.columnCount); resultSet.close(); resultSet = null; done(); @@ -2270,5 +2270,595 @@ describe('rdbResultSetSyncTest', function () { console.log(TAG + "************* testSyncBigData0014 end *************"); }) + + /** + * @tc.name testgetColumnTypeSync001 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0244 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync001', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync001 start *************"); + try { + const asset = { + name: "name4", + uri: "uri4", + createTime: "createTime4", + modifyTime: "modifyTime4", + size: "size4", + path: "path4", + } + const assets = [asset]; + const valueBucket = { + "data1": "hello world", + "data2": 3, + "data3": 10.5, + "data4": new Uint8Array([1, 2, 3]), + "data5":asset, + "data6":assets, + "data7":new Float32Array([1.5, 2.5]), + "data8":100n + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync001 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("id")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data1")); + expect(type).assertEqual(relationalStore.ColumnType.TEXT); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data2")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data3")); + expect(type).assertEqual(relationalStore.ColumnType.REAL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data4")); + expect(type).assertEqual(relationalStore.ColumnType.BLOB); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data5")); + expect(type).assertEqual(relationalStore.ColumnType.ASSET); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data6")); + expect(type).assertEqual(relationalStore.ColumnType.ASSETS); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data7")); + expect(type).assertEqual(relationalStore.ColumnType.FLOAT_VECTOR); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data8")); + expect(type).assertEqual(relationalStore.ColumnType.UNLIMITED_INT); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync001 getColumnTypeSync failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync001 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync002 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0245 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync002', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync002 start *************"); + try { + const valueBucket = { + "data1": null, + "data2": null, + "data3": null, + "data4": null, + "data5": null, + "data6": null, + "data7": null, + "data8": null + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync002 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("id")); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data1")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data2")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data3")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data4")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data5")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data6")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data7")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("data8")); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync002 getColumnTypeSync failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync002 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync003 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0246 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync003', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync003 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync003 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.close(); + resultSet?.getColumnTypeSync(resultSet?.getColumnIndex("id")); + console.error(TAG + "testgetColumnTypeSync003 getColumnTypeSync success"); + expect(true).assertFail(); + } catch(err) { + expect('14800014').assertEqual(err.code); + console.error(TAG + "testgetColumnTypeSync003 getColumnTypeSync failed. " + err); + } + console.log(TAG + "************* testgetColumnTypeSync003 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync004 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0247 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync004', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync004 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync004 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + resultSet?.getColumnTypeSync(0); + console.error(TAG + "testgetColumnTypeSync004 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect(14800012).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync004 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync004 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync005 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0248 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync005', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync005 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync005 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + resultSet?.getColumnTypeSync(columnCount); + console.error(TAG + "testgetColumnTypeSync005 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect(14800013).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync005 getColumnType failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync005 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync006 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0249 + * @tc.desc querySharingResource test + */ + it('testgetColumnTypeSync006', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync006 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync006 insert failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.getColumnTypeSync(-1); + console.error(TAG + "testgetColumnTypeSync006 getColumnType success "); + expect(true).assertFail(); + } catch(err) { + expect('401').assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync006 getColumnType failed. " + err); + } + console.log(TAG + "************* testgetColumnTypeSync006 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync007 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0250 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync007', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync007 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync007 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = resultSet?.getColumnTypeSync(Number.MAX_SAFE_INTEGER); + console.error(TAG + "testgetColumnTypeSync007 failed type " + type); + expect(true).assertFail(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync007 success " + err); + expect('401').assertEqual(err.code); + resultSet?.close(); + } + console.log(TAG + "************* testgetColumnTypeSync007 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync008 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0251 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync008', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync008 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync008 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = resultSet?.getColumnTypeSync(Number.MAX_VALUE); + console.error(TAG + "testgetColumnTypeSync008 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync008 failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync008 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync009 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0252 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync009', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync009 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync009 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = resultSet?.getColumnTypeSync(Number.MIN_VALUE); + console.error(TAG + "testgetColumnTypeSync009 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync009 failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync009 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync010 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0253 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync010', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync010 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync010 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = resultSet?.getColumnTypeSync(Number.MIN_SAFE_INTEGER); + console.error(TAG + "testgetColumnTypeSync010 success type " + type); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync010 failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync010 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync011 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0254 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync011', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync011 start *************"); + try { + const asset = { + name: "name4", + uri: "uri4", + createTime: "createTime4", + modifyTime: "modifyTime4", + size: "size4", + path: "path4", + } + const assets = [asset]; + const valueBucket = { + "data1": "hello world", + "data2": 3, + "data3": 100000, + "data4": new Uint8Array([1, 2, 3]), + "data5":asset, + "data6":assets, + "data7":new Float32Array([1.5, 2.5]), + "data8":100n + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync011 insert failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = resultSet?.getColumnTypeSync("id"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync("data1"); + expect(type).assertEqual(relationalStore.ColumnType.TEXT); + type = resultSet?.getColumnTypeSync("data2"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync("data3"); + expect(type).assertEqual(relationalStore.ColumnType.REAL); + type = resultSet?.getColumnTypeSync("data4"); + expect(type).assertEqual(relationalStore.ColumnType.BLOB); + type = resultSet?.getColumnTypeSync("data5"); + expect(type).assertEqual(relationalStore.ColumnType.ASSET); + type = resultSet?.getColumnTypeSync("data6"); + expect(type).assertEqual(relationalStore.ColumnType.ASSETS); + type = resultSet?.getColumnTypeSync("data7"); + expect(type).assertEqual(relationalStore.ColumnType.FLOAT_VECTOR); + type = resultSet?.getColumnTypeSync("data8"); + resultSet?.close(); + expect(type).assertEqual(relationalStore.ColumnType.UNLIMITED_INT); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync011 querySql failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync011 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync012 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0255 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync012', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync012 start *************"); + try { + const valueBucket = { + "data1": null, + "data2": null, + "data3": null, + "data4": null, + "data5": null, + "data6": null, + "data7": null, + "data8": null + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync012 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let type = relationalStore.ColumnType.NULL; + type = resultSet?.getColumnTypeSync("id"); + expect(type).assertEqual(relationalStore.ColumnType.INTEGER); + type = resultSet?.getColumnTypeSync("data1"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data2"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data3"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data4"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data5"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data6"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data7"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + type = resultSet?.getColumnTypeSync("data8"); + expect(type).assertEqual(relationalStore.ColumnType.NULL); + resultSet?.close(); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync012 failed " + err); + } + console.log(TAG + "************* testgetColumnTypeSync012 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync013 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0256 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync013', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync013 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync013 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + resultSet?.getColumnTypeSync(""); + console.error(TAG + "testgetColumnTypeSync013 failed "); + expect(true).assertFail(); + + } catch(err) { + expect('401').assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync013 success " + err); + } + console.log(TAG + "************* testgetColumnTypeSync013 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync014 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0257 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync014', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync014 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync014 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + let columnCount = resultSet?.columnCount; + expect(9).assertEqual(columnCount); + resultSet?.getColumnTypeSync("columnName"); + console.error(TAG + "testgetColumnType014 failed "); + expect(true).assertFail(); + } catch(err) { + expect(401).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync014 success " + err); + } + console.log(TAG + "************* testgetColumnTypeSync014 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync015 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0258 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync015', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync015 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync015 failed " + err); + } + try { + let resultSet = await rdbStore?.querySql("SELECT * FROM test"); + let count = resultSet?.rowCount; + expect(true).assertEqual(resultSet?.goToRow(count - 1)); + resultSet?.close(); + resultSet?.getColumnTypeSync("id"); + console.error(TAG + "testgetColumnTypeSync015 failed "); + expect(true).assertFail(); + } catch(err) { + expect('14800014').assertEqual(err.code); + console.error(TAG + "testgetColumnTypeSync015 success " + err); + } + console.log(TAG + "************* testgetColumnTypeSync015 end *************"); + }); + + /** + * @tc.name testgetColumnTypeSync016 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_ResultSet_0259 + * @tc.desc getColumnTypeSync test + */ + it('testgetColumnTypeSync016', 0, async () => { + console.log(TAG + "************* testgetColumnTypeSync016 start *************"); + try { + const valueBucket = { + "data1": null, + } + await rdbStore?.insert("test", valueBucket); + } catch(err) { + console.error(TAG + "testgetColumnTypeSync016 failed " + err); + } + let resultSet; + try { + resultSet = await rdbStore?.querySql("SELECT * FROM test"); + resultSet?.getColumnTypeSync("id"); + console.error(TAG + "testgetColumnTypeSync016 failed "); + expect(true).assertFail(); + } catch(err) { + expect(14800012).assertEqual(err.code); + resultSet?.close(); + console.error(TAG + "testgetColumnTypeSync016 success " + err); + } + console.log(TAG + "************* testgetColumnTypeSync016 end *************"); + }); console.log(TAG + "*************Unit Test End*************"); }) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreTransaction.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreTransaction.test.js index a973a7a2..3bd8700e 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbStoreTransaction.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreTransaction.test.js @@ -1185,11 +1185,11 @@ describe('rdbStoreTransactionTest', function () { transactionType: data_relationalStore.TransactionType.EXCLUSIVE }) console.log(TAG + "begin EXCLUSIVE success abnormal"); - exclusiveTrans.rollback(); + await exclusiveTrans.rollback(); } catch (e) { console.log(TAG + e); - expect(e.code).assertEqual(14800024) console.log(TAG + "begin EXCLUSIVE failed"); + expect(true).assertFail(); } var rowId = await deferredTrans.insert("test", valueBucket); console.log(TAG + "testTransactionIsolation0008 deferredTrans.insert row " + rowId) @@ -1994,7 +1994,7 @@ describe('rdbStoreTransactionTest', function () { expect(null).assertFail() } catch (e) { console.log(TAG + e + " code: " + e.code); - expect(e.code).assertEqual(14800014); + expect(e.code).assertEqual(14800021); } console.log(TAG + "testON_CONFLICT_ROLLBACKInTransaction0021 success"); } @@ -2038,7 +2038,7 @@ describe('rdbStoreTransactionTest', function () { expect(null).assertFail() } catch (e) { console.log(TAG + e + " code: " + e.code); - expect(e.code).assertEqual(14800014); + expect(e.code).assertEqual(14800021); } console.log(TAG + "testON_CONFLICT_ROLLBACKInTransaction0022 success"); } @@ -2076,7 +2076,7 @@ describe('rdbStoreTransactionTest', function () { expect(null).assertFail() } catch (e) { console.log(TAG + e + " code: " + e.code); - expect(e.code).assertEqual(14800014); + expect(e.code).assertEqual(14800021); } console.log(TAG + "testON_CONFLICT_ROLLBACKInTransaction0023 success"); } diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbStoreVectorJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbStoreVectorJsunit.test.js new file mode 100644 index 00000000..e1822a68 --- /dev/null +++ b/relational_store/test/js/relationalstore/unittest/src/RdbStoreVectorJsunit.test.js @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from 'deccjsunit/index' +import relationalStore from '@ohos.data.relationalStore' +import featureAbility from '@ohos.ability.featureAbility' + +var context = featureAbility.getContext(); + +const TAG = "[RELATIONAL_STORE_VECTOR_JSKITS_TEST]" + +const STORE_CONFIG = { + name: "VectorTest.db", + securityLevel: relationalStore.SecurityLevel.S1, + vector: true +} +var store = undefined; +var isSupported = false; +describe('ActsRdbStoreVectorTest', function () { + beforeAll(async function () { + console.info(TAG + 'beforeAll'); + isSupported = relationalStore.isVectorSupported(); + if (!isSupported) { + return; + } + store = await relationalStore.getRdbStore(context, STORE_CONFIG); + expect(store != null).assertTrue(); + }) + + beforeEach(async function () { + if (!isSupported) { + return; + } + await store.execute("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 floatvector(2));"); + let floatVector = Float32Array.from([1.2, 2.3]); + await store.execute("INSERT INTO test (id, data1) VALUES (?, ?);", 0, [1, floatVector]); + console.info(TAG + 'beforeEach'); + }) + + afterEach(async function () { + if (!isSupported) { + return; + } + await store.execute("DROP TABLE IF EXISTS test;"); + console.info(TAG + 'afterEach'); + }) + + afterAll(async function () { + if (!isSupported) { + return; + } + console.info(TAG + 'afterAll'); + store = null; + await relationalStore.deleteRdbStore(context, STORE_CONFIG); + }) + + console.log(TAG + "*************Unit Test Begin*************"); + + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0001 + * @tc.desc Vector test + */ + it('testVectorStore0001', 0, async function () { + console.log(TAG + "************* testVectorStore0001 start *************"); + if (!isSupported) { + return; + } + try { + let resultSet = await store.querySql("select * from test where id = ?;", [1]); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let floats = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(floats[0] > 1).assertTrue(); + expect(floats[1] > 1).assertTrue(); + } + resultSet.close(); + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0002 + * @tc.desc Vector test + */ + it('testVectorStore0002', 0, async function () { + console.log(TAG + "************* testVectorStore0002 start *************"); + if (!isSupported) { + return; + } + try { + let floatVector = Float32Array.from([1.2, 2.3]); + let resultSet = await store.querySql("select id, data1 <-> ? as distance from test ORDER BY " + + "data1 <-> '[1.5,5.6]' limit 5;", [floatVector]); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let distance = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(distance == 0).assertTrue(); + } + resultSet.close(); + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0003 + * @tc.desc Vector test + */ + it('testVectorStore0003', 0, async function () { + console.log(TAG + "************* testVectorStore0003 start *************"); + if (!isSupported) { + return; + } + try { + let floatVector = Float32Array.from([1.5, 2.7]); + let resultSet = await store.querySql("select id, data1 <-> ? as distance from test where id = ? and " + + "data1 <-> ? > 0.5 ORDER BY data1 <-> ? limit 10;", [floatVector, 1, floatVector, floatVector]); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let distance = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(distance > 0).assertTrue(); + } + resultSet.close(); + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0004 + * @tc.desc Vector test + */ + it('testVectorStore0004', 0, async function () { + console.log(TAG + "************* testVectorStore0004 start *************"); + if (!isSupported) { + return; + } + try { + let floatVector = Float32Array.from([1.5, 2.7]); + let resultSet = await store.querySql("select id, data1 from test where id > ? group by " + + "id, data1 having max(data1<=>?);", [0, floatVector]); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let floats = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(floats[0] > 0).assertTrue(); + expect(floats[1] > 0).assertTrue(); + } + resultSet.close(); + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0005 + * @tc.desc Vector test + */ + it('testVectorStore0005', 0, async function () { + console.log(TAG + "************* testVectorStore0005 start *************"); + if (!isSupported) { + return; + } + try { + let resultSet = await store.querySql("select id, data1 <-> '[1.5, 3.0]' as distance from test union select " + + "id, data1 <-> '[1.5, 3.0]' as distance from test order by distance limit 5;"); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let distance = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(distance > 0).assertTrue(); + } + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + + /** + * @tc.name Vector test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Vector_0006 + * @tc.desc Vector test + */ + it('testVectorStore0006', 0, async function () { + console.log(TAG + "************* testVectorStore0006 start *************"); + if (!isSupported) { + return; + } + try { + await this.store.execute("create view v1 as select * from test where id > 0;"); + let resultSet = await store.querySql("select * from v1 where id > ?", [0]); + expect(1).assertEqual(resultSet.rowCount); + while (resultSet.goToNextRow()) { + let dataId = resultSet.getLong(0); + let floats = resultSet.getValue(1); + expect(1).assertEqual(dataId); + expect(floats[0] > 0).assertTrue(); + expect(floats[1] > 0).assertTrue(); + } + } catch (err) { + console.log(TAG + `Query failed,code is ${err.code},message is ${err.message}`); + } + }) + + console.log(TAG + "*************Unit Test End*************"); +}) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbstoreConcurrentQueryWithCrud.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbstoreConcurrentQueryWithCrud.test.js new file mode 100644 index 00000000..cf01c404 --- /dev/null +++ b/relational_store/test/js/relationalstore/unittest/src/RdbstoreConcurrentQueryWithCrud.test.js @@ -0,0 +1,704 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from 'deccjsunit/index' +import relationalStore from '@ohos.data.relationalStore'; +import ability_featureAbility from '@ohos.ability.featureAbility' + +var context = ability_featureAbility.getContext(); +let rdbStore; +const TAG = "[QuerWithCrud]" +const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "name TEXT NOT NULL, " + "age INTEGER, " + "salary REAL, " + "blobType BLOB)"; + +const STORE_CONFIG = { + name: "CrudTest.db", + securityLevel: relationalStore.SecurityLevel.S1, +} +describe('QueryWithCrudTest', function () { + beforeAll(async function () { + console.info(TAG + 'beforeAll'); + rdbStore = await relationalStore.getRdbStore(context, STORE_CONFIG); + }) + + beforeEach(async function () { + console.info(TAG + 'beforeEach'); + await rdbStore.executeSql(CREATE_TABLE_TEST); + }) + + afterEach(async function () { + console.info(TAG + 'afterEach'); + await rdbStore.executeSql("DROP TABLE IF EXISTS test"); + }) + + afterAll(async function () { + console.info(TAG + 'afterAll'); + await relationalStore.deleteRdbStore(context, STORE_CONFIG.name); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete001 + * @tc.desc 10 records with a total of less than 2M data, + * delete 3 records, rowCount equals 7 records. + */ + it('QueryWithDelete001', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete001 start *************"); + let u8 = new Uint8Array(Array(1).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 5, 7); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(7); + done(); + console.log(TAG + "************* QueryWithDelete001 end *************"); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete002 + * @tc.desc 10 pieces of 1M data, delete 3 pieces, rowCount equals 7 pieces. + */ + it('QueryWithDelete002', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete002 start *************"); + let u8 = new Uint8Array(Array(1024 * 1024).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 5, 7); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(7); + done(); + console.log(TAG + "************* QueryWithDelete002 end *************"); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete003 + * @tc.desc Single data exceeding 2M can return an error normally + */ + it('QueryWithDelete003', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete003 start *************"); + let u8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + for (let i = 0; i < 2; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.equalTo('age', 1) + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(1); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + expect().assertFail(); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + expect(err.code).assertEqual('14800000'); + done(); + } + console.log(TAG + "rowCount:" + resultSet.rowCount); + done(); + console.log(TAG + "************* QueryWithDelete003 end *************"); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete004 + * @tc.desc 2 pieces of single 2M data, delete 2 pieces, rowCount equals 0. + */ + it('QueryWithDelete004', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete004 start *************"); + let u8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + for (let i = 0; i < 2; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age',0,1) + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(2); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(0); + done(); + console.log(TAG + "************* QueryWithDelete004 end *************"); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete005 + * @tc.desc 10 pieces of data, one of which is greater than 2M, the other 1M, + * delete 9 pieces of data less than 1M, expect an error. + */ + it('QueryWithDelete005', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete005 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(2).fill(1)); + for (let i = 0; i < 9; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + const value = { + "name": "lisi", + "age": 30, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", value); + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 0, 8); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(9); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + expect().assertFail(); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + expect(err.code).assertEqual('14800000'); + done(); + } + done(); + console.log(TAG + "************* QueryWithDelete005 end *************"); + }) + + /** + * @tc.name Concurrent query and delet test + * @tc.number QueryWithDelete006 + * @tc.desc 10 pieces of data, one of which is greater than 2M, the other 1M, + * delete data greater than 2M, rowCount is 9. + */ + it('QueryWithDelete006', 0, async function (done) { + console.log(TAG + "************* QueryWithDelete006 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(2).fill(1)); + for (let i = 0; i < 9; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + const value = { + "name": "lisi", + "age": 30, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", value); + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.equalTo('age', 30); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(1); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(9); + done(); + console.log(TAG + "************* QueryWithDelete006 end *************"); + }) + + /** + * @tc.name Concurrent query and insert test + * @tc.number QueryWithInsert001 + * @tc.desc 10 pieces of data with a total size less than 2M, delete 4 pieces, + * insert 1 piece of data with a single size less than 1M, rowCount is equal to 7. + */ + it('QueryWithInsert001', 0, async function (done) { + console.log(TAG + "************* QueryWithInsert001 start *************"); + let u8 = new Uint8Array(Array(1).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 1, 4); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(4); + const value = { + "name": "lisi", + "age": 30, + "salary": 100.5, + "blobType": u8, + }; + let rowId = await rdbStore.insert('test', value); + expect(rowId).assertEqual(11); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(7); + done(); + console.log(TAG + "************* QueryWithInsert001 end *************"); + }) + + /** + * @tc.name Concurrent query and insert test + * @tc.number QueryWithInsert002 + * @tc.desc 10 pieces of data with a total size less than 2M, delete 4 pieces, + * insert 3 pieces of data with a single size less than 2M, rowCount is equal to 9. + */ + it('QueryWithInsert002', 0, async function (done) { + console.log(TAG + "************* QueryWithInsert002 start *************"); + let u8 = new Uint8Array(Array(1024 * 1024).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 1, 4); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(4); + const value = Array(3).fill(0).map(() => { + return { + "name": "zhangsan", + "age": 20, + "salary": 100.5, + "blobType": u8, + }; + }); + let insertRows = await rdbStore.batchInsert('test', value); + expect(insertRows).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(9); + done(); + console.log(TAG + "************* QueryWithInsert002 end *************"); + }) + + /** + * @tc.name Concurrent query and insert test + * @tc.number QueryWithInsert003 + * @tc.desc 3 pieces of data larger than 2M, delete 2 pieces, + * insert 1 piece of data smaller than 1M, expect an error. + */ + it('QueryWithInsert003', 0, async function (done) { + console.log(TAG + "************* QueryWithInsert003 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(2).fill(1)); + for (let i = 0; i < 3; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 0, 1); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(2); + const value = { + "name": "zhangsan", + "age": 30, + "salary": 100.5, + "blobType": u8, + }; + let insertRows = await rdbStore.insert('test', value); + expect(insertRows).assertEqual(4); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + expect().assertFail(); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + expect(err.code).assertEqual('14800000'); + done(); + } + done(); + console.log(TAG + "************* QueryWithInsert003 end *************"); + }) + + /** + * @tc.name Concurrent query and insert test + * @tc.number QueryWithInsert004 + * @tc.desc 2 pieces of data larger than 2M, delete 2 pieces, + * insert 1 piece of data smaller than 1M, rowCount is equal to 1. + */ + it('QueryWithInsert004', 0, async function (done) { + console.log(TAG + "************* QueryWithInsert004 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(2).fill(1)); + for (let i = 0; i < 2; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 0, 1); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(2); + const value = { + "name": "zhangsan", + "age": 30, + "salary": 100.5, + "blobType": u8, + }; + let rowId = await rdbStore.insert('test', value); + expect(rowId).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(1); + done(); + console.log(TAG + "************* QueryWithInsert004 end *************"); + }) + /** + * @tc.name Concurrent query and insert test + * @tc.number QueryWithInsert005 + * @tc.desc 10 pieces of data with a total of less than 2M, delete 6 pieces, + * insert 3 pieces of data with a single value greater than 2M, expect an error. + */ + it('QueryWithInsert005', 0, async function (done) { + console.log(TAG + "************* QueryWithInsert005 start *************"); + let u8 = new Uint8Array(Array(2).fill(1)); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + predicates.between('age', 0, 5); + let deleteRows = await rdbStore.delete(predicates); + expect(deleteRows).assertEqual(6); + const value = { + "name": "zhangsan", + "age": 30, + "salary": 100.5, + "blobType": bigU8, + }; + let rowId = await rdbStore.insert('test', value); + expect(rowId).assertEqual(11); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + expect(err.code).assertEqual('14800000'); + done(); + } + console.log(TAG + "rowCount:" + resultSet.rowCount); + done(); + console.log(TAG + "************* QueryWithInsert005 end *************"); + }) + + /** + * @tc.name Concurrent query and update test + * @tc.number QueryWithUpdate001 + * @tc.desc 10 records with a total of less than 2M data, updated 3 records, rowCount is 10. + */ + it('QueryWithUpdate001', 0, async function (done) { + console.log(TAG + "************* QueryWithUpdate001 start *************"); + let u8 = new Uint8Array(Array(1).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + const value = { + "name": "lisi", + "age": 18, + "salary": 200.5, + "blobType": u8, + }; + predicates.between("age", 5, 7) + + let updateRows = await rdbStore.update(value, predicates); + expect(updateRows).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(10); + done(); + console.log(TAG + "************* QueryWithUpdate001 end *************"); + }) + + /** + * @tc.name Concurrent query and update test + * @tc.number QueryWithUpdate002 + * @tc.desc 10 pieces of single data less than 1M, update 3 pieces, rowCount is 10. + */ + it('QueryWithUpdate002', 0, async function (done) { + console.log(TAG + "************* QueryWithUpdate002 start *************"); + let u8 = new Uint8Array(Array(1024 * 1024).fill(1)); + for (let i = 0; i < 10; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": u8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + const value = { + "name": "lisi", + "age": 18, + "salary": 200.5, + "blobType": new Uint8Array(Array(1).fill(2)), + }; + predicates.between("age", 5, 7) + let updateRows = await rdbStore.update(value, predicates); + expect(updateRows).assertEqual(3); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(10); + done(); + console.log(TAG + "************* QueryWithUpdate002 end *************"); + }) + + /** + * @tc.name Concurrent query and update test + * @tc.number QueryWithUpdate003 + * @tc.desc 2 pieces of data larger than 2M, update 2 pieces of data smaller than 1M, + * rowCount is 2 + */ + it('QueryWithUpdate003', 0, async function (done) { + console.log(TAG + "************* QueryWithUpdate003 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(3).fill(1)); + for (let i = 0; i < 2; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + const value = { + "name": "lisi", + "age": 18, + "salary": 200.5, + "blobType": u8, + }; + predicates.between("age", 0, 1); + let updateRows = await rdbStore.update(value, predicates); + expect(updateRows).assertEqual(2); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + expect(resultSet.rowCount).assertEqual(2); + done(); + console.log(TAG + "************* QueryWithUpdate003 end *************"); + }) + + /** + * @tc.name Concurrent query and update test + * @tc.number QueryWithUpdate004 + * @tc.desc 4 pieces of single data less than 1M, update 2 pieces of single data greater than 2M, + * expect an error + */ + it('QueryWithUpdate004', 0, async function (done) { + console.log(TAG + "************* QueryWithUpdate004 start *************"); + let bigU8 = new Uint8Array(Array(1024 * 1024 * 2).fill(1)); + let u8 = new Uint8Array(Array(3).fill(1)); + for (let i = 0; i < 2; i++) { + const valueBucket = { + "name": "zhangsan" + String(i), + "age": i, + "salary": 100.5, + "blobType": bigU8, + }; + await rdbStore.insert("test", valueBucket); + } + let predicates = new relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + const value = { + "name": "lisi", + "age": 18, + "salary": 200.5, + "blobType": u8, + }; + predicates.between("age", 0, 1); + let updateRows = await rdbStore.update(value, predicates); + expect(updateRows).assertEqual(2); + try { + while (resultSet.goToNextRow()) { + const age = resultSet.getString(resultSet.getColumnIndex("age")); + console.log(TAG + "age:" + age); + } + } catch (err) { + console.log(TAG + "catch err: failed, err: code=" + err.code + " message=" + err.message); + done(); + expect().assertFail(); + } + console.log(TAG + "rowCount:" + resultSet.rowCount); + done(); + console.log(TAG + "************* QueryWithUpdate004 end *************"); + }) + console.log(TAG + "*************Unit Test End*************"); +}) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbstoreMemoryDbJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbstoreMemoryDbJsunit.test.js new file mode 100644 index 00000000..da7b8913 --- /dev/null +++ b/relational_store/test/js/relationalstore/unittest/src/RdbstoreMemoryDbJsunit.test.js @@ -0,0 +1,1617 @@ +/* + * Copyright (C) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from 'deccjsunit/index' +import data_relationalStore from '@ohos.data.relationalStore'; +import ability_featureAbility from '@ohos.ability.featureAbility' + +const TAG = "[RELATIONAL_STORE_JSKITS_MEMORY_DB_TEST]" +const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "name TEXT NOT NULL, " + "age INTEGER, " + "salary REAL, " + "blobType BLOB)"; + +const STORE_CONFIG = { + name: "MemoryDbTest.db", + securityLevel: data_relationalStore.SecurityLevel.S1, + persist: false, +} + +var rdbStore = undefined +var context = ability_featureAbility.getContext() +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }) +} +describe('rdbStoreMemoryDbTest', function () { + beforeAll(async function () { + console.info(TAG + 'beforeAll') + rdbStore = await data_relationalStore.getRdbStore(context, STORE_CONFIG); + }) + + beforeEach(async function () { + console.info(TAG + 'beforeEach') + await rdbStore.executeSql(CREATE_TABLE_TEST, null); + }) + + afterEach(async function () { + console.info(TAG + 'afterEach') + await rdbStore.executeSql("DROP TABLE IF EXISTS test") + }) + + afterAll(async function () { + console.info(TAG + 'afterAll') + rdbStore = null + await data_relationalStore.deleteRdbStore(context, "MemoryDbTest.db"); + }) + + console.log(TAG + "*************Unit Test Begin*************"); + + /** + * @tc.number SUB_DDM_JSRDB_MEMORY_DB_NOT_SUPPORT_0001 + * @tc.name nor support test case of memory db + * @tc.desc + */ + it('testMemoryDbNotSupport0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbNotSupport0001 start *************"); + let predicates = new data_relationalStore.RdbPredicates("test"); + try { + await rdbStore.cleanDirtyData("test"); + expect(null).assertFail() + } catch (err) { + console.log("cleanDirtyData catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + await rdbStore.backup("memoryBackup"); + expect(null).assertFail(); + } catch (err) { + console.log("backup catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + await rdbStore.restore("memoryBackup"); + expect(null).assertFail(); + } catch (err) { + console.log("restore catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + await rdbStore.setDistributedTables(["test"]); + expect(null).assertFail(); + } catch (err) { + console.log("setDistributedTables catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + await rdbStore.sync(data_relationalStore.SyncMode.SYNC_MODE_PUSH, predicates); + expect(null).assertFail(); + } catch (err) { + console.log("sync catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + done(); + console.log(TAG + "************* testMemoryDbNotSupport0001 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_MEMORY_DB_NOT_SUPPORT_0002 + * @tc.name nor support on test case of memory db + * @tc.desc + */ + it('testMemoryDbNotSupport0002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbNotSupport0002 start *************"); + let predicates = new data_relationalStore.RdbPredicates("test"); + function storeObserver(devices) { + console.info(TAG + devices + " dataChange"); + expect(devices).assertEqual(null) + } + try { + rdbStore.on("dataChange", data_relationalStore.SubscribeType.SUBSCRIBE_TYPE_REMOTE, storeObserver); + expect(null).assertFail(); + } catch (err) { + console.log("on dataChange SUBSCRIBE_TYPE_REMOTE catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + rdbStore.on("dataChange", data_relationalStore.SubscribeType.SUBSCRIBE_TYPE_CLOUD, storeObserver); + expect(null).assertFail(); + } catch (err) { + console.log("on dataChange SUBSCRIBE_TYPE_CLOUD catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + rdbStore.on("dataChange", data_relationalStore.SubscribeType.SUBSCRIBE_TYPE_CLOUD_DETAILS, storeObserver); + expect(null).assertFail(); + } catch (err) { + console.log("on dataChange SUBSCRIBE_TYPE_CLOUD_DETAILS catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + rdbStore.on("dataChange", data_relationalStore.SubscribeType.SUBSCRIBE_TYPE_LOCAL_DETAILS, storeObserver); + expect(null).assertFail(); + } catch (err) { + console.log("on dataChange SUBSCRIBE_TYPE_LOCAL_DETAILS catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + rdbStore.on("autoSyncProgress",function (detail) { + console.log(TAG + `Progress:` + JSON.stringify(detail)); + }); + expect(null).assertFail(); + } catch (err) { + console.log("on autoSyncProgress catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + rdbStore.on("shareEvent", true, function () { + }); + expect(null).assertFail(); + } catch (err) { + console.log("on interProcess catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + done(); + console.log(TAG + "************* testMemoryDbNotSupport0002 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_MEMORY_DB_NOT_SUPPORT_0003 + * @tc.name nor support getRdbStore test case of memory db + * @tc.desc + */ + it('testMemoryDbNotSupport0003', 0, async function (done) { + console.log(TAG + "************* testMemoryDbNotSupport0003 start *************"); + let config = { + name: "testMemoryDbNotSupport0003.db", + securityLevel: data_relationalStore.SecurityLevel.S1, + persist: false, + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.encrypt = true; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore encrypt catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.customDir = "/mem"; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore customDir catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.rootDir = "/data/mem"; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore rootDir catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.isSearchable = true; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore isSearchable catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.vector = true; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore vector catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.isReadOnly = true; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore isReadOnly catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.haMode = data_relationalStore.HAMode.MAIN_REPLICA; + let store = await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore HAMode catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + try { + let tmp = JSON.parse(JSON.stringify(config)) + tmp.cryptoParam = { + encryptionKey: new Uint8Array(['t', 'e', 's', 't', 'k', 'e', 'y']), + iterationCount: 25000, + encryptionAlgo: data_relationalStore.EncryptionAlgo.AES_256_CBC, + hmacAlgo: data_relationalStore.HmacAlgo.SHA512, + kdfAlgo: data_relationalStore.KdfAlgo.KDF_SHA512, + cryptoPageSize: 1024 + }; + await data_relationalStore.getRdbStore(context, tmp); + expect(null).assertFail(); + } catch (err) { + console.log("getRdbStore cryptoParam catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 801); + } + done(); + console.log(TAG + "************* testMemoryDbNotSupport0003 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_Insert_0001 + * @tc.name Normal test case of insert + * @tc.desc 1.Insert data + * 2.Query data + */ + it('testMemoryDbInsert0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbInsert0001 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 20, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + + let predicates = new data_relationalStore.RdbPredicates("test"); + predicates.equalTo("name", "zhangsan") + let resultSet = await rdbStore.query(predicates) + try { + console.log(TAG + "resultSet query done"); + expect(true).assertEqual(resultSet.goToFirstRow()) + const id = resultSet.getLong(resultSet.getColumnIndex("id")) + const name = resultSet.getString(resultSet.getColumnIndex("name")) + const age = resultSet.getLong(resultSet.getColumnIndex("age")) + const salary = resultSet.getDouble(resultSet.getColumnIndex("salary")) + const blobType = resultSet.getBlob(resultSet.getColumnIndex("blobType")) + console.log(TAG + "id=" + id + ", name=" + name + ", age=" + age + ", salary=" + salary + ", blobType=" + blobType); + expect(1).assertEqual(id); + expect("zhangsan").assertEqual(name) + expect(18).assertEqual(age) + expect(100.5).assertEqual(salary) + expect(1).assertEqual(blobType[0]) + expect(2).assertEqual(blobType[1]) + expect(3).assertEqual(blobType[2]) + expect(false).assertEqual(resultSet.goToNextRow()) + } catch (e) { + console.log("insert1 error " + e); + } + resultSet.close() + resultSet = null + done() + console.log(TAG + "************* testMemoryDbInsert0001 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_Insert_0002 + * @tc.name Abnormal test case of insert, if TABLE name is wrong + * @tc.desc 1.Create value + * 2.Execute insert (with wrong table) + */ + it('testMemoryDbInsert0002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbInsert0002 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + try{ + let ret = await rdbStore.insert("wrong", valueBucket); + console.log(TAG + "insert wrong success: " + ret) + expect(null).assertFail() + } catch (err) { + console.log("testMemoryDbInsert0002 insert wrong catch err: failed, err: code=" + err.code + " message=" + err.message) + expect(true).assertEqual(err.code == 14800021); + } + done() + console.log(TAG + "************* testMemoryDbInsert0002 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_Insert_0003 + * @tc.name Normal test case of insert (Chinese and long string) + * @tc.desc 1.Insert data + * 2.Configure predicates + * 3.Query data + */ + it('testMemoryDbInsert0003', 0, async function (done) { + console.log(TAG + "************* testMemoryDbInsert0003 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + var nameStr = "苹果是水果" + "e".repeat(2000) + const valueBucket = { + "name": nameStr, + "age": 20, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + let predicates = new data_relationalStore.RdbPredicates("test"); + predicates.equalTo("age", 20) + let resultSet = await rdbStore.query(predicates) + try { + console.log(TAG + "resultSet query done"); + expect(true).assertEqual(resultSet.goToFirstRow()) + const name = resultSet.getString(resultSet.getColumnIndex("name")) + console.log(TAG + "id=" + id + ", name=" + name + ", age=" + age + ", salary=" + salary + ", blobType=" + blobType); + expect(nameStr).assertEqual(name) + } catch (e) { + console.log("insert error " + e); + } + resultSet.close() + resultSet = null + done() + console.log(TAG + "************* testMemoryDbInsert0003 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_Insert_With_Conflict_resolution_0001 + * @tc.name Abnormal test case of insert with ON_CONFLICT_ROLLBACK, if primary key conflict + * @tc.desc 1.Insert data with ON_CONFLICT_ROLLBACK + * 2.Create value (conflict "id") + * 3.Begin Transaction + * 4.Insert data + * 5.Insert data with ON_CONFLICT_ROLLBACK (conflict "id") + * 6.Query data + */ + it('testMemoryDbInsertWithConflictResolution0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbInsertWithConflictResolution0001 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + { + const valueBucket = { + "id": 1, + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket, data_relationalStore.ConflictResolution.ON_CONFLICT_ROLLBACK); + } + + { + const valueBucket = { + "id": 1, + "name": "zhangsan", + "age": 18, + "salary": 200.5, + "blobType": u8, + } + + rdbStore.beginTransaction() + const valueBucketInsert = { + "name": "wangwu", + "age": 30, + "salary": 400.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucketInsert) + try { + await rdbStore.insert("test", valueBucket, data_relationalStore.ConflictResolution.ON_CONFLICT_ROLLBACK); + expect(null).assertFail(); + } catch (err) { + console.log("catch err: failed, err: code=" + err.code + " message=" + err.message); + expect(14800032).assertEqual(err.code); + } + } + + { + let predicates = await new data_relationalStore.RdbPredicates("test"); + let resultSet = await rdbStore.query(predicates); + + expect(1).assertEqual(resultSet.rowCount); + resultSet.close(); + done(); + } + + console.log(TAG + "************* testMemoryDbInsertWithConflictResolution0001 end *************"); + }) + + /** + * @tc.number SUB_DDM_JSRDB_BatchInsert_0001 + * @tc.name Normal test case of batchInsert + * @tc.desc 1.Create valueBucket + * 2.Execute push + * 3.BatchInsert data + * 4.Query data + */ + it('testMemoryDbBatchInsert0001', 0, async function () { + console.log(TAG + "************* testMemoryDbBatchInsert0001 start *************"); + + var u8 = new Uint8Array([1, 2, 3]) + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + let valueBucketArray = new Array(); + for (let i = 0; i < 100; i++) { + valueBucketArray.push(valueBucket); + } + await rdbStore.batchInsert("test", valueBucketArray); + let resultSet = await rdbStore.querySql("SELECT * FROM test"); + let count = resultSet.rowCount; + expect(100).assertEqual(count); + resultSet.close() + console.log(TAG + "************* testMemoryDbBatchInsert0001 end *************"); + }) + + /** + * @tc.number testMemoryDbBatchInsertWithConflictResolution001 + * @tc.name batch insert with conflict resolution + * @tc.desc normal batch insert with conflict resolution + */ + it('testMemoryDbBatchInsertWithConflictResolution001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbBatchInsertWithConflictResolution001 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + try { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + let valueBucketArray = new Array(); + for (let i = 0; i < 2; i++) { + valueBucketArray.push(valueBucket); + } + var num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_NONE) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num1 " + num) + expect(2).assertEqual(num); + + num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_ROLLBACK) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num2 " + num) + expect(2).assertEqual(num); + + num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_ABORT) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num3 " + num) + expect(2).assertEqual(num); + + num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_FAIL) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num4 " + num) + expect(2).assertEqual(num); + + num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_IGNORE) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num5 " + num) + expect(2).assertEqual(num); + + num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_REPLACE) + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 batch num6 " + num) + expect(2).assertEqual(num); + + let resultSet = await rdbStore.querySql("select * from test") + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 result count " + resultSet.rowCount) + expect(12).assertEqual(resultSet.rowCount) + resultSet.close() + } catch (e) { + console.log(TAG + e + " code: " + e.code); + expect(null).assertFail() + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution001 failed"); + } + done() + console.log(TAG + "************* testMemoryDbBatchInsertWithConflictResolution001 end *************"); + }) + + /** + * @tc.number testMemoryDbBatchInsertWithConflictResolution002 + * @tc.name batch insert with conflict resolution + * @tc.desc conflict when batch insert with conflict resolution + */ + it('testMemoryDbBatchInsertWithConflictResolution002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbBatchInsertWithConflictResolution002 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + try { + const valueBucket = { + "id" : 2, + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket); + let valueBucketArray = new Array(); + for (let i = 0; i < 5; i++) { + let val = { + "id" : i, + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + valueBucketArray.push(val); + } + try { + await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_NONE); + expect(null).assertFail(); + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_NONE code: " + e.code); + expect(14800032).assertEqual(e.code) + } + try { + await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_ROLLBACK); + expect(null).assertFail(); + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_ROLLBACK code: " + e.code); + expect(14800032).assertEqual(e.code) + } + try { + await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_ABORT); + expect(null).assertFail(); + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_ABORT code: " + e.code); + expect(14800032).assertEqual(e.code) + } + try { + await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_FAIL); + expect(null).assertFail(); + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_FAIL code: " + e.code); + expect(14800032).assertEqual(e.code) + } + try { + let num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_IGNORE); + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_IGNORE num " + num) + expect(2).assertEqual(num) + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_IGNORE code: " + e.code); + expect(null).assertFail(); + } + try { + let num = await rdbStore.batchInsertWithConflictResolution("test", valueBucketArray, + data_relationalStore.ConflictResolution.ON_CONFLICT_REPLACE); + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_REPLACE num " + num) + expect(5).assertEqual(num) + } catch (e) { + console.log(TAG + e + "testMemoryDbBatchInsertWithConflictResolution002 ON_CONFLICT_REPLACE code: " + e.code); + expect(null).assertFail(); + } + let resultSet = await rdbStore.querySql("select * from test") + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution002 result count " + resultSet.rowCount) + expect(5).assertEqual(resultSet.rowCount) + resultSet.close() + } catch (e) { + console.log(TAG + e + " code: " + e.code); + expect(null).assertFail() + console.log(TAG + "testMemoryDbBatchInsertWithConflictResolution002 failed"); + } + done() + console.log(TAG + "************* testMemoryDbBatchInsertWithConflictResolution002 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0001 + * @tc.name Normal test case of Execute, check integrity for store + * @tc.desc 1. Execute sql: PRAGMA integrity_check + * 2. Check returned value + */ + it('testMemoryDbExecute0001', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0001 start *************"); + try { + let ret = await rdbStore.execute("PRAGMA integrity_check"); + console.error("integrity_check result:" + ret); + expect("ok").assertEqual(ret); + ret = await rdbStore.execute("PRAGMA quick_check"); + console.error("quick_check result:" + ret); + expect("ok").assertEqual(ret); + } catch (err) { + expect(null).assertFail(); + console.error(`check failed, code:${err.code}, message: ${err.message}`); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0001 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0002 + * @tc.name Normal test case of Execute, get user_version of store + * @tc.desc 1. Execute sql: PRAGMA user_version + * 2. Check returned value + */ + it('testMemoryDbExecute0002', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0002 start *************"); + try { + // set user_version as 5 + rdbStore.version = 5; + let ret = await rdbStore.execute("PRAGMA user_version"); + // get user_version 5 + expect(5).assertEqual(ret); + } catch (err) { + expect(null).assertFail(); + console.error(`get user_version failed, code:${err.code}, message: ${err.message}`); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0002 end *************"); + }) + + /** + * @tc.number SUB_DDM_AppDataFWK_JSRDB_Execute_0004 + * @tc.name AbNormal test case of Execute, execute select sql + * @tc.desc 1. Execute select sql + * 2. Check returned value + */ + it('testMemoryDbExecute0003', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0003 start *************"); + try { + await rdbStore.execute("SELECT * FROM test"); + expect(null).assertFail(); + } catch (err) { + // 14800021: SQLite: Generic error. + expect(14800021).assertEqual(err.code); + console.error(`execute select sql failed, code:${err.code}, message: ${err.message}`); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0003 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0004 + * @tc.name Normal test case of Execute, execute sql for inserting data + * @tc.desc 1. Execute insert sql + * 2. Check returned value + */ + it('testMemoryDbExecute0004', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0004 start *************"); + try { + let ret = await rdbStore.execute("INSERT INTO test(name, age, salary) VALUES ('tt', 28, 50000)"); + // 1 represent that the last data is inserted in the first row + expect(1).assertEqual(ret); + } catch (err) { + console.error(`execute select sql failed, code:${err.code}, message: ${err.message}`); + expect(null).assertFail(); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0004 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0005 + * @tc.name Normal test case of Execute, execute sql for inserting data + * @tc.desc 1. Execute insert sql + * 2. Check returned value + */ + it('testMemoryDbExecute0005', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0005 start *************"); + try { + let ret = await rdbStore.execute("INSERT INTO test(name, age, salary) VALUES (?, ?, ?)", ['tt', 28, 50000]); + // 1 represent that the last data is inserted in the first row + expect(1).assertEqual(ret); + } catch (err) { + console.error(`execute insert sql failed, code:${err.code}, message: ${err.message}`); + expect(null).assertFail(); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0005 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0006 + * @tc.name Normal test case of Execute, execute sql for updating data + * @tc.desc 1. Execute update sql + * 2. Check returned value + */ + it('testMemoryDbExecute0006', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0006 start *************"); + try { + let ret = await rdbStore.execute("INSERT INTO test(name, age, salary) VALUES (?, ?, ?), (?, ? ,?)", + ['tt', 28, 50000, 'ttt', 278, 500800]); + // 2 represent that the last data is inserted in the second row + expect(2).assertEqual(ret); + + ret = await rdbStore.execute("UPDATE test SET name='dd' WHERE id = 1"); + // 1 represent that effected row id + expect(1).assertEqual(ret); + } catch (err) { + console.error(`execute update sql failed, code:${err.code}, message: ${err.message}`); + expect(null).assertFail(); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0006 end *************"); + }) + + /** + * @tc.number testMemoryDbExecute0007 + * @tc.name Normal test case of Execute, execute sql for deleting data + * @tc.desc 1. Execute delete sql + * 2. Check returned value + */ + it('testMemoryDbExecute0007', 0, async function (done) { + console.info(TAG + "************* testMemoryDbExecute0007 start *************"); + try { + let ret = await rdbStore.execute("INSERT INTO test(name, age, salary) VALUES (?, ?, ?), (?, ? ,?)", + ['tt', 28, 50000, 'ttt', 278, 500800]); + // 2 represent that the last data is inserted in the second row + expect(2).assertEqual(ret); + + ret = await rdbStore.execute("DELETE FROM test"); + // 2 represent that effected row id + expect(2).assertEqual(ret); + } catch (err) { + console.error(`execute delete sql failed, code:${err.code}, message: ${err.message}`); + expect(null).assertFail(); + } + done(); + console.info(TAG + "************* testMemoryDbExecute0007 end *************"); + }) + + /** + * @tc.number testMemoryDbUpdate0001 + * @tc.name Normal test case of update + * @tc.desc 1.Insert data + * 2.Update data + * 3.Query data + */ + it('testMemoryDbUpdate0001', 0, async function () { + console.log(TAG + "************* testMemoryDbUpdate0001 start *************"); + var u8 = new Uint8Array([1, 2, 3]); + try { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + let ret = await rdbStore.insert("test", valueBucket) + expect(1).assertEqual(ret); + } catch (err) { + console.log(TAG + `failed, err: ${JSON.stringify(err)}`) + expect().assertFail() + } + try { + var u8 = new Uint8Array([4, 5, 6]) + const valueBucket = { + "name": "lisi", + "age": 20, + "salary": 200.5, + "blobType": u8, + } + let predicates = new data_relationalStore.RdbPredicates("test") + predicates.equalTo("id", "1") + let ret = await rdbStore.update(valueBucket, predicates) + await expect(1).assertEqual(ret); + await console.log(TAG + "update done: " + ret); + + predicates = new data_relationalStore.RdbPredicates("test") + let resultSet = await rdbStore.query(predicates) + try { + expect(true).assertEqual(resultSet.goToFirstRow()) + const id = await resultSet.getLong(resultSet.getColumnIndex("id")) + const name = await resultSet.getString(resultSet.getColumnIndex("name")) + const age = await resultSet.getLong(resultSet.getColumnIndex("age")) + const salary = await resultSet.getDouble(resultSet.getColumnIndex("salary")) + const blobType = await resultSet.getBlob(resultSet.getColumnIndex("blobType")) + + expect(1).assertEqual(id); + expect("lisi").assertEqual(name); + expect(20).assertEqual(age); + expect(200.5).assertEqual(salary); + expect(4).assertEqual(blobType[0]); + expect(5).assertEqual(blobType[1]); + expect(6).assertEqual(blobType[2]); + expect(false).assertEqual(resultSet.goToNextRow()) + } finally { + resultSet.close() + resultSet = null + } + } catch (err) { + console.log(TAG + `failed, err: ${JSON.stringify(err)}`) + expect().assertFail() + } + + console.log(TAG + "************* testMemoryDbUpdate0001 end *************"); + }) + + /** + * @tc.number testMemoryDbUpdate0002 + * @tc.name Normal test case of update, value is long string and special characters + * @tc.desc 1.Insert data + * 2.Update data + * 3.Query data + */ + it('testMemoryDbUpdate0002', 0, async function () { + console.log(TAG + "************* testMemoryDbUpdate0002 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + try { + const valueBucket = { + "name": "xiaoming", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } catch (err) { + console.log(TAG + `insert failed, err: ${JSON.stringify(err)}`) + expect().assertFail(); + } + try { + var u8 = new Uint8Array([4, 5, 6]) + var nameStr = "abcd" + "e".repeat(2000) + "./&*$!@()" + const valueBucket = { + "name": nameStr, + "age": 20, + "salary": 200.5, + "blobType": u8, + } + let predicates = new data_relationalStore.RdbPredicates("test") + predicates.equalTo("name", "xiaoming") + let ret = await rdbStore.update(valueBucket, predicates) + await expect(1).assertEqual(ret); + await console.log(TAG + "update done: " + ret); + + predicates = new data_relationalStore.RdbPredicates("test") + predicates.equalTo("age", 20) + let resultSet = await rdbStore.query(predicates) + try { + expect(true).assertEqual(resultSet.goToFirstRow()) + const name = await resultSet.getString(resultSet.getColumnIndex("name")) + await expect(nameStr).assertEqual(name); + } finally { + resultSet.close() + resultSet = null + } + } catch (err) { + console.log(TAG + `failed, err: ${JSON.stringify(err)}`) + expect().assertFail() + } + console.log(TAG + "************* testMemoryDbUpdate0002 end *************"); + }) + + /** + * @tc.number testMemoryDbUpdate0003 + * @tc.name Normal test case of insert with ON_CONFLICT_REPLACE + * @tc.desc 1.Insert data + * 2.Create value + * 3.Execute update with ON_CONFLICT_REPLACE + * 4.Query data + */ + it('testMemoryDbUpdate0003', 0, async function () { + console.log(TAG + "************* testMemoryDbUpdate0003 start *************"); + try { + var u8 = new Uint8Array([1, 2, 3]) + const valueBucket = { + "id": 1, + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } catch (err) { + console.log(TAG + `insert1 failed, err: ${JSON.stringify(err)}`) + expect().assertFail(); + } + try { + var u8 = new Uint8Array([4, 5, 6]) + const valueBucket = { + "id": 2, + "name": "lisi", + "age": 19, + "salary": 200.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } catch (err) { + console.log(TAG + `insert2 failed, err: ${JSON.stringify(err)}`) + expect().assertFail(); + } + try { + var u8 = new Uint8Array([7, 8, 9]) + const valueBucket = { + "id": 3, + "name": "wangjing", + "age": 20, + "salary": 300.5, + "blobType": u8, + } + let predicates = new data_relationalStore.RdbPredicates("test") + predicates.equalTo("age", "19") + let ret = rdbStore.updateSync(valueBucket, predicates, data_relationalStore.ConflictResolution.ON_CONFLICT_REPLACE); + await expect(1).assertEqual(ret); + await console.log(TAG + "update done: " + ret); + predicates = new data_relationalStore.RdbPredicates("test") + let resultSet = await rdbStore.query(predicates) + try { + + expect(true).assertEqual(resultSet.goToFirstRow()) + const id = await resultSet.getLong(resultSet.getColumnIndex("id")) + const name = await resultSet.getString(resultSet.getColumnIndex("name")) + const age = await resultSet.getLong(resultSet.getColumnIndex("age")) + const salary = await resultSet.getDouble(resultSet.getColumnIndex("salary")) + const blobType = await resultSet.getBlob(resultSet.getColumnIndex("blobType")) + console.log(TAG + "{id=" + id + ", name=" + name + ", age=" + age + ", salary=" + + salary + ", blobType=" + blobType); + + await expect(1).assertEqual(id); + await expect("zhangsan").assertEqual(name); + await expect(18).assertEqual(age); + await expect(100.5).assertEqual(salary); + await expect(1).assertEqual(blobType[0]); + await expect(2).assertEqual(blobType[1]); + await expect(3).assertEqual(blobType[2]); + + await expect(true).assertEqual(resultSet.goToNextRow()) + const id_1 = await resultSet.getLong(resultSet.getColumnIndex("id")) + const name_1 = await resultSet.getString(resultSet.getColumnIndex("name")) + const age_1 = await resultSet.getLong(resultSet.getColumnIndex("age")) + const salary_1 = await resultSet.getDouble(resultSet.getColumnIndex("salary")) + const blobType_1 = await resultSet.getBlob(resultSet.getColumnIndex("blobType")) + console.log(TAG + "{id=" + id_1 + ", name=" + name_1 + ", age=" + age_1 + ", salary=" + + salary_1 + ", blobType=" + blobType_1); + + await expect(3).assertEqual(id_1); + await expect("wangjing").assertEqual(name_1); + await expect(20).assertEqual(age_1); + await expect(300.5).assertEqual(salary_1); + await expect(7).assertEqual(blobType_1[0]); + await expect(8).assertEqual(blobType_1[1]); + await expect(9).assertEqual(blobType_1[2]); + await expect(false).assertEqual(resultSet.goToNextRow()) + } finally { + resultSet.close() + resultSet = null + } + } catch (err) { + console.log(TAG + `failed, err: ${JSON.stringify(err)}`) + expect().assertFail(); + } + console.log(TAG + "************* testMemoryDbUpdate0003 end *************"); + }) + + /** + * @tc.number testMemoryDbDelete0001 + * @tc.name Normal test case of delete + * @tc.desc 1.Insert data + * 2.Execute delete + */ + it('testMemoryDbDelete0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbDelete0001 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 28, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 38, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + let predicates = await new data_relationalStore.RdbPredicates("test") + let deletePromise = rdbStore.delete(predicates) + deletePromise.then(async (ret) => { + expect(3).assertEqual(ret) + console.log(TAG + "Delete done: " + ret) + }).catch((err) => { + expect(null).assertFail() + }) + await deletePromise + } + done() + console.log(TAG + "************* testMemoryDbDelete0001 end *************"); + }) + + /** + * @tc.number testMemoryDbDelete0002 + * @tc.name Abnormal test case of delete, if column is invalid + * @tc.desc 1.Insert data + * 2.Configure predicates ("aaa id", 1) + * 3.Execute delete + */ + it('testMemoryDbDelete0002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbDelete0002 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 28, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + const valueBucket = { + "name": "lisi", + "age": 38, + "salary": 100.5, + "blobType": u8, + } + await rdbStore.insert("test", valueBucket) + } + { + let predicates = await new data_relationalStore.RdbPredicates("test") + predicates.equalTo("aaa id", 1) + try { + let ret = rdbStore.deleteSync(predicates) + expect(null).assertFail() + } catch (err) { + console.log(TAG + "delete with wrong conditions") + } + } + done() + console.log(TAG + "************* testMemoryDbDelete0002 end *************"); + }) + + /** + * @tc.name Normal case for Statistics insert data execution time + * @tc.number testMemoryDbStatistics0001 + * @tc.desc 1. Register callback for statistics + * 2. Insert data + * 3. UnRegister callback + */ + it('testMemoryDbStatistics0001', 0, async function (done) { + console.info(TAG + "************* testMemoryDbStatistics0001 start *************"); + let sql = ""; + try { + rdbStore.on('statistics', (SqlExeInfo) => { + sql = SqlExeInfo.sql[0]; + console.info(TAG + "on statistics success, sql:" + sql); + }) + } catch (err) { + console.error(TAG + `on statistics fail, code:${err.code}, message: ${err.message}`); + expect().assertFail(); + done() + } + + try { + const valueBucket1 = { + 'name': 'zhangsan', + 'age': 18, + 'salary': 25000, + 'blobType': new Uint8Array([1, 2, 3]), + }; + let rowId = await rdbStore.insert('test', valueBucket1); + expect(1).assertEqual(rowId); + await sleep(500); + expect('INSERT INTO test(age,blobType,name,salary) VALUES (?,?,?,?)').assertEqual(sql) + } catch (error) { + console.error(TAG + `testMemoryDbStatistics0001 fail, code:${error.code}, message: ${error.message}`); + expect().assertFail(); + } + done(); + console.info(TAG + "************* testMemoryDbStatistics0001 end *************"); + }) + + /** + * @tc.number testMemoryDbQueryByStep0001 + * @tc.name Normal test case of queryByStep, query all data + * @tc.desc 1. Execute queryByStep, sql is 'select * from test' + * @tc.size MediumTest + * @tc.type Function + * @tc.level Level 2 + */ + it('testMemoryDbQueryByStep0001', 0, async function (done) { + console.info(TAG + "************* testMemoryDbQueryByStep0001 start *************"); + try { + let u8 = new Uint8Array([1, 2, 3]); + let valuesBucket1 = { + "name": "lisi", + "age": 15, + "salary": 153.3, + "blobType": u8, + } + await rdbStore.insert("test", valuesBucket1); + + let valuesBucket2 = { + "name": "tom", + "age": 56, + "salary": 1503.3, + } + await rdbStore.insert("test", valuesBucket2); + + let valuesBucket3 = { + "name": "bob", + "age": 116, + "salary": 5503.3, + } + await rdbStore.insert("test", valuesBucket3); + console.info(TAG, "testMemoryDbQueryByStep0001 insertTest data end"); + let resultSet = await rdbStore.queryByStep('select * from test'); + // resultSet.rowCount is 3 + expect(3).assertEqual(resultSet.rowCount); + resultSet.close(); + } catch (err) { + console.error(TAG + `query failed, err code:${err.code}, message:${err.message}`) + expect().assertFail(); + } + + console.info(TAG + "************* testMemoryDbQueryByStep0001 end *************"); + done(); + }) + + /** + * @tc.number testMemoryDbExecuteSqlTest0006 + * @tc.name Normal test case of executeSql and querySql, PRAGMA table_info + * @tc.desc 1.Get table_info + * 2.Check table_info + */ + it('testMemoryDbExecuteSqlTest0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbExecuteSqlTest0001 start *************"); + let resultSet = await rdbStore.querySql("PRAGMA table_info(test)"); + try{ + resultSet.goToFirstRow(); + expect(0).assertEqual(resultSet.getLong(0)) + expect("id").assertEqual(resultSet.getString(1)) + expect("INTEGER").assertEqual(resultSet.getString(2)) + resultSet.goToNextRow(); + expect(1).assertEqual(resultSet.getLong(0)) + expect("name").assertEqual(resultSet.getString(1)) + expect("TEXT").assertEqual(resultSet.getString(2)) + expect(1).assertEqual(resultSet.getLong(3)) + resultSet.goToNextRow(); + expect(2).assertEqual(resultSet.getLong(0)) + expect("age").assertEqual(resultSet.getString(1)) + expect("INTEGER").assertEqual(resultSet.getString(2)) + resultSet.goToNextRow(); + expect(3).assertEqual(resultSet.getLong(0)) + expect("salary").assertEqual(resultSet.getString(1)) + expect("REAL").assertEqual(resultSet.getString(2)) + resultSet.goToNextRow(); + expect(4).assertEqual(resultSet.getLong(0)) + expect("blobType").assertEqual(resultSet.getString(1)) + expect("BLOB").assertEqual(resultSet.getString(2)) + resultSet.close(); + }catch (err) { + console.error(TAG + `testMemoryDbExecuteSqlTest0001 failed, err code:${err.code}, message:${err.message}`) + expect().assertFail(); + } + console.log(TAG + "************* testMemoryDbExecuteSqlTest0001 end *************"); + done(); + }) + + /** + * @tc.number testMemoryDbExecuteSqlTest0002 + * @tc.name Normal test case of ExecuteSql + * @tc.desc 1.Insert data (param is long string) + * 2.Query data + * 3.ExecuteSql (delete age = 19 AND name = nameStr) + * 4.Query data + */ + it('testMemoryDbExecuteSqlTest0002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbExecuteSqlTest0002 start *************"); + var u8 = new Uint8Array([3, 4, 5]) + var nameStr = "lisi" + "e".repeat(2000) + "zhangsan" + { + const valueBucket = { + "name": "zhangsan", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + let ret = await rdbStore.insert("test", valueBucket) + expect(1).assertEqual(ret); + } + { + const valueBucket = { + "name": nameStr, + "age": 19, + "salary": 100.5, + "blobType": u8, + } + let ret = await rdbStore.insert("test", valueBucket) + expect(2).assertEqual(ret); + } + { + const valueBucket = { + "name": nameStr, + "age": 28, + "salary": 100.5, + "blobType": u8, + } + let ret = await rdbStore.insert("test", valueBucket) + expect(3).assertEqual(ret); + } + { + let predicates = await new data_relationalStore.RdbPredicates("test") + predicates.equalTo("name", nameStr) + let querySqlPromise = rdbStore.query(predicates) + querySqlPromise.then(async (resultSet) => { + await expect(2).assertEqual(resultSet.rowCount) + resultSet.close() + }).catch((err) => { + console.error(TAG + `testMemoryDbExecuteSqlTest0002 failed, err code:${err.code}, message:${err.message}`) + expect(null).assertFail(); + }) + await querySqlPromise + } + { + let executeSqlPromise = rdbStore.executeSql("DELETE FROM test WHERE age = 19 AND name ='" + nameStr + "'") + executeSqlPromise.then(async () => { + await console.log(TAG + "executeSql done."); + }).catch((err) => { + console.error(TAG + `testMemoryDbExecuteSqlTest0002 failed, err code:${err.code}, message:${err.message}`) + expect(null).assertFail(); + }) + await executeSqlPromise + } + { + let querySqlPromise = rdbStore.querySql("SELECT * FROM test WHERE name ='" + nameStr + "'") + querySqlPromise.then(async (resultSet) => { + await expect(1).assertEqual(resultSet.rowCount) + expect(true).assertEqual(resultSet.goToFirstRow()) + const name = resultSet.getString(resultSet.getColumnIndex("name")) + const age = resultSet.getLong(resultSet.getColumnIndex("age")) + const salary = resultSet.getDouble(resultSet.getColumnIndex("salary")) + const blobType = resultSet.getBlob(resultSet.getColumnIndex("blobType")) + expect(nameStr).assertEqual(name) + expect(2012).assertEqual(name.length) + expect(28).assertEqual(age) + expect(100.5).assertEqual(salary) + expect(3).assertEqual(blobType[0]) + resultSet.close(); + done(); + }).catch((err) => { + console.error(TAG + `testMemoryDbExecuteSqlTest0002 failed, err code:${err.code}, message:${err.message}`) + expect(null).assertFail(); + }) + await querySqlPromise + } + console.log(TAG + "************* testMemoryDbExecuteSqlTest0002 end *************"); + }) + + /** + * @tc.name testMemoryDbPluginLibs0001 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_plugin_Libs_0001 + * @tc.desc Test pluginLibs are empty + */ + it('testMemoryDbPluginLibs0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbPluginLibs0001 start *************"); + try { + const testPluginLibsConfig = { + name: "testPluginLibs0001.db", + securityLevel: data_relationalStore.SecurityLevel.S1, + persist: false, + } + testPluginLibsConfig.pluginLibs = ["", ""] + await data_relationalStore.getRdbStore(context, testPluginLibsConfig); + } catch (e) { + console.log("testMemoryDbPluginLibs0001 getRdbStore err: failed, err: code=" + e.code + " message=" + e.message) + expect().assertFail(); + } + done() + console.log(TAG + "************* testMemoryDbPluginLibs0001 end *************"); + }) + + /** + * @tc.name testMemoryDbTokenizer0001 + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_plugin_Libs_0001 + * @tc.desc Test pluginLibs are empty + */ + it('testMemoryDbTokenizer0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbTokenizer0001 start *************"); + try { + const testPluginLibsConfig = { + name: "testMemoryDbTokenizer0001.db", + securityLevel: data_relationalStore.SecurityLevel.S1, + persist: false, + tokenizer:data_relationalStore.Tokenizer.ICU_TOKENIZER + } + await data_relationalStore.getRdbStore(context, testPluginLibsConfig); + } catch (e) { + console.log("testMemoryDbTokenizer0001 getRdbStore err: failed, err: code=" + e.code + " message=" + e.message) + expect().assertFail(); + } + done() + console.log(TAG + "************* testMemoryDbTokenizer0001 end *************"); + }) + + /** + * @tc.number testMemoryDbTransaction0001 + * @tc.name Normal test case of transactions, insert a row of data + * @tc.desc 1.Execute beginTransaction + * 2.BatchInsertSync data + * 3.InsertSync data + * 4.UpdateSync data + * 5.DeleteSync data + * 6.Execute commit + * 7.querySqlSync + * 7.ExecuteSync + */ + it('testMemoryDbTransaction0001', 0, async function (done) { + console.log(TAG + "************* testMemoryDbTransaction0001 start *************"); + var u8 = new Uint8Array([1, 2, 3]) + var transaction = await rdbStore.createTransaction({ + transactionType: data_relationalStore.TransactionType.DEFERRED + }) + try { + const valueBucket = { + "name": "lisi", + "age": 18, + "salary": 100.5, + "blobType": u8, + } + var num = await transaction.insertSync("test", valueBucket, data_relationalStore.ConflictResolution.ON_CONFLICT_REPLACE); + expect(1).assertEqual(num); + const updateValueBucket = { + "name": "update", + "age": 28, + "salary": 25, + "blobType": u8, + } + let predicates = new data_relationalStore.RdbPredicates("test"); + predicates.equalTo("name", "lisi") + num = await transaction.updateSync(updateValueBucket, predicates) + expect(1).assertEqual(num); + + let deletePredicates = new data_relationalStore.RdbPredicates("test"); + predicates.equalTo("name", "update") + num = await transaction.deleteSync(deletePredicates); + expect(1).assertEqual(num); + + let resultSet = await transaction.querySqlSync("select * from test") + console.log(TAG + "testMemoryDbTransaction0001 result count " + resultSet.rowCount) + expect(0).assertEqual(resultSet.rowCount) + resultSet.close() + + await transaction.commit() + } catch (e) { + await transaction.rollback() + console.log(TAG + e); + expect(null).assertFail() + console.log(TAG + "testMemoryDbTransaction0001 failed"); + } + done() + console.log(TAG + "************* testMemoryDbTransaction0001 end *************"); + }) + + /** + * @tc.number testMemoryDbTransactionIsolation0002 + * @tc.name testTransactionIsolation. DEFERRED and EXCLUSIVE + * @tc.desc 1.begin DEFERRED Transaction + * 2.begin EXCLUSIVE Transaction again + * 3.insert data with EXCLUSIVE Transaction + * 4.query data with DEFERRED Transaction -> no data + * 5.execute commit with EXCLUSIVE Transaction + * 6.query data with DEFERRED Transaction -> no data -> why? step 4 start isolation + * 7.query data with Rdb -> has data + */ + it('testMemoryDbTransactionIsolation0002', 0, async function (done) { + console.log(TAG + "************* testMemoryDbTransactionIsolation0002 start *************"); + var deferredTrans = await rdbStore.createTransaction({ + transactionType: data_relationalStore.TransactionType.DEFERRED + }) + try { + var exclusiveTrans = await rdbStore.createTransaction({ + transactionType: data_relationalStore.TransactionType.IMMEDIATE + }) + try { + const valueBucket = { + "name": "lisi", + "age": 18, + "salary": 100.5, + } + var insertRow = await exclusiveTrans.insert("test", valueBucket); + console.log(TAG + "testMemoryDbTransactionIsolation0002 exclusiveTrans.insert row " + insertRow) + expect(1).assertEqual(insertRow) + + var resultSet = deferredTrans.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0002 deferredTrans querySqlSync before exclusiveTrans commit count " + resultSet.rowCount); + expect(-1).assertEqual(resultSet.rowCount); + resultSet.close() + + await exclusiveTrans.commit(); + + resultSet = deferredTrans.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0002 deferredTrans querySqlSync after exclusiveTrans commit count " + resultSet.rowCount); + expect(1).assertEqual(resultSet.rowCount); + + resultSet = rdbStore.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0002 rdbStore querySqlSync after exclusiveTrans commit count " + resultSet.rowCount); + expect(1).assertEqual(resultSet.rowCount); + resultSet.close() + + } catch (e) { + exclusiveTrans.rollback(); + console.log(TAG + e); + expect(null).assertFail() + console.log(TAG + "insert failed"); + } + await deferredTrans.commit(); + } catch (e) { + await deferredTrans.rollback(); + console.log(TAG + e); + expect(null).assertFail() + console.log(TAG + "testMemoryDbTransactionIsolation0002 failed"); + } + done() + console.log(TAG + "************* testMemoryDbTransactionIsolation0002 end *************"); + }) + + /** + * @tc.number testMemoryDbTransactionIsolation0003 + * @tc.name testTransactionIsolation. IMMEDIATE and rdbStore + * @tc.desc 1.begin IMMEDIATE Transaction + * 2.insert data with rdbStore -> busy + * 3.insert data with IMMEDIATE Transaction + * 4.execute commit with IMMEDIATE Transaction + * 5.query data with rdbStore -> has data + */ + it('testMemoryDbTransactionIsolation0003', 0, async function (done) { + console.log(TAG + "************* testMemoryDbTransactionIsolation0003 start *************"); + var immediateTrans = await rdbStore.createTransaction({ + transactionType: data_relationalStore.TransactionType.IMMEDIATE + }) + try { + const valueBucket = { + "name": "lisi", + "age": 18, + "salary": 100.5, + } + try { + await rdbStore.insert("test", valueBucket); + console.log(TAG + "testMemoryDbTransactionIsolation0003 rdbStore.insert success "); + expect(null).assertFail() + } catch (e) { + console.log(TAG + e); + expect(e.code).assertEqual(14800025) + console.log(TAG + "insert failed"); + } + var insertRow = await immediateTrans.insert("test", valueBucket); + console.log(TAG + "testMemoryDbTransactionIsolation0003 immediateTrans.insert row " + insertRow); + expect(insertRow).assertEqual(1); + + await immediateTrans.commit(); + + var resultSet = rdbStore.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0003 querySqlSync count " + resultSet.rowCount); + expect(1).assertEqual(resultSet.rowCount); + resultSet.close() + } catch (e) { + await immediateTrans.rollback(); + console.log(TAG + e); + expect(null).assertFail() + console.log(TAG + "testMemoryDbTransactionIsolation0003 failed"); + } + done() + console.log(TAG + "************* testMemoryDbTransactionIsolation0003 end *************"); + }) + + /** + * @tc.number testMemoryDbTransactionIsolation0004 + * @tc.name testTransactionIsolation. DEFERRED and rdbStore + * @tc.desc 1.begin DEFERRED Transaction + * 2.insert data with rdbStore + * 3.insert data with DEFERRED Transaction + * 4.query data with rdbStore -> has 2 row + * 5.insert data with rdbStore again -> busy + * 6.query data with DEFERRED Transaction -> has 2 row + * 7.execute commit with DEFERRED Transaction + * 8.insert data with rdbStore again + * 9.query data with rdbStore -> has 3 row + */ + it('testMemoryDbTransactionIsolation0004', 0, async function (done) { + console.log(TAG + "************* testMemoryDbTransactionIsolation0004 start *************"); + var deferredTrans = await rdbStore.createTransaction({ + transactionType: data_relationalStore.TransactionType.DEFERRED + }) + try { + const valueBucket = { + "name": "lisi", + "age": 18, + "salary": 100.5, + } + await rdbStore.insert("test", valueBucket); + + await deferredTrans.insert("test", valueBucket); + + var resultSet = rdbStore.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0004 rdbStore.querySqlSync count " + resultSet.rowCount); + // because sqlite_locked + expect(-1).assertEqual(resultSet.rowCount); + + try { + await rdbStore.insert("test", valueBucket); + console.log(TAG + "testMemoryDbTransactionIsolation0004 insert success "); + expect(null).assertFail() + } catch (e) { + console.log(TAG + e); + expect(e.code).assertEqual(14800025) + console.log(TAG + "insert failed"); + } + resultSet = deferredTrans.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0004 deferredTrans.querySqlSync count " + resultSet.rowCount); + expect(2).assertEqual(resultSet.rowCount); + + await deferredTrans.commit(); + + await rdbStore.insert("test", valueBucket); + + resultSet = rdbStore.querySqlSync("select * from test where name = ?", ["lisi"]); + console.log(TAG + "testMemoryDbTransactionIsolation0004 rdbStore.querySqlSync after deferredTrans commit count " + resultSet.rowCount); + expect(3).assertEqual(resultSet.rowCount); + resultSet.close() + } catch (e) { + await deferredTrans.rollback(); + console.log(TAG + e); + expect(null).assertFail() + console.log(TAG + "testMemoryDbTransactionIsolation0004 failed"); + } + done() + console.log(TAG + "************* testMemoryDbTransactionIsolation0004 end *************"); + }) + + console.log(TAG + "*************Unit Test End*************"); +}) \ No newline at end of file diff --git a/relational_store/test/js/relationalstore/unittest/src/RdbstoreRdbstoreJsunit.test.js b/relational_store/test/js/relationalstore/unittest/src/RdbstoreRdbstoreJsunit.test.js index 04599cd2..72b830c5 100644 --- a/relational_store/test/js/relationalstore/unittest/src/RdbstoreRdbstoreJsunit.test.js +++ b/relational_store/test/js/relationalstore/unittest/src/RdbstoreRdbstoreJsunit.test.js @@ -309,55 +309,58 @@ describe('rdbStoreTest', function () { */ it('testRdbStore0012', 0, async function (done) { console.log(TAG + "************* testRdbStore0012 start *************"); - try { - const rowCount = 18; - const rdbStore = await data_relationalStore.getRdbStore(context, { - name: "walTest", - securityLevel: data_relationalStore.SecurityLevel.S3, - encrypt: true, - }) - const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" - + "id INTEGER PRIMARY KEY AUTOINCREMENT, " - + "blobType BLOB)"; - - rdbStore.executeSync(CREATE_TABLE_TEST); - const valueBuckets = Array(rowCount).fill(0).map(() => { - return { - blobType: new Uint8Array(Array(1024 * 1024).fill(1)), - } - }) - - rdbStore.batchInsertSync('test', valueBuckets); - - const predicates = new data_relationalStore.RdbPredicates('test'); - const resultSet = rdbStore.querySync(predicates); - expect(resultSet.rowCount).assertEqual(rowCount); - resultSet.goToFirstRow() - const value = new Uint8Array(Array(1024 * 1024).fill(1)); - const startTime = new Date().getTime(); - rdbStore.insertSync('test', { - blobType: new Uint8Array(Array(1024 * 1024).fill(1)), - }) - const middleTime = new Date().getTime(); - console.log(TAG + "testRdbStore0012, startTime:" + startTime + " middleTime:" + middleTime + " costTime" + (middleTime-startTime)); - expect((middleTime - startTime) > 500).assertTrue(); - - rdbStore.insertSync('test', { - blobType: value, - }) - const endTime = new Date().getTime(); - console.log(TAG + "testRdbStore0012, endTime:" + endTime + " middleTime:" + middleTime + " costTime" + (endTime-middleTime)); - expect((endTime - middleTime) < 500).assertTrue(); - - console.log(TAG + "************* testRdbStore0012 end *************"); - done(); + const rowCount = 18; + const rdbStore = await data_relationalStore.getRdbStore(context, { + name: "walTest", + securityLevel: data_relationalStore.SecurityLevel.S3, + encrypt: true, + }); + const CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "blobType BLOB)"; + rdbStore.executeSync(CREATE_TABLE_TEST); + const valueTest = Array(rowCount).fill(0).map(() => { + return { + blobType: new Uint8Array(Array(1024).fill(1)), + }; + }); + rdbStore.batchInsertSync('test', valueTest); + const predicates = new data_relationalStore.RdbPredicates('test'); + const resultSet = rdbStore.querySync(predicates); + expect(resultSet.rowCount).assertEqual(rowCount); + resultSet.goToFirstRow(); + const value = new Uint8Array(Array(1024 * 1024).fill(1)); + const valueBuckets = Array(rowCount).fill(0).map(() => { + return { + blobType: value, + }; + }); + rdbStore.batchInsertSync('test', valueBuckets); + const startTime = new Date().getTime(); + rdbStore.insertSync('test', { + blobType: value, + }); + const middleTime = new Date().getTime(); + console.log(TAG + "testRdbStore0012, startTime:" + startTime + " middleTime:" + middleTime + " costTime" + + (middleTime - startTime)); + expect((middleTime - startTime) > 500).assertTrue(); + rdbStore.insertSync('test', { + blobType: value, + }); + const endTime = new Date().getTime(); + console.log(TAG + "testRdbStore0012, endTime:" + endTime + " middleTime:" + middleTime + " costTime" + + (endTime - middleTime)); + expect((endTime - middleTime) < 500).assertTrue(); + resultSet.close(); + console.log(TAG + "************* testRdbStore0012 end *************"); + done(); } catch (e) { - console.log(TAG + "testRdbStore0012 failed " + JSON.stringify(e)); - done(); - expect().assertFail(); + console.log(TAG + "testRdbStore0012 failed " + JSON.stringify(e)); + done(); + expect().assertFail(); } - }) + }) /** * @tc.name rdb store update after deleteRdbStore @@ -1456,5 +1459,62 @@ describe('rdbStoreTest', function () { done(); } }) + + /** + * @tc.name tokenizer supported test + * @tc.number SUB_DDM_AppDataFWK_JSRDB_RdbStore_0055 + * @tc.desc invalid tokenizer test callback + */ + it('testRdbStore0055', 0, async (done) => { + console.log(TAG + "************* testRdbStore0055 start *************"); + let storeConfig = { + name: "testSupportTokenizer.db", + securityLevel: data_relationalStore.SecurityLevel.S1, + tokenizer: data_relationalStore.Tokenizer.CUSTOM_TOKENIZER, + } + try { + await data_relationalStore.getRdbStore(context, storeConfig); + expect().assertFail(); + } catch (e) { + console.log("catch err: failed, err: code=" + e.code + " message=" + e.message); + expect(String(e.code)).assertEqual(String(801)); + console.info(TAG + "************* testRdbStore0055 end *************"); + done(); + } + }) + + /** + * @tc.number testCrypt + * @tc.name testCrypt0001 + * @tc.desc + */ + it('testCrypt0001', 0, async () => { + console.log(TAG + "************* testCrypt0001 start *************"); + let cryptoParam = { + encryptionKey: new Uint8Array([1, 2, 3, 4, 5, 6]), + } + let storeConfig = { + name: "testCrypt0001.db", + securityLevel: data_relationalStore.SecurityLevel.S2, + cryptoParam: cryptoParam + } + try { + let store = await data_relationalStore.getRdbStore(context, storeConfig); + await store.executeSql(CREATE_TABLE_TEST); + store.close(); + console.log(TAG + "getRdbStore success 1"); + cryptoParam.encryptionKey = new Uint8Array([6, 5, 4, 3, 2, 1]); + store = await data_relationalStore.getRdbStore(context, storeConfig); + store.close(); + console.log(TAG + "getRdbStore success 2"); + expect(false).assertTrue(); + } catch (e) { + console.log(TAG + e + " code: " + e.code); + expect(e.code).assertEqual(14800011) + console.log(TAG + "testCorrupt0001 success"); + } + await data_relationalStore.deleteRdbStore(context, storeConfig); + console.log(TAG + "************* testCorrupt0001 end *************"); + }) console.log(TAG + "*************Unit Test End*************"); }) \ No newline at end of file diff --git a/relational_store/test/native/appdatafwk/BUILD.gn b/relational_store/test/native/appdatafwk/BUILD.gn index 46046cf6..3d7893bf 100644 --- a/relational_store/test/native/appdatafwk/BUILD.gn +++ b/relational_store/test/native/appdatafwk/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_appdatafwk" +module_output_path = "relational_store/relational_store/native_appdatafwk" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/native/clouddata/BUILD.gn b/relational_store/test/native/clouddata/BUILD.gn index b1d17d72..357e7582 100644 --- a/relational_store/test/native/clouddata/BUILD.gn +++ b/relational_store/test/native/clouddata/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_clouddata" +module_output_path = "relational_store/relational_store/native_clouddata" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/native/dataability/BUILD.gn b/relational_store/test/native/dataability/BUILD.gn index 85ae91a7..375dcf41 100644 --- a/relational_store/test/native/dataability/BUILD.gn +++ b/relational_store/test/native/dataability/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "dataability/native_dataability" +module_output_path = "relational_store/relational_store/native_dataability" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/native/dataability/unittest/data_ability_predicates_test.cpp b/relational_store/test/native/dataability/unittest/data_ability_predicates_test.cpp index d1c664d7..1174106d 100644 --- a/relational_store/test/native/dataability/unittest/data_ability_predicates_test.cpp +++ b/relational_store/test/native/dataability/unittest/data_ability_predicates_test.cpp @@ -44,7 +44,6 @@ void DataAbilityPredicatesTest::TearDownTestCase(void) * @tc.name: DataAbilityPredicates_001 * @tc.desc: test DataAbilityPredicates() * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_001, TestSize.Level1) { @@ -56,7 +55,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_001, TestSize.Level1) * @tc.name: DataAbilityPredicates_002 * @tc.desc: test DataAbilityPredicates() * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_002, TestSize.Level1) { @@ -94,7 +92,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_002, TestSize.Level1) * @tc.name: DataAbilityPredicates_003 * @tc.desc: test DataAbilityPredicates * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_003, TestSize.Level1) { @@ -127,7 +124,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_003, TestSize.Level1) * @tc.name: DataAbilityPredicates_004 * @tc.desc: test DataAbilityPredicates(std::string rawSelection); * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_004, TestSize.Level1) { @@ -141,7 +137,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_004, TestSize.Level1) * @tc.name: DataAbilityPredicates_005 * @tc.desc: test DataAbilityPredicates(std::string rawSelection); * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_005, TestSize.Level1) { @@ -155,7 +150,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_005, TestSize.Level1) * @tc.name: DataAbilityPredicates_006 * @tc.desc: test DataAbilityPredicates(std::string rawSelection); * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_006, TestSize.Level1) { @@ -169,7 +163,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_006, TestSize.Level1) * @tc.name: DataAbilityPredicates_007 * @tc.desc: test DataAbilityPredicates(OHOS::Parcel *source) * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_007, TestSize.Level1) { @@ -181,7 +174,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_007, TestSize.Level1) * @tc.name: DataAbilityPredicates_008 * @tc.desc: test DataAbilityPredicates(OHOS::Parcel *source) * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_008, TestSize.Level1) { @@ -200,7 +192,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_008, TestSize.Level1) * @tc.name: DataAbilityPredicates_009 * @tc.desc: test DataAbilityPredicates(OHOS::Parcel *source) * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_009, TestSize.Level1) { @@ -227,7 +218,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_009, TestSize.Level1) * @tc.name: DataAbilityPredicates_010 * @tc.desc: test DataAbilityPredicates Marshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_010, TestSize.Level1) { @@ -273,7 +263,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_010, TestSize.Level1) * @tc.name: DataAbilityPredicates_011 * @tc.desc: test DataAbilityPredicates Marshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_011, TestSize.Level1) { @@ -313,7 +302,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_011, TestSize.Level1) * @tc.name: DataAbilityPredicates_012 * @tc.desc: test DataAbilityPredicates Marshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_012, TestSize.Level1) { @@ -353,7 +341,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_012, TestSize.Level1) * @tc.name: DataAbilityPredicates_013 * @tc.desc: test DataAbilityPredicates Unmarshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_013, TestSize.Level1) { @@ -370,7 +357,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_013, TestSize.Level1) * @tc.name: DataAbilityPredicates_014 * @tc.desc: test DataAbilityPredicates Unmarshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_014, TestSize.Level1) { @@ -390,7 +376,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_014, TestSize.Level1) * @tc.name: DataAbilityPredicates_015 * @tc.desc: test DataAbilityPredicates Unmarshalling * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_015, TestSize.Level1) { @@ -407,7 +392,6 @@ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_015, TestSize.Level1) * @tc.name: DataAbilityPredicates_016 * @tc.desc: test SetAttributes * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(DataAbilityPredicatesTest, DataAbilityPredicates_016, TestSize.Level1) { diff --git a/relational_store/test/native/gdb/BUILD.gn b/relational_store/test/native/gdb/BUILD.gn index 3162965e..4b1f8e90 100644 --- a/relational_store/test/native/gdb/BUILD.gn +++ b/relational_store/test/native/gdb/BUILD.gn @@ -13,7 +13,34 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_gdb" +module_output_path = "relational_store/relational_store/native_gdb" + +base_sources = [ + "${relational_store_native_path}/gdb/adapter/src/grd_adapter.cpp", + "${relational_store_native_path}/gdb/adapter/src/grd_adapter_manager.cpp", + "${relational_store_native_path}/gdb/src/connection.cpp", + "${relational_store_native_path}/gdb/src/connection_pool.cpp", + "${relational_store_native_path}/gdb/src/db_helper.cpp", + "${relational_store_native_path}/gdb/src/db_store_impl.cpp", + "${relational_store_native_path}/gdb/src/db_store_manager.cpp", + "${relational_store_native_path}/gdb/src/edge.cpp", + "${relational_store_native_path}/gdb/src/full_result.cpp", + "${relational_store_native_path}/gdb/src/gdb_utils.cpp", + "${relational_store_native_path}/gdb/src/graph_connection.cpp", + "${relational_store_native_path}/gdb/src/graph_statement.cpp", + "${relational_store_native_path}/gdb/src/path.cpp", + "${relational_store_native_path}/gdb/src/path_segment.cpp", + "${relational_store_native_path}/gdb/src/store_config.cpp", + "${relational_store_native_path}/gdb/src/trans_db.cpp", + "${relational_store_native_path}/gdb/src/transaction.cpp", + "${relational_store_native_path}/gdb/src/transaction_impl.cpp", + "${relational_store_native_path}/gdb/src/vertex.cpp", + "${relational_store_native_path}/rdb/src/rdb_security_manager.cpp", + "${relational_store_native_path}/rdb/src/rdb_store_config.cpp", + "${relational_store_native_path}/rdb/src/rdb_time_utils.cpp", + "${relational_store_native_path}/rdb/src/sqlite_utils.cpp", + "${relational_store_native_path}/rdb/src/string_utils.cpp", +] ############################################################################### @@ -29,7 +56,8 @@ ohos_unittest("NativeGdbTest") { "${relational_store_native_path}/rdb/include", ] - sources = [ + sources = base_sources + sources += [ "unittest/gdb_encrypt_test.cpp", "unittest/gdb_execute_test.cpp", "unittest/gdb_function_test.cpp", @@ -41,13 +69,18 @@ ohos_unittest("NativeGdbTest") { external_deps = [ "c_utils:utils", + "file_api:securitylabel", "googletest:gtest_main", "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + "huks:libhukssdk", "json:nlohmann_json_static", "kv_store:distributeddata_inner", ] - - deps = [ "${relational_store_innerapi_path}/gdb:framework_graphstore" ] + if (arkdata_db_core_is_exists) { + defines = [ "ARKDATA_DB_CORE_IS_EXISTS" ] + } } ohos_unittest("NativeGdbAdaptTest") { @@ -62,17 +95,108 @@ ohos_unittest("NativeGdbAdaptTest") { "${relational_store_native_path}/rdb/include", ] - sources = [ "unittest/gdb_adapt_test.cpp" ] + sources = base_sources + sources += [ "unittest/gdb_adapt_test.cpp" ] + + external_deps = [ + "c_utils:utils", + "file_api:securitylabel", + "googletest:gmock", + "googletest:gtest_main", + "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + "huks:libhukssdk", + "json:nlohmann_json_static", + "kv_store:distributeddata_inner", + ] + if (arkdata_db_core_is_exists) { + defines = [ "ARKDATA_DB_CORE_IS_EXISTS" ] + } +} + +ohos_unittest("NativeGdbGrdAdapterTest") { + module_out_path = module_output_path + + include_dirs = [ + "${relational_store_common_path}/include", + "${relational_store_innerapi_path}/gdb/include", + "${relational_store_innerapi_path}/rdb/include", + "${relational_store_native_path}/gdb/include", + "${relational_store_native_path}/gdb/adapter/include", + "${relational_store_native_path}/rdb/include", + ] + + sources = [ + "${relational_store_native_path}/gdb/adapter/src/grd_adapter_manager.cpp", + "${relational_store_native_path}/gdb/src/connection.cpp", + "${relational_store_native_path}/gdb/src/connection_pool.cpp", + "${relational_store_native_path}/gdb/src/db_helper.cpp", + "${relational_store_native_path}/gdb/src/db_store_impl.cpp", + "${relational_store_native_path}/gdb/src/db_store_manager.cpp", + "${relational_store_native_path}/gdb/src/edge.cpp", + "${relational_store_native_path}/gdb/src/full_result.cpp", + "${relational_store_native_path}/gdb/src/gdb_utils.cpp", + "${relational_store_native_path}/gdb/src/graph_connection.cpp", + "${relational_store_native_path}/gdb/src/graph_statement.cpp", + "${relational_store_native_path}/gdb/src/path.cpp", + "${relational_store_native_path}/gdb/src/path_segment.cpp", + "${relational_store_native_path}/gdb/src/store_config.cpp", + "${relational_store_native_path}/gdb/src/trans_db.cpp", + "${relational_store_native_path}/gdb/src/transaction.cpp", + "${relational_store_native_path}/gdb/src/transaction_impl.cpp", + "${relational_store_native_path}/gdb/src/vertex.cpp", + "${relational_store_native_path}/rdb/src/rdb_security_manager.cpp", + "${relational_store_native_path}/rdb/src/rdb_store_config.cpp", + "${relational_store_native_path}/rdb/src/rdb_time_utils.cpp", + "${relational_store_native_path}/rdb/src/sqlite_utils.cpp", + "${relational_store_native_path}/rdb/src/string_utils.cpp", + "mock/grd_adapter.cpp", + "unittest/gdb_grd_adapter_test.cpp", + ] external_deps = [ "c_utils:utils", + "file_api:securitylabel", "googletest:gmock", "googletest:gtest_main", "hilog:libhilog", + "hisysevent:libhisysevent", + "hitrace:hitrace_meter", + "huks:libhukssdk", "json:nlohmann_json_static", ] - deps = [ "${relational_store_innerapi_path}/gdb:framework_graphstore" ] + if (arkdata_db_core_is_exists) { + defines = [ "ARKDATA_DB_CORE_IS_EXISTS" ] + } +} + +ohos_unittest("NativeGdbStoreConfigTest") { + module_out_path = module_output_path + + include_dirs = [ + "${relational_store_common_path}/include", + "${relational_store_innerapi_path}/gdb/include", + "${relational_store_innerapi_path}/rdb/include", + "${relational_store_native_path}/gdb/include", + "${relational_store_native_path}/rdb/include", + ] + + sources = [ + "${relational_store_native_path}/gdb/src/store_config.cpp", + "unittest/gdb_store_config_test.cpp", + ] + + external_deps = [ + "c_utils:utils", + "googletest:gtest_main", + "hilog:libhilog", + "huks:libhukssdk", + ] + if (arkdata_db_core_is_exists) { + defines = [ "ARKDATA_DB_CORE_IS_EXISTS" ] + } } ############################################################################### @@ -81,6 +205,8 @@ group("unittest") { deps = [ ":NativeGdbAdaptTest", + ":NativeGdbGrdAdapterTest", + ":NativeGdbStoreConfigTest", ":NativeGdbTest", ] } diff --git a/relational_store/test/native/gdb/fuzztest/gdbstore_fuzzer/BUILD.gn b/relational_store/test/native/gdb/fuzztest/gdbstore_fuzzer/BUILD.gn index 464b24ec..cc3e3cb8 100644 --- a/relational_store/test/native/gdb/fuzztest/gdbstore_fuzzer/BUILD.gn +++ b/relational_store/test/native/gdb/fuzztest/gdbstore_fuzzer/BUILD.gn @@ -39,7 +39,7 @@ ohos_fuzztest("GdbStoreFuzzTest") { sources = [ "gdbstore_fuzzer.cpp" ] - deps = [ "${relational_store_innerapi_path}/gdb:framework_graphstore" ] + deps = [ "${relational_store_innerapi_path}/gdb:native_graphstore" ] external_deps = [ "hilog:libhilog", diff --git a/relational_store/test/native/gdb/mock/grd_adapter.cpp b/relational_store/test/native/gdb/mock/grd_adapter.cpp new file mode 100644 index 00000000..35d4e81f --- /dev/null +++ b/relational_store/test/native/gdb/mock/grd_adapter.cpp @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "grd_adapter.h" + +#include +#include + +#include "gdb_errors.h" +#include "gdb_utils.h" +#include "grd_adapter_manager.h" +#include "grd_error.h" + +namespace OHOS::DistributedDataAip { + +static GrdAdapterHolder g_adapterHolder; + +int32_t GrdAdapter::errorCode_[FuncName::ALL] = { GRD_OK }; + +std::map GrdAdapter::GRD_ERRNO_MAP = { + { GRD_OK, E_OK }, + { GRD_REBUILD_DATABASE, E_OK }, + { GRD_NO_DATA, E_GRD_NO_DATA }, + { GRD_DATA_CORRUPTED, E_GRD_DATA_CORRUPTED }, + { GRD_INVALID_FILE_FORMAT, E_GRD_INVALID_FILE_FORMAT }, + { GRD_PRIMARY_KEY_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_RESTRICT_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_CONSTRAINT_CHECK_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_NOT_SUPPORT, E_GRD_NOT_SUPPORT }, + { GRD_OVER_LIMIT, E_GRD_OVER_LIMIT }, + { GRD_INVALID_ARGS, E_GRD_INVALID_ARGS }, + { GRD_FAILED_FILE_OPERATION, E_GRD_FAILED_FILE_OPERATION }, + { GRD_INSUFFICIENT_SPACE, E_GRD_DISK_SPACE_FULL }, + { GRD_RESOURCE_BUSY, E_DATABASE_BUSY }, + { GRD_DB_BUSY, E_DATABASE_BUSY }, + { GRD_FAILED_MEMORY_ALLOCATE, E_GRD_FAILED_MEMORY_ALLOCATE }, + { GRD_CRC_CHECK_DISABLED, E_GRD_CRC_CHECK_DISABLED }, + { GRD_DISK_SPACE_FULL, E_GRD_DISK_SPACE_FULL }, + + { GRD_PERMISSION_DENIED, E_GRD_PERMISSION_DENIED }, + { GRD_PASSWORD_UNMATCHED, E_GRD_PASSWORD_UNMATCHED }, + { GRD_PASSWORD_NEED_REKEY, E_GRD_PASSWORD_NEED_REKEY }, + + { GRD_NAME_TOO_LONG, E_GRD_INVALID_NAME }, + { GRD_INVALID_TABLE_DEFINITION, E_GRD_SEMANTIC_ERROR }, + { GRD_SEMANTIC_ERROR, E_GRD_SEMANTIC_ERROR }, + { GRD_SYNTAX_ERROR, E_GRD_SYNTAX_ERROR }, + { GRD_WRONG_STMT_OBJECT, E_GRD_WRONG_STMT_OBJECT }, + { GRD_DATA_CONFLICT, E_GRD_DATA_CONFLICT }, + + { GRD_INNER_ERR, E_GRD_INNER_ERR }, + { GRD_FAILED_MEMORY_RELEASE, E_GRD_FAILED_MEMORY_RELEASE }, + { GRD_NOT_AVAILABLE, E_GRD_NOT_AVAILABLE }, + { GRD_INVALID_FORMAT, E_GRD_SEMANTIC_ERROR }, + { GRD_TIME_OUT, E_DATABASE_BUSY }, + { GRD_DB_INSTANCE_ABNORMAL, E_GRD_DB_INSTANCE_ABNORMAL }, + { GRD_CIPHER_ERROR, E_GRD_CIPHER_ERROR }, + { GRD_DUPLICATE_TABLE, E_GRD_DUPLICATE_PARAM }, + { GRD_DUPLICATE_OBJECT, E_GRD_DUPLICATE_PARAM }, + { GRD_DUPLICATE_COLUMN, E_GRD_DUPLICATE_PARAM }, + { GRD_UNDEFINE_COLUMN, E_GRD_UNDEFINED_PARAM }, + { GRD_UNDEFINED_OBJECT, E_GRD_UNDEFINED_PARAM }, + { GRD_UNDEFINED_TABLE, E_GRD_UNDEFINED_PARAM }, + { GRD_INVALID_CONFIG_VALUE, E_CONFIG_INVALID_CHANGE }, + { GRD_REQUEST_TIME_OUT, E_DATABASE_BUSY }, + { GRD_DATATYPE_MISMATCH, E_GRD_SEMANTIC_ERROR }, + { GRD_UNIQUE_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_INVALID_BIND_VALUE, E_GRD_INVALID_BIND_VALUE }, + { GRD_JSON_OPERATION_NOT_SUPPORT, E_GRD_SEMANTIC_ERROR }, + { GRD_MODEL_NOT_SUPPORT, E_GRD_SEMANTIC_ERROR }, + { GRD_FEATURE_NOT_SUPPORTED, E_GRD_SEMANTIC_ERROR }, + { GRD_JSON_LEN_LIMIT, E_GRD_DATA_CONFLICT }, + { GRD_SUBSCRIPTION_EXCEEDED_LIMIT, E_GRD_INNER_ERR }, + { GRD_SYNC_EXCEED_TASK_QUEUE_LIMIT, E_DATABASE_BUSY }, + { GRD_SHARED_OBJ_ENABLE_UNDO_EXCEED_LIMIT, E_GRD_INNER_ERR }, + { GRD_TABLE_LIMIT_EXCEEDED, E_GRD_OVER_LIMIT }, + { GRD_FIELD_TYPE_NOT_MATCH, E_GRD_SEMANTIC_ERROR }, + { GRD_LARGE_JSON_NEST, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_JSON_TYPE, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_OPERATOR, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_PROJECTION_FIELD, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_PROJECTION_VALUE, E_GRD_SEMANTIC_ERROR }, + { GRD_DB_NOT_EXIST, E_GRD_DB_NOT_EXIST }, + { GRD_INVALID_VALUE, E_GRD_INVALID_ARGS }, + { GRD_SHARED_OBJ_NOT_EXIST, E_GRD_DATA_NOT_FOUND }, + { GRD_SUBSCRIBE_NOT_EXIST, E_GRD_DATA_NOT_FOUND }, + { GRD_COLLECTION_NOT_EXIST, E_GRD_DATA_NOT_FOUND }, + { GRD_RESULTSET_BUSY, E_DATABASE_BUSY }, + { GRD_RECORD_NOT_FOUND, E_GRD_DATA_NOT_FOUND }, + { GRD_FIELD_NOT_FOUND, E_GRD_DATA_NOT_FOUND }, + { GRD_ARRAY_INDEX_NOT_FOUND, E_GRD_DATA_NOT_FOUND }, + { GRD_RESULT_SET_NOT_AVAILABLE, E_GRD_DATA_NOT_FOUND }, + { GRD_SHARED_OBJ_UNDO_NOT_AVAILABLE, E_GRD_DATA_NOT_FOUND }, + { GRD_SHARED_OBJ_REDO_NOT_AVAILABLE, E_GRD_DATA_NOT_FOUND }, + { GRD_INVALID_JSON_FORMAT, E_GRD_DATA_CONFLICT }, + { GRD_INVALID_KEY_FORMAT, E_GRD_INVALID_NAME }, + { GRD_INVALID_COLLECTION_NAME, E_GRD_INVALID_NAME }, + { GRD_INVALID_EQUIP_ID, E_GRD_SEMANTIC_ERROR }, + { GRD_KEY_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_FIELD_TYPE_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_SHARED_OBJ_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_SUBSCRIBE_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_EQUIP_ID_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_SHARED_OBJ_ENABLE_UNDO_CONFLICT, E_GRD_DATA_CONFLICT }, + { GRD_SCHEMA_CHANGED, E_CONFIG_INVALID_CHANGE }, + { GRD_DATA_EXCEPTION, E_GRD_DATA_EXCEPTION }, + { GRD_FIELD_OVERFLOW, E_GRD_SEMANTIC_ERROR }, + { GRD_DIVISION_BY_ZERO, E_GRD_SYNTAX_ERROR }, + { GRD_TRANSACTION_ROLLBACK, E_GRD_TRANSACTION_ROLLBACK }, + { GRD_NO_ACTIVE_TRANSACTION, E_GRD_NO_ACTIVE_TRANSACTION }, + { GRD_ACTIVE_TRANSACTION, E_GRD_ACTIVE_TRANSACTION }, +}; + +int GrdAdapter::TransErrno(int err) +{ + if (err > 0) { + return err; + } + auto result = GRD_ERRNO_MAP.find(err); + if (result != GRD_ERRNO_MAP.end()) { + return result->second; + } + return E_GRD_INNER_ERR; +} + +void GrdAdapter::SetErrorCode(FuncName func, int32_t err) +{ + if (func == FuncName::ALL) { + for (int i = FuncName::PREPARE; i < FuncName::ALL; i++) { + errorCode_[i] = err; + } + } else { + errorCode_[func] = err; + } +} + +int32_t GrdAdapter::Prepare(GRD_DB *db, const char *str, uint32_t strLen, GRD_StmtT **stmt, const char **unusedStr) +{ + if (errorCode_[FuncName::PREPARE] == GRD_OK) { + if (g_adapterHolder.Prepare == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Prepare == nullptr) { + return E_NOT_SUPPORT; + } + int32_t ret = g_adapterHolder.Prepare(db, str, strLen, stmt, unusedStr); + return TransErrno(ret); + } + return TransErrno(errorCode_[FuncName::PREPARE]); +} + +int32_t GrdAdapter::Step(GRD_StmtT *stmt) +{ + if (errorCode_[FuncName::STEP] == GRD_OK) { + if (g_adapterHolder.Step == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Step == nullptr) { + return E_NOT_SUPPORT; + } + int32_t ret = g_adapterHolder.Step(stmt); + return TransErrno(ret); + } + return TransErrno(errorCode_[FuncName::STEP]); +} + +int32_t GrdAdapter::Finalize(GRD_StmtT *stmt) +{ + if (errorCode_[FuncName::FINALIZE] == GRD_OK) { + if (g_adapterHolder.Finalize == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Finalize == nullptr) { + return E_NOT_SUPPORT; + } + int32_t ret = g_adapterHolder.Finalize(stmt); + return TransErrno(ret); + } + return TransErrno(errorCode_[FuncName::FINALIZE]); +} + +int32_t GrdAdapter::Rekey(const char *dbFile, const char *configStr, const std::vector &encryptedKey) +{ + if (errorCode_[FuncName::REKEY] == GRD_OK) { + if (g_adapterHolder.Rekey == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Rekey == nullptr) { + return E_NOT_SUPPORT; + } + if (encryptedKey.empty()) { + return E_GRD_INVALID_ARGS; + } + int32_t ret = E_OK; + GRD_CipherInfoT info = { 0 }; + const size_t keySize = encryptedKey.size() * 2 + 1; + std::vector key(keySize); + info.hexPassword = GdbUtils::GetEncryptKey(encryptedKey, key.data(), keySize); + ret = g_adapterHolder.Rekey(dbFile, configStr, &info); + key.assign(keySize, 0); + return TransErrno(ret); + } + return TransErrno(errorCode_[FuncName::REKEY]); +} + +ColumnType GrdAdapter::TransColType(int grdColType) +{ + switch (grdColType) { + case GRD_DB_DATATYPE_INTEGER: + return ColumnType::TYPE_INTEGER; + case GRD_DB_DATATYPE_FLOAT: + return ColumnType::TYPE_FLOAT; + case GRD_DB_DATATYPE_TEXT: + return ColumnType::TYPE_TEXT; + case GRD_DB_DATATYPE_BLOB: + return ColumnType::TYPE_BLOB; + case GRD_DB_DATATYPE_FLOATVECTOR: + return ColumnType::TYPE_FLOATVECTOR; + case GRD_DB_DATATYPE_JSONSTR: + return ColumnType::TYPE_JSONSTR; + case GRD_DB_DATATYPE_NULL: + return ColumnType::TYPE_NULL; + default: + return ColumnType::TYPE_NULL; + } +} + +int GrdAdapter::Open(const char *dbPath, const char *configStr, uint32_t flags, GRD_DB **db) +{ + if (g_adapterHolder.Open == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Open == nullptr) { + return E_NOT_SUPPORT; + } + auto ret = g_adapterHolder.Open(dbPath, configStr, flags, db); + return TransErrno(ret); +} + +int GrdAdapter::Close(GRD_DB *db, uint32_t flags) +{ + if (g_adapterHolder.Close == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Close == nullptr) { + return E_NOT_SUPPORT; + } + auto ret = g_adapterHolder.Close(db, flags); + return TransErrno(ret); +} + +int GrdAdapter::Repair(const char *dbPath, const char *configStr) +{ + if (g_adapterHolder.Repair == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Repair == nullptr) { + return E_NOT_SUPPORT; + } + return E_NOT_SUPPORT; +} + +int GrdAdapter::Backup(GRD_DB *db, const char *backupDbFile, const std::vector &encryptedKey) +{ + if (g_adapterHolder.Backup == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Backup == nullptr) { + return E_NOT_SUPPORT; + } + return E_NOT_SUPPORT; +} + +int GrdAdapter::Restore(const char *dbFile, const char *backupDbFile, const std::vector &encryptedKey) +{ + if (g_adapterHolder.Restore == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Restore == nullptr) { + return E_NOT_SUPPORT; + } + return E_NOT_SUPPORT; +} + +int32_t GrdAdapter::Reset(GRD_StmtT *stmt) +{ + if (g_adapterHolder.Reset == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.Reset == nullptr) { + return E_NOT_SUPPORT; + } + return TransErrno(g_adapterHolder.Reset(stmt)); +} + +uint32_t GrdAdapter::ColumnCount(GRD_StmtT *stmt) +{ + if (g_adapterHolder.ColumnCount == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnCount == nullptr) { + return E_NOT_SUPPORT; + } + return g_adapterHolder.ColumnCount(stmt); +} + +GRD_DbDataTypeE GrdAdapter::ColumnType(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.GetColumnType == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.GetColumnType == nullptr) { + return GRD_DB_DATATYPE_NULL; + } + return g_adapterHolder.GetColumnType(stmt, idx); +} + +uint32_t GrdAdapter::ColumnBytes(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnBytes == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnBytes == nullptr) { + return E_NOT_SUPPORT; + } + return g_adapterHolder.ColumnBytes(stmt, idx); +} + +char *GrdAdapter::ColumnName(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnName == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnName == nullptr) { + return nullptr; + } + return g_adapterHolder.ColumnName(stmt, idx); +} + +GRD_DbValueT GrdAdapter::ColumnValue(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnValue == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnValue == nullptr) { + return {}; + } + return g_adapterHolder.ColumnValue(stmt, idx); +} + +int64_t GrdAdapter::ColumnInt64(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnInt64 == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnInt64 == nullptr) { + return 0; + } + return g_adapterHolder.ColumnInt64(stmt, idx); +} + +int32_t GrdAdapter::ColumnInt(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnInt == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnInt == nullptr) { + return 0; + } + return g_adapterHolder.ColumnInt(stmt, idx); +} + +double GrdAdapter::ColumnDouble(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnDouble == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnDouble == nullptr) { + return 0; + } + return g_adapterHolder.ColumnDouble(stmt, idx); +} + +const char *GrdAdapter::ColumnText(GRD_StmtT *stmt, uint32_t idx) +{ + if (g_adapterHolder.ColumnText == nullptr) { + g_adapterHolder = GetAdapterHolder(); + } + if (g_adapterHolder.ColumnText == nullptr) { + return nullptr; + } + return g_adapterHolder.ColumnText(stmt, idx); +} +} + diff --git a/relational_store/test/native/gdb/mock/grd_adapter.h b/relational_store/test/native/gdb/mock/grd_adapter.h new file mode 100644 index 00000000..c1b10547 --- /dev/null +++ b/relational_store/test/native/gdb/mock/grd_adapter.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_DISTRIBUTED_DATA_NATIVE_MOCK_GRD_ADAPTER_H +#define OHOS_DISTRIBUTED_DATA_NATIVE_MOCK_GRD_ADAPTER_H + +#include +#include +#include +#include + +#include "full_result.h" +#include "grd_error.h" +#include "grd_type_export.h" +#include "statement.h" + +namespace OHOS::DistributedDataAip { +class GrdAdapter { +public: + enum FuncName : int32_t { + PREPARE = 0, + STEP, + FINALIZE, + REKEY, + ALL + }; + + static void SetErrorCode(FuncName func, int32_t err); + + static int32_t Prepare(GRD_DB *db, const char *str, uint32_t strLen, GRD_StmtT **stmt, const char **unusedStr); + static int32_t Step(GRD_StmtT *stmt); + static int32_t Finalize(GRD_StmtT *stmt); + static int32_t Rekey(const char *dbFile, const char *configStr, const std::vector &encryptedKey); + + static ColumnType TransColType(int grdColType); + static int Open(const char *dbPath, const char *configStr, uint32_t flags, GRD_DB **db); + static int Repair(const char *dbPath, const char *configStr); + static int Close(GRD_DB *db, uint32_t flags); + static int32_t Reset(GRD_StmtT *stmt); + static uint32_t ColumnCount(GRD_StmtT *stmt); + static GRD_DbDataTypeE ColumnType(GRD_StmtT *stmt, uint32_t idx); + static uint32_t ColumnBytes(GRD_StmtT *stmt, uint32_t idx); + static char *ColumnName(GRD_StmtT *stmt, uint32_t idx); + static GRD_DbValueT ColumnValue(GRD_StmtT *stmt, uint32_t idx); + static int64_t ColumnInt64(GRD_StmtT *stmt, uint32_t idx); + static int32_t ColumnInt(GRD_StmtT *stmt, uint32_t idx); + static double ColumnDouble(GRD_StmtT *stmt, uint32_t idx); + static const char *ColumnText(GRD_StmtT *stmt, uint32_t idx); + + static int Backup(GRD_DB *db, const char *backupDbFile, const std::vector &encryptedKey); + static int Restore(const char *dbFile, const char *backupDbFile, const std::vector &encryptedKey); + +private: + static int TransErrno(int err); + + static int32_t errorCode_[FuncName::ALL]; + static std::map GRD_ERRNO_MAP; +}; +} +#endif // OHOS_DISTRIBUTED_DATA_NATIVE_MOCK_GRD_ADAPTER_H + diff --git a/relational_store/test/native/gdb/unittest/gdb_encrypt_test.cpp b/relational_store/test/native/gdb/unittest/gdb_encrypt_test.cpp index 5fbce743..14551571 100644 --- a/relational_store/test/native/gdb/unittest/gdb_encrypt_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_encrypt_test.cpp @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_store_impl.h" #include "db_store_manager.h" #include "edge.h" @@ -224,7 +224,7 @@ HWTEST_F(GdbEncryptTest, GdbEncrypt_EncryptToUnencrypt, TestSize.Level1) config.SetEncryptStatus(false); store = GDBHelper::GetDBStore(config, errCode); EXPECT_EQ(store, nullptr); - EXPECT_EQ(errCode, E_GRD_INVALID_ARGS); + EXPECT_EQ(errCode, E_CONFIG_INVALID_CHANGE); GDBHelper::DeleteDBStore(config); } diff --git a/relational_store/test/native/gdb/unittest/gdb_execute_test.cpp b/relational_store/test/native/gdb/unittest/gdb_execute_test.cpp index 49d31263..8db22068 100644 --- a/relational_store/test/native/gdb/unittest/gdb_execute_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_execute_test.cpp @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_store_impl.h" #include "edge.h" #include "grd_adapter.h" @@ -1812,23 +1812,6 @@ HWTEST_F(GdbExecuteTest, GdbStore_Execute_UtilsAnonymousTest, TestSize.Level1) EXPECT_EQ(ret, "/***/el1/***/com.my.hmos.arkwebcore"); } -HWTEST_F(GdbExecuteTest, GdbStore_Execute_UtilsErrorTest, TestSize.Level1) -{ - auto errCode = GrdAdapter::TransErrno(100); - EXPECT_EQ(errCode, 100); - errCode = GrdAdapter::TransErrno(E_PARSE_JSON_FAILED); - EXPECT_EQ(errCode, E_PARSE_JSON_FAILED); - - errCode = GrdAdapter::TransErrno(E_PARSE_JSON_FAILED); - EXPECT_EQ(errCode, E_PARSE_JSON_FAILED); - - errCode = GrdAdapter::TransErrno(E_OK); - EXPECT_EQ(errCode, E_OK); - - errCode = GrdAdapter::TransErrno(-100); - EXPECT_EQ(errCode, E_GRD_INNER_ERR); -} - HWTEST_F(GdbExecuteTest, GdbStore_Execute_StatementTest, TestSize.Level1) { auto errCode = E_OK; @@ -2040,9 +2023,13 @@ HWTEST_F(GdbExecuteTest, GdbStore_Execute_UtilsConfigTest, TestSize.Level1) { std::vector keys = {1, 2, 3, 4, 5, 6, 7, 8}; auto ret = GdbUtils::GetConfigStr(keys, false); - EXPECT_EQ(ret, "{\"pageSize\":4, \"crcCheckEnable\":0}"); + EXPECT_EQ(ret, R"({"pageSize": 8, "crcCheckEnable": 0, "defaultIsolationLevel": 3, )" + R"("redoFlushByTrx": 1, "metaInfoBak": 1, "maxConnNum": 500, "bufferPoolSize": 10240})"); + ret = GdbUtils::GetConfigStr(keys, true); - EXPECT_EQ(ret, "{\"isEncrypted\":1,\"hexPassword\":\"0102030405060708\",\"pageSize\":4, \"crcCheckEnable\":0}"); + EXPECT_EQ(ret, R"({"isEncrypted":1,"hexPassword":"0102030405060708",)" + R"("pageSize": 8, "crcCheckEnable": 0, "defaultIsolationLevel": 3, )" + R"("redoFlushByTrx": 1, "metaInfoBak": 1, "maxConnNum": 500, "bufferPoolSize": 10240})"); } HWTEST_F(GdbExecuteTest, GdbStore_Execute_UtilsKeyTest, TestSize.Level1) diff --git a/relational_store/test/native/gdb/unittest/gdb_function_test.cpp b/relational_store/test/native/gdb/unittest/gdb_function_test.cpp index 289571a2..228bdad4 100644 --- a/relational_store/test/native/gdb/unittest/gdb_function_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_function_test.cpp @@ -18,7 +18,7 @@ #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "connection_pool.h" #include "gdb_helper.h" #include "grd_adapter_manager.h" diff --git a/relational_store/test/native/gdb/unittest/gdb_grd_adapter_test.cpp b/relational_store/test/native/gdb/unittest/gdb_grd_adapter_test.cpp new file mode 100644 index 00000000..3a60a496 --- /dev/null +++ b/relational_store/test/native/gdb/unittest/gdb_grd_adapter_test.cpp @@ -0,0 +1,455 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "gdb_errors.h" +#include "db_store_manager.h" +#include "edge.h" +#include "full_result.h" +#include "gdb_helper.h" +#include "gdb_store.h" +#include "gdb_transaction.h" +#include "grd_adapter_manager.h" +#include "path.h" +#include "result.h" +#include "vertex.h" +#include "../mock/grd_adapter.h" + +using namespace testing::ext; +using namespace OHOS::DistributedDataAip; + +using FuncName = GrdAdapter::FuncName; + +class GdbGrdAdapterTest : public testing::Test { +public: + static void SetUpTestCase(); + static void TearDownTestCase() {} + void SetUp(); + void TearDown() {} + + void CheckPrepareStepErrCode(int32_t grdErr, int32_t gdbErr, FuncName func); + void CheckRekeyErrCode(int32_t grdErr, int32_t gdbErr); + + void GetStore(const std::string &name); + void DeleteStore(const std::string &name); + + static const std::string path; + static const std::string createGraphGql; + static const std::string executeGql; + static const std::string queryGql; + + static std::shared_ptr store_; +}; + +const std::string GdbGrdAdapterTest::path = "/data"; + +const std::string GdbGrdAdapterTest::createGraphGql = "CREATE GRAPH test { " + "(person:Person {name STRING, age INT, sex BOOL DEFAULT false})," + "(dog:Dog {name STRING, age INT}), " + "(person) -[:Friend]-> (person) " + "};"; + +const std::string GdbGrdAdapterTest::executeGql = "INSERT (:Person {name: 'name_1', age: 11});"; +const std::string GdbGrdAdapterTest::queryGql = "MATCH (person:Person {name: 'name_1'}) RETURN person;"; + +std::shared_ptr GdbGrdAdapterTest::store_; + +static constexpr int32_t MAX_LEN = 128; +static constexpr int32_t MAX_PROP_CNT = 1024; + +void GdbGrdAdapterTest::SetUpTestCase() +{ + if (!IsSupportArkDataDb()) { + GTEST_SKIP() << "Current testcase is not compatible from current gdb"; + return; + } +} + +void GdbGrdAdapterTest::SetUp() +{ + if (!IsSupportArkDataDb()) { + GTEST_SKIP() << "Current testcase is not compatible from current gdb"; + return; + } +} + +void GdbGrdAdapterTest::GetStore(const std::string &name) +{ + auto config = StoreConfig(name, path); + GDBHelper::DeleteDBStore(config); + + int32_t errorCode = E_OK; + store_ = GDBHelper::GetDBStore(config, errorCode); + ASSERT_NE(store_, nullptr); + ASSERT_EQ(errorCode, E_OK); + + auto [errCode, result] = store_->ExecuteGql(createGraphGql); + ASSERT_EQ(errCode, E_OK); + + GrdAdapter::SetErrorCode(FuncName::ALL, GRD_OK); +} + +void GdbGrdAdapterTest::DeleteStore(const std::string &name) +{ + auto config = StoreConfig(name, path); + GDBHelper::DeleteDBStore(config); + + store_ = nullptr; +} + +void GdbGrdAdapterTest::CheckPrepareStepErrCode(int32_t grdErr, int32_t gdbErr, FuncName func) +{ + std::string name = "prepare_step_test"; + GetStore(name); + ASSERT_NE(store_, nullptr); + + GrdAdapter::SetErrorCode(func, grdErr); + + auto [errCode, result] = store_->ExecuteGql(executeGql); + EXPECT_EQ(errCode, gdbErr); + + std::tie(errCode, result) = store_->QueryGql(queryGql); + if (func == FuncName::STEP && grdErr == GRD_NO_DATA) { + EXPECT_EQ(errCode, E_OK); + } else { + EXPECT_EQ(errCode, gdbErr); + } + + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, gdbErr); + EXPECT_EQ(trans, nullptr); + + GrdAdapter::SetErrorCode(func, GRD_OK); + + std::tie(err, trans) = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + EXPECT_NE(trans, nullptr); + + GrdAdapter::SetErrorCode(func, grdErr); + + std::tie(errCode, result) = trans->Execute(executeGql); + EXPECT_EQ(errCode, gdbErr); + + std::tie(errCode, result) = trans->Query(queryGql); + if (func == FuncName::STEP && grdErr == GRD_NO_DATA) { + EXPECT_EQ(errCode, E_OK); + } else { + EXPECT_EQ(errCode, gdbErr); + } + + errCode = trans->Commit(); + EXPECT_EQ(errCode, gdbErr); + + GrdAdapter::SetErrorCode(func, GRD_OK); + + std::tie(err, trans) = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + EXPECT_NE(trans, nullptr); + + GrdAdapter::SetErrorCode(func, grdErr); + + errCode = trans->Rollback(); + EXPECT_EQ(errCode, gdbErr); + + GrdAdapter::SetErrorCode(func, GRD_OK); + + DeleteStore(name); +} + +void GdbGrdAdapterTest::CheckRekeyErrCode(int32_t grdErr, int32_t gdbErr) +{ + std::string name = "rekey_test"; + GetStore(name); + ASSERT_NE(store_, nullptr); + + auto errCode = store_->Close(); + EXPECT_EQ(errCode, E_OK); + StoreManager::GetInstance().Clear(); + + GrdAdapter::SetErrorCode(FuncName::REKEY, grdErr); + + auto config = StoreConfig(name, path, DBType::DB_GRAPH, true); + store_ = GDBHelper::GetDBStore(config, errCode); + EXPECT_EQ(errCode, gdbErr); + EXPECT_EQ(store_, nullptr); + + GrdAdapter::SetErrorCode(FuncName::REKEY, GRD_OK); + + DeleteStore(name); +} + +/** + * @tc.name: GdbGrdAdapterTest001 + * @tc.desc: test Prepare returned the following error code + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest001, TestSize.Level1) +{ + std::map errCodeMap = { + { GRD_INVALID_ARGS, E_GRD_INVALID_ARGS }, + { GRD_SYNTAX_ERROR, E_GRD_SYNTAX_ERROR }, + { GRD_SEMANTIC_ERROR, E_GRD_SEMANTIC_ERROR }, + { GRD_OVER_LIMIT, E_GRD_OVER_LIMIT }, + { GRD_FAILED_MEMORY_ALLOCATE, E_GRD_FAILED_MEMORY_ALLOCATE }, + { GRD_MODEL_NOT_SUPPORT, E_GRD_SEMANTIC_ERROR }, + { GRD_FEATURE_NOT_SUPPORTED, E_GRD_SEMANTIC_ERROR }, + { GRD_DATATYPE_MISMATCH, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_VALUE, E_GRD_INVALID_ARGS }, + { GRD_NAME_TOO_LONG, E_GRD_INVALID_NAME }, + { GRD_DUPLICATE_TABLE, E_GRD_DUPLICATE_PARAM }, + { GRD_DUPLICATE_OBJECT, E_GRD_DUPLICATE_PARAM }, + { GRD_INVALID_CONFIG_VALUE, E_CONFIG_INVALID_CHANGE }, + { GRD_DUPLICATE_COLUMN, E_GRD_DUPLICATE_PARAM }, + { GRD_UNDEFINE_COLUMN, E_GRD_UNDEFINED_PARAM }, + { GRD_UNDEFINED_OBJECT, E_GRD_UNDEFINED_PARAM }, + { GRD_UNDEFINED_TABLE, E_GRD_UNDEFINED_PARAM }, + }; + + for (auto [grdErr, gdbErr] : errCodeMap) { + CheckPrepareStepErrCode(grdErr, gdbErr, FuncName::PREPARE); + } +} + +/** + * @tc.name: GdbGrdAdapterTest002 + * @tc.desc: test Step returned the following error code + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest002, TestSize.Level1) +{ + std::map errCodeMap = { + { GRD_NO_DATA, E_GRD_NO_DATA }, + { GRD_INVALID_TABLE_DEFINITION, E_GRD_SEMANTIC_ERROR }, + { GRD_INVALID_ARGS, E_GRD_INVALID_ARGS }, + { GRD_INVALID_CONFIG_VALUE, E_CONFIG_INVALID_CHANGE }, + { GRD_DATATYPE_MISMATCH, E_GRD_SEMANTIC_ERROR }, + { GRD_DIVISION_BY_ZERO, E_GRD_SYNTAX_ERROR }, + { GRD_MODEL_NOT_SUPPORT, E_GRD_SEMANTIC_ERROR }, + { GRD_FEATURE_NOT_SUPPORTED, E_GRD_SEMANTIC_ERROR }, + { GRD_FIELD_OVERFLOW, E_GRD_SEMANTIC_ERROR }, + { GRD_FILE_OPERATE_FAILED, E_GRD_INNER_ERR }, + { GRD_INSUFFICIENT_RESOURCES, E_GRD_INNER_ERR }, + { GRD_RESOURCE_BUSY, E_DATABASE_BUSY }, + { GRD_OVER_LIMIT, E_GRD_OVER_LIMIT }, + { GRD_REQUEST_TIME_OUT, E_DATABASE_BUSY }, + { GRD_RESTRICT_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_NO_ACTIVE_TRANSACTION, E_GRD_NO_ACTIVE_TRANSACTION }, + { GRD_TRANSACTION_ROLLBACK, E_GRD_TRANSACTION_ROLLBACK }, + { GRD_ACTIVE_TRANSACTION, E_GRD_ACTIVE_TRANSACTION }, + { GRD_UNIQUE_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_PRIMARY_KEY_VIOLATION, E_GRD_DATA_CONFLICT }, + { GRD_UNDEFINED_TABLE, E_GRD_UNDEFINED_PARAM }, + { GRD_UNDEFINED_OBJECT, E_GRD_UNDEFINED_PARAM }, + { GRD_DUPLICATE_TABLE, E_GRD_DUPLICATE_PARAM }, + { GRD_DUPLICATE_OBJECT, E_GRD_DUPLICATE_PARAM }, + { GRD_DATA_CORRUPTED, E_GRD_DATA_CORRUPTED }, + { GRD_THIRD_PARTY_FUNCTION_EXECUTE_FAILED, E_GRD_INNER_ERR }, + { GRD_SCHEMA_CHANGED, E_CONFIG_INVALID_CHANGE }, + }; + + for (auto [grdErr, gdbErr] : errCodeMap) { + CheckPrepareStepErrCode(grdErr, gdbErr, FuncName::STEP); + } +} + +/** + * @tc.name: GdbGrdAdapterTest003 + * @tc.desc: test ReKey returned the following error code + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest003, TestSize.Level1) +{ + std::map errCodeMap = { + { GRD_INVALID_ARGS, E_GRD_INVALID_ARGS }, + { GRD_INNER_ERR, E_GRD_INNER_ERR }, + { GRD_NO_DATA, E_GRD_NO_DATA }, + { GRD_RESOURCE_BUSY, E_DATABASE_BUSY }, + { GRD_SYSTEM_ERR, E_GRD_INNER_ERR }, + { GRD_INVALID_FORMAT, E_GRD_SEMANTIC_ERROR }, + { GRD_INSUFFICIENT_SPACE, E_GRD_DISK_SPACE_FULL }, + { GRD_DB_INSTANCE_ABNORMAL, E_GRD_DB_INSTANCE_ABNORMAL }, + { GRD_DB_BUSY, E_DATABASE_BUSY }, + { GRD_DATA_CORRUPTED, E_GRD_DATA_CORRUPTED }, + }; + + for (auto [grdErr, gdbErr] : errCodeMap) { + CheckRekeyErrCode(grdErr, gdbErr); + } +} + +/** + * @tc.name: GdbGrdAdapterTest004 + * @tc.desc: test whether the error code returned by Prepare when using the following gql is correct + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest004, TestSize.Level1) +{ + std::string duplicateObjectGql = "CREATE GRAPH test { " + "(person:Person {name STRING, age INT, age DOUBLE})," + "(person) -[:Friend]-> (person) " + "};"; + std::map errCodeMap = { + { "INSERT (:Person {});", E_GRD_SYNTAX_ERROR }, // GRD_SYNTAX_ERROR + { "INSERT ();", E_GRD_SEMANTIC_ERROR }, // GRD_SEMANTIC_ERROR + { "INSERT (:Person {name: 11, age: 'age_11'});", E_GRD_SEMANTIC_ERROR }, // GRD_DATATYPE_MISMATCH + { "DROP GRAPH nonexistent_graph;", E_GRD_UNDEFINED_PARAM }, // GRD_UNDEFINE_COLUMN + { "MATCH (a: Person {name: 'Alf'}) SET a.nationality='EN'", E_GRD_UNDEFINED_PARAM }, // GRD_UNDEFINED_OBJECT + { "INSERT (:Student {name: 'name_1', age: 11});", E_GRD_UNDEFINED_PARAM }, // GRD_UNDEFINED_TABLE + { duplicateObjectGql, E_GRD_DUPLICATE_PARAM }, // GRD_DUPLICATE_OBJECT + }; + + for (auto [gql, gdbErr] : errCodeMap) { + std::string name = "prepare_test"; + GetStore(name); + ASSERT_NE(store_, nullptr); + + auto [errCode, result] = store_->ExecuteGql(gql); + EXPECT_EQ(errCode, gdbErr); + + DeleteStore(name); + } +} + +/** + * @tc.name: GdbGrdAdapterTest005 + * @tc.desc: test whether the error code returned by Prepare when using the following gql is correct + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest005, TestSize.Level1) +{ + std::string graphName = "test_"; + for (int i = 0; i < MAX_LEN; i++) { + graphName += "a"; + } + std::string nameTooLongGql = "CREATE GRAPH " + graphName + " { " + "(person:Person {name STRING, age INT, sex BOOL DEFAULT false})," + "(person) -[:Friend]-> (person) " + "};"; + std::string duplicateTableGql = "CREATE GRAPH test { " + "(person:Person {name STRING, age INT, sex BOOL DEFAULT false})," + "(student:Person {name STRING, age INT}), " + "(person) -[:Friend]-> (person) " + "};"; + std::string duplicateColumnGql = "CREATE GRAPH test { " + "(person:Person {name STRING, age INT, age DOUBLE})," + "(person) -[:Friend]-> (person) " + "};"; + std::map errCodeMap = { + { nameTooLongGql, E_GRD_INVALID_NAME }, // GRD_NAME_TOO_LONG + { duplicateTableGql, E_GRD_DUPLICATE_PARAM }, // GRD_DUPLICATE_TABLE + { duplicateColumnGql, E_GRD_DUPLICATE_PARAM }, // GRD_DUPLICATE_COLUMN + }; + + for (auto [gql, gdbErr] : errCodeMap) { + std::string name = "prepare_test"; + auto config = StoreConfig(name, path); + GDBHelper::DeleteDBStore(config); + + int32_t errorCode = E_OK; + store_ = GDBHelper::GetDBStore(config, errorCode); + ASSERT_NE(store_, nullptr); + ASSERT_EQ(errorCode, E_OK); + + auto [errCode, result] = store_->ExecuteGql(gql); + EXPECT_EQ(errCode, gdbErr); + + DeleteStore(name); + } +} + +/** + * @tc.name: GdbGrdAdapterTest006 + * @tc.desc: test whether the error code returned by Step when using the following gql is correct + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest006, TestSize.Level1) +{ + std::map errCodeMap = { + { "MATCH (p:Person {age: 11}) RETURN p.age / 0;", E_GRD_SYNTAX_ERROR }, // GRD_DIVISION_BY_ZERO + { "MATCH (p:Person {age: 11}) SET p.age=9223372036854775808;", E_GRD_SEMANTIC_ERROR }, // GRD_FIELD_OVERFLOW + }; + + for (auto [gql, gdbErr] : errCodeMap) { + std::string name = "step_test"; + GetStore(name); + ASSERT_NE(store_, nullptr); + + auto [errCode, result] = store_->ExecuteGql(executeGql); + EXPECT_EQ(errCode, E_OK); + + std::tie(errCode, result) = store_->ExecuteGql(gql); + EXPECT_EQ(errCode, gdbErr); + + DeleteStore(name); + } +} + +/** + * @tc.name: GdbGrdAdapterTest007 + * @tc.desc: test whether the error code returned by Step when using the following gql is correct + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest007, TestSize.Level1) +{ + std::string name = "step_test"; + auto config = StoreConfig(name, path); + GDBHelper::DeleteDBStore(config); + + int32_t errorCode = E_OK; + store_ = GDBHelper::GetDBStore(config, errorCode); + ASSERT_NE(store_, nullptr); + ASSERT_EQ(errorCode, E_OK); + + std::string overLimitGql = "CREATE GRAPH test { (person:Person {"; + for (int i = 0; i < MAX_PROP_CNT; i++) { + overLimitGql += "name" + std::to_string(i) + " STRING, "; + } + overLimitGql += "age INT, sex BOOL DEFAULT false}) };"; + + auto [errCode, result] = store_->ExecuteGql(overLimitGql); + EXPECT_EQ(errCode, E_GRD_OVER_LIMIT); // GRD_OVER_LIMIT + + DeleteStore(name); +} + +/** + * @tc.name: GdbGrdAdapterTest008 + * @tc.desc: test whether the error code returned by Step when using the following gql is correct + * @tc.type: FUNC + */ +HWTEST_F(GdbGrdAdapterTest, GdbGrdAdapterTest008, TestSize.Level1) +{ + std::string name = "step_test"; + GetStore(name); + ASSERT_NE(store_, nullptr); + + auto [errCode, result] = store_->ExecuteGql("CREATE UNIQUE INDEX nameIndex ON Person(name);"); + EXPECT_EQ(errCode, E_OK); + + std::tie(errCode, result) = store_->ExecuteGql("INSERT (:Person {name: 'name_1', age: 11});"); + EXPECT_EQ(errCode, E_OK); + + std::tie(errCode, result) = store_->ExecuteGql("INSERT (:Person {name: 'name_1', age: 22});"); + EXPECT_EQ(errCode, E_GRD_DATA_CONFLICT); // GRD_UNIQUE_VIOLATION + + DeleteStore(name); +} diff --git a/relational_store/test/native/gdb/unittest/gdb_grdapi_test.cpp b/relational_store/test/native/gdb/unittest/gdb_grdapi_test.cpp index 31fda199..f9088256 100644 --- a/relational_store/test/native/gdb/unittest/gdb_grdapi_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_grdapi_test.cpp @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "db_store_manager.h" #include "gdb_helper.h" #include "grd_adapter.h" diff --git a/relational_store/test/native/gdb/unittest/gdb_multi_thread_test.cpp b/relational_store/test/native/gdb/unittest/gdb_multi_thread_test.cpp index 85188595..6eded774 100644 --- a/relational_store/test/native/gdb/unittest/gdb_multi_thread_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_multi_thread_test.cpp @@ -20,7 +20,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "executor_pool.h" #include "grd_adapter_manager.h" #include "gdb_helper.h" diff --git a/relational_store/test/native/gdb/unittest/gdb_query_test.cpp b/relational_store/test/native/gdb/unittest/gdb_query_test.cpp index e1698eac..eac39961 100644 --- a/relational_store/test/native/gdb/unittest/gdb_query_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_query_test.cpp @@ -21,7 +21,7 @@ #include #include -#include "aip_errors.h" +#include "gdb_errors.h" #include "gdb_helper.h" #include "gdb_store.h" #include "grd_adapter_manager.h" diff --git a/relational_store/test/native/gdb/unittest/gdb_store_config_test.cpp b/relational_store/test/native/gdb/unittest/gdb_store_config_test.cpp new file mode 100644 index 00000000..306ddf92 --- /dev/null +++ b/relational_store/test/native/gdb/unittest/gdb_store_config_test.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gdb_store_config.h" +#include + +using namespace testing::ext; +using namespace OHOS::DistributedDataAip; +class GdbStoreConfigTest : public testing::Test { +public: + static void SetUpTestCase() {} + static void TearDownTestCase() {} + void SetUp() {} + void TearDown() {} +}; + +/** + * @tc.name: GdbStoreConfig_Test_SetWriteTime + * @tc.desc: test SetWriteTime + * @tc.type: FUNC + */ +HWTEST_F(GdbStoreConfigTest, GdbStoreConfig_Test_SetWriteTime, TestSize.Level1) +{ + StoreConfig storeCfg; + const int timeout = 10; + storeCfg.SetWriteTime(timeout); + int getTime = storeCfg.GetWriteTime(); + EXPECT_EQ(getTime, timeout); +} \ No newline at end of file diff --git a/relational_store/test/native/gdb/unittest/gdb_transaction_test.cpp b/relational_store/test/native/gdb/unittest/gdb_transaction_test.cpp index 5a964a65..71626cbe 100644 --- a/relational_store/test/native/gdb/unittest/gdb_transaction_test.cpp +++ b/relational_store/test/native/gdb/unittest/gdb_transaction_test.cpp @@ -13,6 +13,8 @@ * limitations under the License. */ +#include "gdb_transaction.h" + #include #include @@ -21,15 +23,14 @@ #include #include -#include "aip_errors.h" #include "edge.h" #include "full_result.h" +#include "gdb_errors.h" #include "gdb_helper.h" #include "gdb_store.h" #include "grd_adapter_manager.h" #include "path.h" #include "result.h" -#include "transaction.h" #include "vertex.h" using namespace testing::ext; @@ -70,6 +71,11 @@ std::shared_ptr GdbTransactionTest::store_; static constexpr int32_t MAX_GQL_LEN = 1024 * 1024; static constexpr int32_t MAX_CNT = 4; +static constexpr int32_t MAX_DATA_CNT = 200; + +static constexpr int32_t BUSY_TIMEOUT = 2; +static constexpr int32_t EXECUTE_INTERVAL = 3; +static constexpr int32_t READ_INTERVAL = 100; void GdbTransactionTest::SetUpTestCase() { @@ -601,3 +607,196 @@ HWTEST_F(GdbTransactionTest, GdbTransactionTest016_TransactionMultiThread, TestS MatchAndVerifyPerson("name_" + std::to_string(i), i); } } + +/** + * @tc.name: GdbTransactionTest017_TransactionMultiThread + * @tc.desc: test Transaction MultiThread write busy(Unable to obtain lock in 2 seconds) + * @tc.type: FUNC + */ +HWTEST_F(GdbTransactionTest, GdbTransactionTest017_TransactionMultiThread, TestSize.Level1) +{ + std::thread th[MAX_CNT]; + std::atomic_int insertCnt = 0; + std::atomic_int insertIndex = -1; + for (int32_t i = 0; i < MAX_CNT; i++) { + th[i] = std::thread([this, i, &insertCnt, &insertIndex] () { + ASSERT_NE(store_, nullptr); + + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + + std::shared_ptr result = std::make_shared(); + std::string age = std::to_string(i); + std::string name = "name_" + age; + std::string gql = "INSERT (:Person {name: '" + name + "', age: " + age + "});"; + std::tie(err, result) = trans->Execute(gql); + if (err == E_OK && insertCnt.load() == 0) { + insertCnt++; + insertIndex.store(i); + } else { + EXPECT_EQ(err, E_DATABASE_BUSY); + } + + std::this_thread::sleep_for(std::chrono::seconds(BUSY_TIMEOUT)); + + err = trans->Commit(); + EXPECT_EQ(err, E_OK); + }); + } + + for (int32_t i = 0; i < MAX_CNT; i++) { + th[i].join(); + } + + for (int32_t i = 0; i < MAX_CNT; i++) { + if (i == insertIndex.load()) { + MatchAndVerifyPerson("name_" + std::to_string(i), i); + } else { + MatchAndVerifyPerson("name_" + std::to_string(i), i, nullptr, false); + } + } + + EXPECT_EQ(insertCnt, 1); +} + +/** + * @tc.name: GdbTransactionTest018_TransactionMultiThread + * @tc.desc: test Transaction MultiThread Trasaction.write and Transaction.read + * @tc.type: FUNC + */ +HWTEST_F(GdbTransactionTest, GdbTransactionTest018_TransactionMultiThread, TestSize.Level1) +{ + ASSERT_NE(store_, nullptr); + + std::thread writeThread([this] () { + for (int i = 0; i < MAX_DATA_CNT; i++) { + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + InsertPerson("name_" + std::to_string(i), i, trans); + err = trans->Commit(); + EXPECT_EQ(err, E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + std::thread readThread([this] () { + std::this_thread::sleep_for(std::chrono::milliseconds(READ_INTERVAL)); + for (int i = 0; i < MAX_DATA_CNT; i++) { + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + MatchAndVerifyPerson("name_" + std::to_string(i), i, trans); + err = trans->Rollback(); + EXPECT_EQ(err, E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + writeThread.join(); + readThread.join(); +} + +/** + * @tc.name: GdbTransactionTest019_TransactionMultiThread + * @tc.desc: test Transaction MultiThread GraphStore.write and Transaction.read + * @tc.type: FUNC + */ +HWTEST_F(GdbTransactionTest, GdbTransactionTest019_TransactionMultiThread, TestSize.Level1) +{ + ASSERT_NE(store_, nullptr); + + std::thread writeThread([this] () { + for (int i = 0; i < MAX_DATA_CNT; i++) { + InsertPerson("name_" + std::to_string(i), i); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + std::thread readThread([this] () { + std::this_thread::sleep_for(std::chrono::milliseconds(READ_INTERVAL)); + for (int i = 0; i < MAX_DATA_CNT; i++) { + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + MatchAndVerifyPerson("name_" + std::to_string(i), i, trans); + err = trans->Rollback(); + EXPECT_EQ(err, E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + writeThread.join(); + readThread.join(); +} + +/** + * @tc.name: GdbTransactionTest020_TransactionMultiThread + * @tc.desc: test Transaction MultiThread Trasaction.write and GraphStore.read + * @tc.type: FUNC + */ +HWTEST_F(GdbTransactionTest, GdbTransactionTest020_TransactionMultiThread, TestSize.Level1) +{ + ASSERT_NE(store_, nullptr); + + std::thread writeThread([this] () { + for (int i = 0; i < MAX_DATA_CNT; i++) { + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + InsertPerson("name_" + std::to_string(i), i, trans); + err = trans->Commit(); + EXPECT_EQ(err, E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + std::thread readThread([this] () { + std::this_thread::sleep_for(std::chrono::milliseconds(READ_INTERVAL)); + for (int i = 0; i < MAX_DATA_CNT; i++) { + MatchAndVerifyPerson("name_" + std::to_string(i), i); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + writeThread.join(); + readThread.join(); +} + +/** + * @tc.name: GdbTransactionTest021_TransactionMultiThread + * @tc.desc: test Transaction MultiThread Trasaction.write and GraphStore.write + * @tc.type: FUNC + */ +HWTEST_F(GdbTransactionTest, GdbTransactionTest021_TransactionMultiThread, TestSize.Level1) +{ + ASSERT_NE(store_, nullptr); + + std::thread transWriteThread([this] () { + for (int i = 0; i < MAX_DATA_CNT; i++) { + auto [err, trans] = store_->CreateTransaction(); + EXPECT_EQ(err, E_OK); + ASSERT_NE(trans, nullptr); + InsertPerson("name_trans_" + std::to_string(i), i, trans); + err = trans->Commit(); + EXPECT_EQ(err, E_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + std::thread writeThread([this] () { + for (int i = 0; i < MAX_DATA_CNT; i++) { + InsertPerson("name_" + std::to_string(i), i); + std::this_thread::sleep_for(std::chrono::milliseconds(EXECUTE_INTERVAL)); + } + }); + + transWriteThread.join(); + writeThread.join(); + + for (int i = 0; i < MAX_DATA_CNT; i++) { + MatchAndVerifyPerson("name_trans_" + std::to_string(i), i); + MatchAndVerifyPerson("name_" + std::to_string(i), i); + } +} diff --git a/relational_store/test/native/rdb/BUILD.gn b/relational_store/test/native/rdb/BUILD.gn index 00a203bc..5a785407 100644 --- a/relational_store/test/native/rdb/BUILD.gn +++ b/relational_store/test/native/rdb/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_rdb" +module_output_path = "relational_store/relational_store/native_rdb" ############################################################################### config("module_private_config") { @@ -120,10 +120,12 @@ ohos_unittest("NativeRdbTest") { "unittest/rdb_get_store_test.cpp", "unittest/rdb_helper_test.cpp", "unittest/rdb_insert_test.cpp", + "unittest/rdb_memory_db_test.cpp", "unittest/rdb_open_callback_test.cpp", "unittest/rdb_predicates_join_b_test.cpp", "unittest/rdb_predicates_join_test.cpp", "unittest/rdb_predicates_test.cpp", + "unittest/rdb_rd_data_aging_test.cpp", "unittest/rdb_read_only_test.cpp", "unittest/rdb_security_manager_test.cpp", "unittest/rdb_sql_utils_test.cpp", diff --git a/relational_store/test/native/rdb/distributedtest/rdb_store_impl_test/BUILD.gn b/relational_store/test/native/rdb/distributedtest/rdb_store_impl_test/BUILD.gn index 6a449841..39778e1a 100644 --- a/relational_store/test/native/rdb/distributedtest/rdb_store_impl_test/BUILD.gn +++ b/relational_store/test/native/rdb/distributedtest/rdb_store_impl_test/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_rdb" +module_output_path = "relational_store/relational_store/native_rdb" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/native/rdb/unittest/common.cpp b/relational_store/test/native/rdb/unittest/common.cpp index 6e5630bd..7aa22377 100644 --- a/relational_store/test/native/rdb/unittest/common.cpp +++ b/relational_store/test/native/rdb/unittest/common.cpp @@ -16,9 +16,7 @@ #include "common.h" #include -#include -#include "logger.h" #include "rdb_errno.h" namespace OHOS { @@ -79,25 +77,6 @@ ValuesBucket UTUtils::SetRowDatas(const RowDatas &rowDatas) return value; } -bool UTUtils::DestoryDb(const std::string &path, uint64_t offset, const std::vector &bytes) -{ - std::fstream file(path, std::ios::in | std::ios::out | std::ios::binary); - if (!file.is_open()) { - return false; - } - file.seekp(offset, std::ios::beg); - if (!file.good()) { - return false; - } - file.write(bytes.data(), sizeof(bytes)); - if (!file.good()) { - return false; - } - file.close(); - file.flush(); - return true; -} - const RowDatas UTUtils::gRowDatas[14] = { { 1001, "SunWuKong", 4, ValueObject(1004), "2000-12-17", 8000.00, ValueObject(), 20 }, { 1002, "LuJunYi", 3, ValueObject(1006), "2001-02-20", 16000.00, ValueObject(3000.00), 30 }, diff --git a/relational_store/test/native/rdb/unittest/common.h b/relational_store/test/native/rdb/unittest/common.h index 9a6a625c..f51aa446 100644 --- a/relational_store/test/native/rdb/unittest/common.h +++ b/relational_store/test/native/rdb/unittest/common.h @@ -51,9 +51,6 @@ public: static ValuesBucket SetRowDatas(const RowDatas &rowDatas); - static bool DestoryDb( - const std::string &path, uint64_t offset = 30, const std::vector &bytes = { 0x0a, 0x0b }); - static const RowData g_rowData[3]; static const RowDatas gRowDatas[14]; diff --git a/relational_store/test/native/rdb/unittest/multiThread/rdb_store_multiprocess_createDB_test.cpp b/relational_store/test/native/rdb/unittest/multiThread/rdb_store_multiprocess_createDB_test.cpp index 37953ba0..6758a23e 100644 --- a/relational_store/test/native/rdb/unittest/multiThread/rdb_store_multiprocess_createDB_test.cpp +++ b/relational_store/test/native/rdb/unittest/multiThread/rdb_store_multiprocess_createDB_test.cpp @@ -215,10 +215,24 @@ void RdbMultiProcessCreateDBTest::Create() */ HWTEST_F(RdbMultiProcessCreateDBTest, Rdb_ConcurrentCreate_001, TestSize.Level1) { - int pid = fork(); + Create(); +} + +int32_t main(int32_t argc, char *argv[]) +{ + testing::GTEST_FLAG(output) = "xml:./"; + testing::InitGoogleTest(&argc, argv); + pid_t pid = fork(); if (pid == 0) { - Create(); - } else { - Create(); + RdbMultiProcessCreateDBTest::Create(); + exit(0); + } else if (pid < 0) { + return 1; + } + int res = RUN_ALL_TESTS(); + if (pid > 0) { + int status; + waitpid(pid, &status, 0); } + return res; } \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_attach_test.cpp b/relational_store/test/native/rdb/unittest/rdb_attach_test.cpp index 84705522..4e8cb5bb 100644 --- a/relational_store/test/native/rdb/unittest/rdb_attach_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_attach_test.cpp @@ -282,6 +282,158 @@ HWTEST_F(RdbAttachTest, RdbStore_Attach_006, TestSize.Level2) EXPECT_EQ(0, ret.second); } +/** + * @tc.name: RdbStore_Attach_007 + * @tc.desc: normal testCase for Attach with memoryDb + * @tc.type: FUNC + */ +HWTEST_F(RdbAttachTest, RdbStore_Attach_007, TestSize.Level2) +{ + const std::string attachedName = "attached"; + RdbStoreConfig config(RdbAttachTest::MAIN_DATABASE_NAME); + MainOpenCallback helper; + int errCode = E_OK; + std::shared_ptr walDb = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(nullptr, walDb); + ASSERT_EQ(E_OK, errCode); + + RdbStoreConfig attachedConfig("/data/test/memoryAttachTest007.db"); + attachedConfig.SetStorageMode(StorageMode::MODE_MEMORY); + std::shared_ptr memDb = RdbHelper::GetRdbStore(attachedConfig, 1, helper, errCode); + + std::string sql = "CREATE TABLE IF NOT EXISTS test(id INTEGER PRIMARY KEY " + "AUTOINCREMENT, name TEXT NOT NULL)"; + auto code = memDb->ExecuteSql(sql); + ASSERT_EQ(E_OK, code); + + ValueObject val; + std::tie(code, val) = memDb->Execute("insert into test (id, name) values(?,?)", { 1, "memDbName" }); + ASSERT_EQ(E_OK, code); + + auto ret = walDb->Attach(attachedConfig, attachedName, BUSY_TIMEOUT); + ASSERT_EQ(E_OK, ret.first); + ASSERT_EQ(1, ret.second); + + auto result = walDb->QuerySql("select * from attached.test"); + ASSERT_NE(result, nullptr); + int count = -1; + ASSERT_EQ(E_OK, result->GetRowCount(count)); + EXPECT_EQ(count, 1); + ASSERT_EQ(E_OK, result->GoToFirstRow()); + + int index = -1; + ASSERT_EQ(E_OK, result->GetColumnIndex("name", index)); + std::string name; + ASSERT_EQ(E_OK, result->GetString(index, name)); + EXPECT_EQ(name, "memDbName"); + result->Close(); + + ret = walDb->Detach(attachedName); + EXPECT_EQ(E_OK, ret.first); + EXPECT_EQ(0, ret.second); +} + +/** + * @tc.name: RdbStore_Attach_008 + * @tc.desc: normal testCase for memoryDb attach with memoryDb + * @tc.type: FUNC + */ +HWTEST_F(RdbAttachTest, RdbStore_Attach_008, TestSize.Level2) +{ + const std::string attachedName = "attached"; + RdbStoreConfig config("/data/test/memoryAttachMain008.db"); + config.SetStorageMode(StorageMode::MODE_MEMORY); + MainOpenCallback helper; + int errCode = E_OK; + std::shared_ptr mainMemDb = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(nullptr, mainMemDb); + ASSERT_EQ(E_OK, errCode); + + RdbStoreConfig attachedConfig("/data/test/memoryAttachTest008.db"); + attachedConfig.SetStorageMode(StorageMode::MODE_MEMORY); + std::shared_ptr memDb = RdbHelper::GetRdbStore(attachedConfig, 1, helper, errCode); + + std::string sql = "CREATE TABLE IF NOT EXISTS test1(id INTEGER PRIMARY KEY " + "AUTOINCREMENT, name TEXT NOT NULL)"; + auto code = memDb->ExecuteSql(sql); + ASSERT_EQ(E_OK, code); + + auto ret = mainMemDb->Attach(attachedConfig, attachedName, BUSY_TIMEOUT); + ASSERT_EQ(E_OK, ret.first); + ASSERT_EQ(1, ret.second); + + ValueObject val; + std::tie(code, val) = mainMemDb->Execute("insert into attached.test1 (id, name) values(?,?)", { 1, "memDbName" }); + ASSERT_EQ(E_OK, ret.first); + auto result = mainMemDb->QuerySql("select * from attached.test1;"); + ASSERT_NE(result, nullptr); + int count = -1; + ASSERT_EQ(E_OK, result->GetRowCount(count)); + EXPECT_EQ(count, 1); + ASSERT_EQ(E_OK, result->GoToFirstRow()); + + int index = -1; + ASSERT_EQ(E_OK, result->GetColumnIndex("name", index)); + std::string name; + ASSERT_EQ(E_OK, result->GetString(index, name)); + EXPECT_EQ(name, "memDbName"); + result->Close(); + + ret = mainMemDb->Detach(attachedName); + EXPECT_EQ(E_OK, ret.first); + EXPECT_EQ(0, ret.second); +} + +/** + * @tc.name: RdbStore_Attach_009 + * @tc.desc: normal testCase for memoryDb attach with walDb + * @tc.type: FUNC + */ +HWTEST_F(RdbAttachTest, RdbStore_Attach_009, TestSize.Level2) +{ + const std::string attachedName = "attached"; + RdbStoreConfig config("/data/test/memoryAttachMain009.db"); + config.SetStorageMode(StorageMode::MODE_MEMORY); + MainOpenCallback helper; + int errCode = E_OK; + std::shared_ptr mainMemDb = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(nullptr, mainMemDb); + ASSERT_EQ(E_OK, errCode); + + RdbStoreConfig attachedConfig("/data/test/AttachTest009.db"); + std::shared_ptr walDb = RdbHelper::GetRdbStore(attachedConfig, 1, helper, errCode); + + std::string sql = "CREATE TABLE IF NOT EXISTS test1(id INTEGER PRIMARY KEY " + "AUTOINCREMENT, name TEXT NOT NULL)"; + auto code = walDb->ExecuteSql(sql); + ASSERT_EQ(E_OK, code); + + auto ret = mainMemDb->Attach(attachedConfig, attachedName, BUSY_TIMEOUT); + ASSERT_EQ(E_OK, ret.first); + ASSERT_EQ(1, ret.second); + + ValueObject val; + std::tie(code, val) = mainMemDb->Execute("insert into attached.test1 (id, name) values(?,?)", { 1, "memDbName" }); + ASSERT_EQ(E_OK, ret.first); + auto result = mainMemDb->QuerySql("select * from attached.test1;"); + ASSERT_NE(result, nullptr); + int count = -1; + ASSERT_EQ(E_OK, result->GetRowCount(count)); + EXPECT_EQ(count, 1); + ASSERT_EQ(E_OK, result->GoToFirstRow()); + + int index = -1; + ASSERT_EQ(E_OK, result->GetColumnIndex("name", index)); + std::string name; + ASSERT_EQ(E_OK, result->GetString(index, name)); + EXPECT_EQ(name, "memDbName"); + result->Close(); + + ret = mainMemDb->Detach(attachedName); + EXPECT_EQ(E_OK, ret.first); + EXPECT_EQ(0, ret.second); +} + void RdbAttachTest::QueryCheck1(std::shared_ptr &store) const { std::shared_ptr resultSet = store->QuerySql("SELECT * FROM test1"); diff --git a/relational_store/test/native/rdb/unittest/rdb_delete_test.cpp b/relational_store/test/native/rdb/unittest/rdb_delete_test.cpp index 6ccbe1a2..b60c4a9b 100644 --- a/relational_store/test/native/rdb/unittest/rdb_delete_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_delete_test.cpp @@ -24,33 +24,39 @@ using namespace testing::ext; using namespace OHOS::NativeRdb; +namespace OHOS::RdbDeleteTest { +struct RdbTestParam { + std::shared_ptr store; + operator std::shared_ptr() + { + return store; + } +}; +static RdbTestParam g_store; +static RdbTestParam g_memDb; -class RdbDeleteTest : public testing::Test { +class RdbDeleteTest : public testing::TestWithParam { public: static void SetUpTestCase(void); static void TearDownTestCase(void); void SetUp(); void TearDown(); + std::shared_ptr store_; static const std::string DATABASE_NAME; - static std::shared_ptr store; }; const std::string RdbDeleteTest::DATABASE_NAME = RDB_TEST_PATH + "delete_test.db"; -std::shared_ptr RdbDeleteTest::store = nullptr; class DeleteTestOpenCallback : public RdbOpenCallback { public: int OnCreate(RdbStore &store) override; int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; - static const std::string CREATE_TABLE_TEST; }; - -std::string const DeleteTestOpenCallback::CREATE_TABLE_TEST = - std::string("CREATE TABLE IF NOT EXISTS test ") + std::string("(id INTEGER PRIMARY KEY AUTOINCREMENT, " - "name TEXT NOT NULL, age INTEGER, salary " - "REAL, blobType BLOB)"); - +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test" + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary " + "REAL, blobType BLOB)"; int DeleteTestOpenCallback::OnCreate(RdbStore &store) { return store.ExecuteSql(CREATE_TABLE_TEST); @@ -64,20 +70,29 @@ int DeleteTestOpenCallback::OnUpgrade(RdbStore &store, int oldVersion, int newVe void RdbDeleteTest::SetUpTestCase(void) { int errCode = E_OK; + RdbHelper::DeleteRdbStore(DATABASE_NAME); RdbStoreConfig config(RdbDeleteTest::DATABASE_NAME); DeleteTestOpenCallback helper; - RdbDeleteTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); - EXPECT_NE(RdbDeleteTest::store, nullptr); + g_store.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store.store, nullptr); + + config.SetStorageMode(StorageMode::MODE_MEMORY); + g_memDb.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_memDb.store, nullptr); } void RdbDeleteTest::TearDownTestCase(void) { - RdbHelper::DeleteRdbStore(RdbDeleteTest::DATABASE_NAME); + RdbStoreConfig config(RdbDeleteTest::DATABASE_NAME); + RdbHelper::DeleteRdbStore(config); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); } void RdbDeleteTest::SetUp(void) { - store->ExecuteSql("DELETE FROM test"); + store_ = *GetParam(); + store_->ExecuteSql("DELETE FROM test"); } void RdbDeleteTest::TearDown(void) @@ -89,9 +104,9 @@ void RdbDeleteTest::TearDown(void) * @tc.desc: test RdbStore update, select id and update one row * @tc.type: FUNC */ -HWTEST_F(RdbDeleteTest, RdbStore_Delete_001, TestSize.Level1) +HWTEST_P(RdbDeleteTest, RdbStore_Delete_001, TestSize.Level1) { - std::shared_ptr &store = RdbDeleteTest::store; + std::shared_ptr store = *GetParam(); int64_t id; int deletedRows; @@ -144,9 +159,9 @@ HWTEST_F(RdbDeleteTest, RdbStore_Delete_001, TestSize.Level1) * @tc.desc: test RdbStore update, select id and update one row * @tc.type: FUNC */ -HWTEST_F(RdbDeleteTest, RdbStore_Delete_002, TestSize.Level1) +HWTEST_P(RdbDeleteTest, RdbStore_Delete_002, TestSize.Level1) { - std::shared_ptr &store = RdbDeleteTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -198,9 +213,9 @@ HWTEST_F(RdbDeleteTest, RdbStore_Delete_002, TestSize.Level1) * @tc.desc: test RdbStore update, select id and update one row * @tc.type: FUNC */ -HWTEST_F(RdbDeleteTest, RdbStore_Delete_003, TestSize.Level1) +HWTEST_P(RdbDeleteTest, RdbStore_Delete_003, TestSize.Level1) { - std::shared_ptr &store = RdbDeleteTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -228,3 +243,6 @@ HWTEST_F(RdbDeleteTest, RdbStore_Delete_003, TestSize.Level1) EXPECT_EQ(ret, E_OK); EXPECT_EQ(deletedRows, 1); } + +INSTANTIATE_TEST_SUITE_P(DeleteTest, RdbDeleteTest, testing::Values(&g_store, &g_memDb)); +} // namespace OHOS::RdbDeleteTest diff --git a/relational_store/test/native/rdb/unittest/rdb_double_write_test.cpp b/relational_store/test/native/rdb/unittest/rdb_double_write_test.cpp index 85d3e8bb..4438c316 100644 --- a/relational_store/test/native/rdb/unittest/rdb_double_write_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_double_write_test.cpp @@ -500,6 +500,8 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_008, TestSize.Level1) file.close(); LOG_INFO("RdbStore_DoubleWrite_008 corrupt db finish"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::DATABASE_NAME + "-dwr"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::SLAVE_DATABASE_NAME + "-dwr"); int errCode = E_OK; RdbStoreConfig config(RdbDoubleWriteTest::DATABASE_NAME); config.SetHaMode(HAMode::MAIN_REPLICA); @@ -559,6 +561,8 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_010, TestSize.Level1) ASSERT_TRUE(file.good() == true); file.close(); LOG_INFO("RdbStore_DoubleWrite_010 corrupt db finish"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::DATABASE_NAME + "-dwr"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::SLAVE_DATABASE_NAME + "-dwr"); int errCode = E_OK; RdbStoreConfig config(RdbDoubleWriteTest::DATABASE_NAME); @@ -603,8 +607,9 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_011, TestSize.Level1) file.close(); LOG_INFO("RdbStore_DoubleWrite_011 corrupt db finish"); - EXPECT_EQ(store->Backup(std::string(""), {}), E_OK); + EXPECT_NE(store->Backup(std::string(""), {}), E_OK); LOG_INFO("RdbStore_DoubleWrite_011 backup db finish"); + EXPECT_EQ(store->Backup(std::string(""), {}), E_OK); RdbStoreConfig slaveConfig(RdbDoubleWriteTest::SLAVE_DATABASE_NAME); DoubleWriteTestOpenCallback slaveHelper; @@ -759,6 +764,8 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_015, TestSize.Level1) ASSERT_TRUE(file.good() == true); file.close(); LOG_INFO("RdbStore_DoubleWrite_015 corrupt db finish"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::DATABASE_NAME + "-dwr"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::SLAVE_DATABASE_NAME + "-dwr"); int errCode = slaveStore->ExecuteSql("CREATE TABLE IF NOT EXISTS xx (id INTEGER PRIMARY KEY AUTOINCREMENT," "name TEXT NOT NULL, age INTEGER, salary REAL, blobType BLOB)"); @@ -1014,7 +1021,7 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_029, TestSize.Level1) EXPECT_EQ(store->Backup(std::string(""), {}), E_OK); RdbDoubleWriteTest::CheckNumber(store, count); - RdbDoubleWriteTest::CheckNumber(slaveStore, -1, E_SQLITE_IOERR); + RdbDoubleWriteTest::CheckNumber(slaveStore, -1, E_SQLITE_CORRUPT); int errCode = E_OK; slaveStore = nullptr; @@ -1105,6 +1112,8 @@ HWTEST_F(RdbDoubleWriteTest, RdbStore_DoubleWrite_033, TestSize.Level1) ASSERT_TRUE(file.good() == true); file.close(); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::DATABASE_NAME + "-dwr"); + SqliteUtils::DeleteFile(RdbDoubleWriteTest::SLAVE_DATABASE_NAME + "-dwr"); int errCode = E_OK; RdbStoreConfig config(RdbDoubleWriteTest::DATABASE_NAME); config.SetHaMode(HAMode::SINGLE); diff --git a/relational_store/test/native/rdb/unittest/rdb_execute_test.cpp b/relational_store/test/native/rdb/unittest/rdb_execute_test.cpp index 821397db..bf769385 100644 --- a/relational_store/test/native/rdb/unittest/rdb_execute_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_execute_test.cpp @@ -24,21 +24,33 @@ using namespace testing::ext; using namespace OHOS::NativeRdb; +namespace OHOS::RdbExecuteTest { +struct RdbTestParam { + std::shared_ptr store; + std::string mode; + operator std::shared_ptr() + { + return store; + } +}; +static RdbTestParam g_store; +static RdbTestParam g_memDb; -class RdbExecuteTest : public testing::Test { +class RdbExecuteTest : public testing::TestWithParam { public: static void SetUpTestCase(void); static void TearDownTestCase(void); void SetUp(); void TearDown(); + std::shared_ptr store_; static const std::string DATABASE_NAME; - static std::shared_ptr store; - static const std::string CREATE_TABLE_TEST; }; - +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary REAL, " + "blobType BLOB)"; const std::string RdbExecuteTest::DATABASE_NAME = RDB_TEST_PATH + "execute_test.db"; -std::shared_ptr RdbExecuteTest::store = nullptr; class ExecuteTestOpenCallback : public RdbOpenCallback { public: @@ -46,11 +58,6 @@ public: int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; }; -const std::string RdbExecuteTest::CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test " - "(id INTEGER PRIMARY KEY AUTOINCREMENT, " - "name TEXT NOT NULL, age INTEGER, salary REAL, " - "blobType BLOB)"; - int ExecuteTestOpenCallback::OnCreate(RdbStore &store) { return E_OK; @@ -64,28 +71,37 @@ int ExecuteTestOpenCallback::OnUpgrade(RdbStore &store, int oldVersion, int newV void RdbExecuteTest::SetUpTestCase(void) { int errCode = E_OK; - RdbHelper::DeleteRdbStore(RdbExecuteTest::DATABASE_NAME); + RdbHelper::DeleteRdbStore(DATABASE_NAME); RdbStoreConfig config(RdbExecuteTest::DATABASE_NAME); ExecuteTestOpenCallback helper; - RdbExecuteTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); - EXPECT_NE(RdbExecuteTest::store, nullptr); - EXPECT_EQ(errCode, E_OK); + g_store.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store.store, nullptr); + g_store.mode = "wal"; + + config.SetStorageMode(StorageMode::MODE_MEMORY); + g_memDb.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_memDb.store, nullptr); + g_memDb.mode = "memory"; } void RdbExecuteTest::TearDownTestCase(void) { - RdbHelper::DeleteRdbStore(RdbExecuteTest::DATABASE_NAME); - store = nullptr; + RdbStoreConfig config(RdbExecuteTest::DATABASE_NAME); + RdbHelper::DeleteRdbStore(config); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); } void RdbExecuteTest::SetUp(void) { - store->ExecuteSql(CREATE_TABLE_TEST); + store_ = *GetParam(); + store_->ExecuteSql(CREATE_TABLE_TEST); } void RdbExecuteTest::TearDown(void) { - store->ExecuteSql("DROP TABLE test"); + store_ = *GetParam(); + store_->ExecuteSql("DROP TABLE test"); } /** @@ -93,9 +109,9 @@ void RdbExecuteTest::TearDown(void) * @tc.desc: test RdbStore Execute * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_001, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_001, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -155,9 +171,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_001, TestSize.Level1) * @tc.desc: test RdbStore Execute * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_002, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_002, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -207,9 +223,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_002, TestSize.Level1) * @tc.desc: test RdbStore Execute * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_003, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_003, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t pageSize; int ret = store->ExecuteAndGetLong(pageSize, "PRAGMA page_size"); @@ -224,7 +240,7 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_003, TestSize.Level1) std::string journalMode; ret = store->ExecuteAndGetString(journalMode, "PRAGMA journal_mode"); EXPECT_EQ(ret, E_OK); - EXPECT_EQ(journalMode, "wal"); + EXPECT_EQ(journalMode, GetParam()->mode); } /** @@ -232,9 +248,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_003, TestSize.Level1) * @tc.desc: Abnormal testCase for ExecuteAndGetString, if sqlstatementtype is special * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_004, TestSize.Level4) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_004, TestSize.Level4) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); std::string outValue; int ret = store->ExecuteAndGetString(outValue, "BEGIN;"); @@ -247,9 +263,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_004, TestSize.Level4) * @tc.type: FUNC * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_005, TestSize.Level4) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_005, TestSize.Level4) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t outValue; int ret = store->ExecuteForLastInsertedRowId(outValue, "", {}); EXPECT_NE(E_OK, ret); @@ -260,9 +276,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_005, TestSize.Level4) * @tc.desc: Abnormal testCase for ExecuteForChangedRowCount, if sql is invalid * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_006, TestSize.Level4) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_006, TestSize.Level4) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t outValue; int ret = store->ExecuteForChangedRowCount(outValue, "", {}); EXPECT_NE(E_OK, ret); @@ -273,9 +289,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_006, TestSize.Level4) * @tc.desc: Normal testCase for ExecuteAndGetString, check integrity for store * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_007, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_007, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); auto [ret, outValue] = store->Execute("PRAGMA integrity_check"); EXPECT_EQ(E_OK, ret); @@ -291,9 +307,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_007, TestSize.Level1) * @tc.desc: Normal testCase for Execute, check integrity for store * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_008, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_008, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); auto [ret, outValue] = store->Execute("PRAGMA quick_check"); EXPECT_EQ(E_OK, ret); @@ -309,9 +325,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_008, TestSize.Level1) * @tc.desc: Normal testCase for Execute, get user_version of store * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_009, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_009, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); // set user_version as 5 store->SetVersion(5); @@ -332,9 +348,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_009, TestSize.Level1) * @tc.desc: AbNormal testCase for Execute, execute select sql * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0010, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0010, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); auto [ret, outValue] = store->Execute("SELECT * FROM test"); EXPECT_EQ(E_NOT_SUPPORT_THE_SQL, ret); @@ -345,9 +361,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0010, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute sql for inserting data * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0011, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0011, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); std::vector args = { ValueObject(std::string("tt")), ValueObject(int(28)), ValueObject(double(50000.0)) }; @@ -366,9 +382,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0011, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute sql for batch insert data * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0012, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0012, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); std::vector args = { ValueObject(std::string("tt")), ValueObject(int(28)), ValueObject(double(50000.0)), ValueObject(std::string("ttt")), ValueObject(int(58)), @@ -389,9 +405,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0012, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute sql for updating data * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0013, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0013, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); std::vector args = { ValueObject(std::string("tt")), ValueObject(int(28)), ValueObject(double(50000.0)), ValueObject(std::string("ttt")), ValueObject(int(58)), @@ -419,9 +435,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0013, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute sql for deleting data * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0014, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0014, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); std::vector args = { ValueObject(std::string("tt")), ValueObject(int(28)), ValueObject(double(50000.0)), ValueObject(std::string("ttt")), ValueObject(int(82)), @@ -449,9 +465,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0014, TestSize.Level1) * @tc.desc: AbNormal testCase for Execute, execute sql for attaching database and transaction * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0015, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0015, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); auto [ret1, outValue1] = store->Execute("ATTACH DATABASE 'execute_attach_test.db' AS 'attach.db'"); EXPECT_EQ(E_NOT_SUPPORT_THE_SQL, ret1); @@ -474,17 +490,18 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0015, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute DDL sql for creating and dropping table * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0016, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0016, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t intOutValue; - const std::string CREATE_TABLE_TEST2 = "CREATE TABLE IF NOT EXISTS test2 " - "(id INTEGER PRIMARY KEY AUTOINCREMENT, " - "name TEXT NOT NULL, age INTEGER, salary REAL, " - "blobType BLOB)"; - const std::string DROP_TABLE_TEST2 = "DROP TABLE test2"; - const std::string TEST_TABLE_IS_EXIST = "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='test2'"; + constexpr const char *CREATE_TABLE_TEST2 = "CREATE TABLE IF NOT EXISTS test2 " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary REAL, " + "blobType BLOB)"; + constexpr const char *DROP_TABLE_TEST2 = "DROP TABLE test2"; + constexpr const char *TEST_TABLE_IS_EXIST = + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='test2'"; auto [ret1, outValue1] = store->Execute(CREATE_TABLE_TEST2); EXPECT_EQ(E_OK, ret1); @@ -518,17 +535,17 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0016, TestSize.Level1) * @tc.desc: Normal testCase for Execute, execute sql for creating table and insert, query data * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0017, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0017, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); int64_t intOutValue; int intOutResultSet; - const std::string CREATE_TABLE_TEST2 = "CREATE TABLE IF NOT EXISTS test2 " + constexpr const char *CREATE_TABLE_TEST2 = "CREATE TABLE IF NOT EXISTS test2 " "(id INTEGER PRIMARY KEY AUTOINCREMENT, " "name TEXT NOT NULL, age INTEGER, salary REAL, " "blobType BLOB)"; - const std::string DROP_TABLE_TEST2 = "DROP TABLE test2"; + constexpr const char *DROP_TABLE_TEST2 = "DROP TABLE test2"; auto [ret1, outValue1] = store->Execute(CREATE_TABLE_TEST2); EXPECT_EQ(E_OK, ret1); @@ -556,9 +573,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0017, TestSize.Level1) * @tc.desc: AbNormal testCase for Execute, execute sql for inserting data but args is [] * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0018, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0018, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); ValueObject outValue; auto [ret1, outValue1] = store->Execute("INSERT INTO test(name, age, salary) VALUES (?, ?, ?), (?, ?, ?)"); @@ -570,9 +587,9 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0018, TestSize.Level1) * @tc.desc: Normal testCase for Execute, set user_version of store * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0019, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0019, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); // set user_version as 5 auto [ret, outValue] = store->Execute("PRAGMA user_version=5"); @@ -590,10 +607,14 @@ HWTEST_F(RdbExecuteTest, RdbStore_Execute_0019, TestSize.Level1) * @tc.desc: AbNormal testCase for Execute, get table_info * @tc.type: FUNC */ -HWTEST_F(RdbExecuteTest, RdbStore_Execute_0020, TestSize.Level1) +HWTEST_P(RdbExecuteTest, RdbStore_Execute_0020, TestSize.Level1) { - std::shared_ptr &store = RdbExecuteTest::store; + std::shared_ptr store = *GetParam(); auto [ret, outValue] = store->Execute("PRAGMA table_info(test)"); EXPECT_EQ(E_NOT_SUPPORT_THE_SQL, ret); -} \ No newline at end of file +} + +INSTANTIATE_TEST_SUITE_P(ExecuteTest, RdbExecuteTest, testing::Values(&g_store, &g_memDb)); + +} // namespace OHOS::RdbExecuteTest \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_helper_test.cpp b/relational_store/test/native/rdb/unittest/rdb_helper_test.cpp index a79afd0c..663d8b33 100644 --- a/relational_store/test/native/rdb/unittest/rdb_helper_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_helper_test.cpp @@ -154,7 +154,7 @@ HWTEST_F(RdbHelperTest, DeleteDatabase_001, TestSize.Level1) int ret1 = RdbHelper::DeleteRdbStore(config1); EXPECT_EQ(ret1, E_OK); int ret2 = RdbHelper::DeleteRdbStore(config2); - EXPECT_EQ(ret2, E_OK); + EXPECT_EQ(ret2, E_INVALID_FILE_PATH); int ret3 = RdbHelper::DeleteRdbStore(config3); EXPECT_EQ(ret3, E_INVALID_FILE_PATH); } @@ -174,7 +174,7 @@ HWTEST_F(RdbHelperTest, DeleteDatabase_002, TestSize.Level1) remove(rdbStorePath.c_str()); - int ret = RdbHelper::DeleteRdbStore("rdbhelper.db"); + int ret = RdbHelper::DeleteRdbStore(RdbHelperTest::rdbStorePath); EXPECT_EQ(ret, E_OK); std::string shmFileName = rdbStorePath + "-shm"; std::string walFileName = rdbStorePath + "-wal"; @@ -631,7 +631,7 @@ HWTEST_F(RdbHelperTest, GetDatabase_001, TestSize.Level0) HWTEST_F(RdbHelperTest, GetDatabase_002, TestSize.Level0) { - const std::string dbPath = RDB_TEST_PATH + "GetDatabase002.db"; + const std::string dbPath = RDB_TEST_PATH + "GetDatabase.db"; RdbStoreConfig config(dbPath); std::string bundleName = "com.ohos.config.GetDatabase"; config.SetBundleName(bundleName); @@ -656,7 +656,7 @@ HWTEST_F(RdbHelperTest, GetDatabase_002, TestSize.Level0) HWTEST_F(RdbHelperTest, GetDatabase_003, TestSize.Level0) { - const std::string dbPath = RDB_TEST_PATH + "GetDatabase003.db"; + const std::string dbPath = RDB_TEST_PATH + "GetDatabase.db"; RdbStoreConfig config(dbPath); std::string bundleName = "com.ohos.config.GetDatabase"; config.SetBundleName(bundleName); @@ -685,7 +685,7 @@ HWTEST_F(RdbHelperTest, GetDatabase_003, TestSize.Level0) HWTEST_F(RdbHelperTest, GetDatabase_004, TestSize.Level0) { - const std::string dbPath = RDB_TEST_PATH + "GetDatabase004.db"; + const std::string dbPath = RDB_TEST_PATH + "GetDatabase.db"; RdbStoreConfig config(dbPath); std::string bundleName = "com.ohos.config.GetDatabase"; config.SetBundleName(bundleName); @@ -707,68 +707,101 @@ HWTEST_F(RdbHelperTest, GetDatabase_004, TestSize.Level0) EXPECT_NE(errCode, E_OK); EXPECT_EQ(rdbStore2, nullptr); } -// -//HWTEST_F(RdbHelperTest, GetDatabase_007, TestSize.Level0) -//{ -// const std::string dbPath = RDB_TEST_PATH + "GetDatabase007.db"; -// RdbStoreConfig config(dbPath); -// std::string bundleName = "com.ohos.config.GetDatabase"; -// config.SetBundleName(bundleName); -// -// // Ensure that the database returns OK when it is successfully opened -// int errCode = E_ERROR; -// -// RdbHelperTestOpenCallback helper; -// std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// EXPECT_EQ(errCode, E_OK); -// ASSERT_NE(rdbStore, nullptr); -// rdbStore = nullptr; -// -// ASSERT_TRUE(UTUtils::DestoryDb(dbPath)); -// -// config.SetVisitorDir(dbPath); -// config.SetRoleType(RoleType::VISITOR_WRITE); -// config.SetAllowRebuild(true); -// rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// EXPECT_NE(errCode, E_OK); -// ASSERT_EQ(rdbStore, nullptr); -// -// config.SetRoleType(RoleType::OWNER); -// rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// EXPECT_EQ(errCode, E_OK); -// ASSERT_NE(rdbStore, nullptr); -// RebuiltType type; -// EXPECT_EQ(rdbStore->GetRebuilt(type), E_OK); -// EXPECT_EQ(type, RebuiltType::REBUILT); -// EXPECT_FALSE(Reportor::IsFlagExist(dbPath)); -//} -// -//HWTEST_F(RdbHelperTest, GetDatabase_008, TestSize.Level0) -//{ -// const std::string dbPath = RDB_TEST_PATH + "GetDatabase008.db"; -// RdbHelper::DeleteRdbStore(dbPath); -// -// RdbStoreConfig config(dbPath); -// std::string bundleName = "com.ohos.config.GetDatabase"; -// config.SetBundleName(bundleName); -// -// const std::string visitorDb = RDB_TEST_PATH + "visitorDb.db"; -// RdbHelper::DeleteRdbStore(visitorDb); -// -// config.SetVisitorDir(visitorDb); -// config.SetRoleType(RoleType::VISITOR_WRITE); -// config.SetAllowRebuild(true); -// -// // Ensure that the database returns OK when it is successfully opened -// int errCode = E_ERROR; -// RdbHelperTestOpenCallback helper; -// auto visitWriterStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// EXPECT_EQ(errCode, E_OK); -// ASSERT_NE(visitWriterStore, nullptr); -// -// config.SetRoleType(RoleType::OWNER); -// auto ownerStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// EXPECT_NE(errCode, E_OK); -// ASSERT_EQ(ownerStore, nullptr); -// EXPECT_NE(visitWriterStore, ownerStore); -//} \ No newline at end of file + +HWTEST_F(RdbHelperTest, GetDatabase_005, TestSize.Level0) +{ + const std::string dbPath = RDB_TEST_PATH + "GetSubUserDatabase.db"; + RdbStoreConfig config(dbPath); + config.SetName("RdbStoreConfig_test.db"); + std::string bundleName = "com.ohos.config.TestSubUser"; + config.SetBundleName(bundleName); + config.SetSubUser(100); + auto subUser = config.GetSubUser(); + EXPECT_EQ(subUser, 100); + int errCode = E_OK; + + RdbHelperTestOpenCallback helper; + std::shared_ptr rdbStore1 = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore1, nullptr); + + int ret = RdbHelper::DeleteRdbStore(config); + EXPECT_EQ(ret, E_OK); +} + +/** + * @tc.name: GetDatabase_006 + * @tc.desc: Insert after GetRdbStore + * @tc.type: FUNC + */ +HWTEST_F(RdbHelperTest, GetDatabase_006, TestSize.Level0) +{ + const std::string dbPath = RDB_TEST_PATH + "GetDatabase1.db"; + RdbStoreConfig config(dbPath); + config.SetName("RdbStoreConfig_test.db"); + std::string bundleName = "com.ohos.config.TestSubUser"; + config.SetBundleName(bundleName); + config.SetSubUser(100); + auto subUser = config.GetSubUser(); + EXPECT_EQ(subUser, 100); + int errCode = E_OK; + + RdbHelperTestOpenCallback helper; + std::shared_ptr rdbStore1 = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore1, nullptr); + rdbStore1->ExecuteSql("CREATE TABLE IF NOT EXISTS test " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, age INTEGER, salary " + "REAL, blobType BLOB);"); + + int64_t id; + ValuesBucket values; + int deletedRows; + + values.PutInt("id", 1); + values.PutString("name", std::string("zhangsan")); + values.PutInt("age", 18); + values.PutDouble("salary", 100.5); + values.PutBlob("blobType", std::vector{ 1, 2, 3 }); + int ret = rdbStore1->Insert(id, "test", values); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(1, id); + + ret = rdbStore1->Delete(deletedRows, "test", "id = 1", std::vector()); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(deletedRows, 1); + + ret = RdbHelper::DeleteRdbStore(config); + EXPECT_EQ(ret, E_OK); +} + +/** + * @tc.name: GetDatabase_007 + * @tc.desc: Insert after GetRdbStore + * @tc.type: FUNC + */ +HWTEST_F(RdbHelperTest, GetDatabase_007, TestSize.Level0) +{ + const std::string dbPath = RDB_TEST_PATH + "GetDatabase_007.db"; + RdbStoreConfig config(dbPath); + config.SetStorageMode(StorageMode::MODE_MEMORY); + + RdbHelper::DeleteRdbStore(config); + + // Ensure that the database returns OK when it is successfully opened + int errCode = E_ERROR; + + RdbHelperTestOpenCallback helper; + std::shared_ptr rdbStore1 = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore1, nullptr); + + RdbHelper::DeleteRdbStore(dbPath); + std::shared_ptr rdbStore2 = RdbHelper::GetRdbStore(config, 1, helper, errCode); + // Ensure that the database can be opened after the encryption parameters are changed + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore2, nullptr); + + // Ensure that two databases not equal + EXPECT_NE(rdbStore1, rdbStore2); +} \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_insert_test.cpp b/relational_store/test/native/rdb/unittest/rdb_insert_test.cpp index 5e8d09c0..52b9eb3e 100644 --- a/relational_store/test/native/rdb/unittest/rdb_insert_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_insert_test.cpp @@ -25,24 +25,33 @@ using namespace testing::ext; using namespace OHOS::NativeRdb; +namespace OHOS::RdbStoreInsertTest { +struct RdbTestParam { + std::shared_ptr store; + operator std::shared_ptr() + { + return store; + } +}; +static RdbTestParam g_store; +static RdbTestParam g_memDb; -class RdbStoreInsertTest : public testing::Test { +class RdbStoreInsertTest : public testing::TestWithParam { public: static void SetUpTestCase(void); static void TearDownTestCase(void); void SetUp(); void TearDown(); - void CheckResultSet(std::shared_ptr &store); - void CheckAge(std::shared_ptr &resultSet); - void CheckSalary(std::shared_ptr &resultSet); - void CheckBlob(std::shared_ptr &resultSet); + static void CheckResultSet(std::shared_ptr &store); + static void CheckAge(std::shared_ptr &resultSet); + static void CheckSalary(std::shared_ptr &resultSet); + static void CheckBlob(std::shared_ptr &resultSet); + std::shared_ptr store_; static const std::string DATABASE_NAME; - static std::shared_ptr store; }; const std::string RdbStoreInsertTest::DATABASE_NAME = RDB_TEST_PATH + "insert_test.db"; -std::shared_ptr RdbStoreInsertTest::store = nullptr; class InsertTestOpenCallback : public RdbOpenCallback { public: @@ -69,20 +78,29 @@ int InsertTestOpenCallback::OnUpgrade(RdbStore &store, int oldVersion, int newVe void RdbStoreInsertTest::SetUpTestCase(void) { int errCode = E_OK; + RdbHelper::DeleteRdbStore(DATABASE_NAME); RdbStoreConfig config(RdbStoreInsertTest::DATABASE_NAME); InsertTestOpenCallback helper; - RdbStoreInsertTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); - EXPECT_NE(RdbStoreInsertTest::store, nullptr); + g_store.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store.store, nullptr); + + config.SetStorageMode(StorageMode::MODE_MEMORY); + g_memDb.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_memDb.store, nullptr); } void RdbStoreInsertTest::TearDownTestCase(void) { RdbHelper::DeleteRdbStore(RdbStoreInsertTest::DATABASE_NAME); + RdbStoreConfig config(RdbStoreInsertTest::DATABASE_NAME); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); } void RdbStoreInsertTest::SetUp(void) { - store->ExecuteSql("DELETE FROM test"); + store_ = *GetParam(); + store_->ExecuteSql("DELETE FROM test"); } void RdbStoreInsertTest::TearDown(void) @@ -94,9 +112,9 @@ void RdbStoreInsertTest::TearDown(void) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Insert_001, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Insert_001, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -240,9 +258,9 @@ void RdbStoreInsertTest::CheckBlob(std::shared_ptr &resultSet) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Insert_002, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Insert_002, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -263,9 +281,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Insert_002, TestSize.Level1) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Insert_003, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Insert_003, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket emptyBucket; @@ -287,9 +305,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Insert_003, TestSize.Level1) * @tc.desc: test RdbStore replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_001, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_001, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -338,9 +356,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_001, TestSize.Level1) * @tc.desc: test RdbStore replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_002, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_002, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -399,9 +417,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_002, TestSize.Level1) * @tc.desc: test RdbStore Replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_003, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_003, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -422,9 +440,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_003, TestSize.Level1) * @tc.desc: test RdbStore Replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_004, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_004, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket emptyBucket; @@ -446,9 +464,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_004, TestSize.Level1) * @tc.desc: test RdbStore replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_005, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_005, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; @@ -493,9 +511,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_005, TestSize.Level1) * @tc.desc: test RdbStore replace * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_006, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_Replace_006, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -551,9 +569,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_Replace_006, TestSize.Level1) * @tc.desc: test RdbStore InsertWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_001_002, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_001_002, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -584,9 +602,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_001_002, Test * @tc.desc: test RdbStore InsertWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_003_004, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_003_004, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -615,9 +633,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_003_004, Test * @tc.desc: test RdbStore InsertWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_005, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_005, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -647,9 +665,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_005, TestSize * @tc.desc: test RdbStore InsertWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_006, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_006, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -709,9 +727,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_006, TestSize * @tc.desc: test RdbStore InsertWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_007, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_007, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -768,9 +786,9 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_007, TestSize * @tc.desc: Abnormal testCase of InsertWithConflictResolution, if conflictResolution is invalid * @tc.type: FUNC */ -HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_008, TestSize.Level1) +HWTEST_P(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_008, TestSize.Level1) { - std::shared_ptr &store = RdbStoreInsertTest::store; + std::shared_ptr store = *GetParam(); int64_t id = 0; ValuesBucket values; @@ -787,4 +805,29 @@ HWTEST_F(RdbStoreInsertTest, RdbStore_InsertWithConflictResolution_008, TestSize ret = store->InsertWithConflictResolution(id, "test", values, static_cast(-1)); EXPECT_EQ(E_INVALID_CONFLICT_FLAG, ret); EXPECT_EQ(0, id); -} \ No newline at end of file +} + +/** + * @tc.name: OverLimitWithInsert_001 + * @tc.desc: over limit + * @tc.type: FUNC + * @tc.require: + * @tc.author: +*/ +HWTEST_P(RdbStoreInsertTest, OverLimitWithInsert_001, TestSize.Level1) +{ + std::shared_ptr store = *GetParam(); + auto [code, maxPageCount] = store->Execute("PRAGMA max_page_count;"); + auto recover = std::shared_ptr("recover", [defPageCount = maxPageCount, store](const char *) { + store->Execute("PRAGMA max_page_count = " + static_cast(defPageCount) + ";"); + }); + std::tie(code, maxPageCount) = store->Execute("PRAGMA max_page_count = 256;"); + + ValuesBucket row; + row.Put("name", std::string(1024 * 1024, 'e')); + auto result = store->Insert("test", row, ConflictResolution::ON_CONFLICT_NONE); + ASSERT_EQ(result.first, E_SQLITE_FULL); +} + +INSTANTIATE_TEST_SUITE_P(InsertTest, RdbStoreInsertTest, testing::Values(&g_store, &g_memDb)); +} // namespace OHOS::RdbStoreInsertTest \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_memory_db_test.cpp b/relational_store/test/native/rdb/unittest/rdb_memory_db_test.cpp new file mode 100644 index 00000000..db452113 --- /dev/null +++ b/relational_store/test/native/rdb/unittest/rdb_memory_db_test.cpp @@ -0,0 +1,522 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include + +#include + +#include "common.h" +#include "rdb_errno.h" +#include "rdb_helper.h" +#include "rdb_open_callback.h" + +using namespace testing::ext; +using namespace OHOS::NativeRdb; + +class OpenCallback : public RdbOpenCallback { +public: + int OnCreate(RdbStore &store) override + { + return E_OK; + } + int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override + { + return E_OK; + } +}; + +class RdbMemoryDbTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + void InitDb(); + + static const std::string rdbStorePath; + static std::shared_ptr g_store; +}; +const std::string RdbMemoryDbTest::rdbStorePath = RDB_TEST_PATH + std::string("MemoryTest"); +std::shared_ptr RdbMemoryDbTest::g_store = nullptr; + +void RdbMemoryDbTest::SetUpTestCase(void) +{ +} + +void RdbMemoryDbTest::TearDownTestCase(void) +{ +} + +void RdbMemoryDbTest::SetUp(void) +{ + InitDb(); +} + +void RdbMemoryDbTest::TearDown(void) +{ + RdbStoreConfig config(rdbStorePath); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); +} + +class RdbMemoryDbTestWrongSqlOpenCallback : public RdbOpenCallback { +public: + int OnCreate(RdbStore &store) override; + int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; +}; +constexpr const char *WRONG_SQL_TEST = "CREATE TABL IF NOT EXISTS test " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary REAL, " + "blobType BLOB)"; +class RdbMemoryDbTestOpenCallback : public RdbOpenCallback { +public: + int OnCreate(RdbStore &store) override; + int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; +}; +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary REAL, " + "blobType BLOB)"; +void RdbMemoryDbTest::InitDb() +{ + int errCode = E_OK; + RdbStoreConfig config(RdbMemoryDbTest::rdbStorePath); + RdbMemoryDbTestOpenCallback helper; + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbMemoryDbTest::g_store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store, nullptr); + ASSERT_EQ(errCode, E_OK); +} + +int RdbMemoryDbTestWrongSqlOpenCallback::OnCreate(RdbStore &store) +{ + return store.ExecuteSql(WRONG_SQL_TEST); +} + +int RdbMemoryDbTestOpenCallback::OnCreate(RdbStore &store) +{ + return store.ExecuteSql(CREATE_TABLE_TEST); +} + +int RdbMemoryDbTestWrongSqlOpenCallback::OnUpgrade(RdbStore &store, int oldVersion, int newVersion) +{ + return E_OK; +} + +int RdbMemoryDbTestOpenCallback::OnUpgrade(RdbStore &store, int oldVersion, int newVersion) +{ + return E_OK; +} +/** +* @tc.name: GetMemoryDb_001 +* @tc.desc: Get MemoryDb with different config +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, GetMemoryDb_001, TestSize.Level1) +{ + int errCode = E_ERROR; + std::string path = RDB_TEST_PATH + "GetMemoryDb_001"; + RdbStoreConfig config(path.c_str()); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbMemoryDbTestOpenCallback normalCallback; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + EXPECT_NE(rdbStore, nullptr); + ASSERT_NE(access(path.c_str(), F_OK), 0); + rdbStore = nullptr; + + RdbMemoryDbTestWrongSqlOpenCallback abnormalHelper; + rdbStore = RdbHelper::GetRdbStore(config, 1, abnormalHelper, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + rdbStore = nullptr; + + config.SetSearchable(true); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_NOT_SUPPORT); + EXPECT_EQ(rdbStore, nullptr); + rdbStore = nullptr; + config.SetSearchable(false); + + config.SetIsVector(true); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_NOT_SUPPORT); + EXPECT_EQ(rdbStore, nullptr); + rdbStore = nullptr; + config.SetIsVector(false); + + config.SetEncryptStatus(true); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_NOT_SUPPORT); + EXPECT_EQ(rdbStore, nullptr); + rdbStore = nullptr; + config.SetEncryptStatus(false); + + config.SetHaMode(HAMode::MAIN_REPLICA); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_NOT_SUPPORT); + EXPECT_EQ(rdbStore, nullptr); + rdbStore = nullptr; + config.SetHaMode(HAMode::SINGLE); + + config.SetRoleType(RoleType::VISITOR); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_NOT_SUPPORT); + EXPECT_EQ(rdbStore, nullptr); +} + +/** +* @tc.name: GetMemoryDb_002 +* @tc.desc: Get MemoryDb with different config +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, GetMemoryDb_002, TestSize.Level1) +{ + int errCode = E_ERROR; + std::string path = RDB_TEST_PATH + "GetMemoryDb_002"; + RdbStoreConfig config(path); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbMemoryDbTestOpenCallback normalCallback; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + EXPECT_NE(rdbStore, nullptr); + + config.SetPageSize(1024); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_CONFIG_INVALID_CHANGE); + EXPECT_EQ(rdbStore, nullptr); +} + +/** +* @tc.name: GetMemoryDb_003 +* @tc.desc: Get MemoryDb after delete +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, GetMemoryDb_003, TestSize.Level1) +{ + int errCode = E_ERROR; + std::string path = RDB_TEST_PATH + "GetMemoryDb_003"; + RdbStoreConfig config(path); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbMemoryDbTestOpenCallback normalCallback; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore, nullptr); + + ValuesBucket values; + values.PutInt("id", 1); + values.PutString("name", std::string("zhangsan")); + values.PutInt("age", 18); + auto [ret, row] = rdbStore->Insert("test", values); + EXPECT_EQ(ret, E_OK); + auto pred = AbsRdbPredicates("test"); + auto resultSet = rdbStore->Query(pred); + ASSERT_NE(resultSet, nullptr); + int count = -1; + EXPECT_EQ(resultSet->GetRowCount(count), E_OK); + EXPECT_EQ(count, 1); + + auto rdbStore2 = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore2, nullptr); + resultSet = rdbStore2->Query(pred); + ASSERT_NE(resultSet, nullptr); + count = -1; + EXPECT_EQ(resultSet->GetRowCount(count), E_OK); + EXPECT_EQ(count, 1); + resultSet = nullptr; + + RdbHelper::DeleteRdbStore(config); + std::tie(ret, row) = rdbStore->Insert("test", values); + EXPECT_EQ(ret, E_ALREADY_CLOSED); + + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore, nullptr); + resultSet = rdbStore->Query(pred); + ASSERT_NE(resultSet, nullptr); + count = -1; + EXPECT_EQ(resultSet->GetRowCount(count), E_OK); + EXPECT_EQ(count, 0); +} + +/** +* @tc.name: CRUD of MemoryDb_001 +* @tc.desc: CRUD with Transaction of MemoryDb +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, CRUDWithMemoryDb_001, TestSize.Level1) +{ + std::shared_ptr &store = g_store; + + auto [ret, transaction] = store->CreateTransaction(Transaction::IMMEDIATE); + ASSERT_EQ(ret, E_OK); + ASSERT_NE(transaction, nullptr); + + auto result = transaction->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[0])); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(1, result.second); + + result = transaction->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[1])); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(2, result.second); + + result = store->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[2]), RdbStore::NO_ACTION); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + + auto resultSet = transaction->QueryByStep("SELECT * FROM test"); + ASSERT_NE(resultSet, nullptr); + int32_t rowCount{}; + ret = resultSet->GetRowCount(rowCount); + ASSERT_EQ(ret, E_OK); + ASSERT_EQ(rowCount, 2); + + ret = transaction->Commit(); + ASSERT_EQ(ret, E_OK); + + ValueObject value; + ret = resultSet->Get(0, value); + ASSERT_EQ(ret, E_ALREADY_CLOSED); + + result = store->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[2]), RdbStore::NO_ACTION); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(3, result.second); + + result = transaction->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[0])); + ASSERT_EQ(result.first, E_ALREADY_CLOSED); + + resultSet = store->QueryByStep("SELECT * FROM test"); + ASSERT_NE(resultSet, nullptr); + resultSet->GetRowCount(rowCount); + EXPECT_EQ(rowCount, 3); +} + +/** +* @tc.name: CRUD of MemoryDb_002 +* @tc.desc: CRUD with Transaction of MemoryDb +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, CRUDWithMemoryDb_002, TestSize.Level1) +{ + std::shared_ptr &store = g_store; + + auto [ret, transaction] = store->CreateTransaction(Transaction::EXCLUSIVE); + ASSERT_EQ(ret, E_OK); + ASSERT_NE(transaction, nullptr); + + auto result = transaction->Insert("test", UTUtils::SetRowData(UTUtils::g_rowData[0])); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(result.second, 1); + + result = transaction->Update("test", UTUtils::SetRowData(UTUtils::g_rowData[1]), "id=1"); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(result.second, 1); + + auto resultSet = transaction->QueryByStep("SELECT * FROM test"); + ASSERT_NE(resultSet, nullptr); + int32_t rowCount{}; + ret = resultSet->GetRowCount(rowCount); + ASSERT_EQ(ret, E_OK); + ASSERT_EQ(rowCount, 1); + ret = resultSet->GoToFirstRow(); + ASSERT_EQ(ret, E_OK); + int32_t columnIndex{}; + ret = resultSet->GetColumnIndex("id", columnIndex); + ASSERT_EQ(ret, E_OK); + int32_t id{}; + ret = resultSet->GetInt(columnIndex, id); + ASSERT_EQ(ret, E_OK); + ASSERT_EQ(id, 2); + + AbsRdbPredicates predicates("test"); + predicates.EqualTo("id", ValueObject(2)); + result = transaction->Update(UTUtils::SetRowData(UTUtils::g_rowData[2]), predicates); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(result.second, 1); + + ret = transaction->Commit(); + ASSERT_EQ(ret, E_OK); + + resultSet = store->QueryByStep("SELECT * FROM test"); + ASSERT_NE(resultSet, nullptr); + ret = resultSet->GetRowCount(rowCount); + ASSERT_EQ(ret, E_OK); + ASSERT_EQ(rowCount, 1); + ret = resultSet->GoToFirstRow(); + ASSERT_EQ(ret, E_OK); + resultSet->GetColumnIndex("id", columnIndex); + ret = resultSet->GetInt(columnIndex, id); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(id, 3); +} + +/** +* @tc.name: CRUD of MemoryDb_003 +* @tc.desc: CRUD with Transaction of MemoryDb +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, CRUDWithMemoryDb_003, TestSize.Level1) +{ + std::shared_ptr &store = g_store; + store->Execute("DROP TABLE IF EXISTS test1"); + auto res = store->Execute("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)"); + ASSERT_EQ(res.first, E_OK); + auto [ret1, transaction1] = store->CreateTransaction(Transaction::IMMEDIATE); + ASSERT_EQ(ret1, E_OK); + ASSERT_NE(transaction1, nullptr); + + auto [ret, transaction] = store->CreateTransaction(Transaction::DEFERRED); + ASSERT_EQ(ret, E_OK); + ASSERT_NE(transaction, nullptr); + + ValuesBuckets rows; + for (int i = 0; i < 5; i++) { + Transaction::Row row; + row.Put("id", i); + row.Put("name", "Jim_batchInsert"); + rows.Put(row); + } + auto result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_NONE); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); + + result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_ROLLBACK); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); + + result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_ABORT); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); + + result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_FAIL); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); + + result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_IGNORE); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); + + result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_REPLACE); + ASSERT_EQ(result.first, E_SQLITE_LOCKED); + ASSERT_EQ(result.second, -1); +} + +/** +* @tc.name: CRUD of MemoryDb_004 +* @tc.desc: CRUD with Transaction of MemoryDb +* @tc.type: FUNC +* @tc.require: +* @tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, CRUDWithMemoryDb_004, TestSize.Level1) +{ + std::shared_ptr &store = g_store; + + auto [ret1, transaction1] = store->CreateTransaction(Transaction::DEFERRED); + ASSERT_EQ(ret1, E_OK); + ASSERT_NE(transaction1, nullptr); + + auto [ret2, transaction2] = store->CreateTransaction(Transaction::DEFERRED); + ASSERT_EQ(ret2, E_OK); + ASSERT_NE(transaction2, nullptr); + + auto [ret3, transaction3] = store->CreateTransaction(Transaction::EXCLUSIVE); + ASSERT_EQ(ret3, E_OK); + ASSERT_NE(transaction3, nullptr); + + auto [ret4, transaction4] = store->CreateTransaction(Transaction::IMMEDIATE); + ASSERT_EQ(ret4, E_SQLITE_LOCKED); + ASSERT_EQ(transaction4, nullptr); + transaction3 = nullptr; + + std::tie(ret4, transaction4) = store->CreateTransaction(Transaction::IMMEDIATE); + ASSERT_EQ(ret4, E_OK); + ASSERT_NE(transaction4, nullptr); + + std::tie(ret3, transaction3) = store->CreateTransaction(Transaction::EXCLUSIVE); + ASSERT_EQ(ret3, E_SQLITE_LOCKED); + ASSERT_EQ(transaction3, nullptr); +} + +/** +*@tc.name: GetMemoryDb_004 +*@tc.desc: Get MemoryDb with diff name +*@tc.type: FUNC +*@tc.require: +*@tc.author: +*/ +HWTEST_F(RdbMemoryDbTest, GetMemoryDb_004, TestSize.Level1) +{ + int errCode = E_ERROR; + RdbStoreConfig config(RDB_TEST_PATH); + config.SetName("mem_test"); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbMemoryDbTestOpenCallback normalCallback; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + EXPECT_NE(rdbStore, nullptr); + + config.SetName("eme.db"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + EXPECT_NE(rdbStore, nullptr); + + config.SetName("test123-test"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_EQ(errCode, E_OK); + EXPECT_NE(rdbStore, nullptr); + + config.SetName("test%"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + + config.SetName("test:"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + + config.SetName("test?"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + + config.SetName("test$"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + + config.SetName("test("); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); + + config.SetName("test)"); + rdbStore = RdbHelper::GetRdbStore(config, 1, normalCallback, errCode); + EXPECT_NE(errCode, E_OK); + EXPECT_EQ(rdbStore, nullptr); +} \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_open_callback_test.cpp b/relational_store/test/native/rdb/unittest/rdb_open_callback_test.cpp index 6e8e894d..c4fd813c 100644 --- a/relational_store/test/native/rdb/unittest/rdb_open_callback_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_open_callback_test.cpp @@ -17,6 +17,7 @@ #include +#include #include #include "common.h" @@ -391,3 +392,89 @@ HWTEST_F(RdbOpenCallbackTest, RdbOpenCallback_05, TestSize.Level1) ret = helper.OnOpen(*store); EXPECT_EQ(ret, E_OK); } + +/** + * @tc.name: RdbOpenCallback_06 + * @tc.desc: test rdb open with not a database + * @tc.type: FUNC + */ +HWTEST_F(RdbOpenCallbackTest, RdbOpenCallback_06, TestSize.Level1) +{ + int errCode = E_OK; + RdbStoreConfig config(RdbOpenCallbackTest::DATABASE_NAME); + OpenCallbackB helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(store, nullptr); + store->ExecuteSql(OpenCallbackB::CreateTableSQL("test1")); + store.reset(); + store = nullptr; + int fd = open(RdbOpenCallbackTest::DATABASE_NAME.c_str(), O_WRONLY); + char str[] = "3333333333sss333333333333333333"; + lseek(fd, 0, SEEK_SET); + write(fd, str, sizeof(str)); + close(fd); + store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(store, nullptr); + store = nullptr; + RdbHelper::DeleteRdbStore(RdbOpenCallbackTest::DATABASE_NAME); +} + +/** + * @tc.name: RdbOpenCallback_07 + * @tc.desc: test rdb open with not a database and write file is not a database + * @tc.type: FUNC + */ +HWTEST_F(RdbOpenCallbackTest, RdbOpenCallback_07, TestSize.Level1) +{ + int errCode = E_OK; + RdbStoreConfig config(RdbOpenCallbackTest::DATABASE_NAME); + OpenCallbackB helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(store, nullptr); + store->ExecuteSql(OpenCallbackB::CreateTableSQL("test1")); + store.reset(); + store = nullptr; + int fd = open(RdbOpenCallbackTest::DATABASE_NAME.c_str(), O_WRONLY); + char str[] = "3333333333sss333333333333333333"; + lseek(fd, 0, SEEK_SET); + write(fd, str, sizeof(str)); + close(fd); + char writeStr[] = "13333333333sss333333333333333333"; + int writeFd = open((RdbOpenCallbackTest::DATABASE_NAME + "-dwr").c_str(), O_WRONLY); + lseek(writeFd, 0, SEEK_SET); + write(writeFd, writeStr, sizeof(writeStr)); + close(writeFd); + store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(store, nullptr); + RdbHelper::DeleteRdbStore(RdbOpenCallbackTest::DATABASE_NAME); +} + +/** + * @tc.name: RdbOpenCallback_08 + * @tc.desc: test rdb open with not a database and write file is not a database + * @tc.type: FUNC + */ +HWTEST_F(RdbOpenCallbackTest, RdbOpenCallback_08, TestSize.Level1) +{ + int errCode = E_OK; + RdbStoreConfig config(RdbOpenCallbackTest::DATABASE_NAME); + OpenCallbackB helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(store, nullptr); + for (int i = 1; i < 11; ++i) { + std::string testName = "test" + std::to_string(i); + store->ExecuteSql(OpenCallbackB::CreateTableSQL(testName)); + } + store.reset(); + store = nullptr; + int fd = open(RdbOpenCallbackTest::DATABASE_NAME.c_str(), O_WRONLY); + char str[] = "3333333333sss333333333333333333"; + lseek(fd, 0, SEEK_SET); + write(fd, str, sizeof(str)); + close(fd); + store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(store, nullptr); + store = nullptr; + RdbHelper::DeleteRdbStore(RdbOpenCallbackTest::DATABASE_NAME); +} + diff --git a/relational_store/test/native/rdb/unittest/rdb_predicates_test.cpp b/relational_store/test/native/rdb/unittest/rdb_predicates_test.cpp index 05b44dcf..0b4b2506 100644 --- a/relational_store/test/native/rdb/unittest/rdb_predicates_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_predicates_test.cpp @@ -343,7 +343,8 @@ const std::string CREATE_TABLE_ALL_DATA_TYPE_SQL = const std::string CREATE_TABLE_PERSON_SQL = "CREATE TABLE IF NOT EXISTS person " - "(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT , age INTEGER , REAL INTEGER);"; + "(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT , age INTEGER , REAL INTEGER, attachments ASSETS," + "attachment ASSET);"; const std::string ALL_DATA_TYPE_INSERT_SQL = "INSERT INTO AllDataType (id, integerValue, longValue, " @@ -620,7 +621,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_RdbPredicates_002, TestSize.Level1) * @tc.name: RdbStore_EqualTo_001 * @tc.desc: Normal testCase of RdbPredicates for EqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_001, TestSize.Level1) { @@ -635,7 +635,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_001, TestSize.Level1) * @tc.name: RdbStore_EqualTo_002 * @tc.desc: Normal testCase of RdbPredicates for EqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_002, TestSize.Level1) { @@ -671,6 +670,67 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_002, TestSize.Level1) RdbStorePredicateTest::store->ExecuteSql("delete from person where id < 3;"); } +/* * + * @tc.name: RdbStore_EqualTo_003 + * @tc.desc: Normal testCase of RdbPredicates for EqualTo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_003, TestSize.Level1) +{ + ValuesBucket values; + int64_t id; + values.PutInt("id", 1); + std::vector assets; + OHOS::NativeRdb::AssetValue asset{ .name = "asset" }; + assets.push_back(std::move(asset)); + ValueObject object(assets); + values.Put("attachments", object); + int ret = store->Insert(id, "person", values); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(1, id); + + RdbPredicates predicates("person"); + predicates.EqualTo("attachments", object); + + if (predicates.predicates_.operations_.size() != 0) { + EXPECT_EQ( + predicates.predicates_.operations_[0].operator_, OHOS::DistributedRdb::RdbPredicateOperator::ASSETS_ONLY); + } else { + EXPECT_TRUE(false); + } + RdbStorePredicateTest::store->ExecuteSql("delete from person where id < 2;"); +} + +/* * + * @tc.name: RdbStore_EqualTo_004 + * @tc.desc: Normal testCase of RdbPredicates for EqualTo + * @tc.type: FUNC + * @tc.require: + */ +HWTEST_F(RdbStorePredicateTest, RdbStore_EqualTo_004, TestSize.Level1) +{ + ValuesBucket values; + int64_t id; + values.PutInt("id", 1);; + OHOS::NativeRdb::AssetValue asset{ .name = "asset" }; + ValueObject object(asset); + values.Put("attachment", object); + int ret = store->Insert(id, "person", values); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(1, id); + + RdbPredicates predicates("person"); + predicates.EqualTo("attachment", object); + if (predicates.predicates_.operations_.size() != 0) { + EXPECT_EQ( + predicates.predicates_.operations_[0].operator_, OHOS::DistributedRdb::RdbPredicateOperator::ASSETS_ONLY); + } else { + EXPECT_TRUE(false); + } + RdbStorePredicateTest::store->ExecuteSql("delete from person where id < 2;"); +} + void RdbStorePredicateTest::CalendarTest(RdbPredicates predicates1) { std::vector columns; @@ -686,6 +746,7 @@ void RdbStorePredicateTest::CalendarTest(RdbPredicates predicates1) allDataTypes9->GetInt(0, valueInt); EXPECT_EQ(2, valueInt); } + void RdbStorePredicateTest::BasicDataTypeTest(RdbPredicates predicates1) { std::vector columns; @@ -763,7 +824,6 @@ int RdbStorePredicateTest::ResultSize(std::shared_ptr &resultSet) * @tc.name: RdbStore_NotEqualTo_001 * @tc.desc: Abnormal testCase of RdbPredicates for NotEqualTo, if field is "" * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotEqualTo_001, TestSize.Level1) { @@ -779,7 +839,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_NotEqualTo_001, TestSize.Level1) * @tc.name: RdbStore_NotEqualTo_002 * @tc.desc: Normal testCase of RdbPredicates for NotEqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotEqualTo_002, TestSize.Level1) { @@ -794,7 +853,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_NotEqualTo_002, TestSize.Level1) * @tc.name: RdbStore_NotEqualTo_003 * @tc.desc: Normal testCase of RdbPredicates for EqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotEqualTo_003, TestSize.Level1) { @@ -907,7 +965,6 @@ void RdbStorePredicateTest::BasicDataTypeTest002(RdbPredicates predicates1) * @tc.name: RdbStore_IsNull_003 * @tc.desc: Normal testCase of RdbPredicates for IsNull * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_IsNull_003, TestSize.Level1) { @@ -922,7 +979,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_IsNull_003, TestSize.Level1) * @tc.name: RdbStore_NotNull_004 * @tc.desc: Normal testCase of RdbPredicates for NotNull * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotNull_003, TestSize.Level1) { @@ -937,7 +993,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_NotNull_003, TestSize.Level1) * @tc.name: RdbStore_GreaterThan_005 * @tc.desc: Normal testCase of RdbPredicates for GreaterThan * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_GreaterThan_005, TestSize.Level1) { @@ -985,7 +1040,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_GreaterThan_005, TestSize.Level1) * @tc.name: RdbStore_GreaterThanOrEqualTo_006 * @tc.desc: Normal testCase of RdbPredicates for GreaterThanOrEqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_GreaterThanOrEqualTo_006, TestSize.Level1) { @@ -1039,7 +1093,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_GreaterThanOrEqualTo_006, TestSize.Leve * @tc.name: RdbStore_lessThan_007 * @tc.desc: Normal testCase of RdbPredicates for LessThan * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_lessThan_007, TestSize.Level1) { @@ -1087,7 +1140,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_lessThan_007, TestSize.Level1) * @tc.name: RdbStore_LessThanOrEqualTo_008 * @tc.desc: Normal testCase of RdbPredicates for LessThanOrEqualTo * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_LessThanOrEqualTo_008, TestSize.Level1) { @@ -1135,7 +1187,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_LessThanOrEqualTo_008, TestSize.Level1) * @tc.name: RdbStore_Between_009 * @tc.desc: Normal testCase of RdbPredicates for Between * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Between_009, TestSize.Level1) { @@ -1188,7 +1239,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Between_009, TestSize.Level1) * @tc.name: RdbStore_Contain_010 * @tc.desc: Normal testCase of RdbPredicates for Contain * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Contain_010, TestSize.Level1) { @@ -1204,7 +1254,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Contain_010, TestSize.Level1) * @tc.name: RdbStore_BeginsWith_011 * @tc.desc: Normal testCase of RdbPredicates for BeginsWith * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_BeginsWith_011, TestSize.Level1) { @@ -1220,7 +1269,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_BeginsWith_011, TestSize.Level1) * @tc.name: RdbStore_EndsWith_012 * @tc.desc: Normal testCase of RdbPredicates for EndsWith * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_EndsWith_012, TestSize.Level1) { @@ -1236,7 +1284,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_EndsWith_012, TestSize.Level1) * @tc.name: RdbStore_Like_013 * @tc.desc: Normal testCase of RdbPredicates for Like * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Like_013, TestSize.Level1) { @@ -1252,7 +1299,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Like_013, TestSize.Level1) * @tc.name: RdbStore_BeginEndWrap_014 * @tc.desc: Normal testCase of RdbPredicates for BeginEndWrap * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_BeginEndWrap_014, TestSize.Level1) { @@ -1278,7 +1324,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_BeginEndWrap_014, TestSize.Level1) * @tc.name: RdbStore_AndOR_015 * @tc.desc: Normal testCase of RdbPredicates for AndOR * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_AndOR_015, TestSize.Level1) { @@ -1305,7 +1350,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_AndOR_015, TestSize.Level1) * @tc.name: RdbStore_Order_016 * @tc.desc: Normal testCase of RdbPredicates for Order * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Order_016, TestSize.Level1) { @@ -1343,7 +1387,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Order_016, TestSize.Level1) * @tc.name: RdbStore_Limit_017 * @tc.desc: Normal testCase of RdbPredicates for Limit * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Limit_017, TestSize.Level1) { @@ -1359,7 +1402,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Limit_017, TestSize.Level1) * @tc.name: RdbStore_JoinTypes_018 * @tc.desc: Normal testCase of RdbPredicates for JoinTypes * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_JoinTypes_018, TestSize.Level1) { @@ -1388,7 +1430,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_JoinTypes_018, TestSize.Level1) * @tc.name: RdbStore_Glob_019 * @tc.desc: Normal testCase of RdbPredicates for Glob * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_Glob_019, TestSize.Level1) { @@ -1429,7 +1470,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_Glob_019, TestSize.Level1) * @tc.name: RdbStore_NotBetween_020 * @tc.desc: Normal testCase of RdbPredicates for NotBetween * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotBetween_020, TestSize.Level1) { @@ -1482,7 +1522,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_NotBetween_020, TestSize.Level1) * @tc.name: RdbStore_ComplexPredicate_021 * @tc.desc: Normal testCase of RdbPredicates for complex combine sql * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_ComplexPredicate_021, TestSize.Level1) { @@ -1509,7 +1548,6 @@ void RdbStorePredicateTest::SetJionList(RdbPredicates &predicates1) * @tc.name: RdbStore_ClearMethod_022 * @tc.desc: Normal testCase of RdbPredicates for Clear Method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_ClearMethod_022, TestSize.Level1) { @@ -1573,7 +1611,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_ClearMethod_022, TestSize.Level1) * @tc.name: RdbStore_InMethod_023 * @tc.desc: Normal testCase of RdbPredicates for in method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_InMethod_023, TestSize.Level1) { @@ -1620,7 +1657,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_InMethod_023, TestSize.Level1) * @tc.name: RdbStore_NotInMethod_023 * @tc.desc: Normal testCase of RdbPredicates for notIn method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotInMethod_023, TestSize.Level1) { @@ -1668,7 +1704,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_NotInMethod_023, TestSize.Level1) * @tc.name: RdbStore_KeywordMethod_024 * @tc.desc: Normal testCase of RdbPredicates for clear method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_KeywordMethod_024, TestSize.Level1) { @@ -1733,7 +1768,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_KeywordMethod_024, TestSize.Level1) * @tc.name: RdbStore_ToString_025 * @tc.desc: Normal testCase of RdbPredicates for clear method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_ToString_025, TestSize.Level1) { @@ -1757,7 +1791,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_ToString_025, TestSize.Level1) * @tc.name: RdbStore_InDevices_InAllDevices_026 * @tc.desc: Normal testCase of RdbPredicates for InDevices and InAllDevices method * @tc.type: FUNC - * @tc.require: AR */ HWTEST_F(RdbStorePredicateTest, RdbStore_InDevices_InAllDevices_026, TestSize.Level1) { @@ -1794,7 +1827,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_GetDistributedPredicates_027, TestSize. * @tc.name: RdbStore_NotInMethod_028 * @tc.desc: Abnormal testCase of RdbPredicates for notIn method * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_NotInMethod_028, TestSize.Level1) { @@ -2416,7 +2448,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_GetStatement_GetBnidArgs_002, TestSize. * @tc.name: RdbStore_GetString_001 * @tc.desc: Normal testCase of RdbPredicates for GetString * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_GetString_001, TestSize.Level1) { @@ -2456,7 +2487,6 @@ HWTEST_F(RdbStorePredicateTest, RdbStore_GetString_001, TestSize.Level1) * @tc.name: RdbStore_GetString_002 * @tc.desc: Normal testCase of RdbPredicates for GetString * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStorePredicateTest, RdbStore_GetString_002, TestSize.Level1) { diff --git a/relational_store/test/native/rdb/unittest/rdb_rd_data_aging_test.cpp b/relational_store/test/native/rdb/unittest/rdb_rd_data_aging_test.cpp new file mode 100644 index 00000000..c6212f18 --- /dev/null +++ b/relational_store/test/native/rdb/unittest/rdb_rd_data_aging_test.cpp @@ -0,0 +1,218 @@ +/* +* Copyright (c) 2025 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#include + +#include +#include +#include + +#include "common.h" +#include "grd_api_manager.h" +#include "rdb_errno.h" +#include "rdb_helper.h" + +using namespace testing::ext; +using namespace OHOS::NativeRdb; + +class RdbRdDataAgingTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + void InsertData(uint64_t second, int startId, int endId); + + static const std::string databaseName; + static std::shared_ptr store; +}; + +class ExecuteTestOpenRdCallback : public RdbOpenCallback { +public: + int OnCreate(RdbStore &store) override; + int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; +}; + +const std::string RdbRdDataAgingTest::databaseName = RDB_TEST_PATH + "data_aging_test.db"; +std::shared_ptr RdbRdDataAgingTest::store = nullptr; + +void RdbRdDataAgingTest::SetUpTestCase(void) +{ +} + +void RdbRdDataAgingTest::TearDownTestCase(void) +{ +} + +void RdbRdDataAgingTest::SetUp(void) +{ + if (!IsUsingArkData()) { + GTEST_SKIP() << "Current testcase is not compatible from current rdb"; + } + int errCode = E_OK; + RdbHelper::DeleteRdbStore(RdbRdDataAgingTest::databaseName); + RdbStoreConfig config(RdbRdDataAgingTest::databaseName); + config.SetIsVector(true); + ExecuteTestOpenRdCallback helper; + RdbRdDataAgingTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(RdbRdDataAgingTest::store, nullptr); + ASSERT_EQ(errCode, E_OK); + string sql1 = "create table test (id integer, start_time integer not null) with " + "(time_col='start_time', ttl='1 hour', data_limit='100 KB', interval='5 second', max_num='100');"; + store->Execute(sql1); +} + +void RdbRdDataAgingTest::TearDown(void) +{ + RdbRdDataAgingTest::store = nullptr; + RdbHelper::DeleteRdbStore(RdbRdDataAgingTest::databaseName); +} + +void RdbRdDataAgingTest::InsertData(uint64_t second, int startId, int endId) +{ + for (int i = startId; i <= endId; i++) { + struct timeval timestamp; + (void)gettimeofday(×tamp, nullptr); + uint64_t startTime = timestamp.tv_sec - second; + string sql = "insert into test values(" + std::to_string(i) + ", " + std::to_string(startTime) + ");"; + auto ret = store->Execute(sql); + ASSERT_EQ(ret.first, E_OK); + } +} + +/** +@tc.name: RdbStore_Data_Aging_001 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_001, TestSize.Level1) +{ + InsertData(3595, 1, 100); + sleep(2); + InsertData(0, 101, 101); + sleep(2); + auto resultSet = store->QueryByStep("select * from test;"); + int count = 0; + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 101); +} + +/** +@tc.name: RdbStore_Data_Aging_002 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_002, TestSize.Level1) +{ + InsertData(3600, 1, 99); + sleep(5); + InsertData(0, 100, 100); + sleep(2); + auto resultSet = store->QueryByStep("select * from test;"); + int count = 0; + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 1); +} + +/** +@tc.name: RdbStore_Data_Aging_003 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_003, TestSize.Level1) +{ + InsertData(3595, 1, 50); + InsertData(0, 51, 100); + sleep(5); + auto resultSet = store->QueryByStep("select * from test;"); + int count = 0; + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 100); +} + +/** +@tc.name: RdbStore_Data_Aging_004 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_004, TestSize.Level1) +{ + InsertData(3595, 1, 100); + InsertData(0, 101, 149); + sleep(5); + auto resultSet = store->QueryByStep("select * from test;"); + int count = 0; + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 149); + + InsertData(0, 150, 150); + sleep(2); + + resultSet = store->QueryByStep("select * from test;"); + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 50); +} + +/** +@tc.name: RdbStore_Data_Aging_005 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_005, TestSize.Level1) +{ + InsertData(3595, 1, 100); + InsertData(3592, 101, 130); + InsertData(3590, 131, 200); + + auto resultSet = store->QueryByStep("select * from test;"); + int count = 0; + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 200); + sleep(5); + InsertData(0, 201, 201); + + sleep(2); + resultSet = store->QueryByStep("select * from test;"); + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 71); + + sleep(5); + InsertData(0, 202, 202); + + sleep(2); + resultSet = store->QueryByStep("select * from test;"); + resultSet->GetRowCount(count); + resultSet->Close(); + ASSERT_EQ(count, 2); +} + +/** +@tc.name: RdbStore_Data_Aging_006 +@tc.desc: test RdbStore_Data_Aging +@tc.type: FUNC +*/ +HWTEST_F(RdbRdDataAgingTest, RdbStore_Data_Aging_006, TestSize.Level1) +{ + InsertData(3600, 1, 99); + InsertData(0, 100, 100); + auto ret = store->Execute("drop table test;"); + ASSERT_EQ(ret.first, E_OK); +} diff --git a/relational_store/test/native/rdb/unittest/rdb_sql_utils_test.cpp b/relational_store/test/native/rdb/unittest/rdb_sql_utils_test.cpp index fbc56309..b6578623 100644 --- a/relational_store/test/native/rdb/unittest/rdb_sql_utils_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_sql_utils_test.cpp @@ -22,6 +22,8 @@ #include "grd_type_export.h" #include "rd_utils.h" +#include "sqlite_sql_builder.h" +#include "values_buckets.h" using namespace testing::ext; using namespace OHOS::NativeRdb; @@ -57,3 +59,25 @@ HWTEST_F(RdbSqlUtilsTest, RdbSqlUtils_Test_001, TestSize.Level1) EXPECT_EQ(dataBasePath1, "/data/test/rdb/myself/RdbTest.db"); EXPECT_EQ(errCode1, E_OK); } + +/** + * @tc.name: RdbStore_UpdateSqlBuilder_001 + * @tc.desc: test RdbStore UpdateSqlBuilder + * @tc.type: FUNC + */ +HWTEST_F(RdbSqlUtilsTest, RdbSqlUtils_UpdateSqlBuilder_001, TestSize.Level1) +{ + ValuesBucket values; + values.PutString("name", std::string("zhangsan")); + values.PutInt("age", 20); + values.PutDouble("salary", 300.5); + + std::vector bindArgs; + std::string updateSql = SqliteSqlBuilder::BuildUpdateString(values, "test", std::vector{ "19" }, "", + "age = ?", "", "", INT_MIN, INT_MIN, bindArgs, ConflictResolution::ON_CONFLICT_NONE); + EXPECT_EQ(updateSql, "UPDATE test SET age=?,name=?,salary=? WHERE age = ?"); + + updateSql = SqliteSqlBuilder::BuildUpdateString(values, "test", std::vector{}, "", "", "", "", + INT_MIN, INT_MIN, bindArgs, ConflictResolution::ON_CONFLICT_NONE); + EXPECT_EQ(updateSql, "UPDATE test SET age=?,name=?,salary=?"); +} diff --git a/relational_store/test/native/rdb/unittest/rdb_sqlite_shared_result_set_test.cpp b/relational_store/test/native/rdb/unittest/rdb_sqlite_shared_result_set_test.cpp index fb785a1c..5ac955a2 100644 --- a/relational_store/test/native/rdb/unittest/rdb_sqlite_shared_result_set_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_sqlite_shared_result_set_test.cpp @@ -57,7 +57,8 @@ public: std::string const SqliteSharedOpenCallback::CREATE_TABLE_TEST = "CREATE TABLE test (id INTEGER PRIMARY KEY " "AUTOINCREMENT, data1 TEXT,data2 INTEGER, data3 " - "FLOAT, data4 BLOB, data5 ASSET, data6 ASSETS);"; + "FLOAT, data4 BLOB, data5 ASSET, data6 ASSETS, " + "data7 floatvector(128), data8 UNLIMITED INT);"; int SqliteSharedOpenCallback::OnCreate(RdbStore &rdbStore) { @@ -99,27 +100,40 @@ void RdbSqliteSharedResultSetTest::GenerateDefaultTable() int64_t id; ValuesBucket values; + AssetValue asset { + .version = 0, + .name = "123", + .uri = "my test path", + .createTime = "12", + .modifyTime = "12", + }; + vector assets; + assets.push_back(asset); values.PutInt("id", 1); values.PutString("data1", std::string("hello")); - values.PutInt("data2", 10); + values.PutInt("data2", 10); // set int value 10 values.PutDouble("data3", 1.0); - values.PutBlob("data4", std::vector{ 66 }); + values.PutBlob("data4", std::vector{ 66 }); // set uint8_t value 66 + values.Put("data5", asset); + values.Put("data6", assets); + values.Put("data7", std::vector(1, 0.5)); // set float value 0.5 + values.Put("data8", BigInteger(0)); store->Insert(id, "test", values); values.Clear(); - values.PutInt("id", 2); + values.PutInt("id", 2); // set int value 2 values.PutString("data1", std::string("2")); - values.PutInt("data2", -5); - values.PutDouble("data3", 2.5); + values.PutInt("data2", -5); // set int value -5 + values.PutDouble("data3", 2.5); // set float value 2.5 values.PutBlob("data4", std::vector{}); store->Insert(id, "test", values); values.Clear(); - values.PutInt("id", 3); + values.PutInt("id", 3); // set int value 3 values.PutString("data1", std::string("hello world")); - values.PutInt("data2", 3); - values.PutDouble("data3", 1.8); + values.PutInt("data2", 3); // set int value 3 + values.PutDouble("data3", 1.8); // set float value 1.8 values.PutBlob("data4", std::vector{}); store->Insert(id, "test", values); } @@ -205,7 +219,6 @@ void RdbSqliteSharedResultSetTest::CheckResultSetAttribute( * @tc.name: Sqlite_Shared_Result_Set_Asset_Timeout * @tc.desc: normal testcase of SqliteSharedResultSet for move * @tc.type: FUNC - * @tc.require: AR000134UL */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_Asset_Timeout, TestSize.Level1) { @@ -234,7 +247,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_Asset_Timeout, T * @tc.name: Sqlite_Shared_Result_Set_Asset * @tc.desc: normal testcase of SqliteSharedResultSet for asset and assets * @tc.type: FUNC - * @tc.require: AR000134UL */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_Asset, TestSize.Level1) { @@ -300,7 +312,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_Asset, TestSize. * @tc.name: RdbStore_Delete_001 * @tc.desc: normal testcase of SqliteSharedResultSet for move * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_001, TestSize.Level1) { @@ -399,7 +410,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_001, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_002 * @tc.desc: normal testcase of SqliteSharedResultSet for goToNextRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_002, TestSize.Level1) { @@ -434,7 +444,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_002, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_003 * @tc.desc: normal testcase of SqliteSharedResultSet for moveFirst * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_003, TestSize.Level1) { @@ -487,7 +496,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_003, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_004 * @tc.desc: normal testcase of SqliteSharedResultSet for getInt * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_004, TestSize.Level1) { @@ -530,7 +538,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_004, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_005 * @tc.desc: normal testcase of SqliteSharedResultSet for getString * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_005, TestSize.Level1) @@ -585,7 +592,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_005, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_006 * @tc.desc: normal testcase of SqliteSharedResultSet for getDouble * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_006, TestSize.Level1) { @@ -638,7 +644,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_006, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_007 * @tc.desc: normal testcase of SqliteSharedResultSet for getBlob * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_007, TestSize.Level1) { @@ -677,7 +682,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_007, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_008 * @tc.desc: normal testcase of SqliteSharedResultSet for getColumnTypeForIndex * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_008, TestSize.Level1) @@ -696,24 +700,22 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_008, TestSize.Le rstSet->GetColumnType(0, colType); EXPECT_EQ(colType, ColumnType::TYPE_INTEGER); - - bool isColNull = true; - rstSet->IsColumnNull(0, isColNull); - EXPECT_EQ(isColNull, false); - rstSet->GetColumnType(1, colType); EXPECT_EQ(colType, ColumnType::TYPE_STRING); - - isColNull = true; - rstSet->IsColumnNull(0, isColNull); - EXPECT_EQ(isColNull, false); - rstSet->GetColumnType(2, colType); EXPECT_EQ(colType, ColumnType::TYPE_INTEGER); rstSet->GetColumnType(3, colType); EXPECT_EQ(colType, ColumnType::TYPE_FLOAT); rstSet->GetColumnType(4, colType); EXPECT_EQ(colType, ColumnType::TYPE_BLOB); + rstSet->GetColumnType(5, colType); + EXPECT_EQ(colType, ColumnType::TYPE_ASSET); + rstSet->GetColumnType(6, colType); + EXPECT_EQ(colType, ColumnType::TYPE_ASSETS); + rstSet->GetColumnType(7, colType); + EXPECT_EQ(colType, ColumnType::TYPE_FLOAT32_ARRAY); + rstSet->GetColumnType(8, colType); + EXPECT_EQ(colType, ColumnType::TYPE_BIGINT); int colCnt = 0; rstSet->GetColumnCount(colCnt); @@ -729,7 +731,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_008, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_009 * @tc.desc: normal testcase of SqliteSharedResultSet for getColumnIndexForName * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_009, TestSize.Level1) { @@ -765,7 +766,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_009, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_010 * @tc.desc: normal testcase of SqliteSharedResultSet for getColumnNameForIndex * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_010, TestSize.Level1) { @@ -806,7 +806,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_010, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_011 * @tc.desc: normal testcase of SqliteSharedResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_011, TestSize.Level1) { @@ -840,7 +839,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_011, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_012 * @tc.desc: normal testcase of SqliteSharedResultSet for getLong * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_012, TestSize.Level1) { @@ -890,7 +888,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_012, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_013 * @tc.desc: normal testcase of SqliteSharedResultSet for fillBlock * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_013, TestSize.Level1) { @@ -912,7 +909,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_013, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_014 * @tc.desc: normal testcase of SqliteSharedResultSet for getBlock * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_014, TestSize.Level1) { @@ -945,7 +941,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_014, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_015 * @tc.desc: normal testcase of SqliteSharedResultSet for setBlock * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_015, TestSize.Level1) { @@ -978,7 +973,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_015, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_016 * @tc.desc: normal testcase of SqliteSharedResultSet for setFillWindowForwardOnly * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_016, TestSize.Level1) { @@ -1019,7 +1013,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_016, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_017 * @tc.desc: normal testcase of SqliteSharedResultSet for setExtensions and getExtensions * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_017, TestSize.Level1) { @@ -1075,7 +1068,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_018, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_019 * @tc.desc: normal testcase of SqliteSharedResultSet for GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_019, TestSize.Level1) { @@ -1119,7 +1111,6 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_019, TestSize.Le * @tc.name: Sqlite_Shared_Result_Set_020 * @tc.desc: normal testcase of SqliteSharedResultSet for GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_020, TestSize.Level1) { @@ -1606,4 +1597,50 @@ HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_038, TestSize.Le EXPECT_EQ("TEXT", strValue); resultSet->Close(); +} + +HWTEST_F(RdbSqliteSharedResultSetTest, Sqlite_Shared_Result_Set_039, TestSize.Level1) +{ + GenerateDefaultTable(); + int64_t id; + ValuesBucket values; + values.PutNull("data1"); + values.PutNull("data2"); + values.PutNull("data3"); + values.PutNull("data4"); + values.PutNull("data5"); + values.PutNull("data6"); + values.PutNull("data7"); + values.PutNull("data8"); + store->Insert(id, "test", values); + std::vector selectionArgs; + std::shared_ptr rstSet = + RdbSqliteSharedResultSetTest::store->QuerySql("SELECT * FROM test", selectionArgs); + EXPECT_NE(rstSet, nullptr); + + ColumnType colType; + int ret = rstSet->GetColumnType(0, colType); + EXPECT_EQ(ret, E_ROW_OUT_RANGE); + int retF = rstSet->GoTo(4); + EXPECT_EQ(retF, E_OK); + + rstSet->GetColumnType(0, colType); + EXPECT_EQ(colType, ColumnType::TYPE_INTEGER); + rstSet->GetColumnType(1, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(2, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(3, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(4, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(5, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(6, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(7, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->GetColumnType(8, colType); + EXPECT_EQ(colType, ColumnType::TYPE_NULL); + rstSet->Close(); } \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_step_result_get_row_test.cpp b/relational_store/test/native/rdb/unittest/rdb_step_result_get_row_test.cpp index 6b47817c..c350be73 100644 --- a/relational_store/test/native/rdb/unittest/rdb_step_result_get_row_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_step_result_get_row_test.cpp @@ -87,7 +87,6 @@ void RdbStepResultSetGetRowTest::TearDown(void) * @tc.name: RdbStore_StepResultSet_GetRow_001 * @tc.desc: test StepResultSet GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_001, TestSize.Level1) { @@ -135,7 +134,6 @@ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_001, TestSize * @tc.name: RdbStore_StepResultSet_GetRow_002 * @tc.desc: test StepResultSet GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_002, TestSize.Level1) { @@ -173,7 +171,6 @@ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_002, TestSize * @tc.name: RdbStore_StepResultSet_GetRow_003 * @tc.desc: test StepResultSet GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_003, TestSize.Level1) { @@ -231,7 +228,6 @@ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_003, TestSize * @tc.name: RdbStore_StepResultSet_GetRow_004 * @tc.desc: test StepResultSet GetRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetGetRowTest, RdbStore_StepResultSet_GetRow_004, TestSize.Level1) { diff --git a/relational_store/test/native/rdb/unittest/rdb_step_result_set_test.cpp b/relational_store/test/native/rdb/unittest/rdb_step_result_set_test.cpp index 1a9b1dc9..f6b63608 100644 --- a/relational_store/test/native/rdb/unittest/rdb_step_result_set_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_step_result_set_test.cpp @@ -106,31 +106,58 @@ void RdbStepResultSetTest::TearDown(void) void RdbStepResultSetTest::GenerateDefaultTable() { - std::string createTableSql = std::string("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, ") + - std::string("data2 INTEGER, data3 FLOAT, data4 BLOB);"); + std::string createTableSql = + "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, data2 INTEGER, data3 FLOAT, data4 BLOB, " + "data5 ASSET, data6 ASSETS, data7 floatvector(128), data8 UNLIMITED INT);"; store->ExecuteSql(createTableSql); - std::string insertSql = "INSERT INTO test (data1, data2, data3, data4) VALUES (?, ?, ?, ?);"; + std::string insertSql = "INSERT INTO test (data1, data2, data3, data4, data5, data6, data7, data8) VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?);"; /* insert first entry data */ - uint8_t uValue = 66; - std::vector typeBlob; - typeBlob.push_back(uValue); - store->ExecuteSql(insertSql, std::vector{ ValueObject(std::string("hello")), ValueObject((int)10), - ValueObject((double)1.0), ValueObject((std::vector)typeBlob) }); + AssetValue asset { + .version = 0, + .name = "123", + .uri = "my test path", + .createTime = "12", + .modifyTime = "12", + }; + vector assets; + assets.push_back(asset); + std::vector args; + args.push_back("hello"); + args.push_back(10); // set int value 10 + args.push_back(1.0); + args.push_back(std::vector(1, 66)); // set uint8_t value 66 + args.push_back(asset); + args.push_back(assets); + args.push_back(std::vector(1, 0.5)); // set float value 0.5 + args.push_back(BigInteger(0)); + store->ExecuteSql(insertSql, args); /* insert second entry data */ - typeBlob.clear(); - store->ExecuteSql(insertSql, std::vector{ - ValueObject(std::string("2")), ValueObject((int)-5), ValueObject((double)2.5), - ValueObject() // set double value 2.5 - }); + args.clear(); + args.push_back("2"); + args.push_back(-5); // set int value -5 + args.push_back(2.5); // set float value 2.5 + args.push_back(ValueObject()); + args.push_back(asset); + args.push_back(assets); + args.push_back(std::vector(1, 0.5)); // set float value 0.5 + args.push_back(BigInteger(0)); + store->ExecuteSql(insertSql, args); /* insert third entry data */ - store->ExecuteSql(insertSql, std::vector{ - ValueObject(std::string("hello world")), ValueObject((int)3), - ValueObject((double)1.8), ValueObject() // set int value 3, double 1.8 - }); + args.clear(); + args.push_back("hello world"); + args.push_back(3); // set int value 3 + args.push_back(1.8); // set float value 1.8 + args.push_back(ValueObject()); + args.push_back(asset); + args.push_back(assets); + args.push_back(std::vector(1, 0.5)); // set float value 0.5 + args.push_back(BigInteger(0)); + store->ExecuteSql(insertSql, args); } void RdbStepResultSetTest::GenerateDefaultEmptyTable() @@ -204,12 +231,10 @@ void RdbStepResultSetTest::CheckResultSetData( * @tc.name: RdbStore_StepResultSet_001 * @tc.desc: test StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_001, TestSize.Level1) { GenerateDefaultTable(); - std::shared_ptr resultSet = store->QueryByStep("SELECT * FROM test"); EXPECT_NE(resultSet, nullptr); bool bResultSet = true; @@ -233,6 +258,14 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_001, TestSize.Level1) CheckColumnType(resultSet, 4, ColumnType::TYPE_BLOB); + CheckColumnType(resultSet, 5, ColumnType::TYPE_ASSET); + + CheckColumnType(resultSet, 6, ColumnType::TYPE_ASSETS); + + CheckColumnType(resultSet, 7, ColumnType::TYPE_FLOAT32_ARRAY); + + CheckColumnType(resultSet, 8, ColumnType::TYPE_BIGINT); + EXPECT_EQ(E_OK, resultSet->GoToFirstRow()); EXPECT_EQ(E_OK, resultSet->GoToNextRow()); @@ -252,7 +285,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_001, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_002 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_002, TestSize.Level1) { @@ -308,7 +340,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_002, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_003 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_003, TestSize.Level1) { @@ -340,7 +371,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_003, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_004 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_004, TestSize.Level1) { @@ -365,7 +395,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_004, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_005 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_005, TestSize.Level1) { @@ -398,7 +427,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_005, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_006 * @tc.desc: normal testcase of StepResultSet for moveFirstWithoutEntry * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_006, TestSize.Level1) { @@ -428,7 +456,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_006, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_007 * @tc.desc: normal testcase of StepResultSet for goToNextRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_007, TestSize.Level1) { @@ -465,7 +492,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_007, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_008 * @tc.desc: normal testcase of StepResultSet for moveNextWithoutEntry * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_008, TestSize.Level1) { @@ -499,7 +525,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_008, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_009 * @tc.desc: normal testcase of StepResultSet for getInt * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_009, TestSize.Level1) { @@ -560,7 +585,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_009, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_010 * @tc.desc: normal testcase of StepResultSet for getString * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_010, TestSize.Level1) { @@ -609,7 +633,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_010, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_011 * @tc.desc: normal testcase of StepResultSet for GetDouble * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_011, TestSize.Level1) { @@ -657,7 +680,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_011, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_012 * @tc.desc: normal testcase of StepResultSet for getBlob * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_012, TestSize.Level1) { @@ -711,7 +733,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_012, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_013 * @tc.desc: normal testcase of StepResultSet for getBlob * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_013, TestSize.Level1) { @@ -736,7 +757,7 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_013, TestSize.Level1) int columnCount = 0; iRet = resultSet->GetColumnCount(columnCount); - EXPECT_EQ(5, columnCount); + EXPECT_EQ(9, columnCount); iRet = resultSet->GetColumnType(columnCount, type); EXPECT_NE(E_OK, iRet); } @@ -745,7 +766,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_013, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_014 * @tc.desc: normal testcase of StepResultSet for getColumnIndexForName * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_014, TestSize.Level1) { @@ -783,7 +803,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_014, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_015 * @tc.desc: normal testcase of StepResultSet for getColumnNameForIndex * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_015, TestSize.Level1) { @@ -814,7 +833,7 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_015, TestSize.Level1) int columnCount = 0; iRet = resultSet->GetColumnCount(columnCount); - EXPECT_EQ(5, columnCount); + EXPECT_EQ(9, columnCount); iRet = resultSet->GetColumnName(columnCount, columnName); EXPECT_NE(E_OK, iRet); } @@ -823,7 +842,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_015, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_016 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_016, TestSize.Level1) { @@ -857,7 +875,6 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_016, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_017 * @tc.desc: Abnormal testcase of StepResultSet, arguments of GetAsset and GetAssets are invalid * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_017, TestSize.Level1) { @@ -873,11 +890,59 @@ HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_017, TestSize.Level1) EXPECT_EQ(E_COLUMN_OUT_RANGE, resultSet->GetAssets(-1, assets)); } +/* * + * @tc.name: RdbStore_StepResultSet_018 + * @tc.desc: normal testcase of StepResultSet + * @tc.type: FUNC + */ +HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_018, TestSize.Level1) +{ + GenerateDefaultTable(); + std::shared_ptr resultSet = store->QueryByStep("SELECT data1, data2, data3, data4 FROM test", {}, false); + EXPECT_NE(resultSet, nullptr); + + bool bResultSet = false; + EXPECT_EQ(E_OK, resultSet->GoToFirstRow()); + + int count = -1; + int iRet = resultSet->GetRowCount(count); + EXPECT_EQ(E_OK, iRet); + EXPECT_EQ(3, count); + + iRet = resultSet->IsAtLastRow(bResultSet); + EXPECT_EQ(E_OK, iRet); + EXPECT_EQ(bResultSet, false); + + std::string name; + int columnIndex = -1; + int errCode = resultSet->GetColumnIndex("data1", columnIndex); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(E_OK, resultSet->GetString(columnIndex, name)); + EXPECT_EQ(name, "hello"); + + EXPECT_EQ(E_OK, resultSet->GoToNextRow()); + errCode = resultSet->GetColumnIndex("data1", columnIndex); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(E_OK, resultSet->GetString(columnIndex, name)); + EXPECT_EQ(name, "2"); + iRet = resultSet->IsAtLastRow(bResultSet); + EXPECT_EQ(E_OK, iRet); + EXPECT_EQ(bResultSet, false); + + EXPECT_EQ(E_OK, resultSet->GoToNextRow()); + errCode = resultSet->GetColumnIndex("data1", columnIndex); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(E_OK, resultSet->GetString(columnIndex, name)); + EXPECT_EQ(name, "hello world"); + iRet = resultSet->IsAtLastRow(bResultSet); + EXPECT_EQ(E_OK, iRet); + EXPECT_EQ(bResultSet, true); +} + /* * * @tc.name: testGetRowCount003 * @tc.desc: normal testcase of StepResultSet for getRowCount * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testGetRowCount003, TestSize.Level1) { @@ -928,7 +993,6 @@ HWTEST_F(RdbStepResultSetTest, testGetRowCount003, TestSize.Level1) * @tc.name: testGetRowCount004 * @tc.desc: normal testcase of StepResultSet for getRowCount * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testGetRowCount004, TestSize.Level1) { @@ -978,7 +1042,6 @@ HWTEST_F(RdbStepResultSetTest, testGetRowCount004, TestSize.Level1) * @tc.name: testGoToRow005 * @tc.desc: normal testcase of StepResultSet for goToRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testGoToRow005, TestSize.Level1) { @@ -1023,7 +1086,6 @@ HWTEST_F(RdbStepResultSetTest, testGoToRow005, TestSize.Level1) * @tc.name: testGo006 * @tc.desc: normal testcase of StepResultSet for goToRow * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testGo006, TestSize.Level1) { @@ -1063,7 +1125,6 @@ HWTEST_F(RdbStepResultSetTest, testGo006, TestSize.Level1) * @tc.name: testGoToPrevious007 * @tc.desc: normal testcase of StepResultSet for go * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testGoToPrevious007, TestSize.Level1) { @@ -1121,7 +1182,6 @@ HWTEST_F(RdbStepResultSetTest, testGoToPrevious007, TestSize.Level1) * @tc.name: testSqlStep008 * @tc.desc: normal testcase of SqlStep for go * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testSqlStep008, TestSize.Level1) { @@ -1169,7 +1229,6 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep008, TestSize.Level1) * @tc.name: testSqlStep009 * @tc.desc: normal testcase of SqlStep for go * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testSqlStep009, TestSize.Level1) { @@ -1232,7 +1291,6 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep009, TestSize.Level1) * @tc.name: testSqlStep010 * @tc.desc: normal testcase of SqlStep for go * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testSqlStep010, TestSize.Level1) { @@ -1603,7 +1661,7 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep020, TestSize.Level1) int columnCount = 0; ret = resultSet->GetColumnCount(columnCount); - EXPECT_EQ(5, columnCount); + EXPECT_EQ(9, columnCount); ret = resultSet->GetColumnName(columnCount, columnName); EXPECT_NE(E_OK, ret); @@ -1663,7 +1721,6 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep022, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_023 * @tc.desc: normal testcase of StepResultSet * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testSqlStep023, TestSize.Level1) { @@ -1721,7 +1778,6 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep023, TestSize.Level1) * @tc.name: RdbStore_StepResultSet_024 * @tc.desc: normal testcase of StepResultSet for getInt * @tc.type: FUNC - * @tc.require: AR000FKD4F */ HWTEST_F(RdbStepResultSetTest, testSqlStep024, TestSize.Level1) { @@ -1781,6 +1837,49 @@ HWTEST_F(RdbStepResultSetTest, testSqlStep024, TestSize.Level1) resultSet->Close(); } +/* * + * @tc.name: RdbStore_StepResultSet_025 + * @tc.desc: test StepResultSet + * @tc.type: FUNC + */ +HWTEST_F(RdbStepResultSetTest, RdbStore_StepResultSet_025, TestSize.Level1) +{ + GenerateDefaultTable(); + int64_t id; + ValuesBucket values; + values.PutNull("data1"); + values.PutNull("data2"); + values.PutNull("data3"); + values.PutNull("data4"); + values.PutNull("data5"); + values.PutNull("data6"); + values.PutNull("data7"); + values.PutNull("data8"); + store->Insert(id, "test", values); + std::shared_ptr resultSet = store->QueryByStep("SELECT * FROM test"); + EXPECT_NE(resultSet, nullptr); + EXPECT_EQ(resultSet->GoTo(4), E_OK); + + CheckColumnType(resultSet, 0, ColumnType::TYPE_INTEGER); + + CheckColumnType(resultSet, 1, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 2, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 3, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 4, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 5, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 6, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 7, ColumnType::TYPE_NULL); + + CheckColumnType(resultSet, 8, ColumnType::TYPE_NULL); + resultSet->Close(); +} + /** * @tc.name: ResultSetProxy001 * @tc.desc: Abnormal testcase of distributed ResultSetProxy, if resultSet is Empty diff --git a/relational_store/test/native/rdb/unittest/rdb_store_backup_restore_test.cpp b/relational_store/test/native/rdb/unittest/rdb_store_backup_restore_test.cpp index 1dae3381..67efdc0a 100644 --- a/relational_store/test/native/rdb/unittest/rdb_store_backup_restore_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_store_backup_restore_test.cpp @@ -691,7 +691,7 @@ HWTEST_F(RdbStoreBackupRestoreTest, Rdb_BackupRestoreTest_014, TestSize.Level2) auto res = store->ExecuteSql("SELECT import_db_from_path"); - EXPECT_EQ(res, E_SQLITE_SCHEMA); + EXPECT_EQ(res, E_SQLITE_ERROR); auto [code, result] = store->Execute("pragma integrity_check"); std::string val; diff --git a/relational_store/test/native/rdb/unittest/rdb_store_concurrent_test.cpp b/relational_store/test/native/rdb/unittest/rdb_store_concurrent_test.cpp index 5d66160f..0e8b427e 100644 --- a/relational_store/test/native/rdb/unittest/rdb_store_concurrent_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_store_concurrent_test.cpp @@ -25,8 +25,17 @@ using namespace testing::ext; using namespace OHOS::NativeRdb; - -class RdbConcurrentTest : public testing::Test { +namespace OHOS::RdbConcurrentTest { +struct RdbTestParam { + std::shared_ptr store; + operator std::shared_ptr() + { + return store; + } +}; +static RdbTestParam g_store; +static RdbTestParam g_memDb; +class RdbConcurrentTest : public testing::TestWithParam { public: static void SetUpTestCase(void); static void TearDownTestCase(void); @@ -34,7 +43,7 @@ public: void TearDown(); static const std::string DATABASE_NAME; - static std::shared_ptr store; + std::shared_ptr store_; static void InsertThread(int n); static void QueryThread(int n); @@ -50,7 +59,6 @@ public: }; const std::string RdbConcurrentTest::DATABASE_NAME = RDB_TEST_PATH + "concurrent_test.db"; -std::shared_ptr RdbConcurrentTest::store = nullptr; int RdbConcurrentTest::insertResult = E_OK; int RdbConcurrentTest::queryResult = E_OK; @@ -58,15 +66,11 @@ class ConcurrentTestOpenCallback : public RdbOpenCallback { public: int OnCreate(RdbStore &store) override; int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; - static const std::string CREATE_TABLE_TEST; }; - -const std::string ConcurrentTestOpenCallback::CREATE_TABLE_TEST = - std::string("CREATE TABLE IF NOT EXISTS test ") + std::string("(id INTEGER PRIMARY KEY " - "AUTOINCREMENT, name TEXT NOT NULL, " - "age INTEGER, salary REAL, blobType " - "BLOB)"); - +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (id INTEGER PRIMARY KEY " + "AUTOINCREMENT, name TEXT NOT NULL, " + "age INTEGER, salary REAL, blobType " + "BLOB)"; int ConcurrentTestOpenCallback::OnCreate(RdbStore &store) { return store.ExecuteSql(CREATE_TABLE_TEST); @@ -83,19 +87,29 @@ void RdbConcurrentTest::SetUpTestCase(void) RdbHelper::DeleteRdbStore(RdbConcurrentTest::DATABASE_NAME); RdbStoreConfig config(RdbConcurrentTest::DATABASE_NAME); ConcurrentTestOpenCallback helper; - RdbConcurrentTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); - EXPECT_NE(RdbConcurrentTest::store, nullptr); - EXPECT_EQ(errCode, E_OK); + g_store.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store.store, nullptr); + ASSERT_EQ(errCode, E_OK); + + config.SetStorageMode(StorageMode::MODE_MEMORY); + g_memDb.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_memDb.store, nullptr); + ASSERT_EQ(errCode, E_OK); } void RdbConcurrentTest::TearDownTestCase(void) { RdbHelper::DeleteRdbStore(RdbConcurrentTest::DATABASE_NAME); + + RdbStoreConfig config(RdbConcurrentTest::DATABASE_NAME); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); } void RdbConcurrentTest::SetUp(void) { - store->ExecuteSql("DELETE FROM test"); + store_ = *GetParam(); + store_->ExecuteSql("DELETE FROM test"); } void RdbConcurrentTest::TearDown(void) @@ -105,7 +119,7 @@ void RdbConcurrentTest::TearDown(void) void RdbConcurrentTest::InsertThread(int n) { insertResult = E_OK; - std::shared_ptr &store = RdbConcurrentTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int64_t id; @@ -143,7 +157,7 @@ void RdbConcurrentTest::QueryThread(int n) int RdbConcurrentTest::Query() { - std::shared_ptr &store = RdbConcurrentTest::store; + std::shared_ptr store = *GetParam(); std::shared_ptr resultSet = store->QuerySql("SELECT * FROM test"); if (resultSet == nullptr) { return E_ERROR; @@ -306,7 +320,7 @@ int RdbConcurrentTest::CheckBlob(ResultSet &resultSet) * @tc.desc: test RdbStore Execute * @tc.type: FUNC */ -HWTEST_F(RdbConcurrentTest, RdbStore_Concurrent_001, TestSize.Level1) +HWTEST_P(RdbConcurrentTest, RdbStore_Concurrent_001, TestSize.Level1) { std::thread insertThread = std::thread(RdbConcurrentTest::InsertThread, 5); std::thread queryThread = std::thread(RdbConcurrentTest::QueryThread, 5); @@ -314,3 +328,6 @@ HWTEST_F(RdbConcurrentTest, RdbStore_Concurrent_001, TestSize.Level1) queryThread.join(); EXPECT_EQ(insertResult, E_OK); } + +INSTANTIATE_TEST_SUITE_P(InsertTest, RdbConcurrentTest, testing::Values(&g_store, &g_memDb)); +} // namespace OHOS::RdbConcurrentTest \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_store_config_test.cpp b/relational_store/test/native/rdb/unittest/rdb_store_config_test.cpp index 154c8895..ecdbac62 100644 --- a/relational_store/test/native/rdb/unittest/rdb_store_config_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_store_config_test.cpp @@ -22,8 +22,8 @@ #include "rdb_errno.h" #include "rdb_helper.h" #include "rdb_open_callback.h" +#include "sqlite_global_config.h" #include "unistd.h" - using namespace testing::ext; using namespace OHOS::Rdb; using namespace OHOS::NativeRdb; @@ -1041,6 +1041,59 @@ HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_033, TestSize.Level1) RdbHelper::DeleteRdbStore(dbPath); } +/** + * @tc.name: RdbStoreConfig_034 + * @tc.desc: test RdbStoreConfig GetCheckpointSize + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_034, TestSize.Level1) +{ + const std::string dbPath = RDB_TEST_PATH + "config_test.db"; + RdbStoreConfig config(dbPath); + + ssize_t walSize = config.GetWalLimitSize(); + ssize_t checkpointSize = config.GetCheckpointSize(); + ssize_t startCheckpointSize = config.GetStartCheckpointSize(); + EXPECT_EQ(walSize, GlobalExpr::DB_WAL_DEFAULT_SIZE); + EXPECT_EQ(checkpointSize, GlobalExpr::DB_WAL_WARNING_SIZE); + EXPECT_EQ(startCheckpointSize, GlobalExpr::DB_WAL_SIZE_LIMIT_MIN); +} +/** + * @tc.name: RdbStoreConfig_035 + * @tc.desc: test RdbStoreConfig GetPromiseInfo + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_035, TestSize.Level1) +{ + const std::string dbPath = RDB_TEST_PATH + "config_test.db"; + RdbStoreConfig config(dbPath); + PromiseInfo expectedInfo; + expectedInfo.user_ = "zhangsan"; + expectedInfo.tokenIds_ = { 1, 2, 3 }; + expectedInfo.uids_ = { 4, 5, 6 }; + expectedInfo.permissionNames_ = { "lisi", "wangwu" }; + + config.SetPromiseInfo(expectedInfo); + const PromiseInfo &actualInfo = config.GetPromiseInfo(); + + EXPECT_EQ(actualInfo.user_, expectedInfo.user_); + + EXPECT_EQ(actualInfo.tokenIds_.size(), expectedInfo.tokenIds_.size()); + for (size_t i = 0; i < expectedInfo.tokenIds_.size(); ++i) { + EXPECT_EQ(actualInfo.tokenIds_[i], expectedInfo.tokenIds_[i]); + } + + EXPECT_EQ(actualInfo.uids_.size(), expectedInfo.uids_.size()); + for (size_t i = 0; i < expectedInfo.uids_.size(); ++i) { + EXPECT_EQ(actualInfo.uids_[i], expectedInfo.uids_[i]); + } + + EXPECT_EQ(actualInfo.permissionNames_.size(), expectedInfo.permissionNames_.size()); + for (size_t i = 0; i < expectedInfo.permissionNames_.size(); ++i) { + EXPECT_EQ(actualInfo.permissionNames_[i], expectedInfo.permissionNames_[i]); + } +} + /** * @tc.name: RdbStoreConfigVisitor_001 * @tc.desc: test RdbStoreConfigVisitor @@ -1081,4 +1134,146 @@ HWTEST_F(RdbStoreConfigTest, RdbStoreConfigVisitor_001, TestSize.Level1) visitorStore = nullptr; ret = RdbHelper::DeleteRdbStore(dbPath); EXPECT_EQ(ret, E_OK); +} + +/* * + * @tc.name: RdbStoreConfigSetCollatorLocales_001 + * @tc.desc: test RdbStoreConfigSetCollatorLocales + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_036, TestSize.Level2) +{ + const std::string dbPath = RDB_TEST_PATH + "config_test.db"; + int errCode = E_OK; + RdbStoreConfig config(dbPath); + config.SetCollatorLocales("zh_CN"); + EXPECT_EQ(config.GetCollatorLocales(), "zh_CN"); + ConfigTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(E_OK, errCode); + ASSERT_NE(nullptr, store); + store->ExecuteSql("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " + "data2 INTEGER);"); + int64_t id; + ValuesBucket valuesBucket; + valuesBucket.PutString("data1", "张三"); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket); + + ValuesBucket valuesBucket1; + valuesBucket1.PutString("data1", "李四"); + valuesBucket1.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket1); + + std::vector columns; + AbsRdbPredicates predicates("test1"); + predicates.OrderByAsc("data1 COLLATE LOCALES"); + std::shared_ptr resultSet = store->Query(predicates, columns); + ASSERT_NE(nullptr, store); + std::string strValue; + resultSet->GoToNextRow(); + resultSet->GetString(1, strValue); + EXPECT_EQ(strValue, "李四"); + + resultSet->GoToNextRow(); + resultSet->GetString(1, strValue); + EXPECT_EQ(strValue, "张三"); +} + +/* * +* @tc.name: RdbStoreConfigSetCollatorLocales_002 +* @tc.desc: test RdbStoreConfigSetCollatorLocales_002 +* @tc.type: FUNC +*/ +HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_037, TestSize.Level2) +{ + const std::string dbPath = RDB_TEST_PATH + "config_test.db"; + int errCode = E_OK; + RdbStoreConfig config(dbPath); + config.SetCollatorLocales(""); + ConfigTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(E_OK, errCode); + ASSERT_NE(nullptr, store); + store->ExecuteSql("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " + "data2 INTEGER);"); + int64_t id; + ValuesBucket valuesBucket; + valuesBucket.PutString("data1", "张三"); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket); + + ValuesBucket valuesBucket1; + valuesBucket1.PutString("data1", "李四"); + valuesBucket1.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket1); + + std::vector columns; + AbsRdbPredicates predicates("test1"); + predicates.OrderByAsc("data1 COLLATE LOCALES"); + std::shared_ptr resultSet = store->Query(predicates, columns); + ASSERT_NE(resultSet, nullptr); + std::string strValue; + errCode = resultSet->GoToNextRow(); + EXPECT_EQ(errCode, E_SQLITE_ERROR); +} + +/* * + * @tc.name: RdbStoreConfigSetCollatorLocales_003 + * @tc.desc: test RdbStoreConfigSetCollatorLocales + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreConfigTest, RdbStoreConfig_038, TestSize.Level2) +{ + const std::string dbPath = RDB_TEST_PATH + "config_test.db"; + int errCode = E_OK; + RdbStoreConfig config(dbPath); + ConfigTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(E_OK, errCode); + ASSERT_NE(nullptr, store); + store->ExecuteSql("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " + "data2 INTEGER);"); + int64_t id; + ValuesBucket valuesBucket; + valuesBucket.PutString("data1", "张三"); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket); + + ValuesBucket valuesBucket1; + valuesBucket1.PutString("data1", "李四"); + valuesBucket1.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket1); + + std::vector columns; + AbsRdbPredicates predicates("test1"); + predicates.OrderByAsc("data1 COLLATE LOCALES"); + std::shared_ptr resultSet = store->Query(predicates, columns); + ASSERT_NE(resultSet, nullptr); + std::string strValue; + errCode = resultSet->GoToNextRow(); + EXPECT_EQ(errCode, E_SQLITE_ERROR); + + RdbHelper::ClearCache(); + RdbStoreConfig config1(dbPath); + config1.SetVisitorDir(dbPath); + config1.SetCollatorLocales("zh_CN"); + ConfigTestOpenCallback helper1; + std::shared_ptr store1 = RdbHelper::GetRdbStore(config1, 1, helper1, errCode); + EXPECT_EQ(E_OK, errCode); + ASSERT_NE(nullptr, store1); + + std::vector columns1; + AbsRdbPredicates predicates1("test1"); + predicates1.OrderByAsc("data1 COLLATE LOCALES"); + std::shared_ptr resultSet1 = store1->Query(predicates1, columns); + ASSERT_NE(nullptr, resultSet1); + std::string strValue1; + resultSet1->GoToNextRow(); + resultSet1->GetString(1, strValue1); + EXPECT_EQ(strValue1, "李四"); + + resultSet1->GoToNextRow(); + resultSet1->GetString(1, strValue1); + EXPECT_EQ(strValue1, "张三"); } \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_store_impl_test.cpp b/relational_store/test/native/rdb/unittest/rdb_store_impl_test.cpp index 4c64505a..28012329 100644 --- a/relational_store/test/native/rdb/unittest/rdb_store_impl_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_store_impl_test.cpp @@ -47,6 +47,11 @@ protected: std::shared_ptr store_; }; +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test " + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER, salary REAL, " + "blobType BLOB)"; + const std::string RdbStoreImplTest::DATABASE_NAME = RDB_TEST_PATH + "stepResultSet_impl_test.db"; class RdbStoreImplTestOpenCallback : public RdbOpenCallback { @@ -790,26 +795,88 @@ HWTEST_F(RdbStoreImplTest, Abnormal_CleanDirtyDataTest_001, TestSize.Level2) */ HWTEST_F(RdbStoreImplTest, Normal_ClearCacheTest_001, TestSize.Level2) { - store_->ExecuteSql("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " - "data2 INTEGER, data3 FLOAT, data4 BLOB, data5 BOOLEAN);"); + RdbHelper::DeleteRdbStore(RdbStoreImplTest::DATABASE_NAME); int errCode = E_OK; + RdbStoreConfig config(RdbStoreImplTest::DATABASE_NAME); + RdbStoreImplTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(nullptr, store); + EXPECT_EQ(E_OK, errCode); + store->ExecuteSql("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " + "data2 INTEGER, data3 FLOAT, data4 BLOB, data5 BOOLEAN);"); int64_t id; ValuesBucket valuesBucket; - valuesBucket.PutString("data1", std::string("zhangsan")); - valuesBucket.PutInt("data2", 10); - errCode = store_->Insert(id, "test", valuesBucket); + for (int i = 0; i < 1000; ++i) { + valuesBucket.PutString("data1", std::string(1024 * 1024, 'a')); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test", valuesBucket); + } EXPECT_EQ(errCode, E_OK); - EXPECT_EQ(1, id); - + EXPECT_EQ(1000, id); + int rowCount; - std::shared_ptr resultSet = store_->QueryByStep("SELECT * FROM test"); + std::shared_ptr resultSet = store->QueryByStep("SELECT * FROM test"); EXPECT_NE(resultSet, nullptr); resultSet->GetRowCount(rowCount); - EXPECT_EQ(rowCount, 1); + EXPECT_EQ(rowCount, 1000); int64_t currentMemory = sqlite3_memory_used(); EXPECT_EQ(E_OK, resultSet->Close()); EXPECT_LT(sqlite3_memory_used(), currentMemory); } + +/* * +* @tc.name: ClearCacheTest_002 +* @tc.desc: Normal testCase for ClearCache +* @tc.type: FUNC +*/ +HWTEST_F(RdbStoreImplTest, Normal_ClearCacheTest_002, TestSize.Level2) +{ + RdbHelper::DeleteRdbStore(RdbStoreImplTest::DATABASE_NAME); + int errCode = E_OK; + RdbStoreConfig config(RdbStoreImplTest::DATABASE_NAME); + RdbStoreImplTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_NE(nullptr, store); + EXPECT_EQ(E_OK, errCode); + store->ExecuteSql("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, " + "data2 INTEGER, data3 FLOAT, data4 BLOB, data5 BOOLEAN);"); + int64_t id; + ValuesBucket valuesBucket; + valuesBucket.PutString("data1", std::string(0.5 * 1024 * 1024, 'a')); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(1, id); + int64_t currentMemory = sqlite3_memory_used(); + valuesBucket.PutString("data1", std::string(0.5 * 1024 * 1024, 'a')); + valuesBucket.PutInt("data2", 20); + errCode = store->Insert(id, "test1", valuesBucket); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(2, id); + EXPECT_LT(sqlite3_memory_used(), currentMemory); +} + +/* * + * @tc.name: ClearCacheTest_003 + * @tc.desc: Normal testCase for ClearCache + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreImplTest, Normal_ClearCacheTest_003, TestSize.Level2) +{ + RdbHelper::DeleteRdbStore(RdbStoreImplTest::DATABASE_NAME); + int errCode = E_OK; + int clearMemorySize = 1024 * 1024; + RdbStoreConfig config(RdbStoreImplTest::DATABASE_NAME); + config.SetClearMemorySize(1024 * 1024 + 1); + EXPECT_EQ(clearMemorySize, config.GetClearMemorySize()); + config.SetClearMemorySize(-1); + EXPECT_EQ(clearMemorySize, config.GetClearMemorySize()); + config.SetClearMemorySize(10240); + RdbStoreImplTestOpenCallback helper; + std::shared_ptr store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(nullptr, store); + EXPECT_EQ(E_OK, errCode); +} /* * * @tc.name: LockCloudContainerTest @@ -1384,4 +1451,98 @@ HWTEST_F(RdbStoreImplTest, BatchInsertWithConflictResolution_007, TestSize.Level int rowCount; ASSERT_EQ(resultSet->GetRowCount(rowCount), E_OK); ASSERT_EQ(rowCount, 0); +} + +/** + * @tc.name: RdbStore_Execute_001 + * @tc.desc: test RdbStore Execute + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreImplTest, RdbStore_Execute_001, TestSize.Level1) +{ + const std::string dbPath = RDB_TEST_PATH + "GetDatabase1.db"; + RdbStoreConfig config(dbPath); + config.SetName("RdbStoreConfig_test.db"); + std::string bundleName = "com.ohos.config.TestSubUser"; + config.SetBundleName(bundleName); + config.SetSubUser(100); + auto subUser = config.GetSubUser(); + EXPECT_EQ(subUser, 100); + int errCode = E_OK; + + RdbStoreImplTestOpenCallback helper; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore, nullptr); + rdbStore->ExecuteSql(CREATE_TABLE_TEST); + + int64_t id; + ValuesBucket values; + + values.PutInt("id", 1); + values.PutString("name", std::string("zhangsan")); + values.PutInt("age", 18); + values.PutDouble("salary", 100.5); + values.PutBlob("blobType", std::vector{ 1, 2, 3 }); + int ret = rdbStore->Insert(id, "test", values); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(1, id); + + values.Clear(); + values.PutInt("id", 2); + values.PutString("name", std::string("lisi")); + values.PutInt("age", 19); + values.PutDouble("salary", 200.5); + values.PutBlob("blobType", std::vector{ 4, 5, 6 }); + ret = rdbStore->Insert(id, "test", values); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(2, id); + + int64_t count; + ret = rdbStore->ExecuteAndGetLong(count, "SELECT COUNT(*) FROM test"); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(count, 2); + + ret = rdbStore->ExecuteSql("DELETE FROM test WHERE age = 18"); + EXPECT_EQ(ret, E_OK); + + ret = rdbStore->ExecuteAndGetLong(count, "SELECT COUNT(*) FROM test where age = 19"); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(count, 1); + + ret = rdbStore->ExecuteSql("DELETE FROM test WHERE age = 19"); + EXPECT_EQ(ret, E_OK); + + ret = RdbHelper::DeleteRdbStore(config); + EXPECT_EQ(ret, E_OK); +} + +/** + * @tc.name: RdbStore_Crypt_001 + * @tc.desc: test RdbStore Crypt + * @tc.type: FUNC + */ +HWTEST_F(RdbStoreImplTest, RdbStore_Crypt_001, TestSize.Level1) +{ + const std::string dbPath = RDB_TEST_PATH + "GetDatabase1.db"; + RdbStoreConfig::CryptoParam cryptoParam; + cryptoParam.encryptKey_ = std::vector{ 1, 2, 3, 4, 5, 6 }; + RdbStoreConfig config(dbPath); + config.SetName("RdbStoreConfig_test.db"); + std::string bundleName = "com.ohos.config.TestSubUser"; + config.SetBundleName(bundleName); + config.SetCryptoParam(cryptoParam); + int errCode = E_OK; + + RdbStoreImplTestOpenCallback helper; + std::shared_ptr rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(rdbStore, nullptr); + rdbStore->ExecuteSql(CREATE_TABLE_TEST); + rdbStore = nullptr; + cryptoParam.encryptKey_ = std::vector{ 6, 5, 4, 3, 2, 1 }; + config.SetCryptoParam(cryptoParam); + + rdbStore = RdbHelper::GetRdbStore(config, 1, helper, errCode); + EXPECT_EQ(errCode, E_SQLITE_CORRUPT); } \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_store_subscribe_test.cpp b/relational_store/test/native/rdb/unittest/rdb_store_subscribe_test.cpp index 2a8875e8..debd883b 100644 --- a/relational_store/test/native/rdb/unittest/rdb_store_subscribe_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_store_subscribe_test.cpp @@ -566,6 +566,162 @@ HWTEST_F(RdbStoreSubTest, RdbStoreSubscribeLocalDetail005, TestSize.Level1) * @tc.author: */ HWTEST_F(RdbStoreSubTest, RdbStoreSubscribeLocalDetail006, TestSize.Level1) +{ + int num = 20; + EXPECT_NE(store, nullptr) << "store is null"; + EXPECT_NE(observer_, nullptr) << "observer is null"; + constexpr const char *createTableTest = "CREATE TABLE IF NOT EXISTS local_test2" + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER)"; + store->ExecuteSql(createTableTest); + auto status = store->SubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + + observer_->RegisterCallback([num](RdbStoreObserver::ChangeInfo &changeInfo) { + ASSERT_EQ(changeInfo.size(), 1u); + EXPECT_TRUE(changeInfo["local_test2"][1].empty()); + EXPECT_TRUE(changeInfo["local_test2"][2].empty()); // 2 is delete subscript + for (int i = 0; i < num; i++) { + EXPECT_EQ(std::get(changeInfo["local_test2"][0][i]), i); + } + }); + + observer_->count = 0; + std::vector values; + for (int i = 0; i < num; i++) { + ValuesBucket value; + value.PutInt("id", i); + value.PutString("name", std::string("lisi")); + value.PutInt("age", 16); + values.push_back(value); + } + std::shared_ptr trans = nullptr; + std::tie(status, trans) = store->CreateTransaction(Transaction::DEFERRED); + EXPECT_EQ(status, E_OK); + int64_t insertRows = 0; + std::tie(status, insertRows) = trans->BatchInsert("local_test2", values); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(insertRows, num); + EXPECT_EQ(observer_->count, 0); + trans->Commit(); + trans = nullptr; + EXPECT_EQ(observer_->count, 1); + status = store->UnsubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + observer_->RegisterCallback(nullptr); +} + +/** + * @tc.name: RdbStoreSubscribeLocalDetail007 + * @tc.desc: test abnormal parametar subscribe + * @tc.type: FUNC + * @tc.require: + * @tc.author: + */ +HWTEST_F(RdbStoreSubTest, RdbStoreSubscribeLocalDetail007, TestSize.Level1) +{ + int num = 20; + EXPECT_NE(store, nullptr) << "store is null"; + EXPECT_NE(observer_, nullptr) << "observer is null"; + constexpr const char *createTableTest = "CREATE TABLE IF NOT EXISTS local_test3" + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER)"; + store->ExecuteSql(createTableTest); + std::shared_ptr trans1 = nullptr; + int status = 0; + std::tie(status, trans1) = store->CreateTransaction(Transaction::DEFERRED); + EXPECT_EQ(status, E_OK); + status = store->SubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + trans1 = nullptr; + observer_->RegisterCallback([num](RdbStoreObserver::ChangeInfo &changeInfo) { + ASSERT_EQ(changeInfo.size(), 1u); + EXPECT_TRUE(changeInfo["local_test3"][1].empty()); + EXPECT_TRUE(changeInfo["local_test3"][2].empty()); // 2 is delete subscript + for (int i = 0; i < num; i++) { + EXPECT_EQ(std::get(changeInfo["local_test3"][0][i]), i); + } + }); + + observer_->count = 0; + std::vector values; + for (int i = 0; i < num; i++) { + ValuesBucket value; + value.PutInt("id", i); + value.PutString("name", std::string("lisi")); + value.PutInt("age", 16); + values.push_back(value); + } + std::shared_ptr trans2 = nullptr; + std::tie(status, trans2) = store->CreateTransaction(Transaction::DEFERRED); + EXPECT_EQ(status, E_OK); + int64_t insertRows = 0; + std::tie(status, insertRows) = trans2->BatchInsert("local_test3", values); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(insertRows, num); + EXPECT_EQ(observer_->count, 0); + trans2->Commit(); + trans2 = nullptr; + EXPECT_EQ(observer_->count, 1); + status = store->UnsubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + observer_->RegisterCallback(nullptr); +} + +/** + * @tc.name: RdbStoreSubscribeLocalDetail008 + * @tc.desc: test abnormal parametar subscribe + * @tc.type: FUNC + * @tc.require: + * @tc.author: + */ +HWTEST_F(RdbStoreSubTest, RdbStoreSubscribeLocalDetail008, TestSize.Level1) +{ + int num = 20; + EXPECT_NE(store, nullptr) << "store is null"; + EXPECT_NE(observer_, nullptr) << "observer is null"; + constexpr const char *createTableTest = "CREATE TABLE IF NOT EXISTS local_test4" + "(id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT NOT NULL, age INTEGER)"; + store->ExecuteSql(createTableTest); + auto status = store->SubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + + observer_->RegisterCallback([](RdbStoreObserver::ChangeInfo &) { ASSERT_TRUE(false); }); + + observer_->count = 0; + std::vector values; + for (int i = 0; i < num; i++) { + ValuesBucket value; + value.PutInt("id", i); + value.PutString("name", std::string("lisi")); + value.PutInt("age", 16); + values.push_back(value); + } + std::shared_ptr trans = nullptr; + std::tie(status, trans) = store->CreateTransaction(Transaction::DEFERRED); + EXPECT_EQ(status, E_OK); + int64_t insertRows = 0; + std::tie(status, insertRows) = trans->BatchInsert("local_test4", values); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(insertRows, num); + EXPECT_EQ(observer_->count, 0); + status = store->UnsubscribeObserver({ SubscribeMode::LOCAL_DETAIL, "dataChange" }, observer_); + EXPECT_EQ(status, E_OK); + trans->Commit(); + trans = nullptr; + EXPECT_EQ(observer_->count, 0); + observer_->RegisterCallback(nullptr); +} + +/** + * @tc.name: RdbStoreSubscribeLocalDetail009 + * @tc.desc: test abnormal parametar subscribe + * @tc.type: FUNC + * @tc.require: + * @tc.author: + */ +HWTEST_F(RdbStoreSubTest, RdbStoreSubscribeLocalDetail009, TestSize.Level1) { EXPECT_NE(store, nullptr) << "store is null"; EXPECT_NE(observer_, nullptr) << "observer is null"; diff --git a/relational_store/test/native/rdb/unittest/rdb_update_test.cpp b/relational_store/test/native/rdb/unittest/rdb_update_test.cpp index 41989402..585eb590 100644 --- a/relational_store/test/native/rdb/unittest/rdb_update_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_update_test.cpp @@ -26,39 +26,44 @@ using namespace testing::ext; using namespace OHOS::NativeRdb; +namespace OHOS::RdbStoreUpdateTest { +struct RdbTestParam { + std::shared_ptr store; + operator std::shared_ptr() + { + return store; + } +}; +static RdbTestParam g_store; +static RdbTestParam g_memDb; -class RdbStoreUpdateTest : public testing::Test { +class RdbStoreUpdateTest : public testing::TestWithParam { public: static void SetUpTestCase(void); static void TearDownTestCase(void); static void ExpectValue(const std::shared_ptr &resultSet, const RowData &expect); void SetUp(); void TearDown(); + std::shared_ptr store_; static const std::string DATABASE_NAME; - static std::shared_ptr store; }; const std::string RdbStoreUpdateTest::DATABASE_NAME = RDB_TEST_PATH + "update_test.db"; -std::shared_ptr RdbStoreUpdateTest::store = nullptr; class UpdateTestOpenCallback : public RdbOpenCallback { public: int OnCreate(RdbStore &store) override; int OnUpgrade(RdbStore &store, int oldVersion, int newVersion) override; - static const std::string CREATE_TABLE_TEST; }; - -const std::string UpdateTestOpenCallback::CREATE_TABLE_TEST = - std::string("CREATE TABLE IF NOT EXISTS test (") + - std::string("id INTEGER PRIMARY KEY AUTOINCREMENT, ") + - std::string("name TEXT UNIQUE, ") + - std::string("age INTEGER, ") + - std::string("salary REAL, ") + - std::string("blobType BLOB, ") + - std::string("assetType ASSET, ") + - std::string("assetsType ASSETS)"); - +constexpr const char *CREATE_TABLE_TEST = "CREATE TABLE IF NOT EXISTS test (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "name TEXT UNIQUE, " + "age INTEGER, " + "salary REAL, " + "blobType BLOB, " + "assetType ASSET, " + "assetsType ASSETS)"; int UpdateTestOpenCallback::OnCreate(RdbStore &store) { return store.ExecuteSql(CREATE_TABLE_TEST); @@ -75,19 +80,26 @@ void RdbStoreUpdateTest::SetUpTestCase(void) RdbHelper::DeleteRdbStore(DATABASE_NAME); RdbStoreConfig config(RdbStoreUpdateTest::DATABASE_NAME); UpdateTestOpenCallback helper; - RdbStoreUpdateTest::store = RdbHelper::GetRdbStore(config, 1, helper, errCode); - EXPECT_NE(RdbStoreUpdateTest::store, nullptr); + g_store.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_store.store, nullptr); + + config.SetStorageMode(StorageMode::MODE_MEMORY); + g_memDb.store = RdbHelper::GetRdbStore(config, 1, helper, errCode); + ASSERT_NE(g_memDb.store, nullptr); } void RdbStoreUpdateTest::TearDownTestCase(void) { - store = nullptr; - RdbHelper::DeleteRdbStore(RdbStoreUpdateTest::DATABASE_NAME); + RdbStoreConfig config(RdbStoreUpdateTest::DATABASE_NAME); + RdbHelper::DeleteRdbStore(config); + config.SetStorageMode(StorageMode::MODE_MEMORY); + RdbHelper::DeleteRdbStore(config); } void RdbStoreUpdateTest::SetUp(void) { - store->ExecuteSql("DELETE FROM test"); + store_ = *GetParam(); + store_->ExecuteSql("DELETE FROM test"); } void RdbStoreUpdateTest::TearDown(void) @@ -100,9 +112,9 @@ void RdbStoreUpdateTest::TearDown(void) * @tc.desc: test RdbStore update, select id and update one row * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_001, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_001, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -141,9 +153,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_001, TestSize.Level1) * @tc.desc: test RdbStore update, no select and update all rows * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_002, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_002, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); int64_t id; ValuesBucket values; @@ -187,9 +199,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_002, TestSize.Level1) * @tc.desc: test RdbStore update * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_003, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_003, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); int changedRows; ValuesBucket values; @@ -210,9 +222,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_003, TestSize.Level1) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_004, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_004, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); int changedRows; ValuesBucket emptyBucket; @@ -234,9 +246,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_004, TestSize.Level1) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_005, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_005, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -255,9 +267,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_005, TestSize.Level1) * @tc.desc: test RdbStore insert * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_006, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_006, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -296,9 +308,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_006, TestSize.Level1) * @tc.desc: test RdbStore update asset * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_007, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_007, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; AssetValue value{ .version = 1, .name = "123", .uri = "your test path", .createTime = "13", .modifyTime = "13" }; int changedRows; @@ -324,8 +336,7 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_007, TestSize.Level1) .name = "123", .uri = "your test path", .createTime = "13", - .modifyTime = "13" - } }); + .modifyTime = "13" } }); ret = resultSet->Close(); EXPECT_EQ(ret, E_OK); } @@ -335,9 +346,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_007, TestSize.Level1) * @tc.desc: test RdbStore update asset * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_008, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_008, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; AssetValue valueDef{ .version = 0, @@ -375,9 +386,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_008, TestSize.Level1) * @tc.desc: test RdbStore update asset * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_009, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_009, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; AssetValue valueDef{ .version = 0, .status = AssetValue::STATUS_NORMAL, @@ -422,9 +433,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_009, TestSize.Level1) * @tc.desc: test RdbStore update assets * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_010, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_Update_010, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; std::vector assetsDef{ { .version = 0, .name = "123", .uri = "my test path", .createTime = "12", .modifyTime = "12" } @@ -452,9 +463,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_Update_010, TestSize.Level1) * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_001, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_001, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -501,9 +512,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_001, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_002, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_002, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -550,9 +561,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_002, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_003, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_003, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -600,9 +611,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_003, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_004, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_004, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -649,9 +660,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_004, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_005, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_005, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -699,9 +710,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_005, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_006, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_006, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); ValuesBucket values; int changedRows; @@ -744,9 +755,9 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_006, TestSize * @tc.desc: test RdbStore UpdateWithConflictResolution * @tc.type: FUNC */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_007, TestSize.Level1) +HWTEST_P(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_007, TestSize.Level1) { - std::shared_ptr &store = RdbStoreUpdateTest::store; + std::shared_ptr store = *GetParam(); int changedRows = 0; int64_t id = -1; @@ -772,28 +783,6 @@ HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateWithConflictResolution_007, TestSize EXPECT_EQ(0, changedRows); } -/** - * @tc.name: RdbStore_UpdateSqlBuilder_001 - * @tc.desc: test RdbStore UpdateSqlBuilder - * @tc.type: FUNC - */ -HWTEST_F(RdbStoreUpdateTest, RdbStore_UpdateSqlBuilder_001, TestSize.Level1) -{ - ValuesBucket values; - values.PutString("name", std::string("zhangsan")); - values.PutInt("age", 20); - values.PutDouble("salary", 300.5); - - std::vector bindArgs; - std::string updateSql = SqliteSqlBuilder::BuildUpdateString(values, "test", std::vector{ "19" }, "", - "age = ?", "", "", INT_MIN, INT_MIN, bindArgs, ConflictResolution::ON_CONFLICT_NONE); - EXPECT_EQ(updateSql, "UPDATE test SET age=?,name=?,salary=? WHERE age = ?"); - - updateSql = SqliteSqlBuilder::BuildUpdateString(values, "test", std::vector{}, "", "", "", "", - INT_MIN, INT_MIN, bindArgs, ConflictResolution::ON_CONFLICT_NONE); - EXPECT_EQ(updateSql, "UPDATE test SET age=?,name=?,salary=?"); -} - void RdbStoreUpdateTest::ExpectValue( const std::shared_ptr &resultSet, const RowData &expect) { @@ -843,4 +832,38 @@ void RdbStoreUpdateTest::ExpectValue( EXPECT_EQ(expect.blobType[i], blob[i]); } } -} \ No newline at end of file +} + +/** + * @tc.name: OverLimitWithUpdate_001 + * @tc.desc: over limit + * @tc.type: FUNC + * @tc.require: + * @tc.author: + */ +HWTEST_P(RdbStoreUpdateTest, OverLimitWithUpdate_001, TestSize.Level1) +{ + std::shared_ptr store = *GetParam(); + auto [code, maxPageCount] = store->Execute("PRAGMA max_page_count;"); + auto recover = std::shared_ptr("recover", [defPageCount = maxPageCount, store](const char *) { + store->Execute("PRAGMA max_page_count = " + static_cast(defPageCount) + ";"); + }); + std::tie(code, maxPageCount) = store->Execute("PRAGMA max_page_count = 256;"); + + int64_t id = -1; + int ret = store->Insert(id, "test", UTUtils::SetRowData(UTUtils::g_rowData[0])); + ASSERT_EQ(ret, E_OK); + + ValuesBucket row; + row.PutInt("id", id); + row.Put("name", std::string(1024 * 1024, 'e')); + row.PutInt("age", 20); + row.PutDouble("salary", 200.5); + row.PutBlob("blobType", std::vector{ 4, 5, 6 }); + int changedRows; + auto result = store->Update(changedRows, "test", row); + ASSERT_EQ(result, E_SQLITE_FULL); +} + +INSTANTIATE_TEST_SUITE_P(UpdateTest, RdbStoreUpdateTest, testing::Values(&g_store, &g_memDb)); +} // namespace OHOS::RdbStoreUpdateTest \ No newline at end of file diff --git a/relational_store/test/native/rdb/unittest/rdb_wal_limit_test.cpp b/relational_store/test/native/rdb/unittest/rdb_wal_limit_test.cpp index 75ebd95e..95dc9c62 100644 --- a/relational_store/test/native/rdb/unittest/rdb_wal_limit_test.cpp +++ b/relational_store/test/native/rdb/unittest/rdb_wal_limit_test.cpp @@ -200,7 +200,6 @@ ValuesBucket RdbWalLimitTest::MakeValueBucket(const int &id) * @tc.desc: Without reading data or conducting transactions, if data is continuously written, * the WAL size will not exceed the default limit. * @tc.type: FUNC - * @tc.acquire: AR000HR0G5 */ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_001, TestSize.Level1) { @@ -227,7 +226,6 @@ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_001, TestSize.Level1) * @tc.name: RdbStore_WalOverLimit_002 * @tc.desc: Before the wal file exceeds the limit, both read and write can be executed normally. * @tc.type: FUNC - * @tc.acquire: AR000HR0G5 */ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_002, TestSize.Level1) { @@ -250,7 +248,6 @@ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_002, TestSize.Level1) * @tc.name: RdbStore_WalOverLimit_003 * @tc.desc: During transactions, the size of the wal file may exceed the limit. * @tc.type: FUNC - * @tc.acquire: AR000HR0G5 */ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_003, TestSize.Level3) { @@ -267,7 +264,6 @@ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_003, TestSize.Level3) * @tc.name: RdbStore_WalOverLimit_003 * @tc.desc: During transactions, the size of the wal file may exceed the limit. * @tc.type: FUNC - * @tc.acquire: AR000HR0G5 */ HWTEST_F(RdbWalLimitTest, RdbStore_WalOverLimit_004, TestSize.Level3) { diff --git a/relational_store/test/native/rdb/unittest/transaction_test.cpp b/relational_store/test/native/rdb/unittest/transaction_test.cpp index b134d3bf..d649cf0e 100644 --- a/relational_store/test/native/rdb/unittest/transaction_test.cpp +++ b/relational_store/test/native/rdb/unittest/transaction_test.cpp @@ -915,43 +915,45 @@ HWTEST_F(TransactionTest, RdbStore_Transaction_019, TestSize.Level1) EXPECT_EQ(rowCount, 6); } -///** -// * @tc.name: RdbStore_Transaction_017 -// * @tc.desc: BatchInsert with corrupt -// * @tc.type: FUNC -// */ -//HWTEST_F(TransactionTest, RdbStore_Transaction_017, TestSize.Level1) -//{ -// std::string path = RDB_TEST_PATH + "transaction_test_017.db"; -// RdbHelper::DeleteRdbStore(path); -// RdbStoreConfig config(path); -// TransactionTestOpenCallback helper; -// int errCode = 1; -// auto store = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// ASSERT_NE(store, nullptr); -// ASSERT_EQ(errCode, E_OK); -// store->Execute("DROP TABLE IF EXISTS test1"); -// auto res = store->Execute("CREATE TABLE test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)"); -// ASSERT_EQ(res.first, E_OK); -// -// store = nullptr; -// ASSERT_TRUE(UTUtils::DestoryDb(path, 4096 - 100, std::vector(100, 0x11))); -// store = RdbHelper::GetRdbStore(config, 1, helper, errCode); -// ASSERT_NE(store, nullptr); -// ASSERT_EQ(errCode, E_OK); -// auto [ret, transaction] = store->CreateTransaction(Transaction::EXCLUSIVE); -// ASSERT_EQ(ret, E_OK); -// ASSERT_NE(transaction, nullptr); -// -// ValuesBuckets rows; -// for (int i = 0; i < 2000; i++) { -// Transaction::Row row; -// row.Put("id", i); -// row.Put("name", std::string("JimJimJim1", 10240)); -// rows.Put(row); -// } -// auto result = transaction->BatchInsertWithConflictResolution("test1", rows, ConflictResolution::ON_CONFLICT_NONE); -// ASSERT_EQ(result.first, E_SQLITE_CORRUPT); -// -// RdbHelper::DeleteRdbStore(path); -//} +/** + * @tc.name: RdbStore_Transaction_020 + * @tc.desc: After executing the ddl statement, the transaction links cached in the history need to be cleared. + * Continuing to use the old connections will result in errors due to changes in the table structure. + * @tc.type: FUNC + */ +HWTEST_F(TransactionTest, RdbStore_Transaction_020, TestSize.Level1) +{ + std::shared_ptr &store = TransactionTest::store_; + + auto [ret, transaction] = store->CreateTransaction(Transaction::DEFERRED); + ASSERT_EQ(ret, E_OK); + ASSERT_NE(transaction, nullptr); + + Transaction::Row row; + row.Put("id", 1); + row.Put("name", "Jim"); + auto result = transaction->Insert("test", row); + ASSERT_EQ(result.first, E_OK); + ASSERT_EQ(result.second, 1); + + ret = transaction->Commit(); + ASSERT_EQ(ret, E_OK); + transaction = nullptr; + + store->Execute("DROP TABLE IF EXISTS test1"); + auto res = store->Execute( + "CREATE TABLE IF NOT EXISTS test1 (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)"); + ASSERT_EQ(res.first, E_OK); + // After creating the table, the links will be cleared, and creating a new transaction will create a new connection, + // ensuring that the transaction operation does not report errors. + std::tie(ret, transaction) = store->CreateTransaction(Transaction::DEFERRED); + ASSERT_EQ(ret, E_OK); + ASSERT_NE(transaction, nullptr); + + auto resultSet = transaction->QueryByStep("SELECT * FROM test"); + ASSERT_NE(resultSet, nullptr); + int32_t rowCount{}; + ret = resultSet->GetRowCount(rowCount); + EXPECT_EQ(ret, E_OK); + EXPECT_EQ(rowCount, 1); +} diff --git a/relational_store/test/native/rdb_data_ability_adapter/BUILD.gn b/relational_store/test/native/rdb_data_ability_adapter/BUILD.gn index 988dc48a..2f8e645b 100644 --- a/relational_store/test/native/rdb_data_ability_adapter/BUILD.gn +++ b/relational_store/test/native/rdb_data_ability_adapter/BUILD.gn @@ -14,7 +14,8 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/data_share/datashare.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/rdb_data_ability_adapter" +module_output_path = + "relational_store/relational_store/rdb_data_ability_adapter" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/native/rdb_data_share_adapter/BUILD.gn b/relational_store/test/native/rdb_data_share_adapter/BUILD.gn index 0b4d080a..70a8a93b 100644 --- a/relational_store/test/native/rdb_data_share_adapter/BUILD.gn +++ b/relational_store/test/native/rdb_data_share_adapter/BUILD.gn @@ -14,7 +14,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/data_share/datashare.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/rdb_data_share_adapter" +module_output_path = "relational_store/relational_store/rdb_data_share_adapter" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/ndk/BUILD.gn b/relational_store/test/ndk/BUILD.gn index bac7bfd0..dfd58ab1 100644 --- a/relational_store/test/ndk/BUILD.gn +++ b/relational_store/test/ndk/BUILD.gn @@ -13,7 +13,7 @@ import("//build/test.gni") import("//foundation/distributeddatamgr/relational_store/relational_store.gni") -module_output_path = "relational_store/native_rdb_ndk" +module_output_path = "relational_store/relational_store/native_rdb_ndk" ############################################################################### config("module_private_config") { diff --git a/relational_store/test/ndk/unittest/rdb_store_configv2_test.cpp b/relational_store/test/ndk/unittest/rdb_store_configv2_test.cpp index 59536601..9775bd32 100644 --- a/relational_store/test/ndk/unittest/rdb_store_configv2_test.cpp +++ b/relational_store/test/ndk/unittest/rdb_store_configv2_test.cpp @@ -304,3 +304,13 @@ HWTEST_F(RdbNativeStoreConfigV2Test, RDB_Native_store_test_006, TestSize.Level1) } OH_Rdb_DestroyConfig(config); } + +HWTEST_F(RdbNativeStoreConfigV2Test, RDB_Native_store_test_007, TestSize.Level1) +{ + auto config = InitRdbConfig(); + int errCode = OH_Rdb_SetPersistent(config, true); + EXPECT_EQ(errCode, OH_Rdb_ErrCode::RDB_OK); + errCode = OH_Rdb_SetPersistent(config, false); + EXPECT_EQ(errCode, OH_Rdb_ErrCode::RDB_OK); + OH_Rdb_DestroyConfig(config); +} diff --git a/relational_store/test/ndk/unittest/rdb_store_test.cpp b/relational_store/test/ndk/unittest/rdb_store_test.cpp index 48747698..274db390 100644 --- a/relational_store/test/ndk/unittest/rdb_store_test.cpp +++ b/relational_store/test/ndk/unittest/rdb_store_test.cpp @@ -1521,10 +1521,13 @@ HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_034, TestSize.Level1) HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_035, TestSize.Level1) { ASSERT_NE(storeTestRdbStore_, nullptr); + OH_VBucket *valueBucket = OH_Rdb_CreateValuesBucket(); valueBucket->putInt64(valueBucket, "id", 12); valueBucket->putText(valueBucket, "data1", "zhangSan"); int errCode = OH_Rdb_Insert(storeTestRdbStore_, "store_test", valueBucket); + EXPECT_EQ(12, errCode); + OH_Data_VBuckets *rows = OH_VBuckets_Create(); ASSERT_NE(rows, nullptr); OH_VBucket *vbs[5]; @@ -1536,31 +1539,40 @@ HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_035, TestSize.Level1) vbs[i] = row; EXPECT_EQ(OH_VBuckets_PutRow(rows, row), RDB_OK); } + int64_t changes = -1; int ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_NONE, &changes); ASSERT_EQ(ret, RDB_E_SQLITE_CONSTRAINT); ASSERT_EQ(changes, 0); + ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_ROLLBACK, &changes); ASSERT_EQ(ret, RDB_E_SQLITE_CONSTRAINT); ASSERT_EQ(changes, 0); + ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_ABORT, &changes); ASSERT_EQ(ret, RDB_E_SQLITE_CONSTRAINT); ASSERT_EQ(changes, 0); + ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_FAIL, &changes); ASSERT_EQ(ret, RDB_E_SQLITE_CONSTRAINT); ASSERT_EQ(changes, 2); + ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_IGNORE, &changes); ASSERT_EQ(ret, RDB_OK); ASSERT_EQ(changes, 2); + ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_REPLACE, &changes); ASSERT_EQ(ret, RDB_OK); ASSERT_EQ(changes, 5); + for (OH_VBucket *vb : vbs) { vb->destroy(vb); } OH_VBuckets_Destroy(rows); + char querySql[] = "SELECT * FROM store_test"; OH_Cursor *cursor = OH_Rdb_ExecuteQuery(storeTestRdbStore_, querySql); + int rowCount = 0; cursor->getRowCount(cursor, &rowCount); EXPECT_EQ(rowCount, 6); @@ -1575,6 +1587,7 @@ HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_035, TestSize.Level1) HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_036, TestSize.Level1) { ASSERT_NE(storeTestRdbStore_, nullptr); + OH_Data_VBuckets *rows = OH_VBuckets_Create(); ASSERT_NE(rows, nullptr); OH_VBucket *vbs[2]; @@ -1588,6 +1601,7 @@ HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_036, TestSize.Level1) EXPECT_EQ(OH_VBuckets_PutRow(rows, row), RDB_OK); vbs[i] = row; } + int64_t changes = -1; int ret = OH_Rdb_BatchInsert(storeTestRdbStore_, "store_test", rows, RDB_CONFLICT_NONE, &changes); ASSERT_EQ(ret, RDB_OK); @@ -1616,10 +1630,59 @@ HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_036, TestSize.Level1) vb->destroy(vb); } OH_VBuckets_Destroy(rows); + char querySql[] = "SELECT * FROM store_test"; OH_Cursor *cursor = OH_Rdb_ExecuteQuery(storeTestRdbStore_, querySql); + int rowCount = 0; cursor->getRowCount(cursor, &rowCount); EXPECT_EQ(rowCount, 13); cursor->destroy(cursor); +} + +/** + * @tc.name: RDB_Native_store_test_037 + * @tc.desc: normal testCase for memDb. + * @tc.type: FUNC + */ +HWTEST_F(RdbNativeStoreTest, RDB_Native_store_test_037, TestSize.Level1) +{ + OH_Rdb_ConfigV2 *config = OH_Rdb_CreateConfig(); + ASSERT_NE(config, nullptr); + OH_Rdb_SetStoreName(config, "memDb"); + OH_Rdb_SetBundleName(config, "com.ohos.example.distributedndk"); + OH_Rdb_SetPersistent(config, false); + OH_Rdb_SetArea(config, RDB_SECURITY_AREA_EL1); + OH_Rdb_SetSecurityLevel(config, S1); + + int errCode = 0; + OH_Rdb_Store *store = OH_Rdb_CreateOrOpen(config, &errCode); + ASSERT_NE(store, nullptr); + ASSERT_EQ(errCode, OH_Rdb_ErrCode::RDB_OK); + + char createTableSql[] = "CREATE TABLE mem_test (id INTEGER PRIMARY KEY AUTOINCREMENT, data1 TEXT, data2 INTEGER, " + "data3 FLOAT, data4 BLOB, data5 TEXT);"; + errCode = OH_Rdb_Execute(storeTestRdbStore_, createTableSql); + EXPECT_EQ(errCode, 0); + + OH_VBucket *valueBucket = OH_Rdb_CreateValuesBucket(); + valueBucket->putInt64(valueBucket, "id", 1); + valueBucket->putText(valueBucket, "data1", "zhangSan"); + valueBucket->putInt64(valueBucket, "data2", 12800); + valueBucket->putReal(valueBucket, "data3", 100.1); + uint8_t arr[] = { 1, 2, 3, 4, 5 }; + int len = sizeof(arr) / sizeof(arr[0]); + valueBucket->putBlob(valueBucket, "data4", arr, len); + valueBucket->putText(valueBucket, "data5", "ABCDEFG"); + errCode = OH_Rdb_Insert(storeTestRdbStore_, "mem_test", valueBucket); + EXPECT_EQ(errCode, 1); + + char querySql[] = "SELECT * FROM mem_test"; + OH_Cursor *cursor = OH_Rdb_ExecuteQuery(storeTestRdbStore_, querySql); + + int rowCount = 0; + cursor->getRowCount(cursor, &rowCount); + EXPECT_EQ(rowCount, 1); + cursor->destroy(cursor); + valueBucket->destroy(valueBucket); } \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7c5496c6..600b26e0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,12 +1,12 @@ cmake_minimum_required(VERSION 3.11.2) project(LDBTest) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "-std=c++1y -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-access-control") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -fpermissive") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat=0 -fpermissive -Wa,-mbig-obj") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations -Wno-deprecated-declarations -Wattributes") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") @@ -55,12 +55,12 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/dis aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/adapter/account/test DataMgrTestSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/adapter/communicator/test/unittest DataMgrTestSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/test/mock DataMgrTestSrc) -list(REMOVE_ITEM DataMgrTestSrc "../datamgr_service/services/distributeddataservice/service/test/unittest/cloud/device_manger_adapter_mock.cpp") +#list(REMOVE_ITEM DataMgrTestSrc "../datamgr_service/services/distributeddataservice/service/test/unittest/cloud/device_manger_adapter_mock.cpp") aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/test DataMgrTestSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/framework/test DataMgrTestSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/adapter/test DataMgrTestSrc) add_executable(DataMgrServiceTest ${DataMgrTestSrc} ${mainSrc} ${serviceSrc}) -target_link_libraries(DataMgrServiceTest ${links} gtest_main gmock_main gcov relational_store data_share preferences distributeddb data_object kvdb udmf) +target_link_libraries(DataMgrServiceTest ${links} gtest gtest_main gmock_main gcov relational_store data_share preferences distributeddb data_object kvdb udmf) target_include_directories(DataMgrServiceTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/adapter/account/src) target_include_directories(DataMgrServiceTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/adapter/communicator/src) target_include_directories(DataMgrServiceTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/backup/include) @@ -170,7 +170,7 @@ target_include_directories(UdmfTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../kv_st aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/test/mock CloudTestSrc) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/test/unittest/cloud CloudTestSrc) add_executable(CloudTest ${CloudTestSrc} ${mainSrc} ${serviceSrc}) -target_link_libraries(CloudTest gtest_main gcov svcFwk distributeddb kvdb service app) +target_link_libraries(CloudTest gtest_main gmock_main gcov svcFwk distributeddb kvdb service app) target_include_directories(CloudTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/test) target_include_directories(CloudTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/backup/include) target_include_directories(CloudTest PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../datamgr_service/services/distributeddataservice/service/bootstrap/include) diff --git a/udmf/BUILD.gn b/udmf/BUILD.gn index e81866df..5c7a57c8 100644 --- a/udmf/BUILD.gn +++ b/udmf/BUILD.gn @@ -16,6 +16,7 @@ import("//build/ohos.gni") group("udmf_packages") { if (is_standard_system) { deps = [ + "interfaces/components:udmfcomponents", "interfaces/innerkits:udmf_client", "interfaces/innerkits:utd_client", "interfaces/jskits:intelligence_napi", diff --git a/udmf/CMakeLists.txt b/udmf/CMakeLists.txt index 464a9a21..b2e32d46 100644 --- a/udmf/CMakeLists.txt +++ b/udmf/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10.2) project(udmf) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -fPIC -fpic -ffunction-sections -D_GLIBC_MOCK") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-as-needed -ldl") diff --git a/udmf/adapter/BUILD.gn b/udmf/adapter/BUILD.gn index 32d4ff89..8208f369 100644 --- a/udmf/adapter/BUILD.gn +++ b/udmf/adapter/BUILD.gn @@ -43,6 +43,7 @@ config("udmf_napi_config") { "${udmf_interfaces_path}/jskits/data", "${udmf_framework_path}/common", + "${udmf_framework_path}/innerkitsimpl/data", "${udmf_framework_path}/innerkits/service", "${third_party_path}/libuv/include", @@ -70,12 +71,13 @@ config("arkui_x_udmf_config") { "${arkui_x_graphic_surface_path}/interfaces/inner_api/utils", "${third_party_path}/skia", ] + + defines = [ "CROSS_PLATFORM" ] } arkui_x_public_source = [ "${udmf_framework_path}/innerkitsimpl/client/getter_system.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_key.cpp", - "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/audio.cpp", "${udmf_framework_path}/innerkitsimpl/data/file.cpp", @@ -95,6 +97,10 @@ arkui_x_public_source = [ "${udmf_framework_path}/innerkitsimpl/data/unified_data.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/video.cpp", + "${udmf_framework_path}/common/utd_graph.cpp", + "${udmf_framework_path}/common/graph.cpp", + "${udmf_root_path}/adapter/framework/innerkitsimpl/common/unified_meta.cpp", + "${udmf_root_path}/adapter/framework/innerkitsimpl/client/utd_client.cpp", ] ohos_source_set("arkui_x_udmf_data") { @@ -102,9 +108,7 @@ ohos_source_set("arkui_x_udmf_data") { "${udmf_framework_path}/common/base32_utils.cpp", "${udmf_framework_path}/common/custom_utd_json_parser.cpp", "${udmf_framework_path}/common/custom_utd_store.cpp", - "${udmf_framework_path}/common/graph.cpp", "${udmf_framework_path}/common/utd_cfgs_checker.cpp", - "${udmf_framework_path}/common/utd_graph.cpp", "${udmf_root_path}/adapter/framework/common/udmf_utils.cpp", ] @@ -164,6 +168,9 @@ config("arkui_x_udmf_napi_config") { "${arkui_x_graphic_2d_path}/rosen/modules/platform/ipc_core", "${third_party_path}/skia", ] + if (current_os == "ios") { + defines = [ "IOS_PLATFORM" ] + } } ohos_source_set("arkui_x_uniformtypedescriptor") { @@ -181,7 +188,6 @@ ohos_source_set("arkui_x_uniformtypedescriptor") { "${udmf_framework_path}/jskitsimpl/data/uniform_type_descriptor_napi.cpp", "${udmf_interfaces_path}/jskits/module/uniform_type_descriptor_napi_module.cpp", "${udmf_root_path}/adapter/framework/innerkitsimpl/client/udmf_client.cpp", - "${udmf_root_path}/adapter/framework/innerkitsimpl/client/utd_client.cpp", ] deps = [ ":arkui_x_udmf_data", @@ -205,6 +211,7 @@ ohos_source_set("arkui_x_uniformtypedescriptor") { ohos_source_set("arkui_x_unifieddatachannel") { include_dirs = [ "${ability_runtime_cross_platform_path}/frameworks/js/napi/napi_common", + "${kv_store_path}/interfaces/innerkits/distributeddata/include", "${udmf_root_path}/adapter/framework/innerkitsimpl/client", "${third_party_path}/bounds_checking_function/include", ] @@ -262,6 +269,7 @@ ohos_source_set("arkui_x_udmf_data_napi") { include_dirs = [ "${ability_runtime_cross_platform_path}/frameworks/js/napi/napi_common", "${ability_runtime_cross_platform_path}/interfaces/inner_api/want", + "${kv_store_path}/interfaces/innerkits/distributeddata/include", ] sources = [ diff --git a/udmf/adapter/framework/innerkitsimpl/client/udmf_client.h b/udmf/adapter/framework/innerkitsimpl/client/udmf_client.h index 4398f83f..fcc7f872 100644 --- a/udmf/adapter/framework/innerkitsimpl/client/udmf_client.h +++ b/udmf/adapter/framework/innerkitsimpl/client/udmf_client.h @@ -29,22 +29,22 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT UdmfClient { +class UdmfClient { public: - static UdmfClient &GetInstance(); + static UdmfClient API_EXPORT &GetInstance(); - Status SetData(CustomOption &option, UnifiedData &unifiedData, std::string &key); - Status GetData(const QueryOption &query, UnifiedData &unifiedData); - Status GetBatchData(const QueryOption &query, std::vector &unifiedDataSet); - Status UpdateData(const QueryOption &query, UnifiedData &unifiedData); - Status DeleteData(const QueryOption &query, std::vector &unifiedDataSet); - Status GetSummary(const QueryOption &query, Summary& summary); - Status AddPrivilege(const QueryOption &query, Privilege &privilege); - Status Sync(const QueryOption &query, const std::vector &devices); - Status IsRemoteData(const QueryOption &query, bool &result); - Status SetAppShareOption(const std::string &intention, enum ShareOptions shareOption); - Status RemoveAppShareOption(const std::string &intention); - Status GetAppShareOption(const std::string &intention, enum ShareOptions &shareOption); + Status API_EXPORT SetData(CustomOption &option, UnifiedData &unifiedData, std::string &key); + Status API_EXPORT GetData(const QueryOption &query, UnifiedData &unifiedData); + Status API_EXPORT GetBatchData(const QueryOption &query, std::vector &unifiedDataSet); + Status API_EXPORT UpdateData(const QueryOption &query, UnifiedData &unifiedData); + Status API_EXPORT DeleteData(const QueryOption &query, std::vector &unifiedDataSet); + Status API_EXPORT GetSummary(const QueryOption &query, Summary& summary); + Status API_EXPORT AddPrivilege(const QueryOption &query, Privilege &privilege); + Status API_EXPORT Sync(const QueryOption &query, const std::vector &devices); + Status API_EXPORT IsRemoteData(const QueryOption &query, bool &result); + Status API_EXPORT SetAppShareOption(const std::string &intention, enum ShareOptions shareOption); + Status API_EXPORT RemoveAppShareOption(const std::string &intention); + Status API_EXPORT GetAppShareOption(const std::string &intention, enum ShareOptions &shareOption); private: UdmfClient() = default; diff --git a/udmf/adapter/framework/innerkitsimpl/client/utd_client.h b/udmf/adapter/framework/innerkitsimpl/client/utd_client.h index 64563853..2da4ec4b 100644 --- a/udmf/adapter/framework/innerkitsimpl/client/utd_client.h +++ b/udmf/adapter/framework/innerkitsimpl/client/utd_client.h @@ -32,19 +32,19 @@ namespace OHOS { namespace UDMF { class TypeDescriptor; class UtdChangeSubscriber; -class API_EXPORT UtdClient { +class UtdClient { public: - static UtdClient &GetInstance(); - Status GetTypeDescriptor(const std::string &typeId, std::shared_ptr &descriptor); - Status GetUniformDataTypeByFilenameExtension(const std::string &fileExtension, std::string &typeId, + static UtdClient API_EXPORT &GetInstance(); + Status API_EXPORT GetTypeDescriptor(const std::string &typeId, std::shared_ptr &descriptor); + Status API_EXPORT GetUniformDataTypeByFilenameExtension(const std::string &fileExtension, std::string &typeId, std::string belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypesByFilenameExtension(const std::string &fileExtension, + Status API_EXPORT GetUniformDataTypesByFilenameExtension(const std::string &fileExtension, std::vector &typeIds, const std::string &belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypeByMIMEType(const std::string &mimeType, std::string &typeId, + Status API_EXPORT GetUniformDataTypeByMIMEType(const std::string &mimeType, std::string &typeId, std::string belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypesByMIMEType(const std::string &mimeType, std::vector &typeIds, + Status API_EXPORT GetUniformDataTypesByMIMEType(const std::string &mimeType, std::vector &typeIds, const std::string &belongsTo = DEFAULT_TYPE_ID); - Status IsUtd(std::string typeId, bool &result); + Status API_EXPORT IsUtd(std::string typeId, bool &result); private: UtdClient(); diff --git a/udmf/adapter/framework/innerkitsimpl/common/unified_meta.cpp b/udmf/adapter/framework/innerkitsimpl/common/unified_meta.cpp index e1841971..417432d1 100644 --- a/udmf/adapter/framework/innerkitsimpl/common/unified_meta.cpp +++ b/udmf/adapter/framework/innerkitsimpl/common/unified_meta.cpp @@ -18,6 +18,7 @@ #include "logger.h" #include "unified_key.h" +#include "utd_client.h" namespace OHOS { namespace UDMF { @@ -510,9 +511,13 @@ static constexpr UtdType UTD_TYPES[] = { }; static constexpr std::initializer_list NOT_NEED_COUNT_VALUE_LIST = { - UNIFORM_DATA_TYPE, ARRAY_BUFFER_LENGTH, THUMB_DATA_LENGTH, APP_ICON_LENGTH + UNIFORM_DATA_TYPE, ARRAY_BUFFER_LENGTH, THUMB_DATA_LENGTH, APP_ICON_LENGTH, APPLICATION_DEFINED_RECORD_MARK, + FILE_TYPE }; +static const std::set FILE_SUB_TYPES = { + "general.image", "general.video", "general.audio", "general.folder" }; + namespace UtdUtils { bool IsValidUtdId(const std::string &utdId) { @@ -758,5 +763,33 @@ int64_t ObjectUtils::GetAllObjectSize(const std::shared_ptr object) } return size; } + +void ObjectUtils::ProcessFileUriType(UDType &utdType, ValueType &value) +{ + auto fileUri = std::get>(value); + std::string uniformDataType; + if (!fileUri->GetValue(UNIFORM_DATA_TYPE, uniformDataType) || uniformDataType != GENERAL_FILE_URI) { + LOG_INFO(UDMF_FRAMEWORK, "Attribute uniformDataType not equals to 'general.file-uri'!"); + return; + } + utdType = FILE; + std::string fileType; + if (fileUri->GetValue(FILE_TYPE, fileType)) { + std::shared_ptr descriptor; + UtdClient::GetInstance().GetTypeDescriptor(fileType, descriptor); + if (descriptor == nullptr) { + return; + } + bool isFileType = false; + for (const auto &fileSub : FILE_SUB_TYPES) { + descriptor->BelongsTo(fileSub, isFileType); + if (isFileType) { + utdType = static_cast(UtdUtils::GetUtdEnumFromUtdId(fileSub)); + LOG_INFO(UDMF_FRAMEWORK, "Change type to %{public}s", fileSub.c_str()); + return; + } + } + } +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/bundle.json b/udmf/bundle.json index 5b94235e..742654f2 100644 --- a/udmf/bundle.json +++ b/udmf/bundle.json @@ -50,8 +50,7 @@ "build": { "sub_component": [ "//foundation/distributeddatamgr/udmf:udmf_packages", - "//foundation/distributeddatamgr/udmf/interfaces/cj:cj_unified_data_channel_ffi", - "//foundation/distributeddatamgr/udmf/interfaces/cj:cj_uniform_type_descriptor_ffi" + "//foundation/distributeddatamgr/udmf/interfaces/components:udmfcomponents" ], "inner_kits": [ { @@ -172,6 +171,20 @@ ], "header_base":"//foundation/distributeddatamgr/udmf/interfaces/jskits/intelligence" } + }, + { + "name": "//foundation/distributeddatamgr/udmf/interfaces/cj:cj_unified_data_channel_ffi", + "header": { + "header_files": [], + "header_base":"//foundation/distributeddatamgr/udmf/interfaces/cj/include" + } + }, + { + "name": "//foundation/distributeddatamgr/udmf/interfaces/cj:cj_uniform_type_descriptor_ffi", + "header": { + "header_files": [], + "header_base":"//foundation/distributeddatamgr/udmf/interfaces/cj/include" + } } ], "test": [ diff --git a/udmf/framework/common/custom_utd_json_parser.h b/udmf/framework/common/custom_utd_json_parser.h index fe2d259d..c7bd6264 100644 --- a/udmf/framework/common/custom_utd_json_parser.h +++ b/udmf/framework/common/custom_utd_json_parser.h @@ -20,12 +20,11 @@ #include #include #include -#include "visibility.h" #include "cJSON.h" #include "utd_common.h" namespace OHOS { namespace UDMF { -class API_EXPORT CustomUtdJsonParser { +class CustomUtdJsonParser { public: using json = cJSON; CustomUtdJsonParser(); diff --git a/udmf/framework/common/custom_utd_store.h b/udmf/framework/common/custom_utd_store.h index 2934b152..4c20c40f 100644 --- a/udmf/framework/common/custom_utd_store.h +++ b/udmf/framework/common/custom_utd_store.h @@ -20,12 +20,11 @@ #include #include #include -#include "visibility.h" #include "utd_common.h" #include "custom_utd_json_parser.h" namespace OHOS { namespace UDMF { -class API_EXPORT CustomUtdStore { +class CustomUtdStore { public: static CustomUtdStore &GetInstance(); std::vector GetTypeCfgs(int32_t userId); @@ -40,7 +39,7 @@ private: ~CustomUtdStore(); int32_t SavaCfgFile(const std::string &jsonData, const std::string &cfgFilePath); bool CreateDirectory(const std::string &path) const; - void ProcessUtdForSave(const CustomUtdCfgs &utdTypes, std::vector &customTyepCfgs, + static void ProcessUtdForSave(const CustomUtdCfgs &utdTypes, std::vector &customTyepCfgs, const std::string &bundleName); CustomUtdJsonParser utdJsonParser_; }; diff --git a/udmf/framework/common/graph.h b/udmf/framework/common/graph.h index 7b2dc006..4786bf08 100644 --- a/udmf/framework/common/graph.h +++ b/udmf/framework/common/graph.h @@ -17,22 +17,22 @@ #define UDMF_GRAPH_H #include -#include -#include #include +#include #include #include +#include namespace OHOS { namespace UDMF { struct EdgeNode { uint32_t adjIndex; - EdgeNode* next; + EdgeNode *next; }; struct VertexNode { uint32_t value; - EdgeNode* firstEdge; + EdgeNode *firstEdge; }; class Graph { public: @@ -46,10 +46,11 @@ public: bool IsValidType(const std::string &node); int32_t GetIndex(const std::string &node); bool IsDAG(); + private: uint32_t vertexNum_; - std::vector adjList_; // Adjacency List - std::vector visited_; // Determine whether the vertex has been accessed, index=vertex value + std::vector adjList_; // Adjacency List + std::vector visited_; // Determine whether the vertex has been accessed, index=vertex value std::map typeIdIndex_; }; } // namespace UDMF diff --git a/udmf/framework/common/tlv_tag.h b/udmf/framework/common/tlv_tag.h index a1b3b9f2..4511daa0 100644 --- a/udmf/framework/common/tlv_tag.h +++ b/udmf/framework/common/tlv_tag.h @@ -76,6 +76,20 @@ enum class TAG : uint16_t { TAG_RECORD_UTD_ID, TAG_RECORD_ENTRIES, TAG_INNER_ENTRIES, + TAG_URI_ORI, + TAG_URI_DFS, + TAG_URI_AUTH, + TAG_URI_POS, + TAG_RECORD_URIS, + TAG_SUMMARY, + TAG_SUMMARY_MAP, + TAG_SUMMARY_SIZE, + TAG_PROPERTIES, + TAG_UNIFIED_PROPERTIES, + TAG_PROPERTIES_TAG, + TAG_PROPERTIES_TIMESTAMP, + TAG_PROPERTIES_SHARE_OPTIONS, + TAG_PROPERTIES_EXTRAS, }; } #endif //UDMF_TLV_TAG_H diff --git a/udmf/framework/common/tlv_util.cpp b/udmf/framework/common/tlv_util.cpp index e735865c..e67b7251 100644 --- a/udmf/framework/common/tlv_util.cpp +++ b/udmf/framework/common/tlv_util.cpp @@ -231,8 +231,8 @@ template <> bool Reading(UnifiedKey &output, TLVObject &data, const TLVHead &hea template <> size_t CountBufferSize(const UnifiedData &input, TLVObject &data) { - std::string version = UTILS::GetCurrentSdkVersion(); - return data.CountHead() + data.Count(version) + TLVUtil::CountBufferSize(input.GetRecords(), data); + return data.CountHead() + data.Count(input.GetSdkVersion()) + TLVUtil::CountBufferSize(input.GetRecords(), data) + + TLVUtil::CountBufferSize(input.GetProperties(), data); } template <> bool Writing(const UnifiedData &input, TLVObject &data, TAG tag) @@ -240,13 +240,15 @@ template <> bool Writing(const UnifiedData &input, TLVObject &data, TAG tag) InitWhenFirst(input, data); auto tagCursor = data.GetCursor(); data.OffsetHead(); - std::string version = UTILS::GetCurrentSdkVersion(); - if (!data.Write(TAG::TAG_VERSION, version)) { + if (!data.Write(TAG::TAG_VERSION, input.GetSdkVersion())) { return false; } if (!TLVUtil::Writing(input.GetRecords(), data, TAG::TAG_UNIFIED_RECORD)) { return false; } + if (!TLVUtil::Writing(input.GetProperties(), data, TAG::TAG_UNIFIED_PROPERTIES)) { + return false; + } return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); } @@ -259,7 +261,11 @@ template <> bool Reading(UnifiedData &output, TLVObject &data, const TLVHead &he return false; } if (headItem.tag == static_cast(TAG::TAG_VERSION)) { - data.Skip(headItem); + std::string version; + if (!Reading(version, data, headItem)) { + return false; + } + output.SetSdkVersion(version); continue; } if (headItem.tag == static_cast(TAG::TAG_UNIFIED_RECORD)) { @@ -270,17 +276,162 @@ template <> bool Reading(UnifiedData &output, TLVObject &data, const TLVHead &he output.SetRecords(records); continue; } + if (headItem.tag == static_cast(TAG::TAG_UNIFIED_PROPERTIES)) { + auto properties = output.GetProperties(); + if (!Reading(properties, data, headItem)) { + return false; + } + output.SetProperties(std::move(properties)); + continue; + } data.Skip(headItem); } return true; } +template <> size_t CountBufferSize(const UnifiedDataProperties &input, TLVObject &data) +{ + return data.CountHead() + data.Count(input.tag) + data.CountBasic(input.timestamp) + + data.CountBasic(static_cast(input.shareOptions)) + TLVUtil::CountBufferSize(input.extras, data); +} + +template <> bool Writing(const UnifiedDataProperties &input, TLVObject &data, TAG tag) +{ + InitWhenFirst(input, data); + auto tagCursor = data.GetCursor(); + data.OffsetHead(); + if (!data.Write(TAG::TAG_PROPERTIES_TAG, input.tag)) { + return false; + } + if (!data.WriteBasic(TAG::TAG_PROPERTIES_TIMESTAMP, input.timestamp)) { + return false; + } + if (!data.WriteBasic(TAG::TAG_PROPERTIES_SHARE_OPTIONS, static_cast(input.shareOptions))) { + return false; + } + if (!TLVUtil::Writing(input.extras, data, TAG::TAG_PROPERTIES_EXTRAS)) { + return false; + } + return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); +} + +template <> bool Reading(UnifiedDataProperties &output, TLVObject &data, const TLVHead &head) +{ + auto endCursor = data.GetCursor() + head.len; + while (data.GetCursor() < endCursor) { + TLVHead headItem{}; + if (!data.ReadHead(headItem)) { + return false; + } + bool result = true; + switch (headItem.tag) { + case static_cast(TAG::TAG_PROPERTIES_TAG): + result = data.Read(output.tag, headItem); + break; + case static_cast(TAG::TAG_PROPERTIES_TIMESTAMP): + result = data.ReadBasic(output.timestamp, headItem); + break; + case static_cast(TAG::TAG_PROPERTIES_SHARE_OPTIONS): + result = TLVUtil::Reading(output.shareOptions, data, headItem); + break; + case static_cast(TAG::TAG_PROPERTIES_EXTRAS): + result = TLVUtil::Reading(output.extras, data, headItem); + break; + default: + result = data.Skip(headItem); + } + if (!result) { + return false; + } + } + return true; +} + +template <> bool Reading(ShareOptions &output, TLVObject &data, const TLVHead &head) +{ + int32_t shareOptions; + if (!Reading(shareOptions, data, head)) { + return false; + } + if (shareOptions < ShareOptions::IN_APP || shareOptions >= ShareOptions::SHARE_OPTIONS_BUTT) { + return false; + } + output = static_cast(shareOptions); + return true; +} + +template <> size_t CountBufferSize(const OHOS::AAFwk::WantParams &input, TLVObject &data) +{ + Parcel parcel; + if (!input.Marshalling(parcel)) { + LOG_ERROR(UDMF_FRAMEWORK, "Marshalling want error when Count"); + return 0; + } + auto size = parcel.GetDataSize(); + std::vector val(size); + return CountBufferSize(val, data); +} + +template <> bool Writing(const OHOS::AAFwk::WantParams &input, TLVObject &data, TAG tag) +{ + InitWhenFirst(input, data); + Parcel parcel; + if (!input.Marshalling(parcel)) { + LOG_ERROR(UDMF_FRAMEWORK, "Marshalling wantParams error in tlv write. tag=%{public}hu", tag); + return false; + } + auto size = parcel.GetDataSize(); + auto buffer = parcel.GetData(); + std::vector val(size); + if (size != 0) { + auto err = memcpy_s(val.data(), size, reinterpret_cast(buffer), size); + if (err != EOK) { + LOG_ERROR(UDMF_FRAMEWORK, "memcpy error in tlv write wantParams. tag=%{public}hu", tag); + return false; + } + } + return data.Write(tag, val); +} + +template <> bool Reading(OHOS::AAFwk::WantParams &output, TLVObject &data, const TLVHead &head) +{ + std::vector val; + if (!data.Read(val, head)) { + LOG_ERROR(UDMF_FRAMEWORK, "Reading u8 vector error."); + return false; + } + + std::shared_ptr parcel = std::make_shared(); + auto buffer = malloc(val.size()); + if (buffer == nullptr) { + LOG_ERROR(UDMF_FRAMEWORK, "malloc error in tlv read. tag=%{public}hu", head.tag); + return false; + } + auto err = memcpy_s(buffer, val.size(), val.data(), val.size()); + if (err != EOK) { + LOG_ERROR(UDMF_FRAMEWORK, "memcpy_s error in tlv read wantParams. tag=%{public}hu", head.tag); + free(buffer); + return false; + } + if (!parcel->ParseFrom((uintptr_t)buffer, head.len)) { + LOG_ERROR(UDMF_FRAMEWORK, "ParseFrom error in tlv read wantParams. tag=%{public}hu", head.tag); + free(buffer); + return false; + } + auto wantParams = AAFwk::WantParams::Unmarshalling(*parcel); + if (wantParams == nullptr) { + LOG_ERROR(UDMF_FRAMEWORK, "Unmarshalling wantParams error in tlv read. tag=%{public}hu", head.tag); + return false; + } + output = *wantParams; + return true; +} + template <> size_t CountBufferSize(const UnifiedRecord &input, TLVObject &data) { - std::string version = UTILS::GetCurrentSdkVersion(); - return data.CountHead() + data.Count(version) + data.CountBasic(static_cast(input.GetType())) + + return data.CountHead() + data.CountBasic(static_cast(input.GetType())) + data.Count(input.GetUid()) + CountBufferSize(input.GetOriginValue(), data) + data.Count(input.GetUtdId()) + - CountBufferSize(input.GetInnerEntries(), data); + CountBufferSize(input.GetInnerEntries(), data) + CountBufferSize(input.GetUris(), data); } template <> bool Writing(const UnifiedRecord &input, TLVObject &data, TAG tag) @@ -288,10 +439,6 @@ template <> bool Writing(const UnifiedRecord &input, TLVObject &data, TAG tag) InitWhenFirst(input, data); auto tagCursor = data.GetCursor(); data.OffsetHead(); - std::string version = UTILS::GetCurrentSdkVersion(); - if (!data.Write(TAG::TAG_VERSION, version)) { - return false; - } if (!data.WriteBasic(TAG::TAG_UD_TYPE, static_cast(input.GetType()))) { return false; } @@ -307,6 +454,9 @@ template <> bool Writing(const UnifiedRecord &input, TLVObject &data, TAG tag) if (!TLVUtil::Writing(input.GetInnerEntries(), data, TAG::TAG_RECORD_ENTRIES)) { return false; } + if (!TLVUtil::Writing(input.GetUris(), data, TAG::TAG_RECORD_URIS)) { + return false; + } return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); } @@ -323,10 +473,8 @@ template <> bool Reading(UnifiedRecord &output, TLVObject &data, const TLVHead & } std::string utdId; std::shared_ptr> entries; + std::vector uriInfos; switch (headItem.tag) { - case static_cast(TAG::TAG_VERSION): - data.Skip(headItem); - break; case static_cast(TAG::TAG_UD_TYPE): if (!TLVUtil::Reading(dataType, data, headItem)) { return false; @@ -357,6 +505,12 @@ template <> bool Reading(UnifiedRecord &output, TLVObject &data, const TLVHead & } output.SetInnerEntries(entries); break; + case static_cast(TAG::TAG_RECORD_URIS): + if (!TLVUtil::Reading(uriInfos, data, headItem)) { + return false; + } + output.SetUris(std::move(uriInfos)); + break; default: data.Skip(headItem); } @@ -366,14 +520,13 @@ template <> bool Reading(UnifiedRecord &output, TLVObject &data, const TLVHead & template <> size_t CountBufferSize(const Runtime &input, TLVObject &data) { - std::string version = UTILS::GetCurrentSdkVersion(); return data.CountHead() + data.CountBasic(input.isPrivate) + data.CountBasic(input.dataVersion) + data.CountBasic(input.recordTotalNum) + data.CountBasic(input.tokenId) + data.CountBasic(static_cast(input.createTime)) + data.CountBasic(static_cast(input.lastModifiedTime)) + data.CountBasic(static_cast(input.dataStatus)) + data.Count(input.sourcePackage) + data.Count(input.createPackage) + data.Count(input.deviceId) + TLVUtil::CountBufferSize(input.key, data) + - data.Count(version) + TLVUtil::CountBufferSize(input.privileges, data); + data.Count(input.sdkVersion) + TLVUtil::CountBufferSize(input.privileges, data); } template <> bool Writing(const Runtime &input, TLVObject &data, TAG tag) @@ -381,10 +534,6 @@ template <> bool Writing(const Runtime &input, TLVObject &data, TAG tag) InitWhenFirst(input, data); auto tagCursor = data.GetCursor(); data.OffsetHead(); - std::string version = UTILS::GetCurrentSdkVersion(); - if (!TLVUtil::Writing(version, data, TAG::TAG_VERSION)) { - return false; - } if (!TLVUtil::Writing(input.key, data, TAG::TAG_KEY)) { return false; } @@ -421,6 +570,9 @@ template <> bool Writing(const Runtime &input, TLVObject &data, TAG tag) if (!data.WriteBasic(TAG::TAG_TOKEN_ID, input.tokenId)) { return false; } + if (!TLVUtil::Writing(input.sdkVersion, data, TAG::TAG_VERSION)) { + return false; + } return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); } @@ -474,6 +626,9 @@ template <> bool Reading(Runtime &output, TLVObject &data, const TLVHead &head) case static_cast(TAG::TAG_TOKEN_ID): result = data.ReadBasic(output.tokenId, headItem); break; + case static_cast(TAG::TAG_VERSION): + result = data.Read(output.sdkVersion, headItem); + break; default: result = data.Skip(headItem); } @@ -538,6 +693,64 @@ template <> bool Reading(Privilege &output, TLVObject &data, const TLVHead &head return true; } +template <> size_t CountBufferSize(const UriInfo &input, TLVObject &data) +{ + return data.CountHead() + data.CountBasic(input.position) + data.Count(input.oriUri) + + data.Count(input.dfsUri) + data.Count(input.authUri); +} + +template <> bool Writing(const UriInfo &input, TLVObject &data, TAG tag) +{ + InitWhenFirst(input, data); + auto tagCursor = data.GetCursor(); + data.OffsetHead(); + if (!TLVUtil::Writing(input.oriUri, data, TAG::TAG_URI_ORI)) { + return false; + } + if (!TLVUtil::Writing(input.dfsUri, data, TAG::TAG_URI_DFS)) { + return false; + } + if (!TLVUtil::Writing(input.authUri, data, TAG::TAG_URI_AUTH)) { + return false; + } + if (!data.WriteBasic(TAG::TAG_URI_POS, input.position)) { + return false; + } + return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); +} + +template <> bool Reading(UriInfo &output, TLVObject &data, const TLVHead &head) +{ + auto endCursor = data.GetCursor() + head.len; + while (data.GetCursor() < endCursor) { + TLVHead headItem{}; + if (!data.ReadHead(headItem)) { + return false; + } + bool result = true; + switch (headItem.tag) { + case static_cast(TAG::TAG_URI_ORI): + result = TLVUtil::Reading(output.oriUri, data, headItem); + break; + case static_cast(TAG::TAG_URI_DFS): + result = TLVUtil::Reading(output.dfsUri, data, headItem); + break; + case static_cast(TAG::TAG_URI_AUTH): + result = TLVUtil::Reading(output.authUri, data, headItem); + break; + case static_cast(TAG::TAG_URI_POS): + result = data.ReadBasic(output.position, headItem); + break; + default: + result = data.Skip(headItem); + } + if (!result) { + return false; + } + } + return true; +} + template <> size_t CountBufferSize(const std::shared_ptr &input, TLVObject &data) { std::vector val; @@ -650,10 +863,12 @@ template <> bool Reading(std::shared_ptr &output, TLVObject & auto err = memcpy_s(buffer, val.size(), val.data(), val.size()); if (err != EOK) { LOG_ERROR(UDMF_FRAMEWORK, "memcpy_s error in tlv read want. tag=%{public}hu", head.tag); + free(buffer); return false; } if (!parcel->ParseFrom((uintptr_t)buffer, head.len)) { LOG_ERROR(UDMF_FRAMEWORK, "ParseFrom error in tlv read want. tag=%{public}hu", head.tag); + free(buffer); return false; } auto want = AAFwk::Want::Unmarshalling(*parcel); @@ -664,5 +879,50 @@ template <> bool Reading(std::shared_ptr &output, TLVObject & output = std::shared_ptr(want); return true; } + +template <> size_t CountBufferSize(const Summary &input, TLVObject &data) +{ + return data.CountHead() + CountBufferSize(input.summary, data) + data.CountBasic(input.totalSize); +} + +template <> bool Writing(const Summary &input, TLVObject &data, TAG tag) +{ + InitWhenFirst(input, data); + auto tagCursor = data.GetCursor(); + data.OffsetHead(); + if (!TLVUtil::Writing(input.summary, data, TAG::TAG_SUMMARY_MAP)) { + return false; + } + if (!data.WriteBasic(TAG::TAG_SUMMARY_SIZE, input.totalSize)) { + return false; + } + return data.WriteBackHead(static_cast(tag), tagCursor, data.GetCursor() - tagCursor - sizeof(TLVHead)); +} + +template <> bool Reading(Summary &output, TLVObject &data, const TLVHead &head) +{ + auto endCursor = data.GetCursor() + head.len; + while (data.GetCursor() < endCursor) { + TLVHead headItem{}; + if (!data.ReadHead(headItem)) { + return false; + } + switch (headItem.tag) { + case static_cast(TAG::TAG_SUMMARY_MAP): + if (!TLVUtil::Reading(output.summary, data, headItem)) { + return false; + } + break; + case static_cast(TAG::TAG_SUMMARY_SIZE): + if (!data.ReadBasic(output.totalSize, headItem)) { + return false; + } + break; + default: + data.Skip(headItem); + } + } + return true; +} } // namespace TLVUtil } // namespace OHOS diff --git a/udmf/framework/common/tlv_util.h b/udmf/framework/common/tlv_util.h index 834de4c5..605ad4fe 100644 --- a/udmf/framework/common/tlv_util.h +++ b/udmf/framework/common/tlv_util.h @@ -100,6 +100,20 @@ template <> size_t API_EXPORT CountBufferSize(const Privilege &input, TLVObject template <> bool API_EXPORT Writing(const Privilege &input, TLVObject &data, TAG tag); template <> bool API_EXPORT Reading(Privilege &output, TLVObject &data, const TLVHead &head); +template <> size_t API_EXPORT CountBufferSize(const UriInfo &input, TLVObject &data); +template <> bool API_EXPORT Writing(const UriInfo &input, TLVObject &data, TAG tag); +template <> bool API_EXPORT Reading(UriInfo &output, TLVObject &data, const TLVHead &head); + +template <> bool API_EXPORT Reading(ShareOptions &output, TLVObject &data, const TLVHead &head); + +template <> size_t API_EXPORT CountBufferSize(const UnifiedDataProperties &input, TLVObject &data); +template <> bool API_EXPORT Writing(const UnifiedDataProperties &input, TLVObject &data, TAG tag); +template <> bool API_EXPORT Reading(UnifiedDataProperties &output, TLVObject &data, const TLVHead &head); + +template <> size_t API_EXPORT CountBufferSize(const OHOS::AAFwk::WantParams &input, TLVObject &data); +template <> bool API_EXPORT Writing(const OHOS::AAFwk::WantParams &input, TLVObject &data, TAG tag); +template <> bool API_EXPORT Reading(OHOS::AAFwk::WantParams &output, TLVObject &data, const TLVHead &head); + template <> size_t API_EXPORT CountBufferSize(const std::shared_ptr &input, TLVObject &data); template <> bool API_EXPORT Writing(const std::shared_ptr &input, TLVObject &data, TAG tag); template <> @@ -117,6 +131,10 @@ template <> size_t API_EXPORT CountBufferSize(const std::shared_ptr bool API_EXPORT Writing(const std::shared_ptr &input, TLVObject &data, TAG tag); template <> bool API_EXPORT Reading(std::shared_ptr &output, TLVObject &data, const TLVHead &head); +template <> size_t API_EXPORT CountBufferSize(const Summary &input, TLVObject &data); +template <> bool API_EXPORT Writing(const Summary &input, TLVObject &data, TAG tag); +template <> bool API_EXPORT Reading(Summary &output, TLVObject &data, const TLVHead &head); + template bool ReadTlv(T &output, TLVObject &data, TAG tag) { while (data.GetCursor() < data.GetTotal()) { diff --git a/udmf/framework/common/udmf_copy_file.cpp b/udmf/framework/common/udmf_copy_file.cpp index ce892fbc..827baebf 100644 --- a/udmf/framework/common/udmf_copy_file.cpp +++ b/udmf/framework/common/udmf_copy_file.cpp @@ -17,12 +17,14 @@ #include "udmf_copy_file.h" #include +#include #include #include "copy/file_copy_manager.h" #include "copy/file_size_utils.h" #include "file_uri.h" #include "logger.h" #include "udmf_async_client.h" +#include "unified_html_record_process.h" namespace OHOS { namespace UDMF { @@ -58,15 +60,61 @@ Status UdmfCopyFile::Copy(std::unique_ptr &asyncHelper) } } + HandleUris(context); context.asyncHelper->data = context.processedData; LOG_INFO(UDMF_CLIENT, "Copy end."); return context.status; } +void UdmfCopyFile::HandleUris(CopyContext &context) +{ + for (const auto &record : context.asyncHelper->data->GetRecords()) { + if (record == nullptr) { + continue; + } + for (auto &uri : record->GetUris()) { + std::string srcUri = uri.dfsUri.empty() ? uri.authUri : uri.dfsUri; + if (IsDirectory(srcUri, true)) { + LOG_ERROR(UDMF_CLIENT, "Source cannot be directory."); + context.status = E_COPY_FILE_FAILED; + continue; + } + std::string destUri = ConstructDestUri(context.asyncHelper->destUri, srcUri); + if (context.asyncHelper->fileConflictOptions == FileConflictOptions::SKIP && IsFile(destUri, false)) { + LOG_INFO(UDMF_CLIENT, "File has existed, skip."); + continue; + } + if (!CopyFile(srcUri, destUri, record, context)) { + LOG_ERROR(UDMF_CLIENT, "copy cancel"); + return; + } + UpdateUriInfo(destUri, record, uri); + } + } + UnifiedHtmlRecordProcess::RebuildHtmlRecord(*(context.asyncHelper->data)); +} + +void UdmfCopyFile::UpdateUriInfo(const std::string &destUri, const std::shared_ptr &record, + UriInfo uri) +{ + record->ComputeUris([&destUri, &uri] (UriInfo &uriInfo) { + if (uri.dfsUri.empty()) { + if (uri.authUri == uriInfo.authUri) { + uriInfo.dfsUri = destUri; + } + } else { + if (uri.dfsUri == uriInfo.dfsUri) { + uriInfo.dfsUri = destUri; + } + } + return true; + }); +} + bool UdmfCopyFile::HandleRecord(const std::shared_ptr &record, CopyContext &context) { std::string srcUri; - if (!record->HasFileType(srcUri)) { + if (!record->HasFileType(srcUri) && !record->HasType(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML))) { context.processedData->AddRecord(record); return true; } @@ -141,9 +189,9 @@ std::string UdmfCopyFile::ConstructDestUri(const std::string &destUri, const std return destFileUri; } -int64_t UdmfCopyFile::GetTotalSize(const std::vector &uris) +uint64_t UdmfCopyFile::GetTotalSize(const std::vector &uris) { - int64_t totalSize = 0; + uint64_t totalSize = 0; std::string srcUri; for (const auto &srcUri : uris) { if (IsFile(srcUri, true)) { @@ -173,7 +221,7 @@ std::string UdmfCopyFile::GetFileName(const std::string &path) { std::string realSrc = path; if (IsRemote(path)) { - int index = path.rfind("?", 0); + size_t index = path.rfind("?", 0); realSrc = path.substr(0, index); } std::filesystem::path filePath(realSrc); diff --git a/udmf/framework/common/udmf_copy_file.h b/udmf/framework/common/udmf_copy_file.h index a45556c1..9d671dbf 100644 --- a/udmf/framework/common/udmf_copy_file.h +++ b/udmf/framework/common/udmf_copy_file.h @@ -48,12 +48,15 @@ private: ~UdmfCopyFile() = default; bool HandleRecord(const std::shared_ptr &record, CopyContext &context); + void HandleUris(CopyContext &context); + void UpdateUriInfo(const std::string &destUri, const std::shared_ptr &record, + UriInfo uri); bool CopyFile(const std::string &srcUri, const std::string &destFileUri, const std::shared_ptr &record, CopyContext &context); void HandleProgress(const std::string &srcUri, const std::string &destFileUri, CopyContext &context, uint64_t processSize); std::string ConstructDestUri(const std::string &destUri, const std::string &srcUri); - int64_t GetTotalSize(const std::vector &uris); + uint64_t GetTotalSize(const std::vector &uris); bool IsDirectory(const std::string &uri, bool isSource); std::string GetFileName(const std::string &path); bool IsRemote(const std::string &uri); diff --git a/udmf/framework/common/udmf_types_util.cpp b/udmf/framework/common/udmf_types_util.cpp index cabc896d..921f783b 100644 --- a/udmf/framework/common/udmf_types_util.cpp +++ b/udmf/framework/common/udmf_types_util.cpp @@ -196,7 +196,7 @@ bool Unmarshalling(Intention &output, MessageParcel &parcel) } template<> -bool API_EXPORT Marshalling(const AsyncProcessInfo &input, MessageParcel &parcel) +bool Marshalling(const AsyncProcessInfo &input, MessageParcel &parcel) { uint32_t syncStatus = input.syncStatus; uint32_t permStatus = input.permStatus; @@ -205,7 +205,7 @@ bool API_EXPORT Marshalling(const AsyncProcessInfo &input, MessageParcel &parcel } template<> -bool API_EXPORT Unmarshalling(AsyncProcessInfo &output, MessageParcel &parcel) +bool Unmarshalling(AsyncProcessInfo &output, MessageParcel &parcel) { uint32_t syncStatus; uint32_t permStatus; diff --git a/udmf/framework/common/udmf_utils.cpp b/udmf/framework/common/udmf_utils.cpp index 13366997..ac1f66f1 100644 --- a/udmf/framework/common/udmf_utils.cpp +++ b/udmf/framework/common/udmf_utils.cpp @@ -82,7 +82,8 @@ std::string GetSdkVersionByToken(uint32_t tokenId) std::string GetCurrentSdkVersion() { - return GetSdkVersionByToken(IPCSkeleton::GetSelfTokenID()); + static const std::string sdkVersion = GetSdkVersionByToken(IPCSkeleton::GetSelfTokenID()); + return sdkVersion; } bool IsTokenNative() diff --git a/udmf/framework/common/udmf_utils.h b/udmf/framework/common/udmf_utils.h index 63066cf7..3d4fde6d 100644 --- a/udmf/framework/common/udmf_utils.h +++ b/udmf/framework/common/udmf_utils.h @@ -31,6 +31,7 @@ std::string GetSdkVersionByToken(uint32_t tokenId); std::string GetCurrentSdkVersion(); bool API_EXPORT IsTokenNative(); + } // namespace UTILS } // namespace UDMF } // namespace OHOS diff --git a/udmf/framework/common/unittest/BUILD.gn b/udmf/framework/common/unittest/BUILD.gn index 2103a96d..8009d2c2 100644 --- a/udmf/framework/common/unittest/BUILD.gn +++ b/udmf/framework/common/unittest/BUILD.gn @@ -28,7 +28,10 @@ config("module_private_config") { ] } -common_deps = [ "${udmf_interfaces_path}/innerkits:udmf_client" ] +common_deps = [ + "${udmf_interfaces_path}/innerkits:udmf_client", + "${udmf_interfaces_path}/innerkits:utd_client", +] common_external_deps = [ "ability_base:want", @@ -93,7 +96,9 @@ ohos_unittest("UdmfTypesUtilAbnormalTest") { sources = [ "${udmf_framework_path}/common/tlv_object.cpp", "${udmf_framework_path}/common/udmf_types_util.cpp", + "${udmf_framework_path}/common/udmf_utils.cpp", "${udmf_framework_path}/common/unittest/mock/tlv_util_mock.cpp", + "${udmf_framework_path}/innerkitsimpl/client/getter_system.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", "${udmf_framework_path}/innerkitsimpl/convert/udmf_conversion.cpp", "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", @@ -129,7 +134,9 @@ ohos_unittest("TlvUtilTest") { module_out_path = module_output_path sources = [ + "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", "${udmf_framework_path}/innerkitsimpl/convert/udmf_conversion.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", "tlv_util_test.cpp", ] @@ -138,6 +145,7 @@ ohos_unittest("TlvUtilTest") { deps = common_deps external_deps = [ + "ability_base:base", "ability_base:want", "c_utils:utils", "hilog:libhilog", diff --git a/udmf/framework/common/unittest/tlv_util_test.cpp b/udmf/framework/common/unittest/tlv_util_test.cpp index 174663f5..2079cda3 100644 --- a/udmf/framework/common/unittest/tlv_util_test.cpp +++ b/udmf/framework/common/unittest/tlv_util_test.cpp @@ -23,6 +23,7 @@ #include "plain_text.h" #include "html.h" #include "link.h" +#include "int_wrapper.h" #include "system_defined_appitem.h" #include "application_defined_record.h" #include "file.h" @@ -30,6 +31,7 @@ #include "system_defined_pixelmap.h" #include "system_defined_record.h" #include "udmf_conversion.h" +#include "unified_data_helper.h" #include "unified_meta.h" #include "unified_record.h" #include "unified_types.h" @@ -651,4 +653,154 @@ HWTEST_F(TlvUtilTest, WritingAndReadingFile_001, TestSize.Level1) fclose(file); LOG_INFO(UDMF_TEST, "WritingAndReadingFile_001 end."); } + +/* * + * @tc.name: WritingAndReadingSummary_001 + * @tc.desc: test Summary for Writing And Reading + * @tc.type: FUNC + */ +HWTEST_F(TlvUtilTest, WritingAndReadingSummary_001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "WritingAndReadingSummary_001 begin."); + std::map value; + value["fileType"] = "File Type"; + value["fileUri"] = "File Uri"; + std::shared_ptr obj = std::make_shared(); + obj->value_ = value; + std::shared_ptr fileUri = std::make_shared(UDType::FILE_URI, obj); + std::shared_ptr plainText = std::make_shared(UDType::PLAIN_TEXT, "this is a content"); + std::shared_ptr<UnifiedRecord> html = std::make_shared<Html>(UDType::HTML, "this is a HTML content"); + std::vector<std::shared_ptr<UnifiedRecord>> records = { fileUri, plainText, html }; + UnifiedData data1; + data1.SetRecords(records); + Summary summary; + UnifiedDataHelper::GetSummary(data1, summary); + EXPECT_EQ(summary.summary.size(), records.size()); + + std::vector<uint8_t> dataBytes; + auto tlvObject = TLVObject(dataBytes); + EXPECT_TRUE(TLVUtil::Writing(summary, tlvObject, TAG::TAG_SUMMARY)); + + tlvObject.ResetCursor(); + Summary summary2; + TLVUtil::ReadTlv(summary2, tlvObject, TAG::TAG_SUMMARY); + EXPECT_EQ(summary.totalSize, summary2.totalSize); + EXPECT_EQ(summary.summary.size(), summary2.summary.size()); + for (const auto &[key, val] : summary.summary) { + ASSERT_TRUE(summary2.summary.find(key) != summary2.summary.end()); + EXPECT_EQ(summary.summary[key], summary2.summary[key]); + } + LOG_INFO(UDMF_TEST, "WritingAndReadingSummary_001 end."); +} + +/* * + * @tc.name: WritingAndReadingVersion_001 + * @tc.desc: test Version for Writing And Reading + * @tc.type: FUNC + */ +HWTEST_F(TlvUtilTest, WritingAndReadingVersion_001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "WritingAndReadingVersion_001 begin."); + std::shared_ptr<UnifiedRecord> plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, "this is a content"); + std::shared_ptr<UnifiedRecord> html = std::make_shared<Html>(UDType::HTML, "this is a HTML content"); + std::vector<std::shared_ptr<UnifiedRecord>> records = {plainText, html }; + UnifiedData data1; + data1.SetRecords(records); + auto version = data1.GetSdkVersion(); + + std::vector<uint8_t> dataBytes; + auto tlvObject = TLVObject(dataBytes); + std::vector<UnifiedData> datas = {data1}; + EXPECT_TRUE(TLVUtil::Writing(datas, tlvObject, TAG::TAG_UNIFIED_DATA)); + + tlvObject.ResetCursor(); + std::vector<UnifiedData> datas2; + TLVUtil::ReadTlv(datas2, tlvObject, TAG::TAG_UNIFIED_DATA); + EXPECT_EQ(datas2.size(), 1); + EXPECT_EQ(datas2[0].GetSdkVersion(), version); + LOG_INFO(UDMF_TEST, "WritingAndReadingVersion_001 end."); +} + +/* * + * @tc.name: CountBufferSize_005 + * @tc.desc: test UnifiedDataProperties element for countBufferSize + * @tc.type: FUNC + */ +HWTEST_F(TlvUtilTest, CountBufferSize_005, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "CountBufferSize_005 begin."); + + std::vector<uint8_t> dataBytes; + auto tlvObject = TLVObject(dataBytes); + + OHOS::AAFwk::WantParams wantParams; + auto size = TLVUtil::CountBufferSize(wantParams, tlvObject); + EXPECT_EQ(tlvObject.GetTotal(), size); + + UnifiedDataProperties properties; + EXPECT_EQ(5 * sizeof(TLVHead) + sizeof(int64_t) + sizeof(int32_t) + 4, + TLVUtil::CountBufferSize(properties, tlvObject)); + + LOG_INFO(UDMF_TEST, "CountBufferSize_005 end."); +} + +/* * + * @tc.name: WritingAndReading_006 + * @tc.desc: test UnifiedDataProperties for Writing And Reading + * @tc.type: FUNC + */ +HWTEST_F(TlvUtilTest, WritingAndReading_006, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "WritingAndReading_006 begin."); + OHOS::AAFwk::WantParams wantParams; + std::string idKey = "test006"; + int32_t idValue = 123; + wantParams.SetParam(idKey, OHOS::AAFwk::Integer::Box(idValue)); + UnifiedDataProperties properties; + properties.tag = "props"; + properties.shareOptions = CROSS_APP; + properties.timestamp = 10; + properties.extras = wantParams; + + std::vector<uint8_t> dataBytes; + auto tlvObject = TLVObject(dataBytes); + auto result = TLVUtil::Writing(properties, tlvObject, TAG::TAG_UNIFIED_PROPERTIES); + EXPECT_TRUE(result); + + tlvObject.ResetCursor(); + UnifiedDataProperties propertiesResult; + result = TLVUtil::ReadTlv(propertiesResult, tlvObject, TAG::TAG_UNIFIED_PROPERTIES); + EXPECT_TRUE(result); + EXPECT_EQ(properties.tag, propertiesResult.tag); + EXPECT_EQ(properties.shareOptions, propertiesResult.shareOptions); + EXPECT_EQ(properties.timestamp, propertiesResult.timestamp); + EXPECT_EQ(idValue, propertiesResult.extras.GetIntParam(idKey, 0)); + LOG_INFO(UDMF_TEST, "WritingAndReading_006 end."); +} + +/* * + * @tc.name: WritingAndReading_007 + * @tc.desc: test OHOS::AAFwk::WantParams for Writing And Reading + * @tc.type: FUNC + */ +HWTEST_F(TlvUtilTest, WritingAndReading_007, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "WritingAndReading_007 begin."); + OHOS::AAFwk::WantParams wantParams; + std::string idKey = "test007"; + int32_t idValue = 123456; + wantParams.SetParam(idKey, OHOS::AAFwk::Integer::Box(idValue)); + + std::vector<uint8_t> dataBytes; + auto tlvObject = TLVObject(dataBytes); + auto result = TLVUtil::Writing(wantParams, tlvObject, TAG::TAG_PROPERTIES_EXTRAS); + EXPECT_TRUE(result); + + tlvObject.ResetCursor(); + OHOS::AAFwk::WantParams wantParamsResult; + result = TLVUtil::ReadTlv(wantParamsResult, tlvObject, TAG::TAG_PROPERTIES_EXTRAS); + EXPECT_TRUE(result); + EXPECT_EQ(idValue, wantParamsResult.GetIntParam(idKey, 0)); + LOG_INFO(UDMF_TEST, "WritingAndReading_007 end."); +} } diff --git a/udmf/framework/common/unittest/udmf_types_util_abnormal_test.cpp b/udmf/framework/common/unittest/udmf_types_util_abnormal_test.cpp index f0082134..78b7b4de 100644 --- a/udmf/framework/common/unittest/udmf_types_util_abnormal_test.cpp +++ b/udmf/framework/common/unittest/udmf_types_util_abnormal_test.cpp @@ -31,6 +31,7 @@ #include "application_defined_record.h" #include "unified_types.h" #include "udmf_client.h" +#include "udmf_utils.h" using namespace testing::ext; using namespace OHOS::UDMF; diff --git a/udmf/framework/common/unittest/udmf_types_util_test.cpp b/udmf/framework/common/unittest/udmf_types_util_test.cpp index 9d3c1b10..86e94996 100644 --- a/udmf/framework/common/unittest/udmf_types_util_test.cpp +++ b/udmf/framework/common/unittest/udmf_types_util_test.cpp @@ -69,7 +69,6 @@ void UdmfTypesUtilTest::TearDown() HWTEST_F(UdmfTypesUtilTest, Unmarshalling001, TestSize.Level1) { LOG_INFO(UDMF_TEST, "Unmarshalling001 begin."); - UnifiedData input; std::vector<std::shared_ptr<UnifiedRecord>> inputRecords = { std::make_shared<Text>(), diff --git a/udmf/framework/common/utd_cfgs_checker.h b/udmf/framework/common/utd_cfgs_checker.h index 07ef9a80..13eab867 100644 --- a/udmf/framework/common/utd_cfgs_checker.h +++ b/udmf/framework/common/utd_cfgs_checker.h @@ -19,7 +19,7 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT UtdCfgsChecker { +class UtdCfgsChecker { public: static UtdCfgsChecker &GetInstance(); bool CheckTypeDescriptors(CustomUtdCfgs &typeCfgs, const std::vector<TypeDescriptorCfg> &presetCfgs, diff --git a/udmf/framework/common/utd_graph.cpp b/udmf/framework/common/utd_graph.cpp index 429826d0..4203d9dc 100644 --- a/udmf/framework/common/utd_graph.cpp +++ b/udmf/framework/common/utd_graph.cpp @@ -13,8 +13,9 @@ * limitations under the License. */ #define LOG_TAG "UtdGraph" -#include <mutex> #include "utd_graph.h" + +#include <mutex> #include "logger.h" namespace OHOS { namespace UDMF { diff --git a/udmf/framework/common/utd_graph.h b/udmf/framework/common/utd_graph.h index 20586661..0c185849 100644 --- a/udmf/framework/common/utd_graph.h +++ b/udmf/framework/common/utd_graph.h @@ -18,10 +18,10 @@ #include <vector> #include <map> +#include <memory> #include <stack> #include <string> #include <shared_mutex> -#include <memory> #include "graph.h" #include "utd_common.h" #include "preset_type_descriptors.h" diff --git a/udmf/framework/innerkitsimpl/client/udmf_async_client.cpp b/udmf/framework/innerkitsimpl/client/udmf_async_client.cpp index e9eb8186..d7c1e65e 100644 --- a/udmf/framework/innerkitsimpl/client/udmf_async_client.cpp +++ b/udmf/framework/innerkitsimpl/client/udmf_async_client.cpp @@ -16,6 +16,7 @@ #include "udmf_async_client.h" +#include "dataobs_mgr_client.h" #include "logger.h" #include "plain_text.h" #include "progress_callback.h" @@ -46,7 +47,11 @@ static constexpr int32_t PROGRESS_SYNC_FINSHED = 10; static constexpr int32_t PROGRESS_GET_DATA_FINISHED = 20; static constexpr int32_t PROGRESS_ALL_FINISHED = 100; +#ifdef IOS_PLATFORM +UdmfAsyncClient::UdmfAsyncClient() {} +#else UdmfAsyncClient::UdmfAsyncClient() : executor_(MAX_THREADS, MIN_THREADS) {} +#endif UdmfAsyncClient &UdmfAsyncClient::GetInstance() { @@ -56,6 +61,7 @@ UdmfAsyncClient &UdmfAsyncClient::GetInstance() Status UdmfAsyncClient::StartAsyncDataRetrieval(const GetDataParams &params) { +#ifndef IOS_PLATFORM if (!IsParamValid(params)) { return E_INVALID_PARAMETERS; } @@ -71,6 +77,7 @@ Status UdmfAsyncClient::StartAsyncDataRetrieval(const GetDataParams &params) } asyncHelper->getDataTask = executor_.Execute(std::bind(&UdmfAsyncClient::GetDataTask, this, params.query)); asyncHelper->progressTask = executor_.Execute(std::bind(&UdmfAsyncClient::ProgressTask, this, params.query.key)); +#endif return E_OK; } @@ -145,7 +152,7 @@ Status UdmfAsyncClient::InvokeHapTask(const std::string &businessUdKey) { LOG_INFO(UDMF_CLIENT, "InvokeHap start!"); auto &asyncHelper = asyncHelperMap_.at(businessUdKey); - if (asyncHelper->progressQueue.IsCancel()) { + if (asyncHelper->progressQueue.IsCancel() || asyncHelper->progressQueue.IsClear()) { LOG_INFO(UDMF_CLIENT, "Finished, not invoke hap."); Clear(businessUdKey); return E_OK; @@ -156,9 +163,16 @@ Status UdmfAsyncClient::InvokeHapTask(const std::string &businessUdKey) return E_ERROR; } sptr<IRemoteObject> callback = new ProgressSignalCallback(); - auto status = UdmfServiceClient::GetInstance()->InvokeHap(asyncHelper->processKey, callback); - if (status != E_OK) { - LOG_ERROR(UDMF_CLIENT, "Invoke hap failed, status=%{public}d", status); + auto obsMgrClient = AAFwk::DataObsMgrClient::GetInstance(); + if (obsMgrClient == nullptr) { + LOG_ERROR(UDMF_CLIENT, "Get DataObsMgrClient failed"); + Clear(businessUdKey); + return E_ERROR; + } + + auto status = obsMgrClient->NotifyProcessObserver(asyncHelper->processKey, callback); + if (status != AAFwk::SUCCESS) { + LOG_ERROR(UDMF_CLIENT, "Notify process dialog failed, status=%{public}d", status); Clear(businessUdKey); return E_ERROR; } @@ -185,6 +199,7 @@ Status UdmfAsyncClient::RegisterAsyncHelper(const GetDataParams &params) Status UdmfAsyncClient::CheckSync(std::unique_ptr<AsyncHelper> &asyncHelper, const QueryOption &query) { +#ifndef IOS_PLATFORM AsyncProcessInfo processInfo = { .businessUdKey = query.key }; @@ -220,6 +235,7 @@ Status UdmfAsyncClient::CheckSync(std::unique_ptr<AsyncHelper> &asyncHelper, con (asyncHelper->sycnRetryTime)++; asyncHelper->getDataTask = executor_.Schedule(std::chrono::milliseconds(SYNC_INTERVAL), std::bind(&UdmfAsyncClient::GetDataTask, this, query)); +#endif return E_OK; } @@ -300,11 +316,12 @@ Status UdmfAsyncClient::UpdateProgressData(const std::string &progressUdKey, con Status UdmfAsyncClient::CopyFile(std::unique_ptr<AsyncHelper> &asyncHelper) { + auto status = E_OK; if (asyncHelper->destUri.empty()) { LOG_INFO(UDMF_CLIENT, "No dest path, no copy."); - return E_OK; + } else { + status = UdmfCopyFile::GetInstance().Copy(asyncHelper); } - auto status = UdmfCopyFile::GetInstance().Copy(asyncHelper); ProgressInfo progressInfo = { .progress = PROGRESS_ALL_FINISHED, .errorCode = status @@ -337,6 +354,7 @@ void UdmfAsyncClient::CallProgress(std::unique_ptr<AsyncHelper> &asyncHelper, Pr Status UdmfAsyncClient::Clear(const std::string &businessUdKey) { +#ifndef IOS_PLATFORM std::lock_guard<std::mutex> lockMap(mutex_); if (asyncHelperMap_.find(businessUdKey) == asyncHelperMap_.end()) { LOG_ERROR(UDMF_CLIENT, "No task can Clear, key=%{public}s", businessUdKey.c_str()); @@ -354,12 +372,13 @@ Status UdmfAsyncClient::Clear(const std::string &businessUdKey) asyncHelperMap_.erase(businessUdKey); UdmfServiceClient::GetInstance()->ClearAsynProcessByKey(businessUdKey); LOG_INFO(UDMF_CLIENT, "Clear task success, key = %{public}s", businessUdKey.c_str()); +#endif return E_OK; } Status UdmfAsyncClient::ProcessUnifiedData(std::unique_ptr<AsyncHelper> &asyncHelper) { - if (!asyncHelper->data->HasFileType()) { + if (!asyncHelper->data->HasFileType() && !asyncHelper->data->HasUriInfo()) { ProgressInfo progressInfo = { .progress = PROGRESS_ALL_FINISHED, .errorCode = E_OK diff --git a/udmf/framework/innerkitsimpl/client/udmf_client.cpp b/udmf/framework/innerkitsimpl/client/udmf_client.cpp index efefc4ae..dbaa7d3e 100644 --- a/udmf/framework/innerkitsimpl/client/udmf_client.cpp +++ b/udmf/framework/innerkitsimpl/client/udmf_client.cpp @@ -25,6 +25,7 @@ #include "accesstoken_kit.h" #include "ipc_skeleton.h" #include "unified_data_helper.h" +#include "unified_html_record_process.h" namespace OHOS { namespace UDMF { @@ -73,6 +74,9 @@ Status UdmfClient::SetData(CustomOption &option, UnifiedData &unifiedData, std:: BizScene::SET_DATA, SetDataStage::SET_DATA_END, StageRes::SUCCESS, BizState::DFX_END); return E_OK; } + if (unifiedData.HasType(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML))) { + UnifiedHtmlRecordProcess::GetUriFromHtmlRecord(unifiedData); + } } int32_t ret = service->SetData(option, unifiedData, key); if (ret != E_OK) { @@ -114,6 +118,9 @@ Status UdmfClient::GetData(const QueryOption &query, UnifiedData &unifiedData) LOG_ERROR(UDMF_CLIENT, "failed! ret = %{public}d", ret); return static_cast<Status>(ret); } + if (unifiedData.HasType(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML))) { + UnifiedHtmlRecordProcess::RebuildHtmlRecord(unifiedData); + } RadarReporterAdapter::ReportNormal(std::string(__FUNCTION__), BizScene::GET_DATA, GetDataStage::GET_DATA_END, StageRes::SUCCESS, BizState::DFX_END); return E_OK; @@ -132,6 +139,9 @@ Status UdmfClient::GetBatchData(const QueryOption &query, std::vector<UnifiedDat if (ret != E_OK) { LOG_ERROR(UDMF_CLIENT, "failed! ret = %{public}d", ret); } + for (auto &data : unifiedDataSet) { + data.ClearUriInfo(); + } return static_cast<Status>(ret); } diff --git a/udmf/framework/innerkitsimpl/common/progress_queue.cpp b/udmf/framework/innerkitsimpl/common/progress_queue.cpp index 38986f76..b23b2500 100644 --- a/udmf/framework/innerkitsimpl/common/progress_queue.cpp +++ b/udmf/framework/innerkitsimpl/common/progress_queue.cpp @@ -59,4 +59,10 @@ bool ProgressQueue::Clear() return oldClearableFlag; } +bool ProgressQueue::IsClear() const +{ + std::lock_guard<std::mutex> lock(mutex_); + return clearableFlag_; +} + } // namespace OHOS::UDMF \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/common/unified_key.cpp b/udmf/framework/innerkitsimpl/common/unified_key.cpp index 1fa7eed1..1aaf4cb6 100644 --- a/udmf/framework/innerkitsimpl/common/unified_key.cpp +++ b/udmf/framework/innerkitsimpl/common/unified_key.cpp @@ -13,6 +13,7 @@ * limitations under the License. */ #define LOG_TAG "UnifiedKey" +#include <sstream> #include "unified_key.h" #include "logger.h" @@ -26,6 +27,10 @@ static constexpr const char *UNIFIED_KEY_SCHEMA = "udmf://"; static constexpr const char *ALPHA_AGGREGATE = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; static constexpr const char *DIGIT_AGGREGATE = "0123456789"; static constexpr const char *SYMBOL_AGGREGATE = ":;<=>?@[\\]_`"; +static constexpr const char SEPARATOR = '/'; +static constexpr uint32_t PREFIX_LEN = 24; +static constexpr uint32_t SUFIX_LEN = 8; +static constexpr uint32_t INDEX_LEN = 32; UnifiedKey::UnifiedKey(std::string key) { this->key = std::move(key); @@ -47,10 +52,29 @@ std::string UnifiedKey::GetUnifiedKey() return ""; } // Uri-compliant structure, example: udmf://drag/com.ohos.test/012345679abc - this->key = UNIFIED_KEY_SCHEMA + this->intention + "/" + this->bundleName + "/" + this->groupId; + std::ostringstream oss; + oss << UNIFIED_KEY_SCHEMA << this->intention << SEPARATOR << + this->bundleName << SEPARATOR << this->groupId; + this->key = oss.str(); return this->key; } +std::string UnifiedKey::GetPropertyKey() const +{ + if (this->key.size() > INDEX_LEN) { + return this->key.substr(0, key.size() - SUFIX_LEN); + } + if (this->intention.empty() || this->groupId.size() < INDEX_LEN) { + LOG_ERROR(UDMF_FRAMEWORK, "Empty property key.intention:%{public}s, groupId:%{public}s", + intention.c_str(), groupId.c_str()); + return ""; + } + std::ostringstream oss; + oss << UNIFIED_KEY_SCHEMA << this->intention << SEPARATOR << + this->bundleName << SEPARATOR << this->groupId.substr(0, PREFIX_LEN); + return oss.str(); +} + bool UnifiedKey::IsValid() { if (this->key.empty()) { diff --git a/udmf/framework/innerkitsimpl/common/unified_meta.cpp b/udmf/framework/innerkitsimpl/common/unified_meta.cpp index 548944df..04989624 100644 --- a/udmf/framework/innerkitsimpl/common/unified_meta.cpp +++ b/udmf/framework/innerkitsimpl/common/unified_meta.cpp @@ -18,6 +18,7 @@ #include "logger.h" #include "unified_key.h" +#include "utd_client.h" namespace OHOS { namespace UDMF { @@ -509,8 +510,13 @@ static constexpr UtdType UTD_TYPES[] = { { OPENHARMONY_GOCOLOR, "OPENHARMONY_GOCOLOR", "openharmony.gocolor" } }; -static const std::initializer_list<std::string_view> NOT_NEED_COUNT_VALUE_LIST = { UNIFORM_DATA_TYPE, - ARRAY_BUFFER_LENGTH, THUMB_DATA_LENGTH, APP_ICON_LENGTH, APPLICATION_DEFINED_RECORD_MARK }; +static const std::vector<std::string_view> NOT_NEED_COUNT_VALUE_LIST = { + UNIFORM_DATA_TYPE, ARRAY_BUFFER_LENGTH, THUMB_DATA_LENGTH, APP_ICON_LENGTH, APPLICATION_DEFINED_RECORD_MARK, + FILE_TYPE +}; + +static const std::set<std::string> FILE_SUB_TYPES = { + "general.image", "general.video", "general.audio", "general.folder" }; namespace UtdUtils { bool IsValidUtdId(const std::string &utdId) @@ -762,5 +768,34 @@ int64_t ObjectUtils::GetAllObjectSize(const std::shared_ptr<Object> object) } return size; } + + +void ObjectUtils::ProcessFileUriType(UDType &utdType, ValueType &value) +{ + auto fileUri = std::get<std::shared_ptr<Object>>(value); + std::string uniformDataType; + if (!fileUri->GetValue(UNIFORM_DATA_TYPE, uniformDataType) || uniformDataType != GENERAL_FILE_URI) { + LOG_INFO(UDMF_FRAMEWORK, "Attribute uniformDataType not equals to 'general.file-uri'!"); + return; + } + utdType = FILE; + std::string fileType; + if (fileUri->GetValue(FILE_TYPE, fileType)) { + std::shared_ptr<TypeDescriptor> descriptor; + UtdClient::GetInstance().GetTypeDescriptor(fileType, descriptor); + if (descriptor == nullptr) { + return; + } + bool isFileType = false; + for (const auto &fileSub : FILE_SUB_TYPES) { + descriptor->BelongsTo(fileSub, isFileType); + if (isFileType) { + utdType = static_cast<UDType>(UtdUtils::GetUtdEnumFromUtdId(fileSub)); + LOG_INFO(UDMF_FRAMEWORK, "Change type to %{public}s", fileSub.c_str()); + return; + } + } + } +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/convert/udmf_conversion.cpp b/udmf/framework/innerkitsimpl/convert/udmf_conversion.cpp index f1d4a215..f4d0ae62 100644 --- a/udmf/framework/innerkitsimpl/convert/udmf_conversion.cpp +++ b/udmf/framework/innerkitsimpl/convert/udmf_conversion.cpp @@ -58,6 +58,7 @@ void UdmfConversion::ConvertRecordToSubclass(std::shared_ptr<UnifiedRecord> &rec auto uid = record->GetUid(); auto entries = record->GetInnerEntries(); auto utdId = record->GetUtdId(); + auto uris = record->GetUris(); switch (type) { case UDType::TEXT: { record = std::make_shared<Text>(type, value); @@ -122,6 +123,7 @@ void UdmfConversion::ConvertRecordToSubclass(std::shared_ptr<UnifiedRecord> &rec record->SetUid(uid); record->SetUtdId(utdId); record->SetInnerEntries(entries); + record->SetUris(uris); SetValueWhenNotUds(record); } diff --git a/udmf/framework/innerkitsimpl/data/application_defined_record.cpp b/udmf/framework/innerkitsimpl/data/application_defined_record.cpp index 3e53f342..219c1a1c 100644 --- a/udmf/framework/innerkitsimpl/data/application_defined_record.cpp +++ b/udmf/framework/innerkitsimpl/data/application_defined_record.cpp @@ -55,7 +55,7 @@ ApplicationDefinedRecord::ApplicationDefinedRecord(UDType type, ValueType value) int64_t ApplicationDefinedRecord::GetSize() { - return rawData_.size() + GetInnerEntriesSize(); + return static_cast<int64_t>(rawData_.size()) + GetInnerEntriesSize(); } std::string ApplicationDefinedRecord::GetApplicationDefinedType() const diff --git a/udmf/framework/innerkitsimpl/data/audio.cpp b/udmf/framework/innerkitsimpl/data/audio.cpp index a30e5e22..41dafb68 100644 --- a/udmf/framework/innerkitsimpl/data/audio.cpp +++ b/udmf/framework/innerkitsimpl/data/audio.cpp @@ -20,11 +20,13 @@ namespace UDMF { Audio::Audio() : Audio("") { SetType(AUDIO); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(AUDIO); } Audio::Audio(const std::string &uri) : File(uri) { SetType(AUDIO); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(AUDIO); } Audio::Audio(UDType type, ValueType value) : File(type, value) diff --git a/udmf/framework/innerkitsimpl/data/file.cpp b/udmf/framework/innerkitsimpl/data/file.cpp index 445ba1d5..cccc8487 100644 --- a/udmf/framework/innerkitsimpl/data/file.cpp +++ b/udmf/framework/innerkitsimpl/data/file.cpp @@ -12,7 +12,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include "file.h" namespace OHOS { @@ -35,6 +34,11 @@ File::File(UDType type, ValueType value) : UnifiedRecord(type, value) auto object = std::get<std::shared_ptr<Object>>(value); object->GetValue(ORI_URI, oriUri_); object->GetValue(REMOTE_URI, remoteUri_); + object->GetValue(FILE_TYPE, fileType_); + std::string uniformDataType; + if (object->GetValue(UNIFORM_DATA_TYPE, uniformDataType) && uniformDataType == GENERAL_FILE_URI) { + utdId_ = std::move(uniformDataType); + } std::shared_ptr<Object> detailObj = nullptr; if (object->GetValue(DETAILS, detailObj)) { details_ = ObjectUtils::ConvertToUDDetails(detailObj); @@ -43,10 +47,15 @@ File::File(UDType type, ValueType value) : UnifiedRecord(type, value) } } +void File::SetType(const UDType &type) +{ + this->dataType_ = type; +} + int64_t File::GetSize() { - return this->oriUri_.size() + this->remoteUri_.size() + UnifiedDataUtils::GetDetailsSize(this->details_) + - GetInnerEntriesSize(); + return static_cast<int64_t>(this->oriUri_.size() + this->remoteUri_.size() + + UnifiedDataUtils::GetDetailsSize(this->details_)) + GetInnerEntriesSize(); } std::string File::GetUri() const @@ -97,9 +106,10 @@ void File::InitObject() auto value = value_; value_ = std::make_shared<Object>(); auto object = std::get<std::shared_ptr<Object>>(value_); - object->value_[UNIFORM_DATA_TYPE] = UtdUtils::GetUtdIdFromUtdEnum(dataType_); + object->value_[UNIFORM_DATA_TYPE] = GENERAL_FILE_URI; object->value_[ORI_URI] = oriUri_; object->value_[REMOTE_URI] = remoteUri_; + object->value_[FILE_TYPE] = fileType_; object->value_[DETAILS] = ObjectUtils::ConvertToObject(details_); object->value_.insert_or_assign(VALUE_TYPE, std::move(value)); } diff --git a/udmf/framework/innerkitsimpl/data/folder.cpp b/udmf/framework/innerkitsimpl/data/folder.cpp index e6b973f8..2164cec3 100644 --- a/udmf/framework/innerkitsimpl/data/folder.cpp +++ b/udmf/framework/innerkitsimpl/data/folder.cpp @@ -20,11 +20,13 @@ namespace UDMF { Folder::Folder() : Folder("") { SetType(FOLDER); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(FOLDER); } Folder::Folder(const std::string &uri) : File(uri) { SetType(FOLDER); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(FOLDER); } Folder::Folder(UDType type, ValueType value) : File(type, value) diff --git a/udmf/framework/innerkitsimpl/data/html.cpp b/udmf/framework/innerkitsimpl/data/html.cpp index 9f2279a7..924634d2 100644 --- a/udmf/framework/innerkitsimpl/data/html.cpp +++ b/udmf/framework/innerkitsimpl/data/html.cpp @@ -56,8 +56,8 @@ Html::Html(UDType type, ValueType value) : Text(type, value) int64_t Html::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + this->htmlContent_.size() + this->plainContent_.size() + - GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + this->htmlContent_.size() + + this->plainContent_.size()) + GetInnerEntriesSize(); } std::string Html::GetHtmlContent() const diff --git a/udmf/framework/innerkitsimpl/data/image.cpp b/udmf/framework/innerkitsimpl/data/image.cpp index 90846a64..cf9067d6 100644 --- a/udmf/framework/innerkitsimpl/data/image.cpp +++ b/udmf/framework/innerkitsimpl/data/image.cpp @@ -20,11 +20,13 @@ namespace UDMF { Image::Image() : Image("") { SetType(IMAGE); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(IMAGE); } Image::Image(const std::string &uri) : File(uri) { SetType(IMAGE); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(IMAGE); } Image::Image(UDType type, ValueType value) : File(type, value) diff --git a/udmf/framework/innerkitsimpl/data/link.cpp b/udmf/framework/innerkitsimpl/data/link.cpp index 08b6db39..695730c6 100644 --- a/udmf/framework/innerkitsimpl/data/link.cpp +++ b/udmf/framework/innerkitsimpl/data/link.cpp @@ -54,8 +54,8 @@ Link::Link(const std::string &url, const std::string &description) int64_t Link::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + this->url_.size() + this->description_.size() + - GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + this->url_.size() + + this->description_.size()) + GetInnerEntriesSize(); } std::string Link::GetUrl() const diff --git a/udmf/framework/innerkitsimpl/data/plain_text.cpp b/udmf/framework/innerkitsimpl/data/plain_text.cpp index 6d1ecdf5..7928a6da 100644 --- a/udmf/framework/innerkitsimpl/data/plain_text.cpp +++ b/udmf/framework/innerkitsimpl/data/plain_text.cpp @@ -54,8 +54,8 @@ PlainText::PlainText(UDType type, ValueType value) : Text(type, value) int64_t PlainText::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + this->content_.size() + this->abstract_.size() + - GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + this->content_.size() + + this->abstract_.size()) + GetInnerEntriesSize(); } std::string PlainText::GetContent() const diff --git a/udmf/framework/innerkitsimpl/data/preset_type_descriptors.h b/udmf/framework/innerkitsimpl/data/preset_type_descriptors.h index 08e6cbd0..6a57164c 100644 --- a/udmf/framework/innerkitsimpl/data/preset_type_descriptors.h +++ b/udmf/framework/innerkitsimpl/data/preset_type_descriptors.h @@ -20,7 +20,7 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT PresetTypeDescriptors { +class PresetTypeDescriptors { public: static PresetTypeDescriptors &GetInstance(); std::vector<TypeDescriptorCfg> &GetPresetTypes(); diff --git a/udmf/framework/innerkitsimpl/data/system_defined_appitem.cpp b/udmf/framework/innerkitsimpl/data/system_defined_appitem.cpp index c9e68080..2aea430c 100644 --- a/udmf/framework/innerkitsimpl/data/system_defined_appitem.cpp +++ b/udmf/framework/innerkitsimpl/data/system_defined_appitem.cpp @@ -43,9 +43,9 @@ SystemDefinedAppItem::SystemDefinedAppItem(UDType type, ValueType value) : Syste int64_t SystemDefinedAppItem::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + this->appId_.size() + this->appName_.size() + - this->appIconId_.size() + this->appLabelId_.size() + this->bundleName_.size() + this->abilityName_.size() + - GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + this->appId_.size() + + this->appName_.size() + this->appIconId_.size() + this->appLabelId_.size() + this->bundleName_.size() + + this->abilityName_.size()) + GetInnerEntriesSize(); } std::string SystemDefinedAppItem::GetAppId() const diff --git a/udmf/framework/innerkitsimpl/data/system_defined_form.cpp b/udmf/framework/innerkitsimpl/data/system_defined_form.cpp index e17c5475..d7d3c382 100644 --- a/udmf/framework/innerkitsimpl/data/system_defined_form.cpp +++ b/udmf/framework/innerkitsimpl/data/system_defined_form.cpp @@ -42,8 +42,9 @@ SystemDefinedForm::SystemDefinedForm(UDType type, ValueType value) : SystemDefin int64_t SystemDefinedForm::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + sizeof(formId_) + this->formName_.size() - + this->bundleName_.size() + this->abilityName_.size() + this->module_.size() + GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + sizeof(formId_) + + this->formName_.size() + this->bundleName_.size() + this->abilityName_.size() + this->module_.size()) + + GetInnerEntriesSize(); } int32_t SystemDefinedForm::GetFormId() const diff --git a/udmf/framework/innerkitsimpl/data/system_defined_pixelmap.cpp b/udmf/framework/innerkitsimpl/data/system_defined_pixelmap.cpp index 4e713720..e98789df 100644 --- a/udmf/framework/innerkitsimpl/data/system_defined_pixelmap.cpp +++ b/udmf/framework/innerkitsimpl/data/system_defined_pixelmap.cpp @@ -61,7 +61,8 @@ SystemDefinedPixelMap::SystemDefinedPixelMap(UDType type, ValueType value) : Sys int64_t SystemDefinedPixelMap::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + rawData_.size() + GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_) + rawData_.size()) + + GetInnerEntriesSize(); } std::vector<uint8_t> SystemDefinedPixelMap::GetRawData() const diff --git a/udmf/framework/innerkitsimpl/data/system_defined_record.cpp b/udmf/framework/innerkitsimpl/data/system_defined_record.cpp index 623fcff2..4da832b8 100644 --- a/udmf/framework/innerkitsimpl/data/system_defined_record.cpp +++ b/udmf/framework/innerkitsimpl/data/system_defined_record.cpp @@ -23,7 +23,7 @@ SystemDefinedRecord::SystemDefinedRecord() : UnifiedRecord(SYSTEM_DEFINED_RECORD int64_t SystemDefinedRecord::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_)) + GetInnerEntriesSize(); } SystemDefinedRecord::SystemDefinedRecord(UDType type, ValueType value) : UnifiedRecord(type, value) diff --git a/udmf/framework/innerkitsimpl/data/text.cpp b/udmf/framework/innerkitsimpl/data/text.cpp index 718fd399..55323253 100644 --- a/udmf/framework/innerkitsimpl/data/text.cpp +++ b/udmf/framework/innerkitsimpl/data/text.cpp @@ -41,7 +41,7 @@ Text::Text(UDType type, ValueType value) : UnifiedRecord(type, value) int64_t Text::GetSize() { - return UnifiedDataUtils::GetDetailsSize(this->details_) + GetInnerEntriesSize(); + return static_cast<int64_t>(UnifiedDataUtils::GetDetailsSize(this->details_)) + GetInnerEntriesSize(); } void Text::SetDetails(UDDetails &variantMap) diff --git a/udmf/framework/innerkitsimpl/data/unified_data.cpp b/udmf/framework/innerkitsimpl/data/unified_data.cpp index 37d26645..77b50133 100644 --- a/udmf/framework/innerkitsimpl/data/unified_data.cpp +++ b/udmf/framework/innerkitsimpl/data/unified_data.cpp @@ -13,18 +13,24 @@ * limitations under the License. */ #define LOG_TAG "UnifiedData" -#include "unified_data.h" #include "logger.h" +#include "udmf_utils.h" +#include "unified_data.h" +#include "utd_client.h" namespace OHOS { namespace UDMF { -static std::set<std::string> FILE_TYPES = { +static const std::set<std::string> FILE_TYPES = { "general.file", "general.image", "general.video", "general.audio", "general.folder", "general.file-uri" }; +static const std::set<std::string> FILE_SUB_TYPES = { + "general.image", "general.video", "general.audio", "general.folder" }; +static constexpr const char *RECORDS_TANSFER_TAG = "records_to_entries_data_format"; UnifiedData::UnifiedData() { properties_ = std::make_shared<UnifiedDataProperties>(); auto duration = std::chrono::system_clock::now().time_since_epoch(); properties_->timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count(); + sdkVersion_ = UTILS::GetCurrentSdkVersion(); } UnifiedData::UnifiedData(std::shared_ptr<UnifiedDataProperties> properties) @@ -36,6 +42,7 @@ UnifiedData::UnifiedData(std::shared_ptr<UnifiedDataProperties> properties) properties_ = properties; auto duration = std::chrono::system_clock::now().time_since_epoch(); properties_->timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count(); + sdkVersion_ = UTILS::GetCurrentSdkVersion(); } int64_t UnifiedData::GetSize() @@ -60,6 +67,7 @@ std::shared_ptr<Runtime> UnifiedData::GetRuntime() const void UnifiedData::SetRuntime(Runtime &runtime) { this->runtime_ = std::make_shared<Runtime>(runtime); + this->SetSdkVersion(runtime.sdkVersion); } void UnifiedData::AddRecord(const std::shared_ptr<UnifiedRecord> &record) @@ -123,6 +131,31 @@ bool UnifiedData::HasType(const std::string &type) const return false; } +bool UnifiedData::HasHigherFileType(const std::string &type) const +{ + std::set<std::string> types = GetTypIds(); + if (types.find(type) != types.end()) { + return true; + } + auto subTypesIter = FILE_SUB_TYPES.find(type); + if (subTypesIter == FILE_SUB_TYPES.end()) { + return false; + } + for (auto it = types.begin(); it != types.end(); ++it) { + std::shared_ptr<TypeDescriptor> descriptor; + UtdClient::GetInstance().GetTypeDescriptor(*it, descriptor); + if (descriptor == nullptr) { + continue; + } + bool isFileType = false; + descriptor->BelongsTo(type, isFileType); + if (isFileType) { + return true; + } + } + return false; +} + std::vector<std::string> UnifiedData::GetEntriesTypes() const { std::set<std::string> labels; @@ -190,6 +223,23 @@ bool UnifiedData::HasFileType() const return !intersection.empty(); } +bool UnifiedData::HasUriInfo() const +{ + for (auto record : records_) { + if (!record->GetUris().empty()) { + return true; + } + } + return false; +} + +void UnifiedData::ClearUriInfo() const +{ + for (auto record : records_) { + record->ClearUris(); + } +} + void UnifiedData::SetProperties(std::shared_ptr<UnifiedDataProperties> properties) { if (!properties) { @@ -247,5 +297,46 @@ std::vector<std::string> UnifiedData::GetFileUris() const return uris; } +bool UnifiedData::IsNeedTransferToEntries() const +{ + if (records_.size() <= 1) { + return false; + } + if (properties_ == nullptr) { + return false; + } + return properties_->tag == RECORDS_TANSFER_TAG; +} + +void UnifiedData::TransferToEntries(UnifiedData &data) +{ + if (records_.size() <= 1) { + return; + } + std::shared_ptr<UnifiedRecord> recordFirst = records_[0]; + if (recordFirst == nullptr) { + LOG_ERROR(UDMF_FRAMEWORK, "First record is null"); + return; + } + for (size_t i = 1; i < records_.size(); i++) { + auto record = records_[i]; + if (record == nullptr) { + continue; + } + record->InitObject(); + recordFirst->AddEntry(record->GetUtdId(), record->GetValue()); + } + records_.erase(records_.begin() + 1, records_.end()); +} + +std::string UnifiedData::GetSdkVersion() const +{ + return sdkVersion_; +} + +void UnifiedData::SetSdkVersion(const std::string &version) +{ + sdkVersion_ = version; +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/data/unified_data_helper.cpp b/udmf/framework/innerkitsimpl/data/unified_data_helper.cpp index 06423fd5..204127b2 100644 --- a/udmf/framework/innerkitsimpl/data/unified_data_helper.cpp +++ b/udmf/framework/innerkitsimpl/data/unified_data_helper.cpp @@ -18,20 +18,19 @@ #include "common_func.h" #include "directory_ex.h" #include "file_ex.h" +#include "file.h" #include "file_uri.h" #include "error_code.h" #include "logger.h" #include "tlv_util.h" #include "udmf_conversion.h" +#include "udmf_meta.h" #include "udmf_utils.h" -#include "file.h" namespace OHOS { namespace UDMF { constexpr mode_t MODE = 0700; -static constexpr int64_t MAX_HAP_RECORD_SIZE = 2 * 1024 * 1024; -static constexpr int64_t MAX_HAP_DATA_SIZE = 4 * 1024 * 1024; -static constexpr int64_t MAX_IPC_RAW_DATA_SIZE = 128 * 1024 * 1024; +static constexpr int64_t MAX_IPC_RAW_DATA_SIZE = 127 * 1024 * 1024; static constexpr int64_t MAX_SA_DRAG_RECORD_SIZE = 15 * 1024 * 1024 + 512 * 1024; constexpr const char *TEMP_UNIFIED_DATA_ROOT_PATH = "data/storage/el2/base/temp/udata"; @@ -50,14 +49,14 @@ bool UnifiedDataHelper::ExceedKVSizeLimit(UnifiedData &data) int64_t totalSize = 0; for (const auto &record : data.GetRecords()) { auto recordSize = record->GetSize(); - if (recordSize > MAX_HAP_RECORD_SIZE) { - LOG_INFO(UDMF_FRAMEWORK, "Exceeded 2M record limit, recordSize:%{public}" PRId64 "!", recordSize); + if (recordSize > MAX_SA_DRAG_RECORD_SIZE) { + LOG_INFO(UDMF_FRAMEWORK, "Exceeded 15.5M record limit, recordSize:%{public}" PRId64 "!", recordSize); return true; } totalSize += recordSize; } - if (totalSize > MAX_HAP_DATA_SIZE) { - LOG_INFO(UDMF_FRAMEWORK, "Exceeded 4M data limit, totalSize:%{public}" PRId64 " !", totalSize); + if (totalSize > MAX_IPC_RAW_DATA_SIZE) { + LOG_INFO(UDMF_FRAMEWORK, "Exceeded 127M data limit, totalSize:%{public}" PRId64 " !", totalSize); return true; } return false; @@ -174,7 +173,9 @@ bool UnifiedDataHelper::SaveUDataToFile(const std::string &dataFile, UnifiedData UdmfConversion::InitValueObject(data); if (!TLVUtil::Writing(data, recordTlv, TAG::TAG_UNIFIED_DATA)) { LOG_ERROR(UDMF_FRAMEWORK, "TLV Writing failed!"); - (void)fclose(file); + if (fclose(file) == EOF) { + LOG_ERROR(UDMF_FRAMEWORK, "fclose is failed"); + } return false; } (void)fclose(file); @@ -241,17 +242,32 @@ int32_t UnifiedDataHelper::ProcessBigData(UnifiedData &data, Intention intention return E_OK; } +void UnifiedDataHelper::ProcessTypeId(const ValueType &value, std::string &typeId) +{ + if (!std::holds_alternative<std::shared_ptr<Object>>(value)) { + return; + } + auto object = std::get<std::shared_ptr<Object>>(value); + if (object->value_.find(UNIFORM_DATA_TYPE) == object->value_.end()) { + return; + } + if (object->value_.find(APPLICATION_DEFINED_RECORD_MARK) != object->value_.end()) { + typeId = UtdUtils::GetUtdIdFromUtdEnum(APPLICATION_DEFINED_RECORD); + } else if (std::get<std::string>(object->value_[UNIFORM_DATA_TYPE]) == GENERAL_FILE_URI && + object->value_.find(FILE_TYPE) != object->value_.end()) { + std::string fileType = std::get<std::string>(object->value_[FILE_TYPE]); + if (!fileType.empty()) { + typeId = fileType; + } + } +} + void UnifiedDataHelper::CalRecordSummary(std::map<std::string, ValueType> &entry, Summary &summary) { for (const auto &[utdId, value] : entry) { auto typeId = utdId; auto valueSize = ObjectUtils::GetValueSize(value, false); - if (std::holds_alternative<std::shared_ptr<Object>>(value)) { - auto object = std::get<std::shared_ptr<Object>>(value); - if (object->value_.find(APPLICATION_DEFINED_RECORD_MARK) != object->value_.end()) { - typeId = UtdUtils::GetUtdIdFromUtdEnum(APPLICATION_DEFINED_RECORD); - } - } + ProcessTypeId(value, typeId); auto it = summary.summary.find(typeId); if (it == summary.summary.end()) { summary.summary[typeId] = valueSize; diff --git a/udmf/framework/innerkitsimpl/data/unified_html_record_process.cpp b/udmf/framework/innerkitsimpl/data/unified_html_record_process.cpp new file mode 100644 index 00000000..10116871 --- /dev/null +++ b/udmf/framework/innerkitsimpl/data/unified_html_record_process.cpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "UnifiedHtmlRecordProcess" +#include "unified_html_record_process.h" + +#include <regex> + +#include "file_uri.h" +#include "html.h" +#include "logger.h" + +namespace OHOS { +namespace UDMF { +constexpr const char *IMG_TAG_PATTERN = "<img.*?>"; +constexpr const char *IMG_TAG_SRC_PATTERN = "src=(['\"])(.*?)\\1"; +constexpr const char *IMG_TAG_SRC_HEAD = "src=\""; +constexpr const char *IMG_LOCAL_URI = "file:///"; +constexpr const char *IMG_LOCAL_PATH = "://"; +constexpr const char *FILE_SCHEME_PREFIX = "file://"; + +struct Cmp { + bool operator()(const uint32_t &lhs, const uint32_t &rhs) const + { + return lhs > rhs; + } +}; + +void UnifiedHtmlRecordProcess::RebuildHtmlRecord(UnifiedData &unifiedData) +{ + LOG_DEBUG(UDMF_CLIENT, "start"); + auto utdId = UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML); + for (auto &record : unifiedData.GetRecords()) { + if (record == nullptr || record->GetUris().empty()) { + continue; + } + if (!record->HasType(utdId)) { + continue; + } + if (record->GetType() == UDType::HTML) { + auto htmlRecord = std::static_pointer_cast<Html>(record); + auto rebuildContent = RebuildHtmlContent(htmlRecord->GetHtmlContent(), record->GetUris()); + if (!rebuildContent.empty()) { + htmlRecord->SetHtmlContent(rebuildContent); + } + } else { + ProcessEntry(record); + } + } +} + +void UnifiedHtmlRecordProcess::ProcessEntry(const std::shared_ptr<UnifiedRecord> &record) +{ + if (record->GetInnerEntries() == nullptr) { + return; + } + for (auto &entry : *(record->GetInnerEntries())) { + auto udType = static_cast<UDType>(UtdUtils::GetUtdEnumFromUtdId(entry.first)); + if (udType == UDType::HTML && std::holds_alternative<std::shared_ptr<Object>>(entry.second)) { + RebuildEntry(record->GetUris(), entry.second); + } + } +} + +void UnifiedHtmlRecordProcess::RebuildEntry(const std::vector<UriInfo> &uris, const ValueType &value) +{ + auto object = std::get<std::shared_ptr<Object>>(value); + auto iter = object->value_.find(HTML_CONTENT); + if (iter != object->value_.end()) { + if (std::holds_alternative<std::string>(iter->second)) { + auto content = std::get<std::string>(iter->second); + auto rebuildContent = RebuildHtmlContent(content, uris); + if (!rebuildContent.empty()) { + object->value_[HTML_CONTENT] = rebuildContent; + } + } + } +} + +std::string UnifiedHtmlRecordProcess::RebuildHtmlContent(const std::string &str, const std::vector<UriInfo> &uris) +{ + std::map<uint32_t, std::pair<std::string, std::string>, Cmp> replaceUris; + std::string strResult = str; + for (auto &uri : uris) { + if (uri.dfsUri.empty()) { + continue; + } + std::string realUri = uri.dfsUri; + if (realUri.substr(0, strlen(FILE_SCHEME_PREFIX)) == FILE_SCHEME_PREFIX) { + AppFileService::ModuleFileUri::FileUri fileUri(uri.dfsUri); + realUri = FILE_SCHEME_PREFIX; + realUri += fileUri.GetRealPath(); + } + replaceUris[uri.position] = std::make_pair(std::move(uri.oriUri), std::move(realUri)); + } + if (replaceUris.empty()) { + return ""; + } + LOG_INFO(UDMF_CLIENT, "replaceUris size=%{public}zu", replaceUris.size()); + for (auto &replaceUri : replaceUris) { + strResult.replace(replaceUri.first, replaceUri.second.first.size(), replaceUri.second.second); + } + return strResult; +} + +void UnifiedHtmlRecordProcess::GetUriFromHtmlRecord(UnifiedData &unifiedData) +{ + LOG_DEBUG(UDMF_CLIENT, "start"); + auto utdId = UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML); + for (auto &record : unifiedData.GetRecords()) { + if (record == nullptr) { + continue; + } + if (!record->HasType(utdId)) { + continue; + } + auto htmlData = record->GetEntry(utdId); + if (std::holds_alternative<std::shared_ptr<Object>>(htmlData)) { + auto uriInfos = GetValueStr(htmlData); + if (!uriInfos.empty()) { + LOG_INFO(UDMF_CLIENT, "split uris size=%{public}zu", uriInfos.size()); + record->SetUris(std::move(uriInfos)); + } + } + } +} + +std::vector<UriInfo> UnifiedHtmlRecordProcess::GetValueStr(const ValueType &value) +{ + auto object = std::get<std::shared_ptr<Object>>(value); + auto iter = object->value_.find(HTML_CONTENT); + if (iter != object->value_.end()) { + if (std::holds_alternative<std::string>(iter->second)) { + auto content = std::get<std::string>(iter->second); + return SplitHtmlStr(content); + } + } + return {}; +} + +std::vector<UriInfo> UnifiedHtmlRecordProcess::SplitHtmlStr(const std::string &htmlContent) +{ + std::vector<std::pair<std::string, uint32_t>> matchs = SplitHtmlWithImgLabel(htmlContent); + if (matchs.empty()) { + return {}; + } + LOG_INFO(UDMF_CLIENT, "matchs size=%{public}zu", matchs.size()); + return SplitHtmlWithImgSrcLabel(matchs); +} + +std::vector<std::pair<std::string, uint32_t>> UnifiedHtmlRecordProcess::SplitHtmlWithImgLabel( + const std::string &htmlContent) noexcept +{ + std::smatch match; + std::string pattern(IMG_TAG_PATTERN); + std::regex reg(pattern); + std::string::const_iterator iterStart = htmlContent.begin(); + std::string::const_iterator iterEnd = htmlContent.end(); + std::vector<std::pair<std::string, uint32_t>> matchs; + while (std::regex_search(iterStart, iterEnd, match, reg)) { + std::string tmp = match[0]; + iterStart = match[0].second; + uint32_t position = static_cast<uint32_t>(match[0].first - htmlContent.begin()); + matchs.emplace_back(tmp, position); + } + return matchs; +} + +std::vector<UriInfo> UnifiedHtmlRecordProcess::SplitHtmlWithImgSrcLabel( + const std::vector<std::pair<std::string, uint32_t>> &matchs) noexcept +{ + std::vector<UriInfo> splitResult; + std::smatch match; + std::string pattern(IMG_TAG_SRC_PATTERN); + std::regex reg(pattern); + for (const auto &iter : matchs) { + std::string::const_iterator iterStart = iter.first.begin(); + std::string::const_iterator iterEnd = iter.first.end(); + while (std::regex_search(iterStart, iterEnd, match, reg)) { + std::string tmp = match[0]; + iterStart = match[0].second; + uint32_t position = static_cast<uint32_t>(match[0].first - iter.first.begin()); + tmp = tmp.substr(strlen(IMG_TAG_SRC_HEAD)); + tmp.pop_back(); + if (!IsLocalURI(tmp)) { + continue; + } + position += strlen(IMG_TAG_SRC_HEAD) + iter.second; + UriInfo uriInfo = { + .oriUri = tmp, + .position = position, + }; + splitResult.push_back(std::move(uriInfo)); + } + } + return splitResult; +} + +bool UnifiedHtmlRecordProcess::IsLocalURI(const std::string &uri) noexcept +{ + return uri.substr(0, strlen(IMG_LOCAL_URI)) == std::string(IMG_LOCAL_URI) || + uri.find(IMG_LOCAL_PATH) == std::string::npos; +} + +} // namespace UDMF +} // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/data/unified_record.cpp b/udmf/framework/innerkitsimpl/data/unified_record.cpp index 5cb72c1a..5e6bb685 100644 --- a/udmf/framework/innerkitsimpl/data/unified_record.cpp +++ b/udmf/framework/innerkitsimpl/data/unified_record.cpp @@ -18,11 +18,17 @@ #include "file.h" #include "getter_system.h" #include "logger.h" +#include "utd_client.h" namespace OHOS { namespace UDMF { static constexpr UDType FILE_TYPES[] = {FILE, AUDIO, FOLDER, IMAGE, VIDEO}; static constexpr const char *FILE_SCHEME = "file"; +static const std::set<std::string> FILE_SUB_TYPES = { + "general.image", "general.video", "general.audio", "general.folder" }; +static constexpr UDType UDC_RECORDS[] = {APPLICATION_DEFINED_RECORD, AUDIO, FILE, FOLDER, HTML, IMAGE, HYPERLINK, + PLAIN_TEXT, SYSTEM_DEFINED_APP_ITEM, SYSTEM_DEFINED_FORM, SYSTEM_DEFINED_PIXEL_MAP, SYSTEM_DEFINED_RECORD, + TEXT, VIDEO}; UnifiedRecord::UnifiedRecord() { @@ -42,6 +48,9 @@ UnifiedRecord::UnifiedRecord(UDType type, ValueType value) value_ = value; if (std::holds_alternative<std::shared_ptr<Object>>(value_)) { hasObject_ = true; + if (utdId_ == GENERAL_FILE_URI) { + ObjectUtils::ProcessFileUriType(dataType_, value_); + } } } @@ -117,11 +126,14 @@ void UnifiedRecord::AddEntry(const std::string &utdId, ValueType &&value) utdId_ = utdId; value_ = std::move(value); auto udType = static_cast<UDType>(UtdUtils::GetUtdEnumFromUtdId(utdId_)); - if (udType != UD_BUTT) { - dataType_ = udType; - } else { + if (udType == UD_BUTT) { dataType_ = APPLICATION_DEFINED_RECORD; + return; } + if (utdId == GENERAL_FILE_URI && std::holds_alternative<std::shared_ptr<Object>>(value_)) { + ObjectUtils::ProcessFileUriType(udType, value_); + } + dataType_ = udType; } else { entries_->insert_or_assign(utdId, std::move(value)); } @@ -131,6 +143,10 @@ ValueType UnifiedRecord::GetEntry(const std::string &utdId) { std::lock_guard<std::recursive_mutex> lock(mutex_); if (utdId_ == utdId && !(std::holds_alternative<std::monostate>(value_))) { + if (!std::holds_alternative<std::shared_ptr<Object>>(value_) + && std::find(std::begin(UDC_RECORDS), std::end(UDC_RECORDS), dataType_) != std::end(UDC_RECORDS)) { + InitObject(); + } return value_; } auto it = entries_->find(utdId); @@ -147,6 +163,16 @@ ValueType UnifiedRecord::GetEntry(const std::string &utdId) AddEntry(utdId, ValueType(value)); return value; } + if (utdId_ == utdId && std::holds_alternative<std::monostate>(value_)) { + InitObject(); + auto obj = std::get<std::shared_ptr<Object>>(value_); + if (obj->value_.size() == 1) { // value_ size equals 1 means there are no datas + value_ = obj->value_[VALUE_TYPE]; + return std::monostate(); + } else { + return value_; + } + } return std::monostate(); } @@ -194,9 +220,17 @@ std::set<std::string> UnifiedRecord::GetUtdIds() const std::set<std::string> utdIds; if (!utdId_.empty()) { utdIds.emplace(utdId_); + if (utdId_ == GENERAL_FILE_URI && std::holds_alternative<std::shared_ptr<Object>>(value_)) { + auto fileUri = std::get<std::shared_ptr<Object>>(value_); + AddFileUriType(utdIds, fileUri); + } } for (const auto& [key, value] : *entries_) { utdIds.emplace(key); + if (key == GENERAL_FILE_URI && std::holds_alternative<std::shared_ptr<Object>>(value)) { + auto fileUri = std::get<std::shared_ptr<Object>>(value); + AddFileUriType(utdIds, fileUri); + } } return utdIds; } @@ -314,5 +348,42 @@ void UnifiedRecord::SetFileUri(const std::string &fileUri) } } +std::vector<UriInfo> UnifiedRecord::GetUris() const +{ + return uris_; +} + +void UnifiedRecord::SetUris(std::vector<UriInfo> uris) +{ + uris_ = std::move(uris); +} + +void UnifiedRecord::ClearUris() +{ + uris_.clear(); +} + +void UnifiedRecord::ComputeUris(const std::function<bool(UriInfo &)> &action) +{ + for (auto &uri : uris_) { + if (!action(uri)) { + break; + } + } +} + +void UnifiedRecord::AddFileUriType(std::set<std::string> &utdIds, const std::shared_ptr<Object> &fileUri) const +{ + if (fileUri == nullptr) { + return; + } + std::string uniformDataType; + if (fileUri->GetValue(UNIFORM_DATA_TYPE, uniformDataType) && uniformDataType == GENERAL_FILE_URI) { + std::string fileType; + if (fileUri->GetValue(FILE_TYPE, fileType) && !fileType.empty()) { + utdIds.emplace(fileType); + } + } +} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/data/video.cpp b/udmf/framework/innerkitsimpl/data/video.cpp index afb99239..30dde4e3 100644 --- a/udmf/framework/innerkitsimpl/data/video.cpp +++ b/udmf/framework/innerkitsimpl/data/video.cpp @@ -20,11 +20,13 @@ namespace UDMF { Video::Video() : Video("") { SetType(VIDEO); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(VIDEO); } Video::Video(const std::string &uri) : File(uri) { SetType(VIDEO); + utdId_ = UtdUtils::GetUtdIdFromUtdEnum(VIDEO); } Video::Video(UDType type, ValueType value) : File(type, value) diff --git a/udmf/framework/innerkitsimpl/service/distributeddata_udmf_ipc_interface_code.h b/udmf/framework/innerkitsimpl/service/distributeddata_udmf_ipc_interface_code.h index 57f781c5..4d5e1237 100644 --- a/udmf/framework/innerkitsimpl/service/distributeddata_udmf_ipc_interface_code.h +++ b/udmf/framework/innerkitsimpl/service/distributeddata_udmf_ipc_interface_code.h @@ -34,7 +34,6 @@ enum class UdmfServiceInterfaceCode : uint32_t { REMOVE_APP_SHARE_OPTION, OBTAIN_ASYN_PROCESS, CLEAR_ASYN_PROCESS_BY_KEY, - INVOKEN_HAP, CODE_BUTT }; } // namespace OHOS::UDMF diff --git a/udmf/framework/innerkitsimpl/service/udmf_service.h b/udmf/framework/innerkitsimpl/service/udmf_service.h index 4a1ac7d7..1f685fcf 100644 --- a/udmf/framework/innerkitsimpl/service/udmf_service.h +++ b/udmf/framework/innerkitsimpl/service/udmf_service.h @@ -51,7 +51,6 @@ public: virtual int32_t RemoveAppShareOption(const std::string &intention) = 0; virtual int32_t ObtainAsynProcess(AsyncProcessInfo &processInfo) = 0; virtual int32_t ClearAsynProcessByKey(const std::string &businessUdKey) = 0; - virtual int32_t InvokeHap(const std::string &progressKey, const sptr<IRemoteObject> &observer) = 0; }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/framework/innerkitsimpl/service/udmf_service_client.cpp b/udmf/framework/innerkitsimpl/service/udmf_service_client.cpp index af11fc83..809e5816 100644 --- a/udmf/framework/innerkitsimpl/service/udmf_service_client.cpp +++ b/udmf/framework/innerkitsimpl/service/udmf_service_client.cpp @@ -296,10 +296,5 @@ int32_t UdmfServiceClient::ClearAsynProcessByKey(const std::string &businessUdKe { return udmfProxy_->ClearAsynProcessByKey(businessUdKey); } - -int32_t UdmfServiceClient::InvokeHap(const std::string &progressKey, const sptr<IRemoteObject> &observer) -{ - return udmfProxy_->InvokeHap(progressKey, observer); -} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/service/udmf_service_client.h b/udmf/framework/innerkitsimpl/service/udmf_service_client.h index d702cf4d..09969676 100644 --- a/udmf/framework/innerkitsimpl/service/udmf_service_client.h +++ b/udmf/framework/innerkitsimpl/service/udmf_service_client.h @@ -46,7 +46,6 @@ public: int32_t RemoveAppShareOption(const std::string &intention) override; int32_t ObtainAsynProcess(AsyncProcessInfo& processInfo) override; int32_t ClearAsynProcessByKey(const std::string &businessUdKey) override; - int32_t InvokeHap(const std::string &progressKey, const sptr<IRemoteObject> &observer) override; private: class ServiceDeathRecipient : public IRemoteObject::DeathRecipient { diff --git a/udmf/framework/innerkitsimpl/service/udmf_service_proxy.cpp b/udmf/framework/innerkitsimpl/service/udmf_service_proxy.cpp index 95b614a0..b34f6cb5 100644 --- a/udmf/framework/innerkitsimpl/service/udmf_service_proxy.cpp +++ b/udmf/framework/innerkitsimpl/service/udmf_service_proxy.cpp @@ -261,16 +261,5 @@ int32_t UdmfServiceProxy::ClearAsynProcessByKey(const std::string &businessUdKey } return E_OK; } - -int32_t UdmfServiceProxy::InvokeHap(const std::string &progressKey, const sptr<IRemoteObject> &observer) -{ - MessageParcel reply; - int32_t status = IPC_SEND(UdmfServiceInterfaceCode::INVOKEN_HAP, reply, progressKey, observer); - if (status != E_OK) { - LOG_ERROR(UDMF_SERVICE, "status:0x%{public}x", status); - return status; - } - return E_OK; -} } // namespace UDMF } // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/service/udmf_service_proxy.h b/udmf/framework/innerkitsimpl/service/udmf_service_proxy.h index 3f6f5d60..26d01fd9 100644 --- a/udmf/framework/innerkitsimpl/service/udmf_service_proxy.h +++ b/udmf/framework/innerkitsimpl/service/udmf_service_proxy.h @@ -49,7 +49,6 @@ public: int32_t RemoveAppShareOption(const std::string &intention) override; int32_t ObtainAsynProcess(AsyncProcessInfo &processInfo) override; int32_t ClearAsynProcessByKey(const std::string &businessUdKey) override; - int32_t InvokeHap(const std::string &progressKey, const sptr<IRemoteObject> &observer) override; private: static inline BrokerDelegator<UdmfServiceProxy> delegator_; int32_t SendRequest(UdmfServiceInterfaceCode code, MessageParcel &data, diff --git a/udmf/framework/innerkitsimpl/test/fuzztest/udmfclient_fuzzer/udmf_client_fuzzer.cpp b/udmf/framework/innerkitsimpl/test/fuzztest/udmfclient_fuzzer/udmf_client_fuzzer.cpp index f9e9495e..8100bc2a 100644 --- a/udmf/framework/innerkitsimpl/test/fuzztest/udmfclient_fuzzer/udmf_client_fuzzer.cpp +++ b/udmf/framework/innerkitsimpl/test/fuzztest/udmfclient_fuzzer/udmf_client_fuzzer.cpp @@ -41,6 +41,8 @@ using namespace OHOS::Security::AccessToken; using namespace OHOS::UDMF; namespace OHOS { +static constexpr int END_INTERVAL = 3; + void AllocHapToken() { HapInfoParams info = { @@ -86,6 +88,7 @@ void SetUpTestCase() void TearDown() { + std::this_thread::sleep_for(std::chrono::seconds(END_INTERVAL)); } void SetNativeToken() @@ -622,7 +625,6 @@ void StartAsyncDataRetrievalFuzz(const uint8_t *data, size_t size) params.progressIndicator = ProgressIndicator::DEFAULT; params.progressListener = [](ProgressInfo progressInfo, std::shared_ptr<UnifiedData> data) {}; UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); - std::this_thread::sleep_for(std::chrono::seconds(1)); } void CancelAsyncDataRetrievalFuzz(const uint8_t *data, size_t size) @@ -644,7 +646,6 @@ void CancelAsyncDataRetrievalFuzz(const uint8_t *data, size_t size) }; UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); UdmfAsyncClient::GetInstance().Cancel(key); - std::this_thread::sleep_for(std::chrono::seconds(1)); } } diff --git a/udmf/framework/innerkitsimpl/test/unittest/BUILD.gn b/udmf/framework/innerkitsimpl/test/unittest/BUILD.gn index 8a7fcd32..6e4d149b 100644 --- a/udmf/framework/innerkitsimpl/test/unittest/BUILD.gn +++ b/udmf/framework/innerkitsimpl/test/unittest/BUILD.gn @@ -24,11 +24,10 @@ config("module_private_config") { "${udmf_interfaces_path}/innerkits/convert", "${udmf_interfaces_path}/ndk/data", "${udmf_framework_path}/common", - "${udmf_framework_path}/innerkits/service", + "${udmf_framework_path}/innerkitsimpl/data", "${udmf_framework_path}/innerkitsimpl/client/", "${udmf_framework_path}/innerkitsimpl/test/unittest/mock/include", "${udmf_framework_path}/ndkimpl/data", - "${kv_store_path}/frameworks/common", ] } @@ -41,9 +40,11 @@ common_deps = [ common_external_deps = [ "ability_base:base", "ability_base:want", + "ability_base:zuri", "access_token:libaccesstoken_sdk", "access_token:libnativetoken", "access_token:libtoken_setproc", + "app_file_service:fileuri_native", "bundle_framework:appexecfwk_core", "c_utils:utils", "ffmpeg:libohosffmpeg", @@ -67,7 +68,25 @@ ohos_unittest("UdmfClientTest") { sources = [ "${udmf_framework_path}/common/graph.cpp", + "${udmf_framework_path}/common/udmf_utils.cpp", + "${udmf_framework_path}/innerkitsimpl/client/udmf_client.cpp", "${udmf_framework_path}/innerkitsimpl/convert/udmf_conversion.cpp", + "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/audio.cpp", + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/folder.cpp", + "${udmf_framework_path}/innerkitsimpl/data/html.cpp", + "${udmf_framework_path}/innerkitsimpl/data/image.cpp", + "${udmf_framework_path}/innerkitsimpl/data/link.cpp", + "${udmf_framework_path}/innerkitsimpl/data/plain_text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_appitem.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_form.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_pixelmap.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_data.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "${udmf_framework_path}/innerkitsimpl/service/udmf_service_client.cpp", "${udmf_framework_path}/innerkitsimpl/service/udmf_service_proxy.cpp", "udmf_client_test.cpp", ] @@ -141,8 +160,17 @@ ohos_unittest("UtdClientTest") { module_out_path = module_output_path sources = [ + "${udmf_framework_path}/common/base32_utils.cpp", + "${udmf_framework_path}/common/custom_utd_json_parser.cpp", + "${udmf_framework_path}/common/custom_utd_store.cpp", "${udmf_framework_path}/common/graph.cpp", + "${udmf_framework_path}/common/udmf_utils.cpp", + "${udmf_framework_path}/common/utd_cfgs_checker.cpp", "${udmf_framework_path}/common/utd_graph.cpp", + "${udmf_framework_path}/innerkitsimpl/client/utd_client.cpp", + "${udmf_framework_path}/innerkitsimpl/data/flexible_type.cpp", + "${udmf_framework_path}/innerkitsimpl/data/preset_type_descriptors.cpp", + "${udmf_framework_path}/innerkitsimpl/data/type_descriptor.cpp", "custom_utd_json_parser_test.cpp", "custom_utd_store_test.cpp", "graph_test.cpp", @@ -151,9 +179,13 @@ ohos_unittest("UtdClientTest") { configs = [ ":module_private_config" ] - deps = [ "${udmf_interfaces_path}/innerkits:utd_client" ] + use_exceptions = true external_deps = common_external_deps + external_deps += [ + "cJSON:cjson", + "os_account:os_account_innerkits", + ] defines = [ "private=public", @@ -176,7 +208,13 @@ ohos_unittest("NdkDataConversionTest") { ohos_unittest("ApplicationDefineRecordTest") { module_out_path = module_output_path - sources = [ "application_defined_record_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "application_defined_record_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -191,7 +229,14 @@ ohos_unittest("ApplicationDefineRecordTest") { ohos_unittest("AudioTest") { module_out_path = module_output_path - sources = [ "audio_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/audio.cpp", + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "audio_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -206,7 +251,11 @@ ohos_unittest("AudioTest") { ohos_unittest("FileDataTest") { module_out_path = module_output_path - sources = [ "file_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "file_test.cpp", + ] configs = [ ":module_private_config" ] @@ -223,7 +272,12 @@ ohos_unittest("FileDataTest") { ohos_unittest("FlexibleTypeTest") { module_out_path = module_output_path - sources = [ "flexible_type_test.cpp" ] + sources = [ + "${udmf_framework_path}/common/base32_utils.cpp", + "${udmf_framework_path}/common/udmf_utils.cpp", + "${udmf_framework_path}/innerkitsimpl/data/flexible_type.cpp", + "flexible_type_test.cpp", + ] deps = common_deps @@ -238,7 +292,14 @@ ohos_unittest("FlexibleTypeTest") { ohos_unittest("FolderTest") { module_out_path = module_output_path - sources = [ "folder_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/folder.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "folder_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -253,7 +314,14 @@ ohos_unittest("FolderTest") { ohos_unittest("HtmlTest") { module_out_path = module_output_path - sources = [ "html_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/html.cpp", + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "html_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -268,7 +336,14 @@ ohos_unittest("HtmlTest") { ohos_unittest("ImageTest") { module_out_path = module_output_path - sources = [ "image_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/image.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "image_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -283,7 +358,14 @@ ohos_unittest("ImageTest") { ohos_unittest("LinkTest") { module_out_path = module_output_path - sources = [ "link_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/link.cpp", + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "link_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -298,7 +380,14 @@ ohos_unittest("LinkTest") { ohos_unittest("PlainTextTest") { module_out_path = module_output_path - sources = [ "plain_text_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/plain_text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "plain_text_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -313,7 +402,14 @@ ohos_unittest("PlainTextTest") { ohos_unittest("SystemDefinedAppitemTest") { module_out_path = module_output_path - sources = [ "system_defined_appitem_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/system_defined_appitem.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "system_defined_appitem_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -328,7 +424,14 @@ ohos_unittest("SystemDefinedAppitemTest") { ohos_unittest("SystemDefinedFormTest") { module_out_path = module_output_path - sources = [ "system_defined_form_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/system_defined_form.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "system_defined_form_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -343,7 +446,14 @@ ohos_unittest("SystemDefinedFormTest") { ohos_unittest("SystemDefinedPixelMapTest") { module_out_path = module_output_path - sources = [ "system_defined_pixelmap_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/system_defined_pixelmap.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "system_defined_pixelmap_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -358,7 +468,13 @@ ohos_unittest("SystemDefinedPixelMapTest") { ohos_unittest("SystemDefinedRecordTest") { module_out_path = module_output_path - sources = [ "system_defined_record_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "system_defined_record_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -373,7 +489,13 @@ ohos_unittest("SystemDefinedRecordTest") { ohos_unittest("TextTest") { module_out_path = module_output_path - sources = [ "text_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "text_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -388,7 +510,14 @@ ohos_unittest("TextTest") { ohos_unittest("UnifiedDataTest") { module_out_path = module_output_path - sources = [ "unified_data_test.cpp" ] + sources = [ + "${udmf_framework_path}/common/udmf_utils.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_data.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "unified_data_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -418,7 +547,12 @@ ohos_unittest("UnifiedDataHelperTest") { ohos_unittest("UnifiedRecordTest") { module_out_path = module_output_path - sources = [ "unified_record_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "unified_record_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -433,7 +567,13 @@ ohos_unittest("UnifiedRecordTest") { ohos_unittest("VideoTest") { module_out_path = module_output_path - sources = [ "video_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "video_test.cpp", + ] + + configs = [ ":module_private_config" ] deps = common_deps @@ -469,6 +609,7 @@ ohos_unittest("UdmfClientAbnormalTest") { "${udmf_framework_path}/common/udmf_radar_reporter.cpp", "${udmf_framework_path}/common/udmf_types_util.cpp", "${udmf_framework_path}/common/udmf_utils.cpp", + "${udmf_framework_path}/innerkitsimpl/client/getter_system.cpp", "${udmf_framework_path}/innerkitsimpl/client/udmf_client.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_key.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", @@ -486,6 +627,7 @@ ohos_unittest("UdmfClientAbnormalTest") { "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/text.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_data.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_html_record_process.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/video.cpp", "${udmf_framework_path}/innerkitsimpl/service/udmf_service_proxy.cpp", diff --git a/udmf/framework/innerkitsimpl/test/unittest/udmf_async_client_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/udmf_async_client_test.cpp new file mode 100644 index 00000000..9ad5e05d --- /dev/null +++ b/udmf/framework/innerkitsimpl/test/unittest/udmf_async_client_test.cpp @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "UdmfAsyncClientTest" +#include <gtest/gtest.h> + +#include "token_setproc.h" +#include "accesstoken_kit.h" +#include "nativetoken_kit.h" + +#include "image.h" +#include "logger.h" +#include "plain_text.h" +#include "udmf.h" +#include "udmf_async_client.h" +#include "udmf_client.h" +#include "async_task_params.h" +#include "data_params_conversion.h" + +using namespace testing::ext; +using namespace OHOS::Security::AccessToken; +using namespace OHOS::UDMF; +using namespace OHOS; +namespace OHOS::Test { +static constexpr int USER_ID = 100; +static constexpr int END_INTERVAL = 2; +class UdmfAsyncClientTest : public testing::Test { +public: + Status SetDataTest(std::string key, ProgressIndicator progressIndicator); + + static void SetUpTestCase(); + static void TearDownTestCase(); + void SetUp() override; + void TearDown() override; + + static void AllocHapToken(); +}; + +void UdmfAsyncClientTest::SetUpTestCase() +{ + AllocHapToken(); +} + +void UdmfAsyncClientTest::TearDownTestCase() +{ + std::this_thread::sleep_for(std::chrono::seconds(END_INTERVAL)); + std::vector<UnifiedData> unifiedDataSet; + QueryOption query = { + .intention = UDMF::UD_INTENTION_DATA_HUB + }; + auto status = UdmfClient::GetInstance().DeleteData(query, unifiedDataSet); + EXPECT_EQ(E_OK, status); + auto tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.asyncdemo", 0); + AccessTokenKit::DeleteToken(tokenId); +} + +void UdmfAsyncClientTest::SetUp() {} + +void UdmfAsyncClientTest::TearDown() {} + + +void UdmfAsyncClientTest::AllocHapToken() +{ + HapInfoParams info = { + .userID = USER_ID, + .bundleName = "ohos.test.asyncdemo", + .instIndex = 0, + .appIDDesc = "ohos.test.asyncdemo", + .isSystemApp = false, + }; + HapPolicyParams policy = { + .apl = APL_SYSTEM_BASIC, + .domain = "test.domain", + .permList = { + { + .permissionName = "ohos.permission.MANAGE_UDMF_APP_SHARE_OPTION", + .bundleName = "ohos.test.asyncdemo", + .grantMode = 1, + .availableLevel = APL_SYSTEM_BASIC, + .label = "label", + .labelId = 1, + .description = "test2", + .descriptionId = 1 + } + }, + .permStateList = { + { + .permissionName = "ohos.permission.MANAGE_UDMF_APP_SHARE_OPTION", + .isGeneral = true, + .resDeviceID = { "local" }, + .grantStatus = { PermissionState::PERMISSION_GRANTED }, + .grantFlags = { 1 } + } + } + }; + auto tokenID = AccessTokenKit::AllocHapToken(info, policy); + SetSelfTokenID(tokenID.tokenIDEx); +} + +/* * + * @tc.name: StartAsyncDataRetrieval001 + * @tc.desc: Test get data success. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval001 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(E_OK, status); + + GetDataParams params; + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (data == nullptr) { + ASSERT_TRUE(progress.progress != 0); + return; + } + ASSERT_EQ(1, data->GetRecords().size()); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval001 callback end."); + }; + params.query = query; + params.progressIndicator = ProgressIndicator::NONE; + params.progressListener = callback; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval001 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval002 + * @tc.desc: Test CAPI get data success. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval002, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval002 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(E_OK, status); + + OH_UdmfGetDataParams param; + OH_UdmfGetDataParams_SetProgressIndicator(&param, Udmf_ProgressIndicator::UDMF_DEFAULT); + OH_UdmfGetDataParams_SetDestUri(&param, "/test/demo"); + OH_UdmfGetDataParams_SetFileConflictOptions(&param, Udmf_FileConflictOptions::UDMF_SKIP); + OH_Udmf_DataProgressListener dataProgressListener = [](OH_Udmf_ProgressInfo *progressInfo, OH_UdmfData *data) { + auto progress = OH_UdmfProgressInfo_GetProgress(progressInfo); + auto status = OH_UdmfProgressInfo_GetStatus(progressInfo); + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d.", status, progress); + if (data == nullptr) { + ASSERT_TRUE(progress != 0); + return; + } + unsigned int count = 0; + OH_UdmfData_GetRecords(data, &count); + ASSERT_EQ(1, count); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval002 callback end."); + }; + OH_UdmfGetDataParams_SetDataProgressListener(&param, dataProgressListener); + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + GetDataParams dataParams; + status = DataParamsConversion::GetInnerDataParams(param, query, dataParams); + ASSERT_EQ(E_OK, status); + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(dataParams); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval002 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval003 + * @tc.desc: Test get cache data success. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval003, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval003 begin."); + auto status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::IN_APP); + ASSERT_EQ(E_OK, status); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key; + status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(E_OK, status); + + GetDataParams params; + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (data == nullptr) { + ASSERT_TRUE(progress.progress != 0); + return; + } + ASSERT_EQ(1, data->GetRecords().size()); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval003 callback end."); + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + params.progressListener = callback; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_OK, status); + + status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval003 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval004 + * @tc.desc: Test get data cost 1s. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval004, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval004 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(E_OK, status); + + GetDataParams params; + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (data == nullptr) { + ASSERT_TRUE(progress.progress != 0); + return; + } + ASSERT_EQ(1, data->GetRecords().size()); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval003 callback end."); + }; + params.query = query; + params.progressIndicator = ProgressIndicator::NONE; + params.progressListener = callback; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + ASSERT_EQ(E_OK, status); + + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval004 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval005 + * @tc.desc: Test get no data. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval005, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval005 begin."); + + GetDataParams params; + QueryOption query = { + .key = "udmf://drag/com.demo/NotFoundData", + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (progress.progressStatus == 4) { + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval005 demo1 callback end."); + return; + } + }; + params.query = query; + params.progressIndicator = ProgressIndicator::NONE; + params.progressListener = callback; + auto status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval005 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval006 + * @tc.desc: Test invalid key. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval006, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval006 begin."); + + GetDataParams params; + QueryOption query = { + .key = "InvalidKey", + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (progress.progressStatus == 3) { + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval006 demo2 callback end."); + return; + } + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + params.progressListener = callback; + auto status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval006 end."); +} + +Status UdmfAsyncClientTest::SetDataTest(std::string key, ProgressIndicator progressIndicator) +{ + GetDataParams params; + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (data == nullptr) { + ASSERT_TRUE(progress.progress != 0); + return; + } + }; + params.query = query; + params.progressIndicator = progressIndicator; + params.progressListener = callback; + return UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); +} + +/* * + * @tc.name: StartAsyncDataRetrieval007 + * @tc.desc: Test multithreading different key. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval007, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval007 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key1; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key1); + ASSERT_EQ(E_OK, status); + + std::string key2; + customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + status = UdmfClient::GetInstance().SetData(customOption, data, key2); + ASSERT_EQ(E_OK, status); + + std::string key3; + customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + status = UdmfClient::GetInstance().SetData(customOption, data, key3); + ASSERT_EQ(E_OK, status); + + std::thread t1([&]() { + auto status = SetDataTest(key1, ProgressIndicator::DEFAULT); + ASSERT_EQ(E_OK, status); + }); + EXPECT_NO_FATAL_FAILURE(t1.join()); + std::thread t2([&]() { + auto status = SetDataTest(key2, ProgressIndicator::NONE); + ASSERT_EQ(E_OK, status); + }); + EXPECT_NO_FATAL_FAILURE(t2.join()); + std::thread t3([&]() { + auto status = SetDataTest(key3, ProgressIndicator::DEFAULT); + ASSERT_EQ(E_OK, status); + }); + EXPECT_NO_FATAL_FAILURE(t3.join()); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval007 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval008 + * @tc.desc: Test multithreading same key. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval008, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval008 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto plainText = std::make_shared<PlainText>(UDType::PLAIN_TEXT, obj); + plainText->SetContent("content1"); + data.AddRecord(plainText); + std::string key1; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key1); + ASSERT_EQ(E_OK, status); + + std::thread t1([&]() { + auto status = SetDataTest(key1, ProgressIndicator::DEFAULT); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + ASSERT_EQ(E_OK, status); + }); + EXPECT_NO_FATAL_FAILURE(t1.join()); + std::thread t2([&]() { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + auto status = SetDataTest(key1, ProgressIndicator::NONE); + ASSERT_EQ(E_IDEMPOTENT_ERROR, status); + }); + EXPECT_NO_FATAL_FAILURE(t2.join()); + std::thread t3([&]() { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + auto status = SetDataTest(key1, ProgressIndicator::DEFAULT); + ASSERT_EQ(E_IDEMPOTENT_ERROR, status); + }); + EXPECT_NO_FATAL_FAILURE(t3.join()); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval008 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval009 + * @tc.desc: Test Get File type. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval009, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval009 begin."); + + CustomOption customOption = { + .intention = UDMF::UD_INTENTION_DRAG + }; + UnifiedData data; + auto obj = std::make_shared<Object>(); + auto file = std::make_shared<Image>(UDType::IMAGE, obj); + file->SetUri("uri"); + file->SetRemoteUri("remoteUri"); + data.AddRecord(file); + std::string key; + auto status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(E_OK, status); + + GetDataParams params; + QueryOption query = { + .key = key, + .intention = UD_INTENTION_DRAG, + }; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) { + LOG_INFO(UDMF_TEST, "Callback begin status=%{public}d, progress=%{public}d, name=%{public}s.", + progress.progressStatus, progress.progress, progress.srcDevName.c_str()); + if (data == nullptr) { + ASSERT_TRUE(progress.progress != 0); + return; + } + ASSERT_EQ(1, data->GetRecords().size()); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval009 callback end."); + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + params.progressListener = callback; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_OK, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval009 end."); +} + +/* * + * @tc.name: StartAsyncDataRetrieval010 + * @tc.desc: Test Invalid params. + * @tc.type: FUNC + */ +HWTEST_F(UdmfAsyncClientTest, StartAsyncDataRetrieval010, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval010 begin."); + + GetDataParams params; + QueryOption query = { + .intention = UDMF::UD_INTENTION_DRAG + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + auto callback = [this](ProgressInfo progress, std::shared_ptr<UnifiedData> data) {}; + params.progressListener = callback; + auto status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_INVALID_PARAMETERS, status); + + query = { + .key = "udmf://a/b/c", + .intention = UDMF::UD_INTENTION_DATA_HUB, + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + params.progressListener = callback; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_INVALID_PARAMETERS, status); + + query = { + .key = "udmf://a/b/c", + .intention = UDMF::UD_INTENTION_DRAG, + }; + params.query = query; + params.progressIndicator = ProgressIndicator::DEFAULT; + params.progressListener = nullptr; + status = UdmfAsyncClient::GetInstance().StartAsyncDataRetrieval(params); + ASSERT_EQ(E_INVALID_PARAMETERS, status); + LOG_INFO(UDMF_TEST, "StartAsyncDataRetrieval010 end."); +} +} // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/test/unittest/udmf_client_hap_permission_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_hap_permission_test.cpp new file mode 100644 index 00000000..bbc76a3c --- /dev/null +++ b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_hap_permission_test.cpp @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "UdmfClientHapPermissionTest" +#include <gtest/gtest.h> + +#include <unistd.h> +#include <thread> +#include <chrono> + +#include "token_setproc.h" +#include "accesstoken_kit.h" +#include "directory_ex.h" +#include "nativetoken_kit.h" + +#include "logger.h" +#include "udmf_client.h" +#include "application_defined_record.h" +#include "audio.h" +#include "file.h" +#include "folder.h" +#include "html.h" +#include "image.h" +#include "link.h" +#include "plain_text.h" +#include "system_defined_appitem.h" +#include "system_defined_form.h" +#include "system_defined_pixelmap.h" +#include "system_defined_record.h" +#include "text.h" +#include "unified_data_helper.h" +#include "video.h" + +using namespace testing::ext; +using namespace OHOS::Security::AccessToken; +using namespace OHOS::UDMF; +using namespace OHOS; +namespace OHOS::Test { +static constexpr int USER_ID = 100; +static constexpr int INST_INDEX = 0; +class UdmfClientHapPermissionTest : public testing::Test { +public: + static void SetUpTestCase(); + static void TearDownTestCase(); + void SetUp() override; + void TearDown() override; + + void SetNativeToken(const std::string &processName); + static void AllocHapToken1(); + static void AllocHapToken2(); + void SetHapToken1(); + void SetHapToken2(); + void AddPrivilege(QueryOption &option); +}; + +void UdmfClientHapPermissionTest::SetUpTestCase() +{ + AllocHapToken1(); + AllocHapToken2(); +} + +void UdmfClientHapPermissionTest::TearDownTestCase() +{ + auto tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo11", INST_INDEX); + AccessTokenKit::DeleteToken(tokenId); + tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo22", INST_INDEX); + AccessTokenKit::DeleteToken(tokenId); +} + +void UdmfClientHapPermissionTest::SetUp() +{ + SetHapToken1(); +} + +void UdmfClientHapPermissionTest::TearDown() +{ +} + +void UdmfClientHapPermissionTest::SetNativeToken(const std::string &processName) +{ + auto tokenId = AccessTokenKit::GetNativeTokenId(processName); + SetSelfTokenID(tokenId); +} + +void UdmfClientHapPermissionTest::AllocHapToken1() +{ + HapInfoParams info = { + .userID = USER_ID, + .bundleName = "ohos.test.demo11", + .instIndex = INST_INDEX, + .appIDDesc = "ohos.test.demo11", + .isSystemApp = false, + }; + + HapPolicyParams policy = { + .apl = APL_SYSTEM_BASIC, + .domain = "test.domain", + .permList = { + { + .permissionName = "ohos.permission.test", + .bundleName = "ohos.test.demo11", + .grantMode = 1, + .availableLevel = APL_SYSTEM_BASIC, + .label = "label", + .labelId = 1, + .description = "test1", + .descriptionId = 1 + } + }, + .permStateList = { + { + .permissionName = "ohos.permission.test", + .isGeneral = true, + .resDeviceID = { "local" }, + .grantStatus = { PermissionState::PERMISSION_GRANTED }, + .grantFlags = { 1 } + } + } + }; + auto tokenID = AccessTokenKit::AllocHapToken(info, policy); + SetSelfTokenID(tokenID.tokenIDEx); +} + +void UdmfClientHapPermissionTest::AllocHapToken2() +{ + HapInfoParams info = { + .userID = USER_ID, + .bundleName = "ohos.test.demo22", + .instIndex = 0, + .appIDDesc = "ohos.test.demo22", + .isSystemApp = false, + }; + HapPolicyParams policy = { + .apl = APL_SYSTEM_BASIC, + .domain = "test.domain", + .permList = { + { + .permissionName = "ohos.permission.MANAGE_UDMF_APP_SHARE_OPTION", + .bundleName = "ohos.test.demo22", + .grantMode = 1, + .availableLevel = APL_SYSTEM_BASIC, + .label = "label", + .labelId = 1, + .description = "test2", + .descriptionId = 1 + } + }, + .permStateList = { + { + .permissionName = "ohos.permission.MANAGE_UDMF_APP_SHARE_OPTION", + .isGeneral = true, + .resDeviceID = { "local" }, + .grantStatus = { PermissionState::PERMISSION_GRANTED }, + .grantFlags = { 1 } + } + } + }; + auto tokenID = AccessTokenKit::AllocHapToken(info, policy); + SetSelfTokenID(tokenID.tokenIDEx); +} + +void UdmfClientHapPermissionTest::SetHapToken1() +{ + auto tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo11", INST_INDEX); + SetSelfTokenID(tokenId); +} + +void UdmfClientHapPermissionTest::SetHapToken2() +{ + auto tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo22", INST_INDEX); + SetSelfTokenID(tokenId); +} + +void UdmfClientHapPermissionTest::AddPrivilege(QueryOption &option) +{ + Privilege privilege; + privilege.tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo22", INST_INDEX); + privilege.readPermission = "readPermission"; + privilege.writePermission = "writePermission"; + SetNativeToken("msdp_sa"); + auto status = UdmfClient::GetInstance().AddPrivilege(option, privilege); + ASSERT_EQ(status, E_OK); +} + +/** +* @tc.name: SetAppShareOption001 +* @tc.desc: abnormal test, no permission +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, SetAppShareOption001, TestSize.Level1) +{ + SetHapToken1(); + LOG_INFO(UDMF_TEST, "SetAppShareOption001 begin."); + auto status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::IN_APP); + EXPECT_EQ(status, E_NO_PERMISSION); + LOG_INFO(UDMF_TEST, "SetAppShareOption001 end."); +} + +/** +* @tc.name: SetAppShareOption002 +* @tc.desc: normal test, has permission +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, SetAppShareOption002, TestSize.Level1) +{ + SetHapToken2(); + LOG_INFO(UDMF_TEST, "SetAppShareOption002 begin."); + auto status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::IN_APP); + EXPECT_EQ(status, E_OK); + + ShareOptions appShareOptions; + status = UdmfClient::GetInstance().GetAppShareOption("drag", appShareOptions); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(appShareOptions, ShareOptions::IN_APP); + + status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_OK); + + status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::CROSS_APP); + EXPECT_EQ(status, E_OK); + + status = UdmfClient::GetInstance().GetAppShareOption("drag", appShareOptions); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(appShareOptions, ShareOptions::CROSS_APP); + + status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_OK); + + status = UdmfClient::GetInstance().GetAppShareOption("drag", appShareOptions); + EXPECT_EQ(status, E_NOT_FOUND); + LOG_INFO(UDMF_TEST, "SetAppShareOption002 end."); +} + +/** +* @tc.name: SetAppShareOption003 +* @tc.desc: SetAppShareOption IN_APP, get data success when IN_APP +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, SetAppShareOption003, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetAppShareOption003 begin."); + SetHapToken2(); + auto status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::IN_APP); + EXPECT_EQ(status, E_OK); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data1; + auto text1 = std::make_shared<Text>(); + data1.AddRecord(text1); + std::string key; + status = UdmfClient::GetInstance().SetData(option1, data1, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + SetHapToken2(); + UnifiedData data2; + status = UdmfClient::GetInstance().GetData(option2, data2); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> record2 = data2.GetRecordAt(0); + ASSERT_NE(record2, nullptr); + auto type = record2->GetType(); + EXPECT_EQ(type, UDType::TEXT); + + auto text2 = static_cast<Text *>(record2.get()); + ASSERT_NE(text2, nullptr); + + status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_OK); + LOG_INFO(UDMF_TEST, "SetAppShareOption003 end."); +} + +/** +* @tc.name: SetAppShareOption004 +* @tc.desc: SetAppShareOption CROSS_APP, get data success when CROSS_APP +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, SetAppShareOption004, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetAppShareOption004 begin."); + SetHapToken2(); + auto status = UdmfClient::GetInstance().SetAppShareOption("drag", ShareOptions::CROSS_APP); + EXPECT_EQ(status, E_OK); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data1; + auto text1 = std::make_shared<Text>(); + data1.AddRecord(text1); + std::string key; + status = UdmfClient::GetInstance().SetData(option1, data1, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege(option2); + + SetHapToken2(); + UnifiedData data2; + status = UdmfClient::GetInstance().GetData(option2, data2); + ASSERT_EQ(status, E_OK); + std::shared_ptr<UnifiedRecord> record2 = data2.GetRecordAt(0); + ASSERT_NE(record2, nullptr); + auto type = record2->GetType(); + EXPECT_EQ(type, UDType::TEXT); + auto text2 = static_cast<Text *>(record2.get()); + ASSERT_NE(text2, nullptr); + + SetHapToken2(); + status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_OK); + LOG_INFO(UDMF_TEST, "SetAppShareOption004 end."); +} + +/** +* @tc.name: RemoveAppShareOption001 +* @tc.desc: abnormal test, no permission +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, RemoveAppShareOption001, TestSize.Level1) +{ + SetHapToken1(); + LOG_INFO(UDMF_TEST, "RemoveAppShareOption001 begin."); + auto status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_NO_PERMISSION); + LOG_INFO(UDMF_TEST, "RemoveAppShareOption001 end."); +} + +/** +* @tc.name: RemoveAppShareOption002 +* @tc.desc: normal test, has permission +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientHapPermissionTest, RemoveAppShareOption002, TestSize.Level1) +{ + SetHapToken2(); + LOG_INFO(UDMF_TEST, "RemoveAppShareOption002 begin."); + auto status = UdmfClient::GetInstance().RemoveAppShareOption("drag"); + EXPECT_EQ(status, E_OK); + + ShareOptions appShareOptions; + status = UdmfClient::GetInstance().GetAppShareOption("drag", appShareOptions); + EXPECT_EQ(status, E_NOT_FOUND); + LOG_INFO(UDMF_TEST, "RemoveAppShareOption002 end."); +} +} // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/test/unittest/udmf_client_sa_invoke_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_sa_invoke_test.cpp new file mode 100644 index 00000000..edef7d2c --- /dev/null +++ b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_sa_invoke_test.cpp @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "UdmfClientSaInvokeTest" +#include <gtest/gtest.h> + +#include "token_setproc.h" +#include "accesstoken_kit.h" +#include "executor_pool.h" +#include "nativetoken_kit.h" + +#include "logger.h" +#include "image.h" +#include "plain_text.h" +#include "system_defined_pixelmap.h" +#include "udmf_client.h" + +using namespace testing::ext; +using namespace OHOS::Security::AccessToken; +using namespace OHOS::UDMF; +using namespace OHOS; +namespace OHOS::Test { +constexpr const int32_t FIVE = 5; +constexpr const int32_t TEN = 10; + +class UdmfClientSaInvokeTest : public testing::Test { +public: + + void SetDataTest(); + + static void SetUpTestCase(); + static void TearDownTestCase(); + void SetUp() override; + void TearDown() override; + + static void SetHapToken(); +}; + +void UdmfClientSaInvokeTest::SetUpTestCase() +{ + SetHapToken(); +} + +void UdmfClientSaInvokeTest::TearDownTestCase() +{ + auto tokenId = AccessTokenKit::GetNativeTokenId("msdp_sa"); + AccessTokenKit::DeleteToken(tokenId); +} + +void UdmfClientSaInvokeTest::SetUp() {} + +void UdmfClientSaInvokeTest::TearDown() {} + + +void UdmfClientSaInvokeTest::SetHapToken() +{ + auto tokenId = AccessTokenKit::GetNativeTokenId("msdp_sa"); + SetSelfTokenID(tokenId); +} + +/** +* @tc.name: SetDataTest +* @tc.desc: Test set data success. Record size more than 4M and less than 15.5M. +* @tc.type: FUNC +*/ +void UdmfClientSaInvokeTest::SetDataTest() +{ + LOG_INFO(UDMF_TEST, "SetDataTest begin."); + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < FIVE * 1024 * 1024; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + + std::vector<uint8_t> rawData2; + for (int i = 0; i < TEN * 1024 * 1024; i++) { + rawData2.emplace_back(1); + } + auto pixelMap2 = std::make_shared<SystemDefinedPixelMap>(rawData2); + data.AddRecord(pixelMap2); + auto recordSize = data.GetRecords().size(); + + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(recordSize, readData.GetRecords().size()); + auto readPixelMap = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(FIVE * 1024 * 1024, readPixelMap->GetRawData().size()); + + auto readPixelMap2 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(1)); + EXPECT_TRUE(readPixelMap2->GetRawData().size() == TEN * 1024 * 1024); + EXPECT_EQ(TEN * 1024 * 1024, readPixelMap2->GetRawData().size()); + LOG_INFO(UDMF_TEST, "SetDataTest end."); +} + +/** +* @tc.name: SetData001 +* @tc.desc: Test set data success. Record size Less than 2M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData001 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + auto plainText = std::make_shared<PlainText>("something", "anything"); + UnifiedData data; + data.AddRecord(plainText); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(1, readData.GetRecords().size()); + auto readPlainText = std::static_pointer_cast<PlainText>(readData.GetRecordAt(0)); + EXPECT_EQ("something", readPlainText->GetContent()); + EXPECT_EQ("anything", readPlainText->GetAbstract()); + LOG_INFO(UDMF_TEST, "SetData001 end."); +} + +/** +* @tc.name: SetData002 +* @tc.desc: Test set data success. Record size greater than 2M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData002, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData002 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < 2 * 1024 * 1024 + 1; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(1, readData.GetRecords().size()); + auto readPixelMap = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(2 * 1024 * 1024 + 1, readPixelMap->GetRawData().size()); + LOG_INFO(UDMF_TEST, "SetData002 end."); +} + +/** +* @tc.name: SetData003 +* @tc.desc: Test set data success. Every record size greater than 2M, and data greater than 4M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData003, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData003 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData1; + for (int i = 0; i < 3 * 1024 * 1024; i++) { + rawData1.emplace_back(1); + } + auto pixelMap1 = std::make_shared<SystemDefinedPixelMap>(rawData1); + std::vector<uint8_t> rawData2; + for (int i = 0; i < 3 * 1024 * 1024; i++) { + rawData2.emplace_back(2); + } + auto pixelMap2 = std::make_shared<SystemDefinedPixelMap>(rawData2); + std::vector<uint8_t> rawData3; + for (int i = 0; i < 3 * 1024 * 1024; i++) { + rawData3.emplace_back(3); + } + auto pixelMap3 = std::make_shared<SystemDefinedPixelMap>(rawData3); + std::vector<uint8_t> rawData4; + for (int i = 0; i < 3 * 1024 * 1024; i++) { + rawData4.emplace_back(4); + } + auto pixelMap4 = std::make_shared<SystemDefinedPixelMap>(rawData4); + + UnifiedData data; + data.AddRecord(pixelMap1); + data.AddRecord(pixelMap2); + data.AddRecord(pixelMap3); + data.AddRecord(pixelMap4); + LOG_ERROR(UDMF_TEST, "data size = %{public}lld.", data.GetSize()); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(4, readData.GetRecords().size()); + auto readPixelMap1 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(3 * 1024 * 1024, readPixelMap1->GetRawData().size()); + auto readPixelMap2 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(1)); + EXPECT_EQ(3 * 1024 * 1024, readPixelMap2->GetRawData().size()); + auto readPixelMap3 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(2)); + EXPECT_EQ(3 * 1024 * 1024, readPixelMap3->GetRawData().size()); + auto readPixelMap4 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(3)); + EXPECT_EQ(3 * 1024 * 1024, readPixelMap4->GetRawData().size()); + LOG_INFO(UDMF_TEST, "SetData003 end."); +} + +/** +* @tc.name: SetData004 +* @tc.desc: Test set data error. Record size equals 4M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData004, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData004 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < 4 * 1024 * 1024; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(1, readData.GetRecords().size()); + auto readPixelMap1 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(4 * 1024 * 1024, readPixelMap1->GetRawData().size()); + LOG_INFO(UDMF_TEST, "SetData004 end."); +} + +/** +* @tc.name: SetData005 +* @tc.desc: Test set data error. Record type is file. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData005, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData005 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::string uri = "https://image.jpg"; + auto image = std::make_shared<Image>(uri); + UnifiedData data; + data.AddRecord(image); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_INVALID_PARAMETERS, status); + LOG_INFO(UDMF_TEST, "SetData005 end."); +} + +/** +* @tc.name: SetData006 +* @tc.desc: Test set data success. Record size more than 4M and less than 15.5M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData006, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData006 begin."); + SetDataTest(); + LOG_INFO(UDMF_TEST, "SetData006 end."); +} + +/** +* @tc.name: SetData007 +* @tc.desc: Test set data error. Record size Greater than 15.5M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData007, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData007 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < 15.5 * 1024 * 1024 + 1; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_INVALID_PARAMETERS, status); + LOG_INFO(UDMF_TEST, "SetData007 end."); +} + +/** +* @tc.name: SetData008 +* @tc.desc: Test set data success. Record size equals 15.5M. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData008, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData008 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < 15.5 * 1024 * 1024; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + + std::vector<uint8_t> rawData2; + for (int i = 0; i < 15 * 1024 * 1024; i++) { + rawData2.emplace_back(1); + } + auto pixelMap2 = std::make_shared<SystemDefinedPixelMap>(rawData2); + data.AddRecord(pixelMap2); + + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(2, readData.GetRecords().size()); + auto readPixelMap1 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(15.5 * 1024 * 1024, readPixelMap1->GetRawData().size()); + + auto readPixelMap2 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(1)); + EXPECT_EQ(15 * 1024 * 1024, readPixelMap2->GetRawData().size()); + LOG_INFO(UDMF_TEST, "SetData008 end."); +} + +/** +* @tc.name: SetData009 +* @tc.desc: Test set data with diffient types. +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientSaInvokeTest, SetData009, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData009 begin."); + + CustomOption option = { .intention = Intention::UD_INTENTION_DRAG }; + std::vector<uint8_t> rawData; + for (int i = 0; i < FIVE * 1024 * 1024; i++) { + rawData.emplace_back(1); + } + auto pixelMap = std::make_shared<SystemDefinedPixelMap>(rawData); + UnifiedData data; + data.AddRecord(pixelMap); + + std::string content; + for (int i = 0; i < FIVE * 1024 * 1024; i++) { + content.push_back('a'); + } + + auto plain_text = std::make_shared<PlainText>(content, ""); + data.AddRecord(plain_text); + + std::string key; + auto status = UdmfClient::GetInstance().SetData(option, data, key); + ASSERT_EQ(E_OK, status); + + UnifiedData readData; + QueryOption queryOption = {.key = key}; + status = UdmfClient::GetInstance().GetData(queryOption, readData); + ASSERT_EQ(E_OK, status); + ASSERT_EQ(2, readData.GetRecords().size()); + auto readPixelMap1 = std::static_pointer_cast<SystemDefinedPixelMap>(readData.GetRecordAt(0)); + EXPECT_EQ(FIVE * 1024 * 1024, readPixelMap1->GetRawData().size()); + + auto readPlainText = std::static_pointer_cast<PlainText>(readData.GetRecordAt(1)); + EXPECT_EQ(FIVE * 1024 * 1024, readPlainText->GetContent().size()); + LOG_INFO(UDMF_TEST, "SetData008 end."); +} + +/** + * @tc.name: SetData010 + * @tc.desc: Test set data concurrently. + * @tc.type: FUNC + */ +HWTEST_F(UdmfClientSaInvokeTest, SetData010, TestSize.Level0) +{ + LOG_INFO(UDMF_TEST, "SetData010 begin."); + std::thread t1([&]() { + SetDataTest(); + }); + EXPECT_NO_FATAL_FAILURE(t1.join()); + std::thread t2([&]() { + SetDataTest(); + }); + EXPECT_NO_FATAL_FAILURE(t2.join()); + std::thread t3([&]() { + SetDataTest(); + }); + EXPECT_NO_FATAL_FAILURE(t3.join()); + + std::thread t4([&]() { + SetDataTest(); + }); + EXPECT_NO_FATAL_FAILURE(t4.join()); + LOG_INFO(UDMF_TEST, "SetData010 end."); +} + +} // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/test/unittest/udmf_client_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_test.cpp index 415c5ef3..c3816343 100644 --- a/udmf/framework/innerkitsimpl/test/unittest/udmf_client_test.cpp +++ b/udmf/framework/innerkitsimpl/test/unittest/udmf_client_test.cpp @@ -40,6 +40,7 @@ #include "system_defined_record.h" #include "text.h" #include "unified_data_helper.h" +#include "unified_html_record_process.h" #include "video.h" using namespace testing::ext; @@ -48,6 +49,9 @@ using namespace OHOS::UDMF; using namespace OHOS; namespace OHOS::Test { constexpr int SLEEP_TIME = 50; // 50 ms +constexpr int BATCH_SIZE_2K = 2000; +constexpr int BATCH_SIZE_5K = 5000; +constexpr double BASE_CONVERSION = 1000.0; class UdmfClientTest : public testing::Test { public: static void SetUpTestCase(); @@ -62,8 +66,10 @@ public: void SetHapToken2(); void AddPrivilege(QueryOption &option); + void AddPrivilege1(QueryOption &option); void CompareDetails(const UDDetails &details); void GetEmptyData(QueryOption &option); + void GetFileUriUnifiedData(UnifiedData &data); static constexpr int USER_ID = 100; static constexpr int INST_INDEX = 0; @@ -202,6 +208,17 @@ void UdmfClientTest::AddPrivilege(QueryOption &option) ASSERT_EQ(status, E_OK); } +void UdmfClientTest::AddPrivilege1(QueryOption &option) +{ + Privilege privilege; + privilege.tokenId = AccessTokenKit::GetHapTokenID(USER_ID, "ohos.test.demo1", INST_INDEX); + privilege.readPermission = "readPermission"; + privilege.writePermission = "writePermission"; + SetNativeToken("msdp_sa"); + auto status = UdmfClient::GetInstance().AddPrivilege(option, privilege); + ASSERT_EQ(status, E_OK); +} + void UdmfClientTest::CompareDetails(const UDDetails &details) { for (const auto &detail : details) { @@ -221,6 +238,30 @@ void UdmfClientTest::GetEmptyData(QueryOption &option) EXPECT_EQ(status, E_NOT_FOUND); } +void UdmfClientTest::GetFileUriUnifiedData(UnifiedData &data) +{ + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = "http://demo.com"; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, obj); + + std::shared_ptr<Object> obj1 = std::make_shared<Object>(); + obj1->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj1->value_[FILE_URI_PARAM] = "http://demo.com"; + obj1->value_[FILE_TYPE] = "general.image"; + auto record1 = std::make_shared<UnifiedRecord>(FILE_URI, obj1); + + std::shared_ptr<Object> obj2 = std::make_shared<Object>(); + obj2->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj2->value_[FILE_URI_PARAM] = "http://demo.com"; + obj2->value_[FILE_TYPE] = "general.audio"; + auto record2 = std::make_shared<UnifiedRecord>(FILE_URI, obj2); + data.AddRecord(record); + data.AddRecord(record1); + data.AddRecord(record2); +} + /** * @tc.name: SetData001 * @tc.desc: Set data with invalid params @@ -1244,7 +1285,7 @@ HWTEST_F(UdmfClientTest, GetSummary001, TestSize.Level1) EXPECT_EQ(summary.summary["SystemDefinedType"], systemDefinedRecord->GetSize()); EXPECT_EQ(summary.summary["openharmony.form"], systemDefinedForm->GetSize()); EXPECT_EQ(summary.summary["ApplicationDefinedType"], applicationDefinedRecord->GetSize()); - EXPECT_EQ(summary.summary["general.file-uri"], record8->GetSize()); + EXPECT_EQ(summary.summary["abcdefg"], record8->GetSize()); EXPECT_EQ(summary.summary["general.png"], record9->GetSize()); LOG_INFO(UDMF_TEST, "GetSummary001 end."); @@ -2100,6 +2141,694 @@ HWTEST_F(UdmfClientTest, SetData026, TestSize.Level1) LOG_INFO(UDMF_TEST, "SetData026 end."); } +/** +* @tc.name: SetData027 +* @tc.desc: Set Html record value is null str and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData027, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData027 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::string html = ""; + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 0); + + LOG_INFO(UDMF_TEST, "SetData027 end."); +} + +/** +* @tc.name: SetData028 +* @tc.desc: Set Html record include invalid value and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData028, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData028 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::string html = "<img data-ohos='clipboard' src=>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/103.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/104.png'>"; + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 0); + + LOG_INFO(UDMF_TEST, "SetData028 end."); +} + +/** +* @tc.name: SetData029 +* @tc.desc: Set Html record include valid value and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData029, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData029 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::string html = "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/temp.png'><img " + "data-ohos='clipboard' " + "src=\"file:///data/storage/el2/distributedfiles/temp.png\"><img data-ohos='clipboard' " + "src='https://data/storage/el2/distributedfiles/202305301.png'>"; + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 2); + + LOG_INFO(UDMF_TEST, "SetData029 end."); +} + +/** +* @tc.name: SetData030 +* @tc.desc: Set Html record include valid value and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData030, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData030 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::string html = "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/103.png'>"; + std::string uriSrc = "file:///data/storage/el2/distributedfiles/101.png"; + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 1); + EXPECT_EQ(uris[0].oriUri, uriSrc); + EXPECT_TRUE(uris[0].dfsUri.empty()); + + LOG_INFO(UDMF_TEST, "SetData030 end."); +} + +/** +* @tc.name: SetData031 +* @tc.desc: Set Html record add dfsUri and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData031, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData031 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::string html = "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/101.png'>"; + std::string uriSrc1 = "file:///data/storage/el2/distributedfiles/101.png"; + std::string uriSrc2 = "file:///data/storage/el2/distributedfiles/102.png"; + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 3); + EXPECT_EQ(uris[0].oriUri, uriSrc1); + EXPECT_EQ(uris[1].oriUri, uriSrc2); + EXPECT_EQ(uris[2].oriUri, uriSrc1); + std::string strUri1 = "file:///data/1.png"; + std::string strUri2 = "file:///data/2.png"; + std::unordered_map<std::string, std::string> dfsUris; + dfsUris.insert(std::make_pair(uriSrc1, strUri1)); + dfsUris.insert(std::make_pair(uriSrc2, strUri2)); + readRecord->ComputeUris([&dfsUris] (UriInfo &uriInfo) { + auto iter = dfsUris.find(uriInfo.oriUri); + if (iter != dfsUris.end()) { + uriInfo.dfsUri = iter->second; + } + return true; + }); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + std::string htmlResult = "<img data-ohos='clipboard' src='file:///data/1.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='file:///data/2.png'>" + "<img data-ohos='clipboard' src='file:///data/1.png'>"; + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), htmlResult); + auto uris1 = readRecord->GetUris(); + EXPECT_EQ(uris1[0].dfsUri, strUri1); + EXPECT_EQ(uris1[1].dfsUri, strUri2); + EXPECT_EQ(uris1[2].dfsUri, strUri1); + + LOG_INFO(UDMF_TEST, "SetData031 end."); +} + +/** +* @tc.name: SetData032 +* @tc.desc: Set Html entry is null str and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData032, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData032 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = "http://demo.com"; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, obj); + std::string html = ""; + auto htmlRecord = Html(html, "abstract"); + htmlRecord.InitObject(); + record->AddEntry(htmlRecord.GetUtdId(), htmlRecord.GetOriginValue()); + data.AddRecord(record); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto entryValue = readRecord->GetEntry(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML)); + auto object = std::get<std::shared_ptr<Object>>(entryValue); + ASSERT_NE(object, nullptr); + auto iter = object->value_.find(HTML_CONTENT); + ASSERT_TRUE(iter != object->value_.end()); + EXPECT_TRUE(std::holds_alternative<std::string>(iter->second)); + auto content = std::get<std::string>(iter->second); + EXPECT_EQ(content, html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 0); + + LOG_INFO(UDMF_TEST, "SetData032 end."); +} + +/** +* @tc.name: SetData033 +* @tc.desc: Set Html entry include invalid value and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData033, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData033 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = "http://demo.com"; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, obj); + std::string html = "<img data-ohos='clipboard' src=>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/103.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/104.png'>"; + auto htmlRecord = Html(html, "abstract"); + htmlRecord.InitObject(); + record->AddEntry(htmlRecord.GetUtdId(), htmlRecord.GetOriginValue()); + data.AddRecord(record); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto entryValue = readRecord->GetEntry(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML)); + auto object = std::get<std::shared_ptr<Object>>(entryValue); + ASSERT_NE(object, nullptr); + auto iter = object->value_.find(HTML_CONTENT); + ASSERT_TRUE(iter != object->value_.end()); + EXPECT_TRUE(std::holds_alternative<std::string>(iter->second)); + auto content = std::get<std::string>(iter->second); + EXPECT_EQ(content, html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 0); + + LOG_INFO(UDMF_TEST, "SetData033 end."); +} + +/** +* @tc.name: SetData034 +* @tc.desc: Set Html entry include valid value and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData034, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData034 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = "http://demo.com"; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, obj); + std::string html = "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/103.png'>"; + std::string uriSrc = "file:///data/storage/el2/distributedfiles/103.png"; + auto htmlRecord = Html(html, "abstract"); + htmlRecord.InitObject(); + record->AddEntry(htmlRecord.GetUtdId(), htmlRecord.GetOriginValue()); + data.AddRecord(record); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto entryValue = readRecord->GetEntry(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML)); + auto object = std::get<std::shared_ptr<Object>>(entryValue); + ASSERT_NE(object, nullptr); + auto iter = object->value_.find(HTML_CONTENT); + ASSERT_TRUE(iter != object->value_.end()); + EXPECT_TRUE(std::holds_alternative<std::string>(iter->second)); + auto content = std::get<std::string>(iter->second); + EXPECT_EQ(content, html); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 1); + EXPECT_EQ(uris[0].oriUri, uriSrc); + EXPECT_TRUE(uris[0].dfsUri.empty()); + + LOG_INFO(UDMF_TEST, "SetData034 end."); +} + +/** +* @tc.name: SetData035 +* @tc.desc: Set Html entry add dfsUri and get uris data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData035, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData035 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + obj->value_[FILE_URI_PARAM] = "http://demo.com"; + obj->value_[FILE_TYPE] = "abcdefg"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, obj); + std::string html = "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/101.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/202305301.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/202305301.png'>" + "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/102.png'>" + "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/103.png'>"; + std::string uriSrc1 = "file:///data/storage/el2/distributedfiles/101.png"; + std::string uriSrc2 = "file:///data/storage/el2/distributedfiles/102.png"; + std::string uriSrc3 = "file:///data/storage/el2/distributedfiles/103.png"; + auto htmlRecord = Html(html, "abstract"); + htmlRecord.InitObject(); + record->AddEntry(htmlRecord.GetUtdId(), htmlRecord.GetOriginValue()); + data.AddRecord(record); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), 3); + EXPECT_EQ(uris[0].oriUri, uriSrc1); + EXPECT_EQ(uris[1].oriUri, uriSrc2); + EXPECT_EQ(uris[2].oriUri, uriSrc3); + std::string strUri1 = "file:///data/201.png"; + std::string strUri2 = "file:///data/202.png"; + std::string strUri3 = "file:///data/203.png"; + std::unordered_map<std::string, std::string> dfsUris; + dfsUris.insert(std::make_pair(uriSrc1, strUri1)); + dfsUris.insert(std::make_pair(uriSrc2, strUri2)); + dfsUris.insert(std::make_pair(uriSrc3, strUri3)); + readRecord->ComputeUris([&dfsUris] (UriInfo &uriInfo) { + auto iter = dfsUris.find(uriInfo.oriUri); + if (iter != dfsUris.end()) { + uriInfo.dfsUri = iter->second; + } + return true; + }); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + std::shared_ptr<UnifiedRecord> rebuildRecord = readData.GetRecordAt(0); + std::string htmlResult = "<img data-ohos='clipboard' src='file:///data/201.png'>" + "<img data-ohos='clipboard' src='https://data/storage/el2/distributedfiles/202305301.png'>" + "<img data-ohos='clipboard' src='file://data/storage/el2/distributedfiles/202305301.png'>" + "<img data-ohos='clipboard' src='file:///data/202.png'>" + "<img data-ohos='clipboard' src='file:///data/203.png'>"; + auto entryValue = rebuildRecord->GetEntry(UtdUtils::GetUtdIdFromUtdEnum(UDType::HTML)); + auto object = std::get<std::shared_ptr<Object>>(entryValue); + ASSERT_NE(object, nullptr); + auto iter = object->value_.find(HTML_CONTENT); + ASSERT_TRUE(iter != object->value_.end()); + EXPECT_TRUE(std::holds_alternative<std::string>(iter->second)); + auto content = std::get<std::string>(iter->second); + EXPECT_EQ(content, htmlResult); + + LOG_INFO(UDMF_TEST, "SetData035 end."); +} + +/** +* @tc.name: SetData036 +* @tc.desc: split and rebuild html value performance test +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData036, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData036 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + uint32_t kNumImages = 1000; + std::string html = ""; + for (uint32_t i = 0; i < kNumImages; i++) { + html += "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/101.png'>"; + } + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.html"; + obj->value_["htmlContent"] = html; + obj->value_["plainContent"] = "htmlPlainContent"; + auto obj2 = std::make_shared<Object>(); + obj2->value_["detail1"] = "detail1"; + obj2->value_["detail2"] = "detail2"; + obj->value_["details"] = obj2; + auto htmlRecord = std::make_shared<Html>(UDType::HTML, obj); + data.AddRecord(htmlRecord); + auto start = std::chrono::high_resolution_clock::now(); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto uris = readRecord->GetUris(); + EXPECT_EQ(uris.size(), kNumImages); + std::string strUri = "file:///data/1.png"; + readRecord->ComputeUris([&strUri] (UriInfo &uriInfo) { + uriInfo.dfsUri = strUri; + return true; + }); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start); + LOG_INFO(UDMF_TEST, "SetData036 count duration=%{public}lld", duration.count()); + std::string htmlResult = ""; + for (uint32_t i = 0; i < kNumImages; i++) { + htmlResult += "<img data-ohos='clipboard' src='file:///data/1.png'>"; + } + auto readHtmlRecord = std::static_pointer_cast<Html>(readRecord); + EXPECT_EQ(readHtmlRecord->GetHtmlContent(), htmlResult); + + LOG_INFO(UDMF_TEST, "SetData036 end."); +} + +/** +* @tc.name: SetData037 +* @tc.desc: test html record process +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData037, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData037 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data1; + std::shared_ptr<UnifiedRecord> record = std::make_shared<UnifiedRecord>(); + data1.AddRecord(record); + std::string key; + UnifiedHtmlRecordProcess::RebuildHtmlRecord(data1); + UnifiedHtmlRecordProcess::GetUriFromHtmlRecord(data1); + UnifiedData data; + auto plainText = std::make_shared<PlainText>(); + plainText->SetContent("plainContent"); + data.AddRecord(plainText); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + std::shared_ptr<UnifiedRecord> record1 = std::make_shared<PlainText>(UDType::PLAIN_TEXT, "this is a content"); + record1->ClearUris(); + data.AddRecord(record1); + auto file = std::make_shared<File>(); + file->SetRemoteUri("remoteUri"); + UDDetails details; + details.insert({ "udmf_key", "udmf_value" }); + file->SetDetails(details); + data.AddRecord(file); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(data); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + LOG_INFO(UDMF_TEST, "SetData037 end."); +} + +/** +* @tc.name: SetData038 +* @tc.desc: test html record process +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData038, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData038 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + std::shared_ptr<Object> fileObj = std::make_shared<Object>(); + fileObj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + fileObj->value_[FILE_URI_PARAM] = "http://demo.com"; + auto record = std::make_shared<UnifiedRecord>(FILE_URI, fileObj); + std::string html = "<img data-ohos='clipboard' src='file:///data/storage/el2/distributedfiles/103.png'>"; + auto htmlRecord = Html(html, "abstract"); + htmlRecord.InitObject(); + record->AddEntry(htmlRecord.GetUtdId(), htmlRecord.GetOriginValue()); + data.AddRecord(record); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + std::shared_ptr<UnifiedRecord> readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + std::shared_ptr<std::map<std::string, ValueType>> entries = std::make_shared<std::map<std::string, ValueType>>(); + std::string utd = "general.html"; + ValueType value = "htmlContent"; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[HTML_CONTENT] = 10; + std::shared_ptr<Object> obj1 = std::make_shared<Object>(); + obj1->value_[HTML_CONTENT] = "file:///data/storage/el2/distributedfiles/103.png"; + readRecord->SetInnerEntries(entries); + readRecord->SetType(UDType::HTML); + entries->insert_or_assign(std::move(utd), std::move(value)); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + entries->insert_or_assign(std::move(utd), std::move(obj)); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + entries->insert_or_assign(std::move(utd), std::move(obj1)); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(readData); + + LOG_INFO(UDMF_TEST, "SetData038 end."); +} + +/** +* @tc.name: SetData039 +* @tc.desc: test html record process +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetData039, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetData039 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + auto text = std::make_shared<Text>(); + data.AddRecord(text); + auto plainText = std::make_shared<PlainText>(); + plainText->SetContent("plainContent"); + data.AddRecord(plainText); + auto html = std::make_shared<Html>(); + html->SetPlainContent("htmlContent"); + data.AddRecord(html); + std::string key; + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + UnifiedData data1; + std::shared_ptr<Object> obj = std::make_shared<Object>(); + obj->value_[PLAIN_CONTENT] = "http://demo.com"; + obj->value_[HTML_CONTENT] = 10; + plainText->AddEntry("general.html", obj); + data1.AddRecord(plainText); + UnifiedHtmlRecordProcess::GetUriFromHtmlRecord(data1); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(data1); + UnifiedData data2; + text->AddEntry("general.html", ""); + data2.AddRecord(text); + UnifiedHtmlRecordProcess::GetUriFromHtmlRecord(data2); + UnifiedHtmlRecordProcess::RebuildHtmlRecord(data2); + QueryOption option2 = { .key = key }; + AddPrivilege1(option2); + SetHapToken1(); + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + + LOG_INFO(UDMF_TEST, "SetData039 end."); +} + /** * @tc.name: GetSummary003 * @tc.desc: Get summary data @@ -2594,7 +3323,7 @@ HWTEST_F(UdmfClientTest, GetSummary004, TestSize.Level1) status = UdmfClient::GetInstance().GetSummary(option2, summary); ASSERT_EQ(status, E_OK); - EXPECT_EQ(summary.summary["general.file-uri"], size0); + EXPECT_EQ(summary.summary["abcdefg"], size0); EXPECT_EQ(summary.summary["general.plain-text"], size1); EXPECT_EQ(summary.summary["general.image"], size2); EXPECT_EQ(summary.summary["openharmony.pixel-map"], size3); @@ -2739,4 +3468,255 @@ HWTEST_F(UdmfClientTest, GetSummary005, TestSize.Level1) EXPECT_EQ("title", std::get<std::string>(readCotentForm->value_[TITLE])); LOG_INFO(UDMF_TEST, "GetSummary005 end."); } + + +/** +* @tc.name: FileUriCastTest001 +* @tc.desc: Cast file uri type record to verify works well +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, FileUriCastTest001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "FileUriCastTest001 begin."); + UnifiedData data; + GetFileUriUnifiedData(data); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + std::string key; + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + QueryOption option2 = { .key = key }; + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + auto record0 = readData.GetRecordAt(0); + auto fileValue = record0->GetValue(); + std::shared_ptr<Object> fileObj = std::get<std::shared_ptr<Object>>(fileValue); + EXPECT_EQ(std::get<std::string>(fileObj->value_[FILE_URI_PARAM]), "http://demo.com"); + EXPECT_EQ(std::get<std::string>(fileObj->value_[FILE_TYPE]), "abcdefg"); + File *file = reinterpret_cast<File *>(record0.get()); + EXPECT_EQ(file->GetUri(), "http://demo.com"); + auto record1 = readData.GetRecordAt(1); + auto imageValue = record1->GetValue(); + std::shared_ptr<Object> imageObj = std::get<std::shared_ptr<Object>>(imageValue); + EXPECT_EQ(std::get<std::string>(imageObj->value_[FILE_URI_PARAM]), "http://demo.com"); + EXPECT_EQ(std::get<std::string>(imageObj->value_[FILE_TYPE]), "general.image"); + File *image = reinterpret_cast<Image *>(record1.get()); + EXPECT_EQ(image->GetUri(), "http://demo.com"); + auto record2 = readData.GetRecordAt(2); + File *audio = reinterpret_cast<Audio *>(record2.get()); + EXPECT_EQ(audio->GetUri(), "http://demo.com"); + LOG_INFO(UDMF_TEST, "FileUriCastTest001 end."); +} + +/** +* @tc.name: SetBatchData001 +* @tc.desc: Set 2k record and get data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetBatchData001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetBatchData001 begin."); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data1; + std::string key; + for (int32_t i = 0; i < BATCH_SIZE_2K; ++i) { + auto file1 = std::make_shared<File>(); + file1->SetRemoteUri("remoteUri"); + UDDetails details1; + details1.insert({ "udmf_key", "udmf_value" }); + file1->SetDetails(details1); + data1.AddRecord(file1); + } + auto start = std::chrono::high_resolution_clock::now(); + auto status = UdmfClient::GetInstance().SetData(option1, data1, key); + ASSERT_EQ(status, E_OK); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration<double> du = end - start; + double duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "setDataDuration = %{public}.1f ms.", duration); + + QueryOption option2 = { .key = key }; + Summary summary; + start = std::chrono::high_resolution_clock::now(); + status = UdmfClient::GetInstance().GetSummary(option2, summary); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "getSummaryDuration = %{public}.1f ms.", duration); + + start = std::chrono::high_resolution_clock::now(); + AddPrivilege(option2); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "addPrivilegeDuration = %{public}.1f ms.", duration); + + SetHapToken2(); + UnifiedData data2; + start = std::chrono::high_resolution_clock::now(); + status = UdmfClient::GetInstance().GetData(option2, data2); + ASSERT_EQ(status, E_OK); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "getDataDuration = %{public}.1f ms.", duration); + + ASSERT_EQ(data2.GetRecords().size(), BATCH_SIZE_2K); + for (auto record : data2.GetRecords()) { + ASSERT_NE(record, nullptr); + EXPECT_EQ(record->GetType(), UDType::FILE); + } + LOG_INFO(UDMF_TEST, "SetBatchData001 end."); +} + +/** +* @tc.name: SetBatchData002 +* @tc.desc: Set 5k record and get data +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, SetBatchData002, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "SetBatchData002 begin."); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data1; + std::string key; + for (int32_t i = 0; i < BATCH_SIZE_5K; ++i) { + auto file1 = std::make_shared<File>(); + file1->SetRemoteUri("remoteUri"); + UDDetails details1; + details1.insert({ "udmf_key", "udmf_value" }); + file1->SetDetails(details1); + data1.AddRecord(file1); + } + auto start = std::chrono::high_resolution_clock::now(); + auto status = UdmfClient::GetInstance().SetData(option1, data1, key); + ASSERT_EQ(status, E_OK); + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration<double> du = end - start; + double duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "setDataDuration = %{public}.1f ms.", duration); + + QueryOption option2 = { .key = key }; + Summary summary; + start = std::chrono::high_resolution_clock::now(); + status = UdmfClient::GetInstance().GetSummary(option2, summary); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "getSummaryDuration = %{public}.1f ms.", duration); + + start = std::chrono::high_resolution_clock::now(); + AddPrivilege(option2); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "addPrivilegeDuration = %{public}.1f ms.", duration); + + SetHapToken2(); + UnifiedData data2; + start = std::chrono::high_resolution_clock::now(); + status = UdmfClient::GetInstance().GetData(option2, data2); + ASSERT_EQ(status, E_OK); + end = std::chrono::high_resolution_clock::now(); + du = end - start; + duration = BASE_CONVERSION * du.count(); + LOG_INFO(UDMF_TEST, "getDataDuration = %{public}.1f ms.", duration); + + ASSERT_EQ(data2.GetRecords().size(), BATCH_SIZE_5K); + for (auto record : data2.GetRecords()) { + ASSERT_NE(record, nullptr); + EXPECT_EQ(record->GetType(), UDType::FILE); + } + LOG_INFO(UDMF_TEST, "SetBatchData002 end."); +} + +/** +* @tc.name: GetData002 +* @tc.desc: test Marshalling and Unmarshalling properties drag +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, GetData002, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "GetData002 begin."); + + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + UnifiedData data; + std::string key; + auto record = std::make_shared<Html>("htmlContent", "plainContent"); + auto link = Link("url", "descritpion"); + link.InitObject(); + record->AddEntry(link.GetUtdId(), link.GetOriginValue()); + data.AddRecord(record); + std::shared_ptr<UnifiedDataProperties> properties = std::make_shared<UnifiedDataProperties>(); + std::string tag = "this is a tag of test GetData002"; + properties->tag = tag; + properties->shareOptions = CROSS_APP; + data.SetProperties(std::move(properties)); + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + + QueryOption option2 = { .key = key }; + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + ASSERT_EQ(status, E_OK); + ASSERT_EQ(readData.GetRecords().size(), 1); + auto readRecord = readData.GetRecordAt(0); + ASSERT_NE(readRecord, nullptr); + auto entries = readRecord->GetEntries(); + auto readHtml = std::get<std::shared_ptr<Object>>(entries->at("general.html")); + EXPECT_EQ("htmlContent", std::get<std::string>(readHtml->value_[HTML_CONTENT])); + auto readHyperlink = std::get<std::shared_ptr<Object>>(entries->at("general.hyperlink")); + EXPECT_EQ("descritpion", std::get<std::string>(readHyperlink->value_[DESCRIPTION])); + auto readProperties = readData.GetProperties(); + ASSERT_NE(readProperties, nullptr); + EXPECT_EQ(readProperties->tag, tag); + EXPECT_EQ(readProperties->shareOptions, CROSS_APP); + + LOG_INFO(UDMF_TEST, "GetData002 end."); +} + +/** +* @tc.name: GetBatchData001 +* @tc.desc: test Marshalling and Unmarshalling properties datahub +* @tc.type: FUNC +*/ +HWTEST_F(UdmfClientTest, GetBatchData001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "GetBatchData001 begin."); + + QueryOption query = { .intention = Intention::UD_INTENTION_DATA_HUB }; + std::vector<UnifiedData> unifiedDataSet; + auto status = UdmfClient::GetInstance().DeleteData(query, unifiedDataSet); + ASSERT_EQ(status, E_OK); + unifiedDataSet.clear(); + status = UdmfClient::GetInstance().GetBatchData(query, unifiedDataSet); + ASSERT_EQ(status, E_OK); + ASSERT_TRUE(unifiedDataSet.empty()); + + CustomOption customOption = { .intention = Intention::UD_INTENTION_DATA_HUB }; + UnifiedData data; + std::shared_ptr<UnifiedRecord> record = std::make_shared<PlainText>(UDType::PLAIN_TEXT, "plainTextContent"); + data.AddRecord(record); + std::shared_ptr<UnifiedDataProperties> properties = std::make_shared<UnifiedDataProperties>(); + std::string tag = "this is a tag of test GetBatchData001"; + properties->tag = tag; + properties->shareOptions = CROSS_APP; + data.SetProperties(std::move(properties)); + std::string key; + status = UdmfClient::GetInstance().SetData(customOption, data, key); + ASSERT_EQ(status, E_OK); + query = { .key = key, .intention = Intention::UD_INTENTION_DATA_HUB }; + status = UdmfClient::GetInstance().GetBatchData(query, unifiedDataSet); + ASSERT_EQ(status, E_OK); + ASSERT_EQ(unifiedDataSet.size(), 1); + auto record2 = unifiedDataSet[0].GetRecordAt(0); + ValueType value = record2->GetValue(); + ASSERT_NE(std::get_if<std::string>(&value), nullptr); + EXPECT_EQ(std::get<std::string>(value), "plainTextContent"); + auto readProperties = unifiedDataSet[0].GetProperties(); + ASSERT_NE(readProperties, nullptr); + EXPECT_EQ(readProperties->tag, tag); + EXPECT_EQ(readProperties->shareOptions, CROSS_APP); + + LOG_INFO(UDMF_TEST, "GetBatchData001 end."); +} } // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/test/unittest/unified_data_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/unified_data_test.cpp index 7fbff42e..89f6a988 100644 --- a/udmf/framework/innerkitsimpl/test/unittest/unified_data_test.cpp +++ b/udmf/framework/innerkitsimpl/test/unittest/unified_data_test.cpp @@ -18,8 +18,11 @@ #include <gtest/gtest.h> #include <string> +#include "file.h" #include "logger.h" +#include "plain_text.h" #include "udmf_capi_common.h" +#include "udmf_utils.h" #include "unified_data.h" using namespace testing::ext; @@ -34,6 +37,7 @@ public: static void TearDownTestCase(); void SetUp() override; void TearDown() override; + void TransferToEntriesCompareEntries(UnifiedRecord* recordFirst); }; void UnifiedDataTest::SetUpTestCase() @@ -52,6 +56,57 @@ void UnifiedDataTest::TearDown() { } +void UnifiedDataTest::TransferToEntriesCompareEntries(UnifiedRecord* recordFirst) +{ + auto plainTextFirst = static_cast<PlainText*>(recordFirst); + EXPECT_EQ(plainTextFirst->GetAbstract(), "abstract"); + EXPECT_EQ(plainTextFirst->GetContent(), "http://1111/a.img"); + + std::set<std::string> utdIds = recordFirst->GetUtdIds(); + EXPECT_TRUE(utdIds.find("general.plain-text") != utdIds.end()); + EXPECT_TRUE(utdIds.find("general.file") != utdIds.end()); + EXPECT_TRUE(utdIds.find("general.file-uri") != utdIds.end()); + EXPECT_TRUE(utdIds.find("general.img") == utdIds.end()); + EXPECT_TRUE(utdIds.find("general.media") != utdIds.end()); + + auto fileEntry = recordFirst->GetEntry("general.file"); + std::shared_ptr<Object> fileEntryObj = std::get<std::shared_ptr<Object>>(fileEntry); + std::string getUri; + fileEntryObj->GetValue(ORI_URI, getUri); + EXPECT_EQ(getUri, "http://1111/a.img"); + + auto plainTextEntry = recordFirst->GetEntry("general.plain-text"); + EXPECT_TRUE(std::holds_alternative<std::monostate>(plainTextEntry)); + + auto entries = recordFirst->GetEntries(); + EXPECT_NE(entries, nullptr); + int entrySize = 3; + EXPECT_EQ(entries->size(), entrySize); + auto fileEntry1 = (*entries)["general.file"]; + std::shared_ptr<Object> fileEntryObj1 = std::get<std::shared_ptr<Object>>(fileEntry1); + std::string getUri1; + fileEntryObj1->GetValue(ORI_URI, getUri1); + EXPECT_EQ(getUri1, "http://1111/a.img"); + + auto plainTextEntry1 = (*entries)["general.plain-text"]; + std::shared_ptr<Object> plainTextEntryObj1 = std::get<std::shared_ptr<Object>>(plainTextEntry1); + std::string content; + plainTextEntryObj1->GetValue(CONTENT, content); + EXPECT_EQ(content, "http://1111/a.img"); + std::string abstract; + plainTextEntryObj1->GetValue(ABSTRACT, abstract); + EXPECT_EQ(abstract, "abstract"); + + auto fileUriEntry1 = (*entries)["general.file-uri"]; + std::shared_ptr<Object> fileUriEntryObj1 = std::get<std::shared_ptr<Object>>(fileUriEntry1); + std::string oriUri; + fileUriEntryObj1->GetValue(FILE_URI_PARAM, oriUri); + EXPECT_EQ(oriUri, "http://1111/a.img"); + std::string fileType; + fileUriEntryObj1->GetValue(FILE_TYPE, fileType); + EXPECT_EQ(fileType, "general.media"); +} + /** * @tc.name: UnifiedData001 * @tc.desc: Normal testcase of UnifiedData @@ -62,10 +117,7 @@ HWTEST_F(UnifiedDataTest, UnifiedData001, TestSize.Level1) LOG_INFO(UDMF_TEST, "UnifiedData001 begin."); std::shared_ptr<UnifiedDataProperties> properties = std::make_shared<UnifiedDataProperties>(); UnifiedData unifiedData(properties); - auto duration = std::chrono::system_clock::now().time_since_epoch(); EXPECT_EQ(unifiedData.properties_, properties); - EXPECT_EQ(unifiedData.properties_->timestamp, - std::chrono::duration_cast<std::chrono::milliseconds>(duration).count()); LOG_INFO(UDMF_TEST, "UnifiedData001 end."); } @@ -159,4 +211,112 @@ HWTEST_F(UnifiedDataTest, GetRecordAt001, TestSize.Level1) EXPECT_EQ(ret, nullptr); LOG_INFO(UDMF_TEST, "GetRecordAt001 end."); } + +/** +* @tc.name: TransferToEntries001 +* @tc.desc: Normal testcase of TransferToEntries +* @tc.type: FUNC +*/ +HWTEST_F(UnifiedDataTest, TransferToEntries001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "TransferToEntries001 begin."); + UnifiedData unifiedData; + std::shared_ptr<PlainText> plainText = std::make_shared<PlainText>(); + plainText->SetContent("http://1111/a.img"); + plainText->SetAbstract("abstract"); + std::shared_ptr<File> file = std::make_shared<File>(); + file->SetUri("http://1111/a.img"); + unifiedData.AddRecord(plainText); + unifiedData.AddRecord(file); + unifiedData.TransferToEntries(unifiedData); + auto records = unifiedData.GetRecords(); + int recordSize = 1; + EXPECT_EQ(records.size(), recordSize); + auto recordFirst = records[0].get(); + auto plainTextFirst = static_cast<PlainText*>(recordFirst); + EXPECT_EQ(plainTextFirst->GetAbstract(), "abstract"); + EXPECT_EQ(plainTextFirst->GetContent(), "http://1111/a.img"); + std::set<std::string> utdIds = recordFirst->GetUtdIds(); + EXPECT_TRUE(utdIds.find("general.plain-text") != utdIds.end()); + EXPECT_TRUE(utdIds.find("general.file") != utdIds.end()); + auto fileEntry = recordFirst->GetEntry("general.file"); + std::shared_ptr<Object> fileEntryObj = std::get<std::shared_ptr<Object>>(fileEntry); + std::string getUri; + fileEntryObj->GetValue(ORI_URI, getUri); + EXPECT_EQ(getUri, "http://1111/a.img"); + auto plainTextEntry = recordFirst->GetEntry("general.plain-text"); + EXPECT_TRUE(std::holds_alternative<std::monostate>(plainTextEntry)); + auto entries = recordFirst->GetEntries(); + EXPECT_NE(entries, nullptr); + int entrySize = 2; + EXPECT_EQ(entries->size(), entrySize); + auto fileEntry1 = (*entries)["general.file"]; + std::shared_ptr<Object> fileEntryObj1 = std::get<std::shared_ptr<Object>>(fileEntry1); + std::string getUri1; + fileEntryObj1->GetValue(ORI_URI, getUri1); + EXPECT_EQ(getUri1, "http://1111/a.img"); + auto plainTextEntry1 = (*entries)["general.plain-text"]; + std::shared_ptr<Object> plainTextEntryObj1 = std::get<std::shared_ptr<Object>>(plainTextEntry1); + std::string content; + plainTextEntryObj1->GetValue(CONTENT, content); + EXPECT_EQ(content, "http://1111/a.img"); + std::string abstract; + plainTextEntryObj1->GetValue(ABSTRACT, abstract); + EXPECT_EQ(abstract, "abstract"); + LOG_INFO(UDMF_TEST, "TransferToEntries001 end."); +} + +/** +* @tc.name: TransferToEntries002 +* @tc.desc: Normal testcase of TransferToEntries +* @tc.type: FUNC +*/ +HWTEST_F(UnifiedDataTest, TransferToEntries002, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "TransferToEntries002 begin."); + UnifiedData unifiedData; + std::shared_ptr<PlainText> plainText = std::make_shared<PlainText>(); + plainText->SetContent("http://1111/a.img"); + plainText->SetAbstract("abstract"); + + std::shared_ptr<File> file = std::make_shared<File>(); + file->SetUri("http://1111/a.img"); + + std::shared_ptr<Object> fileUriObj = std::make_shared<Object>(); + fileUriObj->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + fileUriObj->value_[FILE_URI_PARAM] = "http://1111/a.img"; + fileUriObj->value_[FILE_TYPE] = "general.img"; + std::shared_ptr<UnifiedRecord> fileUri = std::make_shared<UnifiedRecord>(FILE_URI, fileUriObj); + + std::shared_ptr<Object> fileUriObj1 = std::make_shared<Object>(); + fileUriObj1->value_[UNIFORM_DATA_TYPE] = "general.file-uri"; + fileUriObj1->value_[FILE_URI_PARAM] = "http://1111/a.img"; + fileUriObj1->value_[FILE_TYPE] = "general.media"; + std::shared_ptr<UnifiedRecord> fileUri1 = std::make_shared<UnifiedRecord>(FILE_URI, fileUriObj1); + + bool isNeed = unifiedData.IsNeedTransferToEntries(); + EXPECT_FALSE(isNeed); + unifiedData.AddRecord(plainText); + isNeed = unifiedData.IsNeedTransferToEntries(); + EXPECT_FALSE(isNeed); + unifiedData.AddRecord(file); + isNeed = unifiedData.IsNeedTransferToEntries(); + EXPECT_FALSE(isNeed); + unifiedData.AddRecord(fileUri); + unifiedData.AddRecord(fileUri1); + std::shared_ptr<UnifiedDataProperties> properties = std::make_shared<UnifiedDataProperties>(); + properties->tag = "records_to_entries_data_format"; + unifiedData.SetProperties(properties); + isNeed = unifiedData.IsNeedTransferToEntries(); + EXPECT_TRUE(isNeed); + + unifiedData.TransferToEntries(unifiedData); + auto records = unifiedData.GetRecords(); + int recordSize = 1; + EXPECT_EQ(records.size(), recordSize); + + auto recordFirst = records[0].get(); + TransferToEntriesCompareEntries(recordFirst); + LOG_INFO(UDMF_TEST, "TransferToEntries001 end."); +} } // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/innerkitsimpl/test/unittest/unified_record_test.cpp b/udmf/framework/innerkitsimpl/test/unittest/unified_record_test.cpp index acbf01b5..05775c9e 100644 --- a/udmf/framework/innerkitsimpl/test/unittest/unified_record_test.cpp +++ b/udmf/framework/innerkitsimpl/test/unittest/unified_record_test.cpp @@ -19,8 +19,14 @@ #include <gtest/gtest.h> #include <string> +#include "application_defined_record.h" +#include "file.h" +#include "html.h" +#include "image.h" #include "logger.h" +#include "plain_text.h" #include "udmf_capi_common.h" +#include "udmf_client.h" #include "unified_record.h" using namespace testing::ext; @@ -156,9 +162,9 @@ HWTEST_F(UnifiedRecordTest, Constructor_003, TestSize.Level0) EXPECT_EQ(*originValueStr, "123456"); auto entry = record.GetEntry(utdId); - EXPECT_TRUE(std::holds_alternative<std::string>(entry)); - auto entryStr = std::get_if<std::string>(&entry); - EXPECT_EQ(*entryStr, "123456"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(entry)); + auto entryStr = std::get<std::shared_ptr<Object>>(entry); + EXPECT_EQ(std::get<std::string>(entryStr->value_[VALUE_TYPE]), "123456"); auto entries = record.GetEntries(); auto it = entries->find(utdId); @@ -225,4 +231,159 @@ HWTEST_F(UnifiedRecordTest, AddEntry_001, TestSize.Level0) std::thread t10(&UnifiedRecord::AddEntry, std::ref(unifiedRecord), utdId9, value9); EXPECT_NO_FATAL_FAILURE(t10.join()); } + +/** +* @tc.name: GetEntryTest001 +* @tc.desc: Test set a UDC data, then get data with GetEntry function. +* @tc.type: FUNC +*/ +HWTEST_F(UnifiedRecordTest, GetEntryTest001, TestSize.Level1) +{ + LOG_INFO(UDMF_TEST, "GetEntryTest001 begin."); + UnifiedData data; + std::shared_ptr<File> file = std::make_shared<File>(); + file->SetUri("https://file/txt.txt"); + data.AddRecord(file); + auto image = std::make_shared<Image>(); + image->SetUri("https://file/txt.txt"); + data.AddRecord(image); + auto applicationDefinedRecord = std::make_shared<ApplicationDefinedRecord>(); + applicationDefinedRecord->SetRawData({ 'a', 'b' }); + data.AddRecord(applicationDefinedRecord); + auto html = std::make_shared<Html>(); + html->SetHtmlContent("content"); + data.AddRecord(html); + auto plainText = std::make_shared<PlainText>(); + plainText->SetContent("content"); + data.AddRecord(plainText); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + std::string key; + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + ASSERT_EQ(status, E_OK); + QueryOption option2 = { .key = key }; + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + auto recordFile = readData.GetRecordAt(0); + auto udsValue = recordFile->GetEntry("general.file"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(udsValue)); + std::shared_ptr<Object> fileUds = std::get<std::shared_ptr<Object>>(udsValue); + EXPECT_EQ(std::get<std::string>(fileUds->value_[ORI_URI]), "https://file/txt.txt"); + auto recordImage = readData.GetRecordAt(1); + auto imageValue = recordImage->GetEntry("general.image"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(imageValue)); + std::shared_ptr<Object> imageUds = std::get<std::shared_ptr<Object>>(imageValue); + EXPECT_EQ(std::get<std::string>(imageUds->value_[ORI_URI]), "https://file/txt.txt"); + auto recordAppDefined = readData.GetRecordAt(2); + auto appDefinedValue = recordAppDefined->GetEntry("ApplicationDefinedType"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(appDefinedValue)); + std::shared_ptr<Object> appDefinedUds = std::get<std::shared_ptr<Object>>(appDefinedValue); + EXPECT_EQ(std::get<std::vector<uint8_t>>(appDefinedUds->value_[ARRAY_BUFFER])[0], 'a'); + LOG_INFO(UDMF_TEST, "GetEntryTest001 end."); +} + +/** +* @tc.name: GetEntryTest002 +* @tc.desc: Test set a UDC data, then get data with GetEntry function. +* @tc.type: FUNC +*/ +HWTEST_F(UnifiedRecordTest, GetEntryTest002, TestSize.Level1) +{ + UnifiedData data; + std::shared_ptr<File> file = std::make_shared<File>("https://file/txt.txt"); + data.AddRecord(file); + auto image = std::make_shared<Image>("https://file/txt.txt"); + data.AddRecord(image); + std::vector<uint8_t> value = { 'a', 'b' }; + auto applicationDefinedRecord = std::make_shared<ApplicationDefinedRecord>("test", value); + data.AddRecord(applicationDefinedRecord); + auto html = std::make_shared<Html>("content", "plaintext content"); + data.AddRecord(html); + auto plainText = std::make_shared<PlainText>("content", "abstract"); + data.AddRecord(plainText); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + std::string key; + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + QueryOption option2 = { .key = key }; + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + auto recordFile = readData.GetRecordAt(0); + auto udsValue = recordFile->GetEntry("general.file"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(udsValue)); + std::shared_ptr<Object> fileUds = std::get<std::shared_ptr<Object>>(udsValue); + EXPECT_EQ(std::get<std::string>(fileUds->value_[ORI_URI]), "https://file/txt.txt"); + auto recordImage = readData.GetRecordAt(1); + auto imageValue = recordImage->GetEntry("general.image"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(imageValue)); + std::shared_ptr<Object> imageUds = std::get<std::shared_ptr<Object>>(imageValue); + EXPECT_EQ(std::get<std::string>(imageUds->value_[ORI_URI]), "https://file/txt.txt"); + auto recordAppDefined = readData.GetRecordAt(2); + auto appDefinedValue = recordAppDefined->GetEntry("test"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(appDefinedValue)); + std::shared_ptr<Object> appDefinedUds = std::get<std::shared_ptr<Object>>(appDefinedValue); + EXPECT_EQ(std::get<std::vector<uint8_t>>(appDefinedUds->value_[ARRAY_BUFFER])[0], 'a'); + auto recordHtml = readData.GetRecordAt(3); + auto htmlValue = recordHtml->GetEntry("general.html"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(htmlValue)); + std::shared_ptr<Object> htmlUds = std::get<std::shared_ptr<Object>>(htmlValue); + EXPECT_EQ(std::get<std::string>(htmlUds->value_[HTML_CONTENT]), "content"); + EXPECT_EQ(std::get<std::string>(htmlUds->value_[PLAIN_CONTENT]), "plaintext content"); + auto recordPlainText = readData.GetRecordAt(4); + auto plainTextValue = recordPlainText->GetEntry("general.plain-text"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(plainTextValue)); + std::shared_ptr<Object> plainTextUds = std::get<std::shared_ptr<Object>>(plainTextValue); + EXPECT_EQ(std::get<std::string>(plainTextUds->value_[CONTENT]), "content"); + EXPECT_EQ(std::get<std::string>(plainTextUds->value_[ABSTRACT]), "abstract"); +} + +/** +* @tc.name: GetEntryTest003 +* @tc.desc: Test set a base UnifiedRecord data, then get data with GetEntry function. +* @tc.type: FUNC +*/ +HWTEST_F(UnifiedRecordTest, GetEntryTest003, TestSize.Level1) +{ + UnifiedData data; + std::shared_ptr<UnifiedRecord> record1 = std::make_shared<UnifiedRecord>(VIDEO, "video"); + data.AddRecord(record1); + std::shared_ptr<UnifiedRecord> record2 = std::make_shared<UnifiedRecord>(HTML, "html"); + data.AddRecord(record2); + std::shared_ptr<UnifiedRecord> record3 = std::make_shared<UnifiedRecord>(KINGSOFT_WRITER_WPT, "abc"); + data.AddRecord(record3); + auto obj = std::make_shared<Object>(); + obj->value_["uniformDataType"] = "general.plain-text"; + obj->value_["textContent"] = "plainTextContent"; + std::shared_ptr<UnifiedRecord> record4 = std::make_shared<UnifiedRecord>(PLAIN_TEXT, obj); + data.AddRecord(record4); + std::shared_ptr<UnifiedRecord> record5 = std::make_shared<UnifiedRecord>(HYPERLINK); + data.AddRecord(record5); + CustomOption option1 = { .intention = Intention::UD_INTENTION_DRAG }; + std::string key; + auto status = UdmfClient::GetInstance().SetData(option1, data, key); + QueryOption option2 = { .key = key }; + UnifiedData readData; + status = UdmfClient::GetInstance().GetData(option2, readData); + auto recordFirst = readData.GetRecordAt(0); + auto firstValue = recordFirst->GetEntry("general.video"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(firstValue)); + std::shared_ptr<Object> firstUds = std::get<std::shared_ptr<Object>>(firstValue); + EXPECT_EQ(std::get<std::string>(firstUds->value_[VALUE_TYPE]), "video"); + auto recordSecond = readData.GetRecordAt(1); + auto secondValue = recordSecond->GetEntry("general.html"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(secondValue)); + std::shared_ptr<Object> secondUds = std::get<std::shared_ptr<Object>>(secondValue); + EXPECT_EQ(std::get<std::string>(secondUds->value_[VALUE_TYPE]), "html"); + auto recordThird = readData.GetRecordAt(2); + auto thirdValue = recordThird->GetEntry("com.kingsoft.office.writer.wpt"); + EXPECT_TRUE(std::holds_alternative<std::string>(thirdValue)); + EXPECT_EQ(std::get<std::string>(thirdValue), "abc"); + auto recordFourth = readData.GetRecordAt(3); + auto fourthValue = recordFourth->GetEntry("general.plain-text"); + EXPECT_TRUE(std::holds_alternative<std::shared_ptr<Object>>(fourthValue)); + std::shared_ptr<Object> fourthUds = std::get<std::shared_ptr<Object>>(fourthValue); + EXPECT_EQ(std::get<std::string>(fourthUds->value_["textContent"]), "plainTextContent"); + auto recordFifth = readData.GetRecordAt(4); + auto fifthValue = recordFifth->GetEntry("general.hyperlink"); + EXPECT_TRUE(std::holds_alternative<std::monostate>( + std::get<std::shared_ptr<Object>>(fifthValue)->value_[VALUE_TYPE])); +} } // OHOS::Test \ No newline at end of file diff --git a/udmf/framework/jskitsimpl/common/napi_data_utils.cpp b/udmf/framework/jskitsimpl/common/napi_data_utils.cpp index d404084f..7d7cf89d 100644 --- a/udmf/framework/jskitsimpl/common/napi_data_utils.cpp +++ b/udmf/framework/jskitsimpl/common/napi_data_utils.cpp @@ -20,6 +20,15 @@ namespace UDMF { constexpr int32_t STR_MAX_LENGTH = 4096; constexpr size_t STR_TAIL_LENGTH = 1; +static const std::map<napi_valuetype, ValueType> objectValueTypeMap = { + {napi_valuetype::napi_number, double()}, + {napi_valuetype::napi_string, std::string()}, + {napi_valuetype::napi_boolean, bool()}, + {napi_valuetype::napi_undefined, std::monostate()}, + {napi_valuetype::napi_null, nullptr} +}; + +static const std::set<std::string> udsAttributeKeySet = {"details", "thumbData", "appIcon", "arrayBuffer"}; /* napi_value <-> bool */ napi_status NapiDataUtils::GetValue(napi_env env, napi_value in, bool &out) { @@ -166,18 +175,26 @@ napi_status NapiDataUtils::GetValue(napi_env env, napi_value in, std::vector<uin { out.clear(); LOG_DEBUG(UDMF_KITS_NAPI, "napi_value -> std::vector<uint8_t> "); + + bool isTypedArray = false; + auto status = napi_is_typedarray(env, in, &isTypedArray); + if (status != napi_ok || !isTypedArray) { + return napi_invalid_arg; + } + napi_typedarray_type type = napi_biguint64_array; size_t length = 0; napi_value buffer = nullptr; size_t offset = 0; void *data = nullptr; - napi_status status = napi_get_typedarray_info(env, in, &type, &length, &data, &buffer, &offset); + status = napi_get_typedarray_info(env, in, &type, &length, &data, &buffer, &offset); LOG_DEBUG(UDMF_KITS_NAPI, "array type=%{public}d length=%{public}d offset=%{public}d status=%{public}d", (int)type, (int)length, (int)offset, status); LOG_ERROR_RETURN(status == napi_ok, "napi_get_typedarray_info failed!", napi_invalid_arg); LOG_ERROR_RETURN(type == napi_uint8_array, "is not Uint8Array!", napi_invalid_arg); - LOG_ERROR_RETURN((length > 0) && (data != nullptr), "invalid data!", napi_invalid_arg); - out.assign(reinterpret_cast<uint8_t *>(data), reinterpret_cast<uint8_t *>(data) + length); + if (length > 0) { + out.assign(static_cast<uint8_t *>(data), static_cast<uint8_t *>(data) + length); + } return status; } @@ -445,29 +462,20 @@ napi_status NapiDataUtils::GetValue(napi_env env, napi_value in, std::shared_ptr } napi_valuetype valueType = napi_undefined; NAPI_CALL_BASE(env, napi_typeof(env, attributeValueNapi, &valueType), napi_invalid_arg); - switch (valueType) { - case napi_valuetype::napi_object: - object->value_[attributeName] = std::make_shared<Object>(); - break; - case napi_valuetype::napi_number: - object->value_[attributeName] = double(); - break; - case napi_valuetype::napi_string: - object->value_[attributeName] = std::string(); - break; - case napi_valuetype::napi_boolean: - object->value_[attributeName] = bool(); - break; - case napi_valuetype::napi_undefined: - object->value_[attributeName] = std::monostate(); - break; - case napi_valuetype::napi_null: - object->value_[attributeName] = nullptr; - break; - default: + napi_status status = napi_ok; + if (valueType == napi_valuetype::napi_object) { + status = ProcessNapiObject(env, in, attributeName, attributeValueNapi, object); + if (status != napi_ok) { + return status; + } + } else { + auto it = objectValueTypeMap.find(valueType); + if (it != objectValueTypeMap.end()) { + object->value_[attributeName] = it->second; + } else { return napi_invalid_arg; + } } - napi_status status = napi_ok; std::visit([&](auto &value) {status = NapiDataUtils::GetValue(env, attributeValueNapi, value);}, object->value_[attributeName]); if (status != napi_ok) { @@ -477,13 +485,46 @@ napi_status NapiDataUtils::GetValue(napi_env env, napi_value in, std::shared_ptr return napi_ok; } +napi_status NapiDataUtils::ProcessNapiObject(napi_env env, napi_value in, std::string &attributeName, + napi_value attributeValueNapi, std::shared_ptr<Object> object) +{ + if (attributeName == PIXEL_MAP) { + object->value_[attributeName] = std::shared_ptr<OHOS::Media::PixelMap>(); + return napi_ok; + } + bool isUint8Array = false; + napi_value constructor = nullptr; + NAPI_CALL_BASE(env, napi_get_named_property(env, attributeValueNapi, "constructor", &constructor), + napi_invalid_arg); + napi_value global = nullptr; + NAPI_CALL_BASE(env, napi_get_global(env, &global), napi_invalid_arg); + napi_value uint8ArrayConstructor = nullptr; + NAPI_CALL_BASE(env, napi_get_named_property(env, global, "Uint8Array", &uint8ArrayConstructor), napi_invalid_arg); + NAPI_CALL_BASE(env, napi_strict_equals(env, constructor, uint8ArrayConstructor, &isUint8Array), napi_invalid_arg); + if (isUint8Array && (udsAttributeKeySet.find(attributeName) != udsAttributeKeySet.end())) { + napi_value jsProValue = nullptr; + NAPI_CALL_BASE(env, napi_get_named_property(env, in, attributeName.c_str(), &jsProValue), napi_invalid_arg); + std::vector<uint8_t> array; + auto status = GetValue(env, jsProValue, array); + if (status != napi_ok) { + LOG_ERROR(UDMF_KITS_NAPI, "Get Uint8Array error: %{public}d", status); + return status; + } + object->value_[attributeName] = array; + return napi_ok; + } + object->value_[attributeName] = std::make_shared<Object>(); + return napi_ok; +} + napi_status NapiDataUtils::SetValue(napi_env env, const std::shared_ptr<Object> &object, napi_value &out) { LOG_DEBUG(UDMF_KITS_NAPI, "napi_value -> std::GetValue Object"); napi_create_object(env, &out); for (auto &[key, value] : object->value_) { napi_value valueNapi = nullptr; - if (std::holds_alternative<std::vector<uint8_t>>(value)) { + if (std::holds_alternative<std::vector<uint8_t>>(value) && + (udsAttributeKeySet.find(key) == udsAttributeKeySet.end())) { auto array = std::get<std::vector<uint8_t>>(value); void *data = nullptr; size_t len = array.size(); @@ -492,6 +533,18 @@ napi_status NapiDataUtils::SetValue(napi_env env, const std::shared_ptr<Object> LOG_ERROR(UDMF_KITS_NAPI, "memcpy_s failed"); return napi_generic_failure; } + } else if (std::holds_alternative<std::vector<uint8_t>>(value) && + (udsAttributeKeySet.find(key) != udsAttributeKeySet.end())) { + auto array = std::get<std::vector<uint8_t>>(value); + void *data = nullptr; + napi_value buffer = nullptr; + NAPI_CALL_BASE(env, napi_create_arraybuffer(env, array.size(), &data, &buffer), napi_generic_failure); + if (!array.empty() && memcpy_s(data, array.size(), array.data(), array.size()) != 0) { + LOG_ERROR(UDMF_KITS_NAPI, "memcpy_s failed"); + return napi_generic_failure; + } + NAPI_CALL_BASE(env, napi_create_typedarray(env, napi_uint8_array, array.size(), buffer, 0, &valueNapi), + napi_generic_failure); } else { std::visit([&](const auto &value) {NapiDataUtils::SetValue(env, value, valueNapi);}, value); } @@ -533,11 +586,11 @@ napi_status NapiDataUtils::SetValue(napi_env env, const ProgressInfo &in, napi_v napi_value jsPercentage = nullptr; SetValue(env, in.progress, jsPercentage); - napi_set_named_property(env, out, "percentage", jsPercentage); + NAPI_CALL_BASE(env, napi_set_named_property(env, out, "progress", jsPercentage), napi_invalid_arg); napi_value jsListenerStatus = nullptr; SetValue(env, in.progressStatus, jsListenerStatus); - napi_set_named_property(env, out, "status", jsListenerStatus); + NAPI_CALL_BASE(env, napi_set_named_property(env, out, "status", jsListenerStatus), napi_invalid_arg); return napi_ok; } diff --git a/udmf/framework/jskitsimpl/common/napi_error_utils.cpp b/udmf/framework/jskitsimpl/common/napi_error_utils.cpp index f71939b8..8567babe 100644 --- a/udmf/framework/jskitsimpl/common/napi_error_utils.cpp +++ b/udmf/framework/jskitsimpl/common/napi_error_utils.cpp @@ -15,8 +15,6 @@ #define LOG_TAG "NapiErrorCode" #include "napi_error_utils.h" -#include <algorithm> - namespace OHOS { namespace UDMF { using NapiErrorCode = OHOS::UDMF::NapiErrorCode; diff --git a/udmf/framework/jskitsimpl/data/unified_data_channel_napi.cpp b/udmf/framework/jskitsimpl/data/unified_data_channel_napi.cpp index 579e3513..d01cfbae 100644 --- a/udmf/framework/jskitsimpl/data/unified_data_channel_napi.cpp +++ b/udmf/framework/jskitsimpl/data/unified_data_channel_napi.cpp @@ -141,8 +141,10 @@ napi_value UnifiedDataChannelNapi::UpdateData(napi_env env, napi_callback_info i auto execute = [ctxt]() { QueryOption option = { .key = ctxt->key }; auto status = UdmfClient::GetInstance().UpdateData(option, *(ctxt->unifiedData)); - ASSERT_WITH_ERRCODE(ctxt, status == E_OK, E_INVALID_PARAMETERS, - "Parameter error: The unifiedData parameter is invalid!"); + if (status == E_INVALID_PARAMETERS) { + ASSERT_WITH_ERRCODE(ctxt, status == E_OK, E_INVALID_PARAMETERS, + "Parameter error: The unifiedData parameter is invalid!"); + } }; return NapiQueue::AsyncWork(env, ctxt, std::string(__FUNCTION__), execute); } @@ -185,8 +187,6 @@ napi_value UnifiedDataChannelNapi::QueryData(napi_env env, napi_callback_info in ASSERT_WITH_ERRCODE(ctxt, status == E_OK, status, "QueryData failed!"); }; auto output = [env, ctxt](napi_value &result) { - ASSERT_WITH_ERRCODE(ctxt, !ctxt->unifiedDataSet.empty(), E_INVALID_PARAMETERS, - "Parameter error: The unifiedDataSet parameter is empty!"); ctxt->status = ConvertUnifiedDataSetToNapi(env, ctxt->unifiedDataSet, result); ASSERT_WITH_ERRCODE(ctxt, ctxt->status == napi_ok, E_ERROR, "ConvertUnifiedDataSetToNapi failed!"); }; @@ -357,6 +357,7 @@ napi_status UnifiedDataChannelNapi::ConvertUnifiedDataSetToNapi( for (const UnifiedData &data : dataSet) { std::shared_ptr<UnifiedData> unifiedData = std::make_shared<UnifiedData>(); unifiedData->SetRecords(data.GetRecords()); + unifiedData->SetProperties(data.GetProperties()); napi_value dataNapi = nullptr; UnifiedDataNapi::NewInstance(env, unifiedData, dataNapi); status = napi_set_element(env, output, index++, dataNapi); diff --git a/udmf/framework/jskitsimpl/data/unified_data_napi.cpp b/udmf/framework/jskitsimpl/data/unified_data_napi.cpp index f88a80c7..38714dbe 100644 --- a/udmf/framework/jskitsimpl/data/unified_data_napi.cpp +++ b/udmf/framework/jskitsimpl/data/unified_data_napi.cpp @@ -254,7 +254,7 @@ napi_value UnifiedDataNapi::HasType(napi_env env, napi_callback_info info) auto *uData = static_cast<UnifiedDataNapi *>(ctxt->native); ASSERT_ERR( ctxt->env, (uData != nullptr && uData->value_ != nullptr), Status::E_ERROR, "invalid object!"); - bool ret = uData->value_->HasType(type); + bool ret = uData->value_->HasHigherFileType(type); napi_value result = nullptr; napi_get_boolean(env, ret, &result); return result; @@ -287,6 +287,7 @@ napi_value UnifiedDataNapi::GetProperties(napi_env env, napi_callback_info info) napi_value value; auto uData = GetUnifiedData(env, info); ASSERT_ERR(env, (uData != nullptr && uData->value_ != nullptr), Status::E_ERROR, "invalid object!"); + ASSERT_ERR(env, (uData != nullptr && uData->propertyRef_ != nullptr), Status::E_ERROR, "invalid properties!"); NAPI_CALL(env, napi_get_reference_value(env, uData->propertyRef_, &value)); return value; } diff --git a/udmf/framework/jskitsimpl/data/unified_record_napi.cpp b/udmf/framework/jskitsimpl/data/unified_record_napi.cpp index c7cfd4be..b1948428 100644 --- a/udmf/framework/jskitsimpl/data/unified_record_napi.cpp +++ b/udmf/framework/jskitsimpl/data/unified_record_napi.cpp @@ -26,6 +26,7 @@ #include "system_defined_form.h" #include "system_defined_pixelmap.h" #include "application_defined_record.h" +#include "utd_client.h" namespace OHOS { namespace UDMF { @@ -108,7 +109,9 @@ std::shared_ptr<UnifiedRecord> UnifiedRecordNapi::GenerateNativeRecord(napi_env {APPLICATION_DEFINED_RECORD, [](UDType type, ValueType value) { return std::make_shared<ApplicationDefinedRecord>(type, value); }}, }; - + if (utdType == FILE_URI && std::holds_alternative<std::shared_ptr<Object>>(value)) { + ObjectUtils::ProcessFileUriType(utdType, value); + } auto constructor = constructors.find(utdType); if (constructor == constructors.end()) { return std::make_shared<UnifiedRecord>(utdType, value); @@ -140,13 +143,7 @@ void UnifiedRecordNapi::GetNativeValue(napi_env env, std::string type, napi_valu ASSERT_ERR_VOID(env, status == napi_ok, Status::E_INVALID_PARAMETERS, "Parameter error: parameter value type must be ValueType"); if (valueType == napi_object) { - if (type == "openharmony.pixel-map") { - value = std::shared_ptr<OHOS::Media::PixelMap>(nullptr); - } else if (type == "openharmony.want") { - value = std::shared_ptr<OHOS::AAFwk::Want>(nullptr); - } else { - value = std::make_shared<Object>(); - } + ProcessNapiObject(env, type, valueNapi, value); } else if (valueType == napi_string) { value = std::string(); } else if (valueType == napi_number) { @@ -162,6 +159,31 @@ void UnifiedRecordNapi::GetNativeValue(napi_env env, std::string type, napi_valu ASSERT_ERR_VOID(env, status == napi_ok, Status::E_ERROR, "get unifiedRecord failed"); } +void UnifiedRecordNapi::ProcessNapiObject(napi_env env, std::string type, napi_value valueNapi, ValueType &value) +{ + if (type == "openharmony.pixel-map") { + napi_value attrName = nullptr; + napi_value attrVal = nullptr; + const std::string uniformDataType = "uniformDataType"; + napi_status status = NapiDataUtils::SetValue(env, uniformDataType, attrName); + ASSERT_ERR_VOID(env, status == napi_ok, Status::E_ERROR, "set attrName failed"); + status = napi_get_property(env, valueNapi, attrName, &attrVal); + if (status == napi_ok) { + std::string proVal; + status = NapiDataUtils::GetValue(env, attrVal, proVal); + if (status == napi_ok && proVal == "openharmony.pixel-map") { + value = std::make_shared<Object>(); + } else { + value = std::shared_ptr<OHOS::Media::PixelMap>(nullptr); + } + } + } else if (type == "openharmony.want") { + value = std::shared_ptr<OHOS::AAFwk::Want>(nullptr); + } else { + value = std::make_shared<Object>(); + } +} + void UnifiedRecordNapi::NewInstance(napi_env env, std::shared_ptr<UnifiedRecord> in, napi_value &out) { LOG_DEBUG(UDMF_KITS_NAPI, "UnifiedRecordNapi"); @@ -254,9 +276,13 @@ napi_value UnifiedRecordNapi::AddEntry(napi_env env, napi_callback_info info) ASSERT_ERR(ctxt->env, ctxt->status == napi_ok, Status::E_ERROR, ctxt->error); auto uRecord = static_cast<UnifiedRecordNapi *>(ctxt->native); ASSERT_ERR(ctxt->env, (uRecord != nullptr && uRecord->value_ != nullptr), Status::E_ERROR, "invalid object!"); - ValueType value; - GetNativeValue(env, type, napiValue, value); - uRecord->value_->AddEntry(type, std::move(value)); + if (uRecord->value_->GetType() == UD_BUTT) { + uRecord->value_ = GenerateNativeRecord(env, type, napiValue); + } else { + ValueType value; + GetNativeValue(env, type, napiValue, value); + uRecord->value_->AddEntry(type, std::move(value)); + } return nullptr; } diff --git a/udmf/framework/jskitsimpl/intelligence/aip_napi_error.cpp b/udmf/framework/jskitsimpl/intelligence/aip_napi_error.cpp index 44b455e1..9cdf24ba 100644 --- a/udmf/framework/jskitsimpl/intelligence/aip_napi_error.cpp +++ b/udmf/framework/jskitsimpl/intelligence/aip_napi_error.cpp @@ -24,6 +24,89 @@ namespace OHOS { namespace DataIntelligence { +struct ErrCodeMapping { + int32_t nativeErrCode; + int32_t tsErrCode; +}; + +constexpr ErrCodeMapping NATIVE_ERR_CODE2TS_ERR_CODE[] = { + { NATIVE_DATABASE_INVALID_FILE_PATH, DATABASE_INVALID_FILE_PATH }, + { NATIVE_DATABASE_ALREADY_CLOSED, DATABASE_ALREADY_CLOSED }, + { NATIVE_DATABASE_IS_BUSY, DATABASE_IS_BUSY }, + { NATIVE_DATABASE_WAL_FILE_EXCEEDS_LIMIT, DATABASE_WAL_FILE_EXCEEDS_LIMIT }, + { NATIVE_DATABASE_GENERIC_ERROD, DATABASE_GENERIC_ERROD }, + { NATIVE_DATABASE_CORRUPTED, DATABASE_CORRUPTED }, + { NATIVE_DATABASE_PERMISSION_DENIED, DATABASE_PERMISSION_DENIED }, + { NATIVE_DATABASE_FILE_IS_LOCKED, DATABASE_FILE_IS_LOCKED }, + { NATIVE_DATABASE_OUT_OF_MEMORY, DATABASE_OUT_OF_MEMORY }, + { NATIVE_DATABASE_IO_ERROR, DATABASE_IO_ERROR }, + { NATIVE_E_RECALL_FAILURE, RETRIEVAL_E_RECALL_FAILURE }, + { NATIVE_E_RERANK_FAILURE, RETRIEVAL_E_RERANK_FAILURE }, + { NATIVE_E_INVALID_NUMBER, RETRIEVAL_E_INVALID_NUMBER }, + { NATIVE_E_INVALID_PRIMARY_KEY, RETRIEVAL_E_INVALID_PRIMARY_KEY }, + { NATIVE_E_UNSUPPORTED_COMPOSITE_PRIMARY_KEY, RETRIEVAL_E_UNSUPPORTED_COMPOSITE_PRIMARY_KEY }, + { NATIVE_E_EMPTY_STRING_FIELD, RETRIEVAL_E_EMPTY_STRING_FIELD }, + { NATIVE_E_INVALID_FILTER_INPUT, RETRIEVAL_E_INVALID_FILTER_INPUT }, + { NATIVE_E_INVALID_RECALL_FIELD, RETRIEVAL_E_INVALID_RECALL_FIELD }, + { NATIVE_E_HIGH_VECTOR_SIMILARITY_THRESHOLD, RETRIEVAL_E_HIGH_VECTOR_SIMILARITY_THRESHOLD }, + { NATIVE_E_RERANK_METHOD_MISMATCH, RETRIEVAL_E_RERANK_METHOD_MISMATCH }, + { NATIVE_E_EMPTY_PARAMETER, RETRIEVAL_E_EMPTY_PARAMETER } +}; + +struct ErrMessage { + int32_t code; + const char *message; +}; + +constexpr ErrMessage ERROR_MESSAGES [] = { + { PARAM_EXCEPTION, "Params check failed." }, + { DEVICE_EXCEPTION, "The device does not support this API." }, + { INNER_ERROR, "Inner error." }, + { DATABASE_CORRUPTED, "Database corrupted." }, + { DATABASE_ALREADY_CLOSED, "Already closed." }, + { DATABASE_IS_BUSY, "The database is busy." }, + { DATABASE_OUT_OF_MEMORY, "The database is out of memory." }, + { DATABASE_GENERIC_ERROD, "SQLite: Generic error." }, + { DATABASE_PERMISSION_DENIED, "SQLite: Access permission denied." }, + { DATABASE_FILE_IS_LOCKED, "SQLite: The database file is locked." }, + { DATABASE_IO_ERROR, "SQLite: Some kind of disk I/O error occurred." }, + { DATABASE_WAL_FILE_EXCEEDS_LIMIT, "SQLite: The WAL file size exceeds the default limit." }, + { DATABASE_INVALID_FILE_PATH, "Unable to open the database file." }, + { RETRIEVAL_E_RECALL_FAILURE, "Retrieval: An error occurred during the recall phase." }, + { RETRIEVAL_E_RERANK_FAILURE, "Retrieval: An error occurred during the re-ranking phase." }, + { RETRIEVAL_E_INVALID_NUMBER, + "Retrieval: The value of the numerical parameter is outside the constrained range." }, + { RETRIEVAL_E_INVALID_PRIMARY_KEY, "Retrieval: There are invalid primary keys." }, + { RETRIEVAL_E_UNSUPPORTED_COMPOSITE_PRIMARY_KEY, + "Retrieval: A re-ranking algorithm that does not support composite primary keys was used." }, + { RETRIEVAL_E_EMPTY_STRING_FIELD, "Retrieval: There are fields with empty strings." }, + { RETRIEVAL_E_INVALID_FILTER_INPUT, "Retrieval: The filter input is invalid." }, + { RETRIEVAL_E_INVALID_RECALL_FIELD, "Retrieval: There are invalid recall names in RecallCondition." }, + { RETRIEVAL_E_HIGH_VECTOR_SIMILARITY_THRESHOLD, "Retrieval: The vector similarity threshold in VectorQuery is " + "higher than the tiered threshold in VectorRerankParameter." }, + { RETRIEVAL_E_RERANK_METHOD_MISMATCH, "Retrieval: RerankMethod parameters do not match the channel type." }, + { RETRIEVAL_E_EMPTY_PARAMETER, + "Retrieval: There exists a parameter value that should not be empty but is actually empty." } +}; + +int32_t ConvertErrCodeNative2Ts(int32_t nativeErrCode) +{ + int left = 0; + int right = sizeof(NATIVE_ERR_CODE2TS_ERR_CODE) / sizeof(NATIVE_ERR_CODE2TS_ERR_CODE[0]) - 1; + while (left <= right) { + int mid = left + (right - left) / 2; + if (NATIVE_ERR_CODE2TS_ERR_CODE[mid].nativeErrCode == nativeErrCode) { + return NATIVE_ERR_CODE2TS_ERR_CODE[mid].tsErrCode; + } else if (NATIVE_ERR_CODE2TS_ERR_CODE[mid].nativeErrCode < nativeErrCode) { + left = mid + 1; + } else { + right = mid - 1; + } + } + return DATABASE_GENERIC_ERROD; +} + + napi_value CreateIntelligenceError(const napi_env &env, int32_t errorCode, const std::string &errorMsg) { napi_value businessError = nullptr; @@ -38,9 +121,17 @@ napi_value CreateIntelligenceError(const napi_env &env, int32_t errorCode, const std::optional<std::string> GetIntelligenceErrMsg(int32_t errorCode) { - auto iter = ERROR_MESSAGES.find(errorCode); - if (iter != ERROR_MESSAGES.end()) { - return iter->second; + int left = 0; + int right = sizeof(ERROR_MESSAGES) / sizeof(ERROR_MESSAGES[0]) - 1; + while (left <= right) { + int mid = left + (right - left) / 2; + if (ERROR_MESSAGES[mid].code == errorCode) { + return ERROR_MESSAGES[mid].message; + } else if (ERROR_MESSAGES[mid].code < errorCode) { + left = mid + 1; + } else { + right = mid - 1; + } } AIP_HILOGE("Error, messages not found"); return std::nullopt; diff --git a/udmf/framework/jskitsimpl/intelligence/aip_napi_utils.cpp b/udmf/framework/jskitsimpl/intelligence/aip_napi_utils.cpp index 4733256a..e46bbbc6 100644 --- a/udmf/framework/jskitsimpl/intelligence/aip_napi_utils.cpp +++ b/udmf/framework/jskitsimpl/intelligence/aip_napi_utils.cpp @@ -17,7 +17,10 @@ #include <dlfcn.h> +#include "js_ability.h" #include "aip_log.h" +#include "hilog/log_c.h" +#include "js_native_api_types.h" #undef LOG_TAG #define LOG_TAG "AipNapiUtils" @@ -25,9 +28,55 @@ namespace OHOS { namespace DataIntelligence { namespace { +using namespace OHOS::AppDataMgrJsKit; +static constexpr uint32_t MAX_VALUE_LENGTH = 1024 * 1024 * 8; // the max length of all kand of out string value +static constexpr uint8_t CONFIG_LENGTH_1 = 2; +static constexpr uint8_t CONFIG_LENGTH_2 = 3; static constexpr uint32_t MAX_STR_PARAM_LEN = 512; +static constexpr int32_t MIN_SECURITY_LEVEL = 1; +static constexpr int32_t MAX_SECURITY_LEVEL = 4; +static constexpr uint32_t MAX_STR_CUSTOMDIR_LEN = 128; +static constexpr uint32_t MAX_STR_REAL_PATH_LEN = 1024; +constexpr int E_OK = 0; +constexpr const char *PATH_SPLIT = "/"; +struct OperatorInfo { + const char *operatorStr; + int32_t operatorEnum; +}; +constexpr OperatorInfo OPERATOR_INFO [] = { + { "!=", TsOperator::OP_NE }, + { "<", TsOperator::OP_LT }, + { "<=", TsOperator::OP_LE }, + { "=", TsOperator::OP_EQ }, + { ">", TsOperator::OP_GT }, + { ">=", TsOperator::OP_GE }, + { "BETWEEN", TsOperator::OP_BETWEEN }, + { "IN", TsOperator::OP_IN }, + { "LIKE", TsOperator::OP_LIKE }, + { "NOT_IN", TsOperator::OP_NOT_IN }, + { "NOT_LIKE", TsOperator::OP_NOT_LIKE }, +}; } // namespace -bool AipNapiUtils::LoadAlgoLibrary(const std::string &algoPath, AipCoreManagerHandle &aipMgrHandler) + +int32_t AipNapiUtils::FindOperaotrEnum(std::string operatorStr) +{ + int left = 0; + int right = sizeof(OPERATOR_INFO) / sizeof(OPERATOR_INFO[0]) - 1; + while (left <= right) { + int mid = left + (right - left) / 2; + if (OPERATOR_INFO[mid].operatorStr == operatorStr) { + return OPERATOR_INFO[mid].operatorEnum; + } else if (OPERATOR_INFO[mid].operatorStr < operatorStr) { + left = mid + 1; + } else { + right = mid - 1; + } + } + return TsOperator::BUTT; +} + +bool AipNapiUtils::LoadAlgoLibrary(const std::string &libraryName, AipCoreManagerHandle &aipMgrHandler, + bool isSingleton) { AIP_HILOGD("Enter"); if (aipMgrHandler.handle != nullptr) { @@ -35,17 +84,12 @@ bool AipNapiUtils::LoadAlgoLibrary(const std::string &algoPath, AipCoreManagerHa return false; } - if (algoPath.empty()) { + if (libraryName.empty()) { AIP_HILOGE("algoPath is empty"); return false; } - char libRealPath[PATH_MAX] = {}; - if (realpath(algoPath.c_str(), libRealPath) == nullptr) { - AIP_HILOGE("get absolute algoPath error, %{public}d", errno); - return false; - } - aipMgrHandler.handle = dlopen(libRealPath, RTLD_LAZY); + aipMgrHandler.handle = dlopen(libraryName.c_str(), RTLD_LAZY); if (aipMgrHandler.handle == nullptr) { AIP_HILOGE("cannot load lib error: %{public}s", dlerror()); return false; @@ -59,13 +103,13 @@ bool AipNapiUtils::LoadAlgoLibrary(const std::string &algoPath, AipCoreManagerHa AIP_HILOGE("Failed to create and destroy algo"); return false; } - - aipMgrHandler.pAipManager = aipMgrHandler.create(); + if (isSingleton) { + aipMgrHandler.pAipManager = aipMgrHandler.create(); + } AIP_HILOGD("Exit"); return true; } - bool AipNapiUtils::UnLoadAlgoLibrary(AipCoreManagerHandle &aipMgrHandler) { AIP_HILOGD("Enter"); @@ -249,5 +293,1021 @@ void AipNapiUtils::CreateDoubleData(napi_env env, double value, napi_value *resu return; } } + +void AipNapiUtils::SetInt32Property(napi_env env, napi_value targetObj, int32_t value, const char *propName) +{ + napi_value prop = nullptr; + napi_status ret = napi_create_int32(env, value, &prop); + if (ret != napi_ok) { + AIP_HILOGE("napi_create_int32 failed"); + return; + } + SetPropertyName(env, targetObj, propName, prop); +} + +void AipNapiUtils::SetStringProperty(napi_env env, napi_value targetObj, std::string value, const char *propName) +{ + napi_value prop = nullptr; + napi_status ret = napi_create_string_utf8(env, value.c_str(), value.size(), &prop); + if (ret != napi_ok) { + AIP_HILOGE("napi_create_int32 failed"); + return; + } + SetPropertyName(env, targetObj, propName, prop); +} + +void AipNapiUtils::SetPropertyName(napi_env env, napi_value targetObj, const char *propName, napi_value propValue) +{ + napi_status status = napi_set_named_property(env, targetObj, propName, propValue); + if (status != napi_ok) { + AIP_HILOGE("napi_set_named_property failed"); + return; + } +} + +bool AipNapiUtils::CheckModelConfig(napi_env env, napi_value value) +{ + napi_valuetype valuetype; + napi_status status = napi_typeof(env, value, &valuetype); + if (status != napi_ok || valuetype != napi_object) { + AIP_HILOGE("ModelConfig is not object"); + return false; + } + + napi_value KeysArray; + status = napi_get_property_names(env, value, &KeysArray); + if (status != napi_ok) { + AIP_HILOGE("Failed to get property names"); + return false; + } + + uint32_t length; + status = napi_get_array_length(env, KeysArray, &length); + if (status != napi_ok) { + AIP_HILOGE("Failed to get array length"); + return false; + } + + if (length != CONFIG_LENGTH_1 && length != CONFIG_LENGTH_2) { + AIP_HILOGE("The modelConfig length is failed"); + return false; + } + return true; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, bool &out) +{ + return napi_get_value_bool(env, in, &out); +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, int32_t &out) +{ + return napi_get_value_int32(env, in, &out); +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, uint32_t &out) +{ + return napi_get_value_uint32(env, in, &out); +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, std::vector<uint8_t> &out) +{ + bool isTypedArray = false; + napi_is_typedarray(env, in, &isTypedArray); + if (!isTypedArray) { + return napi_invalid_arg; + } + + napi_typedarray_type type; + napi_value input_buffer = nullptr; + size_t byte_offset = 0; + size_t length = 0; + void *tmp = nullptr; + auto status = napi_get_typedarray_info(env, in, &type, &length, &tmp, &input_buffer, &byte_offset); + if (status != napi_ok || type != napi_uint8_array) { + return napi_invalid_arg; + } + + out = (tmp != nullptr ? std::vector<uint8_t>(static_cast<uint8_t *>(tmp), static_cast<uint8_t *>(tmp) + length) + : std::vector<uint8_t>()); + return status; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, int64_t &out) +{ + return napi_get_value_int64(env, in, &out); +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, std::string &out) +{ + napi_valuetype type = napi_undefined; + napi_status status = napi_typeof(env, in, &type); + LOG_ERROR_RETURN((status == napi_ok) && (type == napi_string), "The parameter type is not a string.", + napi_invalid_arg); + size_t buffSize = 0; + napi_get_value_string_utf8(env, in, nullptr, 0, &buffSize); + // cut down with 0 if more than MAX_VALUE_LENGTH + if (buffSize >= MAX_VALUE_LENGTH - 1) { + buffSize = MAX_VALUE_LENGTH - 1; + } + std::unique_ptr<char[]> buffer = std::make_unique<char[]>(buffSize + 1); + LOG_ERROR_RETURN(buffer, "Buffer data is nullptr.", napi_invalid_arg); + status = napi_get_value_string_utf8(env, in, buffer.get(), buffSize + 1, &buffSize); + LOG_ERROR_RETURN(status == napi_ok, "napi_get_value_string_utf8 failed", napi_invalid_arg); + out = std::string(buffer.get()); + return status; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, std::vector<float> &out) +{ + bool isTypedArray = false; + napi_is_typedarray(env, in, &isTypedArray); + if (!isTypedArray) { + return napi_invalid_arg; + } + + napi_typedarray_type type; + napi_value input_buffer = nullptr; + size_t byte_offset = 0; + size_t length = 0; + void *tmp = nullptr; + auto status = napi_get_typedarray_info(env, in, &type, &length, &tmp, &input_buffer, &byte_offset); + if (status != napi_ok || type != napi_float32_array) { + return napi_invalid_arg; + } + + out = (tmp != nullptr + ? std::vector<float>(static_cast<float *>(tmp), static_cast<float *>(tmp) + length / sizeof(float)) + : std::vector<float>()); + return status; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, float &out) +{ + double tmp; + napi_status status = napi_get_value_double(env, in, &tmp); + out = tmp; + return status; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, double &out) +{ + return napi_get_value_double(env, in, &out); +} + +bool AipNapiUtils::IsNull(napi_env env, napi_value value) +{ + napi_valuetype type = napi_undefined; + napi_status status = napi_typeof(env, value, &type); + return status == napi_ok && (type == napi_undefined || type == napi_null); +} + +std::pair<napi_status, napi_value> AipNapiUtils::GetInnerValue( + napi_env env, napi_value in, const std::string &prop, bool optional) +{ + bool hasProp = false; + napi_status status = napi_has_named_property(env, in, prop.c_str(), &hasProp); + if (status != napi_ok) { + return std::make_pair(napi_generic_failure, nullptr); + } + if (!hasProp) { + status = optional ? napi_ok : napi_generic_failure; + return std::make_pair(status, nullptr); + } + napi_value inner = nullptr; + status = napi_get_named_property(env, in, prop.c_str(), &inner); + if (status != napi_ok || inner == nullptr) { + return std::make_pair(napi_generic_failure, nullptr); + } + if (optional && AipNapiUtils::IsNull(env, inner)) { + return std::make_pair(napi_ok, nullptr); + } + return std::make_pair(napi_ok, inner); +} + +std::string GetCustomDatabasePath( + const std::string &rootDir, const std::string &name, const std::string &customDir) +{ + std::string databasePath; + databasePath.append(rootDir).append(PATH_SPLIT).append(customDir).append(PATH_SPLIT).append(name); + return databasePath; +} + +std::string GetDefaultDatabasePath( + const std::string &baseDir, const std::string &name, const std::string &customDir) +{ + std::string databaseDir; + databaseDir.append(baseDir).append("/rdb/").append(customDir).append(PATH_SPLIT).append(name); + return databaseDir; +} + +napi_status GetRealPath( + napi_env env, napi_value jsValue, RdbConfig &rdbConfig, ContextParam &param) +{ + LOG_ERROR_RETURN(rdbConfig.name.find(PATH_SPLIT) == std::string::npos, + "rdbConfig.name must not contain '/'", napi_invalid_arg); + if (!rdbConfig.customDir.empty()) { + LOG_ERROR_RETURN(rdbConfig.customDir.find_first_of(PATH_SPLIT) != 0, + "rdbConfig.customDir must a relative directory.", napi_invalid_arg); + LOG_ERROR_RETURN(rdbConfig.customDir.length() <= MAX_STR_CUSTOMDIR_LEN, + "rdbConfig.customDir must less than or equal to 128 bytes.", napi_invalid_arg); + } + std::string baseDir = param.baseDir; + if (!rdbConfig.dataGroupId.empty()) { + LOG_ERROR_RETURN(param.isStageMode, + "The operation is supported in the stage model only.", napi_invalid_arg); + auto stageContext = JSAbility::GetStageModeContext(env, jsValue); + LOG_ERROR_RETURN(stageContext != nullptr, "Illegal context.", napi_invalid_arg); + std::string groupDir; + int errCode = stageContext->GetSystemDatabaseDir(rdbConfig.dataGroupId, groupDir); + LOG_ERROR_RETURN(errCode == E_OK || !groupDir.empty(), "Invalid data group ID.", napi_invalid_arg); + baseDir = groupDir; + } + if (!rdbConfig.rootDir.empty()) { + LOG_ERROR_RETURN(rdbConfig.rootDir.find_first_of(PATH_SPLIT) == 0, "rdbConfig.rootDir must start with '/'", + napi_invalid_arg); + auto realPath = GetCustomDatabasePath(rdbConfig.rootDir, rdbConfig.name, rdbConfig.customDir); + rdbConfig.path = realPath; + return napi_ok; + } + auto realPath = GetDefaultDatabasePath(baseDir, rdbConfig.name, rdbConfig.customDir); + LOG_ERROR_RETURN(realPath.length() <= MAX_STR_REAL_PATH_LEN, "database path is a valid path.", napi_invalid_arg); + rdbConfig.path = realPath; + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value in, RetrievalConfigStruct &out) +{ + napi_value channelConfigsNapi; + napi_get_named_property(env, in, "channelConfigs", &channelConfigsNapi); + std::vector<ChannelConfigStruct> channelConfigs; + uint32_t arrayLength; + napi_get_array_length(env, channelConfigsNapi, &arrayLength); + LOG_ERROR_RETURN(arrayLength > 0, "channelConfigs is empty.", napi_invalid_arg); + for (uint32_t i = 0; i < arrayLength; ++i) { + napi_value channelConfigNapi; + napi_get_element(env, channelConfigsNapi, i, &channelConfigNapi); + ChannelConfigStruct channelConfig; + napi_value channTypeNapi; + napi_get_named_property(env, channelConfigNapi, "channelType", &channTypeNapi); + napi_status status = Convert2Value(env, channTypeNapi, channelConfig.channelType); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field channelType.", napi_invalid_arg); + LOG_ERROR_RETURN(channelConfig.channelType >= TsChannelType::VECTOR_DATABASE && + channelConfig.channelType <= TsChannelType::INVERTED_INDEX_DATABASE, + "Field channelType illegal.", napi_invalid_arg); + napi_value dbConfigNapi; + napi_get_named_property(env, channelConfigNapi, "dbConfig", &dbConfigNapi); + status = Convert2Value(env, dbConfigNapi, channelConfig.dbConfig); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field dbConfig.", napi_invalid_arg); + napi_value contextNapi; + napi_get_named_property(env, channelConfigNapi, "context", &contextNapi); + status = Convert2Value(env, contextNapi, channelConfig.context); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field context.", napi_invalid_arg); + status = GetRealPath(env, contextNapi, channelConfig.dbConfig, channelConfig.context); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the realPath.", napi_invalid_arg); + channelConfigs.push_back(channelConfig); + } + out.channelConfigs = channelConfigs; + return napi_ok; +} + +static napi_status ConvertFilterValue(napi_env env, const napi_value &in, std::string &out) +{ + napi_valuetype type = napi_undefined; + napi_status status = napi_typeof(env, in, &type); + LOG_ERROR_RETURN((status == napi_ok), "invalid type", status); + switch (type) { + case napi_bigint: { + int64_t valueInt64 = false; + status = AipNapiUtils::Convert2Value(env, in, valueInt64); + LOG_ERROR_RETURN((status == napi_ok), "Failed to convert the field filterValue.", status); + out = std::to_string(valueInt64); + break; + } + case napi_number: { + double valueDouble; + status = AipNapiUtils::Convert2Value(env, in, valueDouble); + LOG_ERROR_RETURN((status == napi_ok), "Failed to convert the field filterValue.", status); + out = std::to_string(valueDouble); + break; + } + case napi_string: { + status = AipNapiUtils::Convert2Value(env, in, out); + LOG_ERROR_RETURN((status == napi_ok), "Failed to convert the field filterValue.", status); + break; + } + default: + AIP_HILOGE("type is illegal."); + return napi_invalid_arg; + break; + } + return napi_ok; +} + +napi_status ConvertFilterRange(napi_env env, const napi_value &in, FilterInfoStruct &out) +{ + napi_value filterRangeNapi; + napi_get_named_property(env, in, "filterRange", &filterRangeNapi); + napi_value filterRangeMin; + napi_get_named_property(env, filterRangeNapi, "min", &filterRangeMin); + double min; + napi_status status = AipNapiUtils::Convert2Value(env, filterRangeMin, min); + out.range.first = std::to_string(min); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field min in FilterRange.", napi_invalid_arg); + napi_value filterRangeMax; + napi_get_named_property(env, filterRangeNapi, "max", &filterRangeMax); + double max; + status = AipNapiUtils::Convert2Value(env, filterRangeMax, max); + out.range.second = std::to_string(max); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field max in FilterRange.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, FilterInfoStruct &out) +{ + napi_value columnsNapi; + napi_get_named_property(env, in, "columns", &columnsNapi); + std::vector<std::string> columns; + napi_status status = Convert2Value(env, columnsNapi, columns); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field columns.", napi_invalid_arg); + if (columns.size() == 1) { + out.field = columns[0]; + } else { + out.fields = columns; + } + bool isOperatorPresent = false; + napi_has_named_property(env, in, "operator", &isOperatorPresent); + if (isOperatorPresent) { + napi_value operatorNapi; + napi_get_named_property(env, in, "operator", &operatorNapi); + std::string operatorStr; + status = Convert2Value(env, operatorNapi, operatorStr); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field operator.", napi_invalid_arg); + auto operatorNum = FindOperaotrEnum(operatorStr); + if (operatorNum != TsOperator::BUTT) { + out.op = static_cast<TsOperator>(operatorNum); + } else { + AIP_HILOGE("Failed to convert the operator to an enum, operator: %{public}s", operatorStr.c_str()); + return napi_invalid_arg; + } + } + bool isFilterValuePresent = false; + napi_has_named_property(env, in, "filterValue", &isFilterValuePresent); + if (isFilterValuePresent) { + napi_value filterValueNapi; + napi_get_named_property(env, in, "filterValue", &filterValueNapi); + status = ConvertFilterValue(env, filterValueNapi, out.value); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field filterValue.", napi_invalid_arg); + } + bool isFilterRangePresent = false; + napi_has_named_property(env, in, "filterRange", &isFilterRangePresent); + if (isFilterRangePresent) { + status = ConvertFilterRange(env, in, out); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field filterRange.", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, RecallConditionStruct &out) +{ + napi_value fromClauseNapi; + napi_get_named_property(env, in, "fromClause", &fromClauseNapi); + napi_status status = Convert2Value(env, fromClauseNapi, out.fromClause); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field fromClause.", napi_invalid_arg); + LOG_ERROR_RETURN(!out.fromClause.empty(), "Field fromClause can't be empty.", napi_invalid_arg); + napi_value primaryKeysNapi; + napi_get_named_property(env, in, "primaryKey", &primaryKeysNapi); + status = Convert2Value(env, primaryKeysNapi, out.primaryKey); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field primaryKey.", napi_invalid_arg); + LOG_ERROR_RETURN(!out.primaryKey.empty(), "Field primaryKey can't be empty.", napi_invalid_arg); + napi_value responseColumnsNapi; + napi_get_named_property(env, in, "responseColumns", &responseColumnsNapi); + status = Convert2Value(env, responseColumnsNapi, out.responseColumns); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field responseColumns.", napi_invalid_arg); + LOG_ERROR_RETURN(!out.responseColumns.empty(), "Field responseColumns can't be empty.", napi_invalid_arg); + bool isRecallNamePresent = false; + napi_has_named_property(env, in, "recallName", &isRecallNamePresent); + if (isRecallNamePresent) { + napi_value recallNameNapi; + napi_get_named_property(env, in, "recallName", &recallNameNapi); + status = Convert2Value(env, recallNameNapi, out.recallName); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field recallName.", napi_invalid_arg); + } + bool isDeepSizePresent = false; + napi_has_named_property(env, in, "deepSize", &isDeepSizePresent); + if (isDeepSizePresent) { + napi_value deepSizeNapi; + napi_get_named_property(env, in, "deepSize", &deepSizeNapi); + status = Convert2Value(env, deepSizeNapi, out.deepSize); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field deepSize.", napi_invalid_arg); + } + bool isFiltersPresent = false; + napi_has_named_property(env, in, "filters", &isFiltersPresent); + if (isFiltersPresent) { + napi_value filtersNapi; + napi_get_named_property(env, in, "filters", &filtersNapi); + napi_status status = AipNapiUtils::Convert2Value(env, filtersNapi, out.filters); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field filter.", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, InvertedIndexStrategyStruct &out) +{ + bool isColumnWeightPresent = false; + napi_has_named_property(env, in, "columnWeight", &isColumnWeightPresent); + if (isColumnWeightPresent) { + napi_value columnWeightNapi; + napi_get_named_property(env, in, "columnWeight", &columnWeightNapi); + napi_status status = Convert2Value(env, columnWeightNapi, out.fieldWeight); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field columnWeight.", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, Bm25StrategyStruct &out) +{ + napi_value bm25WeightNapi; + napi_get_named_property(env, in, "bm25Weight", &bm25WeightNapi); + napi_status status = Convert2Value(env, bm25WeightNapi, out.bm25Weight); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field bm25Weight.", napi_invalid_arg); + LOG_ERROR_RETURN(out.bm25Weight >= 0, "Field bm25Weight has an illegal value.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, ExactMatchingStrategyStruct &out) +{ + napi_value exactMatchingWeightNapi; + napi_get_named_property(env, in, "exactMatchingWeight", &exactMatchingWeightNapi); + napi_status status = Convert2Value(env, exactMatchingWeightNapi, out.exactMatchingWeight); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field exactMatchingWeight.", napi_invalid_arg); + LOG_ERROR_RETURN(out.exactMatchingWeight >= 0, "Field exactMatchingWeight has an illegal value.", + napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, OutOfOrderStrategyStruct &out) +{ + napi_value proximityWeightNapi; + napi_get_named_property(env, in, "proximityWeight", &proximityWeightNapi); + napi_status status = Convert2Value(env, proximityWeightNapi, out.outOfOrderWeight); + LOG_ERROR_RETURN(status == napi_ok, "convert field proximityWeight failed.", napi_invalid_arg); + LOG_ERROR_RETURN(out.outOfOrderWeight >= 0, "field proximityWeight value illegal.", + napi_invalid_arg); + bool isColumnSlopsPresent = false; + status = napi_has_named_property(env, in, "columnSlops", &isColumnSlopsPresent); + if (isColumnSlopsPresent) { + napi_value columnSlopsNapi; + napi_get_named_property(env, in, "columnSlops", &columnSlopsNapi); + status = Convert2Value(env, columnSlopsNapi, out.fieldSlops); + LOG_ERROR_RETURN(status == napi_ok, "convert field columnSlops failed.", napi_invalid_arg); + } + return napi_ok; +} + +static napi_status ConvertInvIdxStrategySubClass(napi_env env, const napi_value &in, int index, + std::shared_ptr<InvertedIndexStrategyStruct> &out) +{ + napi_value invertedIndexStrategyNapi; + napi_get_element(env, in, index, &invertedIndexStrategyNapi); + bool isBm25WeightPresent = false; + bool isExactMatchingWeightPresent = false; + bool isProximityWeightPresent = false; + napi_has_named_property(env, invertedIndexStrategyNapi, "bm25Weight", &isBm25WeightPresent); + napi_has_named_property(env, invertedIndexStrategyNapi, "exactMatchingWeight", &isExactMatchingWeightPresent); + napi_has_named_property(env, invertedIndexStrategyNapi, "proximityWeight", &isProximityWeightPresent); + napi_status status; + if (isBm25WeightPresent) { + std::shared_ptr<Bm25StrategyStruct> strategyStruct = + std::make_shared<Bm25StrategyStruct>(); + status = AipNapiUtils::Convert2Value(env, invertedIndexStrategyNapi, strategyStruct); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field Bm25Strategy.", napi_invalid_arg); + out = strategyStruct; + } else if (isExactMatchingWeightPresent) { + std::shared_ptr<ExactMatchingStrategyStruct> strategyStruct = + std::make_shared<ExactMatchingStrategyStruct>(); + status = AipNapiUtils::Convert2Value(env, invertedIndexStrategyNapi, strategyStruct); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field ExactMatchingStrategy.", napi_invalid_arg); + out = strategyStruct; + } else if (isProximityWeightPresent) { + std::shared_ptr<OutOfOrderStrategyStruct> strategyStruct = + std::make_shared<OutOfOrderStrategyStruct>(); + status = AipNapiUtils::Convert2Value(env, invertedIndexStrategyNapi, strategyStruct); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field OutOfOrderStrategy.", napi_invalid_arg); + out = strategyStruct; + } + status = AipNapiUtils::Convert2Value(env, invertedIndexStrategyNapi, out); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the InvertedIndexStrategyStruct.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, InvertedIndexRecallConditionStruct &out) +{ + napi_value ftsTableNameNapi; + napi_get_named_property(env, in, "ftsTableName", &ftsTableNameNapi); + napi_status status = Convert2Value(env, ftsTableNameNapi, out.ftsTableName); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field ftsTableName.", napi_invalid_arg); + LOG_ERROR_RETURN(!out.ftsTableName.empty(), "The field named ftsTableName is empty.", napi_invalid_arg); + bool isStrategyPresent = false; + napi_has_named_property(env, in, "invertedIndexStrategies", &isStrategyPresent); + if (!isStrategyPresent) { + return napi_ok; + } + napi_value invertedIndexStrategiesNapi; + napi_get_named_property(env, in, "invertedIndexStrategies", &invertedIndexStrategiesNapi); + uint32_t arrayLength = 0; + napi_get_array_length(env, invertedIndexStrategiesNapi, &arrayLength); + std::vector<std::shared_ptr<InvertedIndexStrategyStruct>> invertedIndexStrategies; + for (uint32_t i = 0; i < arrayLength; ++i) { + std::shared_ptr<InvertedIndexStrategyStruct> invertedIndexStrategyStruct; + status = ConvertInvIdxStrategySubClass(env, invertedIndexStrategiesNapi, i, invertedIndexStrategyStruct); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the InvertedIndexStrategy.", napi_invalid_arg); + invertedIndexStrategies.push_back(invertedIndexStrategyStruct); + } + out.invertedIndexStrategies = invertedIndexStrategies; + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, VectorChannelRerankParamsStruct &out) +{ + napi_value vectorWeightsNapi; + napi_get_named_property(env, in, "vectorWeights", &vectorWeightsNapi); + napi_status status = Convert2Value(env, vectorWeightsNapi, out.recallWeights); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field vectorWeights.", napi_invalid_arg); + bool isThresholdsPresent = false; + napi_has_named_property(env, in, "thresholds", &isThresholdsPresent); + if (isThresholdsPresent) { + napi_value thresholdsNapi; + napi_get_named_property(env, in, "thresholds", &thresholdsNapi); + status = Convert2Value(env, thresholdsNapi, out.rankLevelThresholds); + LOG_ERROR_RETURN(status == napi_ok, "convert field thresholds failed.", napi_invalid_arg); + } + bool isNumberInspectorPresent = false; + napi_has_named_property(env, in, "numberInspector", &isNumberInspectorPresent); + if (isNumberInspectorPresent) { + napi_value numberInspectorNapi; + napi_get_named_property(env, in, "numberInspector", &numberInspectorNapi); + status = Convert2Value(env, numberInspectorNapi, out.numberFilterCheckFields); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field numberInspector.", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, InvertedIndexRerankParamsStruct &out) +{ + napi_value invertedIndexWeightsNapi; + napi_get_named_property(env, in, "invertedIndexWeights", &invertedIndexWeightsNapi); + napi_status status = Convert2Value(env, invertedIndexWeightsNapi, out.recallWeights); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field invertedIndexWeights.", napi_invalid_arg); + return napi_ok; +} + +static napi_status ConvertRerankChannelParams(napi_env env, const napi_value &in, RerankParamsStruct &out) +{ + napi_value parametersNapi; + napi_get_named_property(env, in, "parameters", &parametersNapi); + napi_value keysNapi; + napi_get_property_names(env, parametersNapi, &keysNapi); + uint32_t keysLength; + napi_get_array_length(env, keysNapi, &keysLength); + napi_status status; + std::map<int32_t, std::shared_ptr<ChannelRerankParamsStruct>> channelParameters; + for (uint32_t i = 0; i < keysLength; ++i) { + napi_value keyNapi; + napi_get_element(env, keysNapi, i, &keyNapi); + std::string keyStr; + status = AipNapiUtils::Convert2Value(env, keyNapi, keyStr); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the channelType in parameters.", napi_invalid_arg); + int32_t keyInt = std::stoi(keyStr); + LOG_ERROR_RETURN(keyInt >= TsChannelType::VECTOR_DATABASE && keyInt <= TsChannelType::INVERTED_INDEX_DATABASE, + "channelType is ileagl.", napi_invalid_arg); + napi_value valueNapi; + napi_get_property(env, parametersNapi, keyNapi, &valueNapi); + std::shared_ptr<ChannelRerankParamsStruct> channelParams; + bool isVectorWeightsPresent = false; + napi_has_named_property(env, valueNapi, "vectorWeights", &isVectorWeightsPresent); + if (isVectorWeightsPresent) { + std::shared_ptr<VectorChannelRerankParamsStruct> vectorRerankParam = + std::make_shared<VectorChannelRerankParamsStruct>(); + status = AipNapiUtils::Convert2Value(env, valueNapi, vectorRerankParam); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the VectorChannelRerankParams.", status); + channelParameters[keyInt] = vectorRerankParam; + } else { + std::shared_ptr<InvertedIndexRerankParamsStruct> invIdxRerankParam = + std::make_shared<InvertedIndexRerankParamsStruct>(); + status = AipNapiUtils::Convert2Value(env, valueNapi, invIdxRerankParam); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the InvertedIndexRerankParams.", status); + channelParameters[keyInt] = invIdxRerankParam; + } + } + out.channelParams = channelParameters; + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, RerankParamsStruct &out) +{ + napi_value rankTypeNapi; + napi_get_named_property(env, in, "rerankType", &rankTypeNapi); + napi_status status = Convert2Value(env, rankTypeNapi, out.rankType); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field rerankType.", napi_invalid_arg); + LOG_ERROR_RETURN(out.rankType >= TsRerankType::RRF && out.rankType <= TsRerankType::FUSION_SCORE, + "The field rankType is illegal. Please check its value..", napi_invalid_arg); + + bool isIsSoftmaxPresent = false; + napi_has_named_property(env, in, "isSoftmaxNormalized", &isIsSoftmaxPresent); + if (isIsSoftmaxPresent) { + napi_value isSoftmaxNormalizedNapi; + napi_get_named_property(env, in, "isSoftmaxNormalized", &isSoftmaxNormalizedNapi); + status = Convert2Value(env, isSoftmaxNormalizedNapi, out.useScoreSoftmax); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field isSoftmaxNormalized.", napi_invalid_arg); + } + bool isParametersPresent = false; + napi_has_named_property(env, in, "parameters", &isParametersPresent); + if (isParametersPresent) { + status = ConvertRerankChannelParams(env, in, out); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field parameters", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, VectorQueryStruct &out) +{ + napi_value columnNapi; + napi_get_named_property(env, in, "column", &columnNapi); + napi_status status = Convert2Value(env, columnNapi, out.vectorColumn); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field column.", napi_invalid_arg); + LOG_ERROR_RETURN(!out.vectorColumn.empty(), "Field column cannot be empty.", napi_invalid_arg); + napi_value valueNapi; + napi_get_named_property(env, in, "value", &valueNapi); + status = Convert2Value(env, valueNapi, out.vectorValue); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field value.", napi_invalid_arg); + bool isThresholdPresent = false; + napi_has_named_property(env, in, "similarityThreshold", &isThresholdPresent); + if (isThresholdPresent) { + napi_value similarityThresholdNapi; + napi_get_named_property(env, in, "similarityThreshold", &similarityThresholdNapi); + status = Convert2Value(env, similarityThresholdNapi, out.similarityThreshold); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field similarityThreshold.", napi_invalid_arg); + } + return napi_ok; +} + +static napi_status ConvertRecallConditionSubClass(napi_env env, const napi_value &in, int index, + std::shared_ptr<RecallConditionStruct> &out) +{ + napi_value recallConditionNapi; + napi_get_element(env, in, index, &recallConditionNapi); + bool isFtsTableNamePresent = false; + bool isVectorQueryPresent = false; + napi_has_named_property(env, recallConditionNapi, "ftsTableName", &isFtsTableNamePresent); + napi_has_named_property(env, recallConditionNapi, "vectorQuery", &isVectorQueryPresent); + napi_status status; + if (isFtsTableNamePresent) { + std::shared_ptr<InvertedIndexRecallConditionStruct> recallConditionPtr = + std::make_shared<InvertedIndexRecallConditionStruct>(); + status = AipNapiUtils::Convert2Value(env, recallConditionNapi, recallConditionPtr); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert InvertedIndexRecallCondition.", napi_invalid_arg); + out = recallConditionPtr; + } else if (isVectorQueryPresent) { + std::shared_ptr<VectorRecallConditionStruct> recallConditionPtr = + std::make_shared<VectorRecallConditionStruct>(); + napi_value vecotrQueryNapi; + napi_get_named_property(env, recallConditionNapi, "vectorQuery", &vecotrQueryNapi); + VectorQueryStruct vecotrQuery; + status = AipNapiUtils::Convert2Value(env, vecotrQueryNapi, recallConditionPtr->vectorQuery); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert field vectorQuery.", napi_invalid_arg); + out = recallConditionPtr; + } + status = AipNapiUtils::Convert2Value(env, recallConditionNapi, out); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field recallCondition.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, RetrievalConditionStruct &out) +{ + napi_status status; + bool isResultCountPresent = false; + napi_has_named_property(env, in, "resultCount", &isResultCountPresent); + if (isResultCountPresent) { + napi_value resultCountNapi; + napi_get_named_property(env, in, "resultCount", &resultCountNapi); + status = Convert2Value(env, resultCountNapi, out.topN); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field resultCount.", napi_invalid_arg); + } + + napi_value recallConditionsNapi; + napi_get_named_property(env, in, "recallConditions", &recallConditionsNapi); + uint32_t arrayLength = 0; + napi_get_array_length(env, recallConditionsNapi, &arrayLength); + LOG_ERROR_RETURN(arrayLength > 0, "Field recallConditions in RetrievalCondition has a length of zero.", + napi_invalid_arg); + std::vector<std::shared_ptr<RecallConditionStruct>> recallConditions; + for (uint32_t i = 0; i < arrayLength; ++i) { + std::shared_ptr<RecallConditionStruct> recallCondition; + status = ConvertRecallConditionSubClass(env, recallConditionsNapi, i, recallCondition); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the RecallCondition.", napi_invalid_arg); + recallConditions.push_back(recallCondition); + } + out.recallConditions = recallConditions; + bool isRerankMethodPresent = false; + napi_has_named_property(env, in, "rerankMethod", &isRerankMethodPresent); + if (isRerankMethodPresent) { + RerankParamsStruct rerankParamsStruct; + napi_value rerankMethodNapi; + napi_get_named_property(env, in, "rerankMethod", &rerankMethodNapi); + status = Convert2Value(env, rerankMethodNapi, rerankParamsStruct); + LOG_ERROR_RETURN(status == napi_ok, "Failed to convert the field rerankParamsStruct.", napi_invalid_arg); + out.rerankParameter = rerankParamsStruct; + } + return napi_ok; +} + +napi_status GetCurrentAbilityParam(napi_env env, napi_value in, ContextParam &param) +{ + std::shared_ptr<Context> context = JSAbility::GetCurrentAbility(env, in); + if (context == nullptr) { + return napi_invalid_arg; + } + param.baseDir = context->GetDatabaseDir(); + param.moduleName = context->GetModuleName(); + param.area = context->GetArea(); + param.bundleName = context->GetBundleName(); + param.isSystemApp = context->IsSystemAppCalled(); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, ContextParam &context) +{ + if (in == nullptr) { + AIP_HILOGI("hasProp is false -> fa stage"); + context.isStageMode = false; + return GetCurrentAbilityParam(env, in, context); + } + + int32_t status = GetNamedProperty(env, in, "stageMode", context.isStageMode); + LOG_ERROR_RETURN(status == napi_ok, "get stageMode param failed", napi_invalid_arg); + if (!context.isStageMode) { + AIP_HILOGW("isStageMode is false -> fa stage"); + return GetCurrentAbilityParam(env, in, context); + } + AIP_HILOGD("Stage mode branch"); + status = GetNamedProperty(env, in, "databaseDir", context.baseDir); + LOG_ERROR_RETURN(status == napi_ok, "get databaseDir failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "area", context.area, true); + LOG_ERROR_RETURN(status == napi_ok, "get area failed.", napi_invalid_arg); + + napi_value hapInfo = nullptr; + GetNamedProperty(env, in, "currentHapModuleInfo", hapInfo); + if (hapInfo != nullptr) { + status = GetNamedProperty(env, hapInfo, "name", context.moduleName); + LOG_ERROR_RETURN(status == napi_ok, "get currentHapModuleInfo.name failed.", napi_invalid_arg); + } + + napi_value appInfo = nullptr; + GetNamedProperty(env, in, "applicationInfo", appInfo); + if (appInfo != nullptr) { + status = GetNamedProperty(env, appInfo, "name", context.bundleName); + LOG_ERROR_RETURN(status == napi_ok, "get applicationInfo.name failed.", napi_invalid_arg); + status = GetNamedProperty(env, appInfo, "systemApp", context.isSystemApp, true); + LOG_ERROR_RETURN(status == napi_ok, "get applicationInfo.systemApp failed.", napi_invalid_arg); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, RdbConfig &out) +{ + int32_t status = GetNamedProperty(env, in, "encrypt", out.isEncrypt, true); + LOG_ERROR_RETURN(status == napi_ok, "get encrypt failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "securityLevel", out.securityLevel); + LOG_ERROR_RETURN(status == napi_ok, "get securityLevel failed.", napi_invalid_arg); + LOG_ERROR_RETURN(out.securityLevel >= MIN_SECURITY_LEVEL && out.securityLevel <= MAX_SECURITY_LEVEL, + "securityLevel is invalid.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "dataGroupId", out.dataGroupId, true); + LOG_ERROR_RETURN(status == napi_ok, "get dataGroupId failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "autoCleanDirtyData", out.isAutoClean, true); + LOG_ERROR_RETURN(status == napi_ok, "get autoCleanDirtyData failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "name", out.name); + LOG_ERROR_RETURN(status == napi_ok, "get name failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "customDir", out.customDir, true); + LOG_ERROR_RETURN(status == napi_ok, "get customDir failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "rootDir", out.rootDir, true); + LOG_ERROR_RETURN(status == napi_ok, "get rootDir failed.", napi_invalid_arg); + + GetNamedProperty(env, in, "isSearchable", out.isSearchable, true); + LOG_ERROR_RETURN(status == napi_ok, "get isSearchable failed.", napi_invalid_arg); + + GetNamedProperty(env, in, "vector", out.vector, true); + LOG_ERROR_RETURN(status == napi_ok, "get vector failed.", napi_invalid_arg); + + GetNamedProperty(env, in, "allowRebuild", out.allowRebuild, true); + LOG_ERROR_RETURN(status == napi_ok, "get allowRebuild failed.", napi_invalid_arg); + + GetNamedProperty(env, in, "isReadOnly", out.isReadOnly, true); + LOG_ERROR_RETURN(status == napi_ok, "get isReadOnly failed.", napi_invalid_arg); + + GetNamedProperty(env, in, "pluginLibs", out.pluginLibs, true); + LOG_ERROR_RETURN(status == napi_ok, "get pluginLibs failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "haMode", out.haMode, true); + LOG_ERROR_RETURN(status == napi_ok, "get haMode failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "cryptoParam", out.cryptoParam, true); + LOG_ERROR_RETURN(status == napi_ok, "get cryptoParam failed.", napi_invalid_arg); + + status = GetNamedProperty(env, in, "tokenizer", out.tokenizer, true); + LOG_ERROR_RETURN(status == napi_ok, "get tokenizer failed.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, CryptoParam &cryptoParam) +{ + napi_valuetype type = napi_undefined; + napi_status status = napi_typeof(env, in, &type); + LOG_ERROR_RETURN(status == napi_ok && type == napi_object, "napi_typeof failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "encryptionKey", cryptoParam.encryptKey_); + LOG_ERROR_RETURN(status == napi_ok, "get encryptionKey failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "iterationCount", cryptoParam.iterNum); + LOG_ERROR_RETURN(status == napi_ok, "get iterationCount failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "encryptionAlgo", cryptoParam.encryptAlgo); + LOG_ERROR_RETURN(status == napi_ok, "get encryptionAlgo failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "hmacAlgo", cryptoParam.hmacAlgo); + LOG_ERROR_RETURN(status == napi_ok, "get hmacAlgo failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "kdfAlgo", cryptoParam.kdfAlgo); + LOG_ERROR_RETURN(status == napi_ok, "get kdfAlgo failed.", napi_invalid_arg); + status = GetNamedProperty(env, in, "cryptoPageSize", cryptoParam.cryptoPageSize); + LOG_ERROR_RETURN(status == napi_ok, "get cryptoPageSize failed.", napi_invalid_arg); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2Value(napi_env env, const napi_value &in, napi_value &out) +{ + out = in; + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::vector<uint8_t> &in, napi_value &out) +{ + void *native = nullptr; + napi_value buffer = nullptr; + napi_status status = napi_create_arraybuffer(env, in.size(), &native, &buffer); + if (status != napi_ok) { + return status; + } + for (size_t i = 0; i < in.size(); i++) { + *(static_cast<uint8_t *>(native) + i) = in[i]; + } + status = napi_create_typedarray(env, napi_uint8_array, in.size(), buffer, 0, &out); + if (status != napi_ok) { + return status; + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::vector<float> &in, napi_value &out) +{ + float *native = nullptr; + napi_value buffer = nullptr; + napi_status status = napi_create_arraybuffer(env, in.size() * sizeof(float), (void **)&native, &buffer); + if (status != napi_ok) { + return status; + } + if (native == nullptr) { + return status; + } + for (size_t i = 0; i < in.size(); i++) { + *(native + i) = in[i]; + } + status = napi_create_typedarray(env, napi_float32_array, in.size(), buffer, 0, &out); + if (status != napi_ok) { + return status; + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::string &in, napi_value &out) +{ + return napi_create_string_utf8(env, in.c_str(), in.size(), &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const double &in, napi_value &out) +{ + return napi_create_double(env, in, &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const bool &in, napi_value &out) +{ + return napi_get_boolean(env, in, &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const int32_t &in, napi_value &out) +{ + return napi_create_int32(env, in, &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const int64_t &in, napi_value &out) +{ + return napi_create_int64(env, in, &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const uint64_t &in, napi_value &out) +{ + return napi_create_bigint_uint64(env, in, &out); +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const DataIntelligence::RetrievalResponseStruct &in, + napi_value &out) +{ + napi_create_object(env, &out); + auto items = in.retrievalResults; + napi_value values = nullptr; + Convert2JSValue(env, items, values); + napi_set_named_property(env, out, "records", values); + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const DataIntelligence::ItemInfoStruct &in, napi_value &out) +{ + napi_value jsPrimaryKey = nullptr; + Convert2JSValue(env, in.primaryKey, jsPrimaryKey); + napi_set_named_property(env, out, "primaryKey", jsPrimaryKey); + + napi_value jsScore = nullptr; + Convert2JSValue(env, in.score, jsScore); + napi_set_named_property(env, out, "score", jsScore); + + napi_value jsFields = nullptr; + Convert2JSValue(env, in.fields, jsFields); + napi_set_named_property(env, out, "columns", jsFields); + + napi_value jsRecallScores = nullptr; + Convert2JSValue(env, in.recallScores, jsRecallScores); + napi_set_named_property(env, out, "recallScores", jsRecallScores); + + napi_value jsFeatures = nullptr; + Convert2JSValue(env, in.features, jsFeatures); + napi_set_named_property(env, out, "features", jsFeatures); + + napi_value jsSimilarityLevel = nullptr; + Convert2JSValue(env, in.similarityLevel, jsSimilarityLevel); + napi_set_named_property(env, out, "similarityLevel", jsSimilarityLevel); + + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::map<std::string, FieldType> &in, napi_value &out) +{ + napi_create_object(env, &out); + for (auto const &[key, value] : in) { + napi_value jsValue = nullptr; + Convert2JSValue(env, value, jsValue); + napi_set_named_property(env, out, key.c_str(), jsValue); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, + const std::map<int32_t, std::map<std::string, RecallScoreStruct>> &in, napi_value &out) +{ + napi_create_object(env, &out); + for (auto const &[key, values] : in) { + napi_value jsValues = nullptr; + napi_create_object(env, &jsValues); + for (auto const &[innerKey, value] : values) { + napi_value jsValue = nullptr; + napi_create_object(env, &jsValue); + + napi_value jsRecallScore = nullptr; + Convert2JSValue(env, value.recallScore, jsRecallScore); + napi_set_named_property(env, jsValue, "recallScore", jsRecallScore); + + napi_value jsReverseQuery = nullptr; + Convert2JSValue(env, value.isOriginal, jsReverseQuery); + napi_set_named_property(env, jsValue, "isReverseQuery", jsReverseQuery); + + napi_set_named_property(env, jsValues, innerKey.c_str(), jsValue); + } + napi_value jsKey = nullptr; + Convert2JSValue(env, key, jsKey); + napi_set_property(env, out, jsKey, jsValues); + } + return napi_ok; +} + +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::map<std::string, double> &in, napi_value &out) +{ + napi_create_object(env, &out); + for (auto const &[key, value] : in) { + napi_value jsValue = nullptr; + Convert2JSValue(env, value, jsValue); + napi_set_named_property(env, out, key.c_str(), jsValue); + } + return napi_ok; +} } // namespace DataIntelligence } // namespace OHOS diff --git a/udmf/framework/jskitsimpl/intelligence/i_aip_core_manager_impl.cpp b/udmf/framework/jskitsimpl/intelligence/i_aip_core_manager_impl.cpp index 614a32e9..8866a390 100644 --- a/udmf/framework/jskitsimpl/intelligence/i_aip_core_manager_impl.cpp +++ b/udmf/framework/jskitsimpl/intelligence/i_aip_core_manager_impl.cpp @@ -22,6 +22,17 @@ namespace OHOS { namespace DataIntelligence { +int32_t IAipCoreManagerImpl::InitRetriever(const RetrievalConfigStruct &retrievalConfig) +{ + return DEVICE_EXCEPTION; +} + +int32_t IAipCoreManagerImpl::Retrieve(const std::string query, const RetrievalConditionStruct &condition, + RetrievalResponseStruct &retrievalResponse) +{ + return DEVICE_EXCEPTION; +} + int32_t IAipCoreManagerImpl::InitTextModel(const ModelConfigData &config) { return DEVICE_EXCEPTION; diff --git a/udmf/framework/jskitsimpl/intelligence/image_embedding_napi.cpp b/udmf/framework/jskitsimpl/intelligence/image_embedding_napi.cpp index 71d3ee16..cbe74806 100644 --- a/udmf/framework/jskitsimpl/intelligence/image_embedding_napi.cpp +++ b/udmf/framework/jskitsimpl/intelligence/image_embedding_napi.cpp @@ -33,10 +33,11 @@ static constexpr uint8_t ARG_0 = 0; static constexpr uint8_t ARG_1 = 1; static constexpr uint8_t NUM_0 = 0; static constexpr uint8_t NUM_1 = 1; +static constexpr uint8_t BASIC_MODEL = 0; static const std::string CLASS_NAME = "ImageEmbedding"; const std::vector<std::string> EXPECTED_GET_ARG_TYPES = { "string" }; const std::vector<std::string> EXPECTED_GET_IMG_MODEL_ARG_TYPES = { "object" }; -const std::string AIP_MANAGER_PATH = "/system/lib64/platformsdk/libaip_core.z.so"; +constexpr const char *AIP_MANAGER_PATH = "libaip_core.z.so"; } // namespace AipCoreManagerHandle ImageEmbeddingNapi::imgAipCoreMgrHandle_{}; @@ -122,7 +123,7 @@ static napi_value StartInit(napi_env env, napi_value exports, struct ImageEmbedd napi_value ImageEmbeddingNapi::Init(napi_env env, napi_value exports) { AIP_HILOGD("Enter"); - if (!AipNapiUtils::LoadAlgoLibrary(AIP_MANAGER_PATH, imgAipCoreMgrHandle_)) { + if (!AipNapiUtils::LoadAlgoLibrary(AIP_MANAGER_PATH, imgAipCoreMgrHandle_, true)) { AIP_HILOGE("LoadAlgoLibrary failed"); } @@ -248,28 +249,35 @@ napi_value ImageEmbeddingNapi::GetImageEmbeddingModel(napi_env env, napi_callbac bool ImageEmbeddingNapi::ParseModelConfig(napi_env env, napi_value *args, size_t argc, ModelConfigData *modelConfig) { AIP_HILOGI("Enter"); - napi_value version, isNPUAvailable, cachePath; + if (modelConfig == nullptr) { + AIP_HILOGE("The modelConfig is null"); + return false; + } + if (!AipNapiUtils::CheckModelConfig(env, args[ARG_0])) { + AIP_HILOGE("The modelConfig is failed"); + return false; + } + napi_value version, isNPUAvailable, cachePath; napi_status status = napi_get_named_property(env, args[ARG_0], "version", &version); if (status != napi_ok) { AIP_HILOGE("napi get version property failed"); return false; } - status = napi_get_named_property(env, args[ARG_0], "isNpuAvailable", &isNPUAvailable); - if (status != napi_ok) { - AIP_HILOGE("napi get isNpuAvailable property failed"); + if (!AipNapiUtils::TransJsToInt32(env, version, modelConfig->versionValue)) { + AIP_HILOGE("Trans version failed"); return false; } - status = napi_get_named_property(env, args[ARG_0], "cachePath", &cachePath); - if (status != napi_ok) { - AIP_HILOGE("napi get cachePath property failed"); + if (modelConfig->versionValue != BASIC_MODEL) { + AIP_HILOGE("The version value is invalid"); return false; } - if (!AipNapiUtils::TransJsToInt32(env, version, modelConfig->versionValue)) { - AIP_HILOGE("Trans version failed"); + status = napi_get_named_property(env, args[ARG_0], "isNpuAvailable", &isNPUAvailable); + if (status != napi_ok) { + AIP_HILOGE("napi get isNpuAvailable property failed"); return false; } @@ -278,9 +286,19 @@ bool ImageEmbeddingNapi::ParseModelConfig(napi_env env, napi_value *args, size_t return false; } - if (!AipNapiUtils::TransJsToStr(env, cachePath, modelConfig->cachePathValue)) { - AIP_HILOGE("Trans cachePath failed"); - return false; + if (modelConfig->isNPUAvailableValue) { + status = napi_get_named_property(env, args[ARG_0], "cachePath", &cachePath); + if (status != napi_ok) { + AIP_HILOGE("napi get cachePath property failed"); + return false; + } + + if (!AipNapiUtils::TransJsToStr(env, cachePath, modelConfig->cachePathValue)) { + AIP_HILOGE("Trans cachePath failed"); + return false; + } + } else { + modelConfig->cachePathValue = ""; } return true; } diff --git a/udmf/framework/jskitsimpl/intelligence/js_ability.cpp b/udmf/framework/jskitsimpl/intelligence/js_ability.cpp new file mode 100644 index 00000000..5c36d681 --- /dev/null +++ b/udmf/framework/jskitsimpl/intelligence/js_ability.cpp @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define LOG_TAG "JSAbility" + +#include "aip_log.h" +#include "extension_context.h" +#include "js_ability.h" + +namespace OHOS { +namespace AppDataMgrJsKit { +constexpr int API_VERSION_MOD = 20; +Context::Context(std::shared_ptr<AbilityRuntime::Context> stageContext) +{ + this->stageContext_ = stageContext; + isStageMode_ = true; + databaseDir_ = stageContext->GetDatabaseDir(); + bundleName_ = stageContext->GetBundleName(); + area_ = stageContext->GetArea(); + auto hapInfo = stageContext->GetHapModuleInfo(); + if (hapInfo != nullptr) { + moduleName_ = hapInfo->moduleName; + } + + if (hapInfo == nullptr || hapInfo->proxyDatas.size() != 0) { + hasProxyDataConfig_ = true; + } else { + auto extensionContext = AbilityRuntime::Context::ConvertTo<AbilityRuntime::ExtensionContext>(stageContext); + if (extensionContext != nullptr) { + auto abilityInfo = extensionContext->GetAbilityInfo(); + if (abilityInfo != nullptr) { + uri_ = abilityInfo->uri; + writePermission_ = abilityInfo->writePermission; + readPermission_ = abilityInfo->readPermission; + AIP_HILOGI("QueryAbilityInfo, uri: %{private}s, readPermission: %{public}s, writePermission: " + "%{public}s.", + abilityInfo->uri.c_str(), abilityInfo->readPermission.c_str(), + abilityInfo->writePermission.c_str()); + } + } + } + auto appInfo = stageContext->GetApplicationInfo(); + isSystemAppCalled_ = appInfo == nullptr ? false : appInfo->isSystemApp; +} + +Context::Context(std::shared_ptr<AbilityRuntime::AbilityContext> abilityContext) +{ + databaseDir_ = abilityContext->GetDatabaseDir(); + bundleName_ = abilityContext->GetBundleName(); + area_ = abilityContext->GetArea(); + auto abilityInfo = abilityContext->GetAbilityInfo(); + if (abilityInfo != nullptr) { + moduleName_ = abilityInfo->moduleName; + } +} + +std::string Context::GetDatabaseDir() +{ + return databaseDir_; +} + +int Context::GetSystemDatabaseDir(const std::string &dataGroupId, std::string &databaseDir) +{ + return stageContext_->GetSystemDatabaseDir(dataGroupId, false, databaseDir); +} + +std::string Context::GetBundleName() +{ + return bundleName_; +} + +std::string Context::GetModuleName() +{ + return moduleName_; +} + +int32_t Context::GetArea() const +{ + return area_; +} +std::string Context::GetUri() +{ + return uri_; +} +std::string Context::GetReadPermission() +{ + return readPermission_; +} +std::string Context::GetWritePermission() +{ + return writePermission_; +} +bool Context::IsSystemAppCalled() +{ + return isSystemAppCalled_; +} + +bool Context::IsHasProxyDataConfig() const +{ + return hasProxyDataConfig_; +} + +bool Context::IsStageMode() const +{ + return isStageMode_; +} + +bool JSAbility::CheckContext(napi_env env, napi_callback_info info) +{ + size_t argc = 1; + napi_value args[1] = { 0 }; + bool mode = false; + napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + napi_status status = AbilityRuntime::IsStageContext(env, args[0], mode); + AIP_HILOGD("Check context as stage mode, mode is %{public}d, status is %{public}d", mode, status == napi_ok); + return status == napi_ok; +} + +std::shared_ptr<Context> JSAbility::GetContext(napi_env env, napi_value value) +{ + bool mode = false; + AbilityRuntime::IsStageContext(env, value, mode); + if (mode) { + return GetStageModeContext(env, value); + } + return GetCurrentAbility(env, value); +} + +std::shared_ptr<Context> JSAbility::GetStageModeContext(napi_env env, napi_value value) +{ + AIP_HILOGD("Get context as stage mode."); + auto stageContext = AbilityRuntime::GetStageModeContext(env, value); + if (stageContext == nullptr) { + AIP_HILOGE("GetStageModeContext failed."); + return nullptr; + } + return std::make_shared<Context>(stageContext); +} + +std::shared_ptr<Context> JSAbility::GetCurrentAbility(napi_env env, napi_value value) +{ + AIP_HILOGD("Get context as feature ability mode."); + auto ability = AbilityRuntime::GetCurrentAbility(env); + if (ability == nullptr) { + AIP_HILOGE("GetCurrentAbility failed."); + return nullptr; + } + auto abilityContext = ability->GetAbilityContext(); + if (abilityContext == nullptr) { + AIP_HILOGE("GetAbilityContext failed."); + return nullptr; + } + return std::make_shared<Context>(abilityContext); +} + +int32_t JSAbility::GetHapVersion(napi_env env, napi_value value) +{ + auto stageContext = AbilityRuntime::GetStageModeContext(env, value); + if (stageContext == nullptr) { + AIP_HILOGE("GetStageModeContext failed."); + return INVALID_HAP_VERSION ; + } + auto appInfo = stageContext->GetApplicationInfo(); + if (appInfo != nullptr) { + return appInfo->apiTargetVersion % API_VERSION_MOD; + } + return INVALID_HAP_VERSION ; +} +} // namespace AppDataMgrJsKit +} // namespace OHOS \ No newline at end of file diff --git a/udmf/framework/jskitsimpl/intelligence/native_module_intelligence.cpp b/udmf/framework/jskitsimpl/intelligence/native_module_intelligence.cpp index 9d92b1ca..daf2b026 100644 --- a/udmf/framework/jskitsimpl/intelligence/native_module_intelligence.cpp +++ b/udmf/framework/jskitsimpl/intelligence/native_module_intelligence.cpp @@ -27,6 +27,7 @@ static napi_value Export(napi_env env, napi_value exports) AIP_HILOGD("NativeModuleIntelligence call"); TextEmbeddingNapi::Init(env, exports); ImageEmbeddingNapi::Init(env, exports); + RetrievalNapi::Init(env, exports); return exports; } diff --git a/udmf/framework/jskitsimpl/intelligence/retrieval_napi.cpp b/udmf/framework/jskitsimpl/intelligence/retrieval_napi.cpp new file mode 100644 index 00000000..a442ebd5 --- /dev/null +++ b/udmf/framework/jskitsimpl/intelligence/retrieval_napi.cpp @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "retrieval_napi.h" + +#include <dlfcn.h> +#include <memory> +#include <node_api.h> +#include <string> + +#include "aip_log.h" +#include "aip_napi_error.h" +#include "aip_napi_utils.h" +#include "i_aip_core_manager_impl.h" + +#undef LOG_TAG +#define LOG_TAG "RetrievalNapi" + +namespace OHOS { +namespace DataIntelligence { +namespace { +const int32_t ERR_OK = 0; +constexpr const char *AIP_MANAGER_PATH = "libaip_core.z.so"; +} // namespace + +AipCoreManagerHandle RetrievalNapi::retrievalAipCoreMgrHandle_{}; + +RetrievalNapi::RetrievalNapi() : env_(nullptr) {} + +RetrievalNapi::~RetrievalNapi() +{ + AipNapiUtils::UnLoadAlgoLibrary(retrievalAipCoreMgrHandle_); +} + +napi_status InitEnum(napi_env env, napi_value exports) +{ + napi_value channelTypeEnum; + napi_status status = napi_create_object(env, &channelTypeEnum); + LOG_ERROR_RETURN(status == napi_ok, "create napi object fail.", napi_invalid_arg); + AipNapiUtils::SetInt32Property(env, channelTypeEnum, TsChannelType::VECTOR_DATABASE, "VECTOR_DATABASE"); + AipNapiUtils::SetInt32Property(env, channelTypeEnum, TsChannelType::INVERTED_INDEX_DATABASE, + "INVERTED_INDEX_DATABASE"); + AipNapiUtils::SetPropertyName(env, exports, "ChannelType", channelTypeEnum); + + napi_value operatorEnum; + status = napi_create_object(env, &operatorEnum); + LOG_ERROR_RETURN(status == napi_ok, "create napi object fail.", napi_invalid_arg); + AipNapiUtils::SetStringProperty(env, operatorEnum, "=", "OP_EQ"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "!=", "OP_NE"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "<", "OP_LT"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "<=", "OP_LE"); + AipNapiUtils::SetStringProperty(env, operatorEnum, ">", "OP_GT"); + AipNapiUtils::SetStringProperty(env, operatorEnum, ">=", "OP_GE"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "IN", "OP_IN"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "NOT_IN", "OP_NOT_IN"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "BETWEEN", "OP_BETWEEN"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "LIKE", "OP_LIKE"); + AipNapiUtils::SetStringProperty(env, operatorEnum, "NOT_LIKE", "OP_NOT_LIKE"); + AipNapiUtils::SetPropertyName(env, exports, "Operator", operatorEnum); + + napi_value rerankTypeEnum; + status = napi_create_object(env, &rerankTypeEnum); + LOG_ERROR_RETURN(status == napi_ok, "create napi object fail.", napi_invalid_arg); + AipNapiUtils::SetInt32Property(env, rerankTypeEnum, TsRerankType::RRF, "RRF"); + AipNapiUtils::SetInt32Property(env, rerankTypeEnum, TsRerankType::FUSION_SCORE, "FUSION_SCORE"); + AipNapiUtils::SetPropertyName(env, exports, "RerankType", rerankTypeEnum); + + napi_value similarityLevelEnum; + status = napi_create_object(env, &similarityLevelEnum); + LOG_ERROR_RETURN(status == napi_ok, "create napi object fail.", napi_invalid_arg); + AipNapiUtils::SetInt32Property(env, similarityLevelEnum, TsSimilarityLevel::LOW, "LOW"); + AipNapiUtils::SetInt32Property(env, similarityLevelEnum, TsSimilarityLevel::MEDIUM, "MEDIUM"); + AipNapiUtils::SetInt32Property(env, similarityLevelEnum, TsSimilarityLevel::HIGH, "HIGH"); + AipNapiUtils::SetPropertyName(env, exports, "SimilarityLevel", similarityLevelEnum); + return napi_ok; +} + +napi_value RetrievalNapi::Init(napi_env env, napi_value exports) +{ + AIP_HILOGI("Init retrieval start."); + napi_property_descriptor desc[] = { + DECLARE_NAPI_FUNCTION("getRetriever", GetRetriever), + }; + napi_status status = napi_define_properties(env, exports, sizeof(desc) / sizeof(desc[0]), desc); + if (status != napi_ok) { + AIP_HILOGE("define properties fail, status = %{public}d", status); + return nullptr; + } + status = InitEnum(env, exports); + LOG_ERROR_RETURN(status == napi_ok, "InitEnum fail.", nullptr); + return exports; +} + +napi_value RetrievalNapi::GetRetriever(napi_env env, napi_callback_info info) +{ + AIP_HILOGD("Enter"); + napi_value args[1] = { nullptr }; + size_t argc = 1; + napi_status status = napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + if (status != napi_ok) { + ThrowIntelligenceErr(env, INNER_ERROR, "napi_get_cb_info failed."); + return nullptr; + } + RetrievalConfigStruct retrievalConfig; + status = AipNapiUtils::Convert2Value(env, args[0], retrievalConfig); + if (status != napi_ok) { + ThrowIntelligenceErr(env, PARAM_EXCEPTION, "Parse RetrievalConfig type failed."); + return nullptr; + } + napi_value promise = nullptr; + napi_deferred deferred = nullptr; + status = napi_create_promise(env, &deferred, &promise); + if (status != napi_ok) { + ThrowIntelligenceErr(env, INNER_ERROR, "create promise failed"); + return nullptr; + } + auto asyncGetRetriever = new AsyncGetRetriever{ + .asyncWork = nullptr, + .deferred = deferred, + .config = retrievalConfig, + }; + if (!CreateAsyncGetRetrieverExecution(env, asyncGetRetriever)) { + ThrowIntelligenceErr(env, INNER_ERROR, "create AsyncGetRetrieverExecution failed"); + delete asyncGetRetriever; + return nullptr; + } + return promise; +} + +bool RetrievalNapi::CreateAsyncGetRetrieverExecution(napi_env env, AsyncGetRetriever *asyncGetRetriever) +{ + AIP_HILOGD("Enter"); + napi_value resourceName; + napi_status status = napi_create_string_utf8(env, "GetRetriever", NAPI_AUTO_LENGTH, &resourceName); + if (status != napi_ok) { + AIP_HILOGE(" napi_create_string_utf8 failed."); + return false; + } + status = napi_create_async_work(env, nullptr, resourceName, GetRetrieverExecutionCB, GetRetrieverCompleteCB, + static_cast<void *>(asyncGetRetriever), &asyncGetRetriever->asyncWork); + if (status != napi_ok) { + AIP_HILOGE("napi_create_async_work failed."); + return false; + } + status = napi_queue_async_work_with_qos(env, asyncGetRetriever->asyncWork, napi_qos_default); + if (status != napi_ok) { + AIP_HILOGE("napi_queue_async_work_with_qos failed"); + return false; + } + return true; +} + +napi_value RetrievalNapi::WrapAipCoreManager(napi_env env, IAipCoreManager *retrievalAipCoreManager) +{ + napi_value result; + napi_status status = napi_create_object(env, &result); + if (status != napi_ok) { + return nullptr; + } + status = napi_wrap( + env, result, static_cast<void *>(retrievalAipCoreManager), + [](napi_env env, void *data, void *hint) { delete static_cast<IAipCoreManager *>(data); }, nullptr, nullptr); + if (status != napi_ok) { + AIP_HILOGE("napi_wrap aipCoreMangager failed, erroCode is %{public}d", status); + delete retrievalAipCoreManager; + return nullptr; + } + // 为 Retriever 对象添加 retrieve 方法 + napi_value retrieveFn; + status = napi_create_function(env, nullptr, NAPI_AUTO_LENGTH, Retrieve, nullptr, &retrieveFn); + if (status != napi_ok) { + return nullptr; + } + status = napi_set_named_property(env, result, "retrieveRdb", retrieveFn); + if (status != napi_ok) { + return nullptr; + } + return result; +} + +void RetrievalNapi::GetRetrieverExecutionCB(napi_env env, void *data) +{ + AIP_HILOGD("Enter"); + AsyncGetRetriever *asyncGetRetrieverdata = static_cast<AsyncGetRetriever *>(data); + auto config = asyncGetRetrieverdata->config; + IAipCoreManager *retrievalAipCoreManager = nullptr; + if (retrievalAipCoreMgrHandle_.create == nullptr) { + AipNapiUtils::LoadAlgoLibrary(AIP_MANAGER_PATH, retrievalAipCoreMgrHandle_, false); + } + if (retrievalAipCoreMgrHandle_.create != nullptr) { + AIP_HILOGI("Init retrieval by dlopen."); + retrievalAipCoreManager = retrievalAipCoreMgrHandle_.create(); + } + if (retrievalAipCoreManager == nullptr) { + AIP_HILOGI("Init retrieval by new."); + retrievalAipCoreManager = new IAipCoreManagerImpl(); + } + int32_t ret = retrievalAipCoreManager->InitRetriever(config); + asyncGetRetrieverdata->ret = ret; + if (ret != ERR_OK) { + AIP_HILOGE("init Retriever failed, erroCode is %{public}d", ret); + delete retrievalAipCoreManager; + return; + } + asyncGetRetrieverdata->retrievalAipCoreManagerPtr = retrievalAipCoreManager; +} + +void RetrievalNapi::GetRetrieverCompleteCB(napi_env env, napi_status status, void *data) +{ + AIP_HILOGD("Enter"); + AsyncGetRetriever *asyncGetRetrieverdata = static_cast<AsyncGetRetriever *>(data); + auto ret = asyncGetRetrieverdata->ret; + napi_value result = nullptr; + if (ret != ERR_OK) { + if (ret == DEVICE_EXCEPTION) { + ThrowIntelligenceErrByPromise(env, DEVICE_EXCEPTION, "GetRetrieverCompleteCB failed", result); + } else { + ThrowIntelligenceErrByPromise(env, INNER_ERROR, "GetRetrieverCompleteCB failed", result); + } + napi_reject_deferred(env, asyncGetRetrieverdata->deferred, result); + } else { + result = WrapAipCoreManager(env, asyncGetRetrieverdata->retrievalAipCoreManagerPtr); + if (result == nullptr) { + ThrowIntelligenceErrByPromise(env, INNER_ERROR, "GetRetrieverCompleteCB failed", result); + napi_reject_deferred(env, asyncGetRetrieverdata->deferred, result); + } + status = napi_resolve_deferred(env, asyncGetRetrieverdata->deferred, result); + if (status != napi_ok) { + AIP_HILOGE(" napi_resolve_deferred failed"); + } + } + status = napi_delete_async_work(env, asyncGetRetrieverdata->asyncWork); + if (status != napi_ok) { + AIP_HILOGE("napi_delete_async_work failed"); + } + delete asyncGetRetrieverdata; +} + +bool RetrievalNapi::CreateAsyncRetrieveExecution(napi_env env, AsyncRetrieve *asyncRetrieve) +{ + AIP_HILOGD("Enter"); + napi_value resourceName; + napi_status status = napi_create_string_utf8(env, "Retrieve", NAPI_AUTO_LENGTH, &resourceName); + if (status != napi_ok) { + AIP_HILOGE(" napi_create_string_utf8 failed."); + return false; + } + + status = napi_create_async_work(env, nullptr, resourceName, RetrieveExecutionCB, RetrieveCompleteCB, + static_cast<void *>(asyncRetrieve), &asyncRetrieve->asyncWork); + if (status != napi_ok) { + AIP_HILOGE("napi_create_async_work failed"); + return false; + } + + status = napi_queue_async_work_with_qos(env, asyncRetrieve->asyncWork, napi_qos_default); + if (status != napi_ok) { + AIP_HILOGE("napi_queue_async_work_with_qos failed"); + return false; + } + return true; +} + +void RetrievalNapi::RetrieveExecutionCB(napi_env env, void *data) +{ + AIP_HILOGD("Enter"); + AsyncRetrieve *asyncRetrieve = static_cast<AsyncRetrieve *>(data); + auto retrievalCondition = asyncRetrieve->retrievalCondition; + auto query = asyncRetrieve->query; + auto retrievalAipCoreManagerPtr = asyncRetrieve->retrievalAipCoreManagerPtr; + RetrievalResponseStruct retrievalResponse; + int32_t result = retrievalAipCoreManagerPtr->Retrieve(query, retrievalCondition, retrievalResponse); + AIP_HILOGI("execute retrieve."); + asyncRetrieve->ret = result; + asyncRetrieve->retrievalResponse = retrievalResponse; +} + +void RetrievalNapi::RetrieveCompleteCB(napi_env env, napi_status status, void *data) +{ + AIP_HILOGD("Enter"); + AsyncRetrieve *asyncRetrieve = static_cast<AsyncRetrieve *>(data); + auto ret = asyncRetrieve->ret; + napi_value result = nullptr; + if (ret != ERR_OK) { + if (ret == DEVICE_EXCEPTION) { + ThrowIntelligenceErrByPromise(env, DEVICE_EXCEPTION, "GetRetrieverCompleteCB failed", result); + } else { + int32_t errCode = ConvertErrCodeNative2Ts(ret); + ThrowIntelligenceErrByPromise(env, errCode, "GetRetrieverCompleteCB failed", result); + } + napi_reject_deferred(env, asyncRetrieve->deferred, result); + } else { + status = AipNapiUtils::Convert2JSValue(env, asyncRetrieve->retrievalResponse, result); + if (status != napi_ok) { + ThrowIntelligenceErrByPromise(env, INNER_ERROR, "convert RetrievalResponse to js value failed.", result); + napi_reject_deferred(env, asyncRetrieve->deferred, result); + } else { + napi_resolve_deferred(env, asyncRetrieve->deferred, result); + } + } + status = napi_delete_async_work(env, asyncRetrieve->asyncWork); + if (status != napi_ok) { + AIP_HILOGE("napi_delete_async_work failed."); + } + delete asyncRetrieve; +} + + +napi_value RetrievalNapi::Retrieve(napi_env env, napi_callback_info info) +{ + AIP_HILOGI("Retrieve being call."); + napi_value args[2] = { nullptr }; + size_t argc = 2; + napi_value jsThis = nullptr; + napi_status status = napi_get_cb_info(env, info, &argc, args, &jsThis, nullptr); + if (status != napi_ok) { + ThrowIntelligenceErr(env, INNER_ERROR, "napi_get_cb_info failed."); + return nullptr; + } + std::string query; + status = AipNapiUtils::Convert2Value(env, args[0], query); + if (status != napi_ok) { + ThrowIntelligenceErr(env, PARAM_EXCEPTION, "Failed to convert the field query."); + return nullptr; + } + RetrievalConditionStruct retrievalCondition; + status = AipNapiUtils::Convert2Value(env, args[1], retrievalCondition); + if (status != napi_ok) { + ThrowIntelligenceErr(env, PARAM_EXCEPTION, "Parse RetrievalCondition failed."); + return nullptr; + } + napi_value promise = nullptr; + napi_deferred deferred = nullptr; + status = napi_create_promise(env, &deferred, &promise); + if (status != napi_ok) { + ThrowIntelligenceErr(env, INNER_ERROR, "create promise failed."); + return nullptr; + } + IAipCoreManager *retrievalAipCoreManagerPtr; + status = napi_unwrap(env, jsThis, reinterpret_cast<void **>(&retrievalAipCoreManagerPtr)); + if (status != napi_ok) { + return nullptr; + } + auto asyncRetrieve = new AsyncRetrieve{ + .asyncWork = nullptr, + .deferred = deferred, + .query = query, + .retrievalCondition = retrievalCondition, + .retrievalAipCoreManagerPtr = retrievalAipCoreManagerPtr + }; + if (!CreateAsyncRetrieveExecution(env, asyncRetrieve)) { + ThrowIntelligenceErr(env, INNER_ERROR, "CreateAsyncRetrieveExecution failed"); + delete asyncRetrieve; + } + return promise; +} +} // namespace DataIntelligence +} // namespace OHOS diff --git a/udmf/framework/jskitsimpl/intelligence/text_embedding_napi.cpp b/udmf/framework/jskitsimpl/intelligence/text_embedding_napi.cpp index 4758fbe7..4c708073 100644 --- a/udmf/framework/jskitsimpl/intelligence/text_embedding_napi.cpp +++ b/udmf/framework/jskitsimpl/intelligence/text_embedding_napi.cpp @@ -35,11 +35,12 @@ static constexpr uint8_t ARG_2 = 2; static constexpr uint8_t NUM_0 = 0; static constexpr uint8_t NUM_1 = 1; +static constexpr uint8_t BASIC_MODEL = 0; static constexpr uint32_t MAX_STR_PARAM_LEN = 512; static const std::string CLASS_NAME = "TextEmbedding"; const std::vector<std::string> EXPECTED_SPLITTEXT_ARG_TYPES = { "string", "object" }; const std::vector<std::string> EXPECTED_GET_TEXT_MODEL_ARG_TYPES = { "object" }; -const std::string AIP_MANAGER_PATH = "/system/lib64/platformsdk/libaip_core.z.so"; +constexpr const char *AIP_MANAGER_PATH = "libaip_core.z.so"; } // namespace AipCoreManagerHandle TextEmbeddingNapi::textAipCoreMgrHandle_{}; thread_local napi_ref TextEmbeddingNapi::sConstructor_ = nullptr; @@ -143,7 +144,7 @@ static napi_value StartInit(napi_env env, napi_value exports, struct TextEmbeddi napi_value TextEmbeddingNapi::Init(napi_env env, napi_value exports) { AIP_HILOGD("Enter"); - if (!AipNapiUtils::LoadAlgoLibrary(AIP_MANAGER_PATH, textAipCoreMgrHandle_)) { + if (!AipNapiUtils::LoadAlgoLibrary(AIP_MANAGER_PATH, textAipCoreMgrHandle_, true)) { AIP_HILOGE("LoadAlgoLibrary failed"); } @@ -169,6 +170,16 @@ napi_value TextEmbeddingNapi::Init(napi_env env, napi_value exports) DECLARE_NAPI_STATIC_FUNCTION("splitText", SplitText), }; + napi_value modelVersion; + napi_status status = napi_create_object(env, &modelVersion); + if (status != napi_ok) { + AIP_HILOGE("Failed create object"); + return nullptr; + } + + AipNapiUtils::SetInt32Property(env, modelVersion, BASIC_MODEL, "BASIC_MODEL"); + AipNapiUtils::SetPropertyName(env, exports, "ModelVersion", modelVersion); + struct TextEmbeddingConstructorInfo info = { .className = CLASS_NAME, .classRef = &sConstructor_, @@ -265,28 +276,35 @@ napi_value TextEmbeddingNapi::GetTextEmbeddingModel(napi_env env, napi_callback_ bool TextEmbeddingNapi::ParseModelConfig(napi_env env, napi_value *args, size_t argc, ModelConfigData *textModelConfig) { AIP_HILOGI("Enter"); - napi_value version, isNPUAvailable, cachePath; + if (textModelConfig == nullptr) { + AIP_HILOGE("The modelConfig is null"); + return false; + } + if (!AipNapiUtils::CheckModelConfig(env, args[ARG_0])) { + AIP_HILOGE("The modelConfig is failed"); + return false; + } + napi_value version, isNPUAvailable, cachePath; napi_status status = napi_get_named_property(env, args[ARG_0], "version", &version); if (status != napi_ok) { AIP_HILOGE("napi get version property failed"); return false; } - status = napi_get_named_property(env, args[ARG_0], "isNpuAvailable", &isNPUAvailable); - if (status != napi_ok) { - AIP_HILOGE("napi get isNpuAvailable property failed"); + if (!AipNapiUtils::TransJsToInt32(env, version, textModelConfig->versionValue)) { + AIP_HILOGE("Trans version failed"); return false; } - status = napi_get_named_property(env, args[ARG_0], "cachePath", &cachePath); - if (status != napi_ok) { - AIP_HILOGE("napi get cachePath property failed"); + if (textModelConfig->versionValue != BASIC_MODEL) { + AIP_HILOGE("The version value is invalid"); return false; } - if (!AipNapiUtils::TransJsToInt32(env, version, textModelConfig->versionValue)) { - AIP_HILOGE("Trans version failed"); + status = napi_get_named_property(env, args[ARG_0], "isNpuAvailable", &isNPUAvailable); + if (status != napi_ok) { + AIP_HILOGE("napi get isNpuAvailable property failed"); return false; } @@ -295,9 +313,19 @@ bool TextEmbeddingNapi::ParseModelConfig(napi_env env, napi_value *args, size_t return false; } - if (!AipNapiUtils::TransJsToStr(env, cachePath, textModelConfig->cachePathValue)) { - AIP_HILOGE("Trans cachePath failed"); - return false; + if (textModelConfig->isNPUAvailableValue) { + status = napi_get_named_property(env, args[ARG_0], "cachePath", &cachePath); + if (status != napi_ok) { + AIP_HILOGE("napi get cachePath property failed"); + return false; + } + + if (!AipNapiUtils::TransJsToStr(env, cachePath, textModelConfig->cachePathValue)) { + AIP_HILOGE("Trans cachePath failed"); + return false; + } + } else { + textModelConfig->cachePathValue = ""; } return true; } @@ -409,7 +437,6 @@ napi_value TextEmbeddingNapi::SplitText(napi_env env, napi_callback_info info) ThrowIntelligenceErr(env, PARAM_EXCEPTION, "TransJsToStrUnlimited failed"); return nullptr; } - AIP_HILOGI("string strArg: %{public}s", strArg.c_str()); napi_value cfgSize; napi_value cfgOverlap; @@ -422,8 +449,12 @@ napi_value TextEmbeddingNapi::SplitText(napi_env env, napi_callback_info info) double configOverlap; AipNapiUtils::TransJsToInt32(env, cfgSize, configSize); AipNapiUtils::TransJsToDouble(env, cfgOverlap, configOverlap); - AIP_HILOGI("string strArg: %{public}d", configSize); - AIP_HILOGI("string strArg: %{public}f", configOverlap); + AIP_HILOGD("string strArg: %{public}d", configSize); + AIP_HILOGD("string strArg: %{public}f", configOverlap); + if (configSize <= NUM_0 || configOverlap < NUM_0 || configOverlap >= NUM_1) { + ThrowIntelligenceErr(env, PARAM_EXCEPTION, "The parameter value range is incorrect"); + return nullptr; + } napi_value promise = nullptr; napi_deferred deferred = nullptr; diff --git a/udmf/framework/jskitsimpl/unittest/AipRdbJsTest.js b/udmf/framework/jskitsimpl/unittest/AipRdbJsTest.js new file mode 100644 index 00000000..aac1fe6b --- /dev/null +++ b/udmf/framework/jskitsimpl/unittest/AipRdbJsTest.js @@ -0,0 +1,981 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from 'deccjsunit/index'; +import intelligence from '@ohos.data.intelligence'; +import deviceInfo from '@ohos.deviceInfo'; +import relationalStore from '@ohos.data.relationalStore'; +import featureAbility from '@ohos.ability.featureAbility'; + +const TAG = '[AipIntelligenceJSTest]'; + +let currentDeviceIsPc = false; +let context = featureAbility.getContext(); +let rdbStore1 = undefined; +let rdbStore2 = undefined; + +let storeConfigInvalid = { + name: '', + securityLevel: relationalStore.SecurityLevel.S1 +}; + +let channelConfigInvalid = { + channelType: 0, + context:context, + dbConfig:storeConfigInvalid +}; + +let storeConfigVector = { + name: 'rdb_store_test.db', + encrypt: false, + securityLevel: relationalStore.SecurityLevel.S3, + autoCleanDirtyData: false, + isSearchable: false, + vector: true, + allowRebuild: false +}; + +let storeConfigInvIdx = { + name: 'sqlite_store_test.db', + encrypt: false, + securityLevel: relationalStore.SecurityLevel.S3, + autoCleanDirtyData: false, + isSearchable: false, + allowRebuild: false, + tokenizer: 2 +}; + +let channelConfigVector = { + channelType: intelligence.ChannelType.VECTOR_DATABASE, + context:context, + dbConfig:storeConfigVector +}; + +let channelConfigInvIdx = { + channelType: intelligence.ChannelType.INVERTED_INDEX_DATABASE, + context:context, + dbConfig:storeConfigInvIdx +}; + +describe('AipJSTest', function () { + + beforeAll(async function () { + console.info(TAG + 'beforeAll'); + let deviceTypeInfo = deviceInfo.deviceType; + currentDeviceIsPc = deviceTypeInfo === '2in1' ? true : false; + console.info(TAG + 'the value of the deviceType is : ' + deviceInfo.deviceType); + relationalStore.deleteRdbStore(context, 'rdb_store_test.db', (err) => { + if (err) { + console.error(`Delete RdbStore failed, code is ${err.code},message is ${err.message}`); + return; + } + console.info(TAG + 'Delete RdbStore successfully.'); + }); + rdbStore1 = await relationalStore.getRdbStore(context, storeConfigVector); + if (rdbStore1 !== undefined) { + let createSql = 'CREATE TABLE IF NOT EXISTS vector_table(fileid TEXT PRIMARY KEY, filename_text TEXT, filename FLOATVECTOR(128), keywords_text TEXT, keywords FLOATVECTOR(128), chapter FLOATVECTOR(128), abstract FLOATVECTOR(128));'; + await rdbStore1.execute(createSql, 0, undefined); + await insertVectorDB(); + }; + rdbStore2 = await relationalStore.getRdbStore(context, storeConfigInvIdx); + if (rdbStore2 !== undefined) { + let createSql = 'CREATE VIRTUAL TABLE IF NOT EXISTS invidx_table USING fts5(fileid, filename, keywords, chapter, abstract, content, tokenize = "customtokenizer");'; + await rdbStore2.executeSql(createSql); + await insertInvIdxDB(); + }; + }); + beforeEach(async function () { + console.info(TAG + 'beforeEach'); + }); + afterEach(async function () { + console.info(TAG + 'afterEach'); + }); + afterAll(async function () { + console.info(TAG + 'afterAll'); + }); + + async function insertVectorDB() { + let insertSQL; + for (let i = 0; i < 5; i++) { + let array = new Array(128); + for (let j = 0; j < array.length; j++) { + let randomNumber = Math.random(); + array[j] = randomNumber; + } + let data = array.toString(); + insertSQL = "INSERT INTO vector_table VALUES('" + i + "', '大模型系统概述', '[" + data + "]', '生成式AI, 人工智能, 大模型', '[" + data + "]', '[" + data + "]', '[" + data + "]');"; + console.info(TAG + 'insertVectorDB insertSQL::' + insertSQL); + await rdbStore1.execute(insertSQL, 0, undefined); + } + let arraySpecial = new Array(128).fill(0.1); + let dataSpecial = arraySpecial.toString(); + let insertSQLSpecial = "INSERT INTO vector_table VALUES('5', '大模型系统概述', '[" + dataSpecial + "]', '生成式AI, 人工智能, 大模型', '[" + dataSpecial + "]', '[" + dataSpecial + "]', '[" + dataSpecial + "]');"; + console.info(TAG + 'insertVectorDB insertSQL::' + insertSQLSpecial); + await rdbStore1.execute(insertSQLSpecial, 0, undefined); + } + + async function insertInvIdxDB() { + let insertSQL = 'INSERT INTO invidx_table VALUES("1", "大模型系统概述", "生成式AI, 人工智能, 大模型", "人工智能导论", "这是一篇关于大模型的综述文章", "这是一篇关于大模型的综述文章");'; + console.info(TAG + 'insertInvIdxDB insertSQL::' + insertSQL); + await rdbStore2.executeSql(insertSQL); + insertSQL = 'INSERT INTO invidx_table VALUES("2", "数据库在大模型时代下的发展", "数据库, 内核, 向量", "数据库导论", "这是一篇关于数据库和大模型交叉的综述文章", "这是一篇关于数据库和大模型交叉的综述文章");'; + console.info(TAG + 'insertInvIdxDB insertSQL::' + insertSQL); + await rdbStore2.executeSql(insertSQL); + insertSQL = 'INSERT INTO invidx_table VALUES("3", "社会发展报告", "旅游, 民生, 交通", "社会发展", "这是一篇关于社会发展的综述文章", "这是一篇关于社会发展的综述文章");'; + console.info(TAG + 'insertInvIdxDB insertSQL::' + insertSQL); + await rdbStore2.executeSql(insertSQL); + insertSQL = 'INSERT INTO invidx_table VALUES("4", "鸿蒙", "鸿蒙, 生态, 遥遥领先", "鸿蒙系统", "这是一篇关于鸿蒙系统的综述文章", "这是一篇关于鸿蒙系统的综述文章");'; + console.info(TAG + 'insertInvIdxDB insertSQL::' + insertSQL); + await rdbStore2.executeSql(insertSQL); + insertSQL = 'INSERT INTO invidx_table VALUES("5", "AI系统", "鸿蒙, 生态, 大模型", "AI系统", "这是一篇关于AI系统的综述文章", "这是一篇关于AI系统的综述文章");'; + console.info(TAG + 'insertInvIdxDB insertSQL::' + insertSQL); + await rdbStore2.executeSql(insertSQL); + console.info(TAG + 'insertInvIdxDB end'); + } + + async function sleep(time) { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, time); + }).then(() => { + console.info(`sleep ${time} over...`); + }); + } + + console.log('******************************Aip API Test Begin******************************'); + + /** + * @tc.name Aip_001 + * @tc.desc Test getRetriever. rdbStoreName is empty. + * @tc.type: FUNC + */ + it('Aip_001', 0, async function (done) { + console.info(TAG, 'Aip_001 start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvalid] + }; + if (currentDeviceIsPc) { + try { + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + expect(data != null).assertEqual(true); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_001 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_002 + * @tc.desc Test getRetriever. channelConfigs is empty. + * @tc.type: FUNC + */ + it('Aip_002', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [] + }; + if (currentDeviceIsPc) { + try { + await intelligence.getRetriever(retrievalConfig); + } catch (err) { + console.info(TAG, 'Aip_002 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + }); + + /** + * @tc.name Aip_003 + * @tc.desc Test retrieveRdb. The value of deepSize is too large. + * @tc.type: FUNC + */ + it('Aip_003', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + filters: [], + deepSize: 1000000 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition) + .then((rdbdata) => { + let length = rdbdata.records.length; + console.info(TAG, 'Aip_003 get result::' + length); + let ret = length >= 0; + expect(ret).assertEqual(true); + done(); + }); + } catch (err) { + console.info(TAG, 'catch::' + err.code); + done(); + } + }); + } catch (err) { + console.info(TAG, 'catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_004 + * @tc.desc Test retrieveRdb.fromClause is empty. + * @tc.type: FUNC + */ + it('Aip_004', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: '', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition); + } catch (err) { + console.info(TAG, 'Aip_004 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_004 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch11::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_005 + * @tc.desc Test retrieveRdb. primaryKey is empty. + * @tc.type: FUNC + */ + it('Aip_005', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: [], + responseColumns: ['filename_text'], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition); + } catch (err) { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_005 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch11::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_006 + * @tc.desc Test retrieveRdb. responseColumns is empty. + * @tc.type: FUNC + */ + it('Aip_006', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: [], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition); + } catch (err) { + console.info(TAG, 'Aip_006 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_007 + * @tc.desc Test retrieveRdb. column is invalid. + * @tc.type: FUNC + */ + it('Aip_007', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'invalidField', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition) + .catch((err) => { + console.info(TAG, 'Aip_007 catch::' + err.code); + expect(err.code).assertEqual(31300100); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_007 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_008 + * @tc.desc Test retrieveRdb. One-way vector normal recall. + * @tc.type: FUNC + */ + it('Aip_008', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let floatArray = new Float32Array(128).fill(0.1); + const str = floatArray.toString(); + console.log(TAG, str); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallCondition = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '软件设计与建模'; + data.retrieveRdb(query, retrievalCondition) + .then((rdbdata) => { + let length = rdbdata.records.length; + console.info(TAG, 'Aip_008 get result length::' + length); + expect(length > 0).assertEqual(true); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_008 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_008 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_008 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_009 + * @tc.desc Test retrieveRdb. One-way invertedindex normal recall. + * @tc.type: FUNC + */ + it('Aip_009', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvIdx] + }; + if (currentDeviceIsPc) { + try { + let fieldWeight = { + 'filename': 4.0 + }; + let fieldSlops = { + 'filename': 5 + }; + let bm25Strategy = { + bm25Weight: 1.5, + columnWeight: fieldWeight + }; + let exactStrategy = { + exactMatchingWeight: 1.2, + columnWeight: fieldWeight + }; + let outOfOrderStrategy = { + proximityWeight: 1.2, + columnWeight: fieldWeight, + columnSlops: fieldSlops + }; + let invertedIndexStrategies = [bm25Strategy, exactStrategy, outOfOrderStrategy]; + let recallCondition = { + ftsTableName: 'invidx_table', + fromClause: 'invidx_table', + primaryKey: ['fileid'], + responseColumns: ['filename', 'keywords'], + deepSize: 500, + invertedIndexStrategies: invertedIndexStrategies + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + console.info(TAG, 'Aip_009 get result::' + data); + try { + let query = '数据库'; + data.retrieveRdb(query, retrievalCondition) + .then((rdbdata) => { + let length = rdbdata.records.length; + console.info(TAG, 'Aip_009 get result::' + length); + expect(length).assertEqual(1); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_009 catch::' + err.code); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_009 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_009 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_010 + * @tc.desc Test retrieveRdb. ftsTableName is empty. + * @tc.type: FUNC + */ + it('Aip_010', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvIdx] + }; + if (currentDeviceIsPc) { + try { + let fieldWeight = { + 'filename': 4.0 + }; + let fieldSlops = { + 'filename': 5 + }; + let bm25Strategy = { + bm25Weight: 1.5, + columnWeight: fieldWeight + }; + let exactStrategy = { + exactMatchingWeight: 1.2, + columnWeight: fieldWeight + }; + let outOfOrderStrategy = { + proximityWeight: 1.2, + columnWeight: fieldWeight, + columnSlops: fieldSlops + }; + let invertedIndexStrategies = [bm25Strategy, exactStrategy, outOfOrderStrategy]; + let recallCondition = { + ftsTableName: '', + fromClause: 'invidx_table', + primaryKey: ['fileid'], + responseColumns: ['filename', 'keywords'], + deepSize: 500, + invertedIndexStrategies: invertedIndexStrategies + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '数据库'; + data.retrieveRdb(query, retrievalCondition); + } catch (err) { + console.info(TAG, 'Aip_010 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_010 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_010 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_011 + * @tc.desc Test retrieveRdb. invertedindex match field is invalid. + * @tc.type: FUNC + */ + it('Aip_011', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvIdx] + }; + if (currentDeviceIsPc) { + try { + let fieldWeight = { + 'invalid': 4.0 + }; + let fieldSlops = { + 'filename': 5 + }; + let bm25Strategy = { + bm25Weight: 1.5, + columnWeight: fieldWeight + }; + let exactStrategy = { + exactMatchingWeight: 1.2, + columnWeight: fieldWeight + }; + let outOfOrderStrategy = { + proximityWeight: 1.2, + columnWeight: fieldWeight, + columnSlops: fieldSlops + }; + let invertedIndexStrategies = [bm25Strategy, exactStrategy, outOfOrderStrategy]; + let recallCondition = { + ftsTableName: 'invidx_table', + fromClause: 'invidx_table', + primaryKey: ['fileid'], + responseColumns: ['filename', 'keywords'], + deepSize: 500, + invertedIndexStrategies: invertedIndexStrategies + }; + let retrievalCondition = { + recallConditions : [recallCondition] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '数据库'; + data.retrieveRdb(query, retrievalCondition) + .catch((err) => { + console.info(TAG, 'Aip_011 catch::' + err.code); + expect(err.code).assertEqual(31301010); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_011 catch::' + err.code); + expect(err.code).assertEqual(401); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_011 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_011 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + + /** + * @tc.name Aip_012 + * @tc.desc Test retrieveRdb. Two-way normal recall. + * @tc.type: FUNC + */ + it('Aip_012', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvIdx, channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let fieldWeight = { + 'filename': 4.0 + }; + + let fieldSlops = { + 'filename': 5 + }; + let bm25Strategy = { + bm25Weight: 1.5, + columnWeight: fieldWeight + }; + let exactStrategy = { + exactMatchingWeight: 1.2, + columnWeight: fieldWeight + }; + let outOfOrderStrategy = { + proximityWeight: 1.2, + columnWeight: fieldWeight, + columnSlops: fieldSlops + }; + let invertedIndexStrategies = [bm25Strategy, exactStrategy, outOfOrderStrategy]; + let recallConditionInvIdx = { + ftsTableName: 'invidx_table', + fromClause: 'invidx_table', + primaryKey: ['fileid'], + responseColumns: ['filename', 'keywords'], + deepSize: 500, + invertedIndexStrategies: invertedIndexStrategies + }; + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallConditionVector = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + deepSize: 500 + }; + let retrievalCondition = { + recallConditions : [recallConditionInvIdx, recallConditionVector] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '数据库'; + data.retrieveRdb(query, retrievalCondition) + .then((rdbdata) => { + let length = rdbdata.records.length; + console.info(TAG, 'Aip_012 get result::' + length); + expect(length >= 2).assertEqual(true); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_012 catch::' + err.code); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_012 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_012 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name Aip_013 + * @tc.desc Test retrieveRdb. Two-way normal recall and configing rerankMethod. + * @tc.type: FUNC + */ + it('Aip_013', 0, async function (done) { + console.info(TAG, 'start'); + let retrievalConfig = { + channelConfigs: [channelConfigInvIdx, channelConfigVector] + }; + if (currentDeviceIsPc) { + try { + let fieldWeight = { + 'filename': 4.0, + 'keywords':4.0 + }; + let fieldSlops = { + 'filename': 5, + 'keywords':5 + }; + let bm25Strategy = { + bm25Weight: 1.5, + columnWeight: fieldWeight + }; + let exactStrategy = { + exactMatchingWeight: 1.2, + columnWeight: fieldWeight + }; + let outOfOrderStrategy = { + proximityWeight: 1.2, + columnWeight: fieldWeight, + columnSlops: fieldSlops + }; + let invertedIndexStrategies = [bm25Strategy, exactStrategy, outOfOrderStrategy]; + let recallConditionInvIdx = { + ftsTableName: 'invidx_table', + fromClause: 'invidx_table', + primaryKey: ['fileid'], + responseColumns: ['filename', 'keywords'], + deepSize: 500, + invertedIndexStrategies: invertedIndexStrategies + }; + let floatArray = new Float32Array(128).fill(0.1); + let vectorQuery = { + column: 'filename', + value: floatArray, + similarityThreshold: 0.1 + }; + let recallConditionVector = { + vectorQuery: vectorQuery, + fromClause: 'vector_table', + primaryKey: ['fileid'], + responseColumns: ['filename_text'], + recallName: 'vectorRecall', + deepSize: 500 + }; + let vectorWeights = { + 'vectorRecall': 1 + }; + let numberInspector = { + 'vector_query':'filename_text' + }; + let vectorRerankParameter = { + vectorWeights:vectorWeights, + thresholds: [0.55, 0.45, 0.35], + numberInspector: numberInspector + }; + let parameters = { + 0: vectorRerankParameter + }; + let rerankMethod = { + rerankType: 1, + parameters: parameters, + isSoftmaxNormalized: true, + }; + let retrievalCondition = { + rerankMethod: rerankMethod, + recallConditions : [recallConditionInvIdx, recallConditionVector] + }; + await intelligence.getRetriever(retrievalConfig) + .then((data) => { + try { + let query = '数据库'; + data.retrieveRdb(query, retrievalCondition) + .then((rdbdata) => { + let length = rdbdata.records.length; + console.info(TAG, 'Aip_013 get result::' + length); + expect(length >= 2).assertEqual(true); + done(); + }); + } catch (err) { + console.info(TAG, 'Aip_013 catch::' + err.code); + done(); + } + }); + } catch (err) { + console.info(TAG, 'Aip_013 catch::' + err.code); + done(); + } + } else { + await intelligence.getRetriever(retrievalConfig) + .catch((err) => { + console.info(TAG, 'Aip_013 catch::' + err.code); + expect(err.code).assertEqual(801); + done(); + }); + } + console.info(TAG, 'end'); + }); +}); \ No newline at end of file diff --git a/udmf/framework/jskitsimpl/unittest/UdmfCallbackJsTest.js b/udmf/framework/jskitsimpl/unittest/UdmfCallbackJsTest.js index ce022919..f263204b 100644 --- a/udmf/framework/jskitsimpl/unittest/UdmfCallbackJsTest.js +++ b/udmf/framework/jskitsimpl/unittest/UdmfCallbackJsTest.js @@ -251,7 +251,7 @@ describe('UdmfCallbackJSTest', function () { console.info(TAG, 'delete success.'); UDC.queryData(optionsValid, (err, data) => { console.info(TAG, 'query has no data.'); - expect(data).assertUndefined(); + expect(err).assertUndefined(); UDC.insertData(optionsValid, unifiedData01, (err, data) => { expect(err).assertUndefined(); console.info(TAG, `insert success. The key: ${data}`); @@ -328,7 +328,7 @@ describe('UdmfCallbackJSTest', function () { expect(records[0].getType()).assertEqual(UTD.UniformDataType.PLAIN_TEXT); expect(records[0].textContent).assertEqual(TEXT_CONTEXT_01); UDC.queryData(options, (err, data) => { - expect(data).assertUndefined(); + expect(err).assertUndefined(); console.info(TAG, 'query has no data.'); done(); }); diff --git a/udmf/framework/jskitsimpl/unittest/UdmfJsTest.js b/udmf/framework/jskitsimpl/unittest/UdmfJsTest.js index 3ae48d28..dce53307 100644 --- a/udmf/framework/jskitsimpl/unittest/UdmfJsTest.js +++ b/udmf/framework/jskitsimpl/unittest/UdmfJsTest.js @@ -1788,10 +1788,28 @@ describe('UdmfJSTest', function () { abstract: 'this is abstract', details: plainTextDetails }; + + const buffer = new ArrayBuffer(128); + const opt = { + size: { height: 5, width: 5 }, + pixelFormat: 3, + editable: true, + alphaType: 1, + scaleMode: 1, + }; + const pixelMap = await image.createPixelMap(buffer, opt); + let pixelMapUds = { + uniformDataType: 'openharmony.pixel-map', + pixelMap: pixelMap, + details: { + 'key1': 'value1', + 'key2': 'value2', + } + }; let record1 = new UDC.UnifiedRecord(UTD.UniformDataType.PLAIN_TEXT, plainText); record1.addEntry(UTD.UniformDataType.HYPERLINK, hyperLink); record1.addEntry('openharmony.app-item', systemDefined); - + record1.addEntry('openharmony.pixel-map', pixelMapUds); let entries = record1.getEntries(); let unifiedData = new UDC.UnifiedData(record1); @@ -1801,6 +1819,7 @@ describe('UdmfJSTest', function () { let types = records[0].getTypes(); let systemDefined1 = records[0].getEntry('openharmony.app-item'); let hyperlink1 = records[0].getEntry(UTD.UniformDataType.HYPERLINK); + let pixelMapUds1 = records[0].getEntry('openharmony.pixel-map'); expect(value1.uniformDataType).assertEqual(plainText.uniformDataType); expect(value1.textContent).assertEqual(plainText.textContent); @@ -1808,12 +1827,18 @@ describe('UdmfJSTest', function () { expect(value1.details.key1).assertEqual(plainText.details.key1); expect(value1.details.key2).assertEqual(plainText.details.key2); expect(hyperlink1.url).assertEqual('www.xxx'); + pixelMapUds1.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(pixelMapUds1.details.key1).assertEqual('value1'); expect(hyperlink1.details.key1).assertEqual('value1'); expect(systemDefined1.appId).assertEqual('app-itemAppId'); expect(systemDefined1.details.key1).assertEqual('value1'); - expect(types.length).assertEqual(3); + expect(types.length).assertEqual(4); expect(types.includes(UTD.UniformDataType.PLAIN_TEXT)).assertTrue(); expect(types.includes(UTD.UniformDataType.HYPERLINK)).assertTrue(); + expect(types.includes('openharmony.pixel-map')).assertTrue(); expect(types.includes('openharmony.app-item')).assertTrue(); expect(entries['openharmony.app-item'].appIconId).assertEqual('app-itemAppIconId'); expect(entries['openharmony.app-item'].details.key1).assertEqual('value1'); @@ -1821,6 +1846,11 @@ describe('UdmfJSTest', function () { expect(entries['general.hyperlink'].details.key1).assertEqual('value1'); expect(entries['general.plain-text'].textContent).assertEqual('This is plainText textContent example'); expect(entries['general.plain-text'].details.key1).assertEqual('value1'); + entries['openharmony.pixel-map'].pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(entries['openharmony.pixel-map'].details.key1).assertEqual('value1'); try { UDC.insertData(optionsValid, unifiedData).then((data) => { @@ -1846,9 +1876,10 @@ describe('UdmfJSTest', function () { expect(hyperlinkQuery.details.key1).assertEqual('value1'); expect(systemDefinedQuery.appId).assertEqual('app-itemAppId'); expect(systemDefinedQuery.details.key1).assertEqual('value1'); - expect(types1.length).assertEqual(3); + expect(types1.length).assertEqual(4); expect(types1.includes(UTD.UniformDataType.PLAIN_TEXT)).assertTrue(); expect(types1.includes(UTD.UniformDataType.HYPERLINK)).assertTrue(); + expect(types1.includes('openharmony.pixel-map')).assertTrue(); expect(types1.includes('openharmony.app-item')).assertTrue(); expect(entriesQuery['openharmony.app-item'].appIconId).assertEqual('app-itemAppIconId'); @@ -1857,6 +1888,11 @@ describe('UdmfJSTest', function () { expect(entriesQuery['general.hyperlink'].details.key1).assertEqual('value1'); expect(entriesQuery['general.plain-text'].textContent).assertEqual('This is plainText textContent example'); expect(entriesQuery['general.plain-text'].details.key1).assertEqual('value1'); + entriesQuery['openharmony.pixel-map'].pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(entriesQuery['openharmony.pixel-map'].details.key1).assertEqual('value1'); UDC.deleteData(options).then((data) => { console.info(TAG, 'delete success.'); @@ -2080,4 +2116,990 @@ describe('UdmfJSTest', function () { done(); } }); + + /** + * @tc.name MultiEntryTest004 + * @tc.desc + * @tc.type: FUNC + * @tc.require: + */ + it('MultiEntryTest004', 0, async function (done) { + const TAG = 'MultiEntryTest004'; + console.info(TAG, 'start'); + const buffer = new ArrayBuffer(128); + const opt = { + size: { height: 5, width: 5 }, + pixelFormat: 3, + editable: true, + alphaType: 1, + scaleMode: 1, + }; + const pixelMap = await image.createPixelMap(buffer, opt); + let pixelMapUds = { + uniformDataType: 'openharmony.pixel-map', + pixelMap: pixelMap, + details: { + 'key1': 'value1', + 'key2': 'value2', + } + }; + let record = new UDC.UnifiedRecord('openharmony.pixel-map', pixelMapUds); + let unifiedData = new UDC.UnifiedData(record); + + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + console.info(TAG, `query start. The options: ${JSON.stringify(options)}`); + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + expect(data.length).assertEqual(1); + let records = data[0].getRecords(); + expect(records.length).assertEqual(1); + const valueQuery = records[0].getValue(); + expect(valueQuery.uniformDataType).assertEqual('openharmony.pixel-map'); + valueQuery.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(valueQuery.details.key1).assertEqual('value1'); + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name MultiEntryTest005 + * @tc.desc + * @tc.type: FUNC + * @tc.require: + */ + it('MultiEntryTest005', 0, async function () { + const TAG = 'MultiEntryTest005'; + console.info(TAG, 'start'); + try { + let plaintextValue = 'plaintextValue'; + let systemDefinedValue = 'systemDefinedValue'; + let record = new UDC.UnifiedRecord(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + record.addEntry(undefined, systemDefinedValue); + } catch (e) { + console.error(TAG, `get e. code is ${e.code},message is ${e.message} `); + expect(e.code === ERROR_PARAMETER).assertTrue(); + } + try { + let plaintextValue = 'plaintextValue'; + let systemDefinedValue = 'systemDefinedValue'; + let record = new UDC.UnifiedRecord(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + record.addEntry(null, systemDefinedValue); + } catch (e) { + console.error(TAG, `get e. code is ${e.code},message is ${e.message} `); + expect(e.code === ERROR_PARAMETER).assertTrue(); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name MultiEntryTest006 + * @tc.desc + * @tc.type: FUNC + * @tc.require: + */ + it('MultiEntryTest006', 0, async function () { + const TAG = 'MultiEntryTest006'; + console.info(TAG, 'start'); + try { + let plaintextValue = 'plaintextValue'; + let systemDefinedValue = 'systemDefinedValue'; + let record = new UDC.UnifiedRecord(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + record.addEntry('openharmony.app-item', systemDefinedValue); + record.getEntry(undefined); + } catch (e) { + console.error(TAG, `get e. code is ${e.code},message is ${e.message} `); + expect(e.code === ERROR_PARAMETER).assertTrue(); + } + try { + let plaintextValue = 'plaintextValue'; + let systemDefinedValue = 'systemDefinedValue'; + let record = new UDC.UnifiedRecord(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + record.addEntry('openharmony.app-item', systemDefinedValue); + record.getEntry(null); + } catch (e) { + console.error(TAG, `get e. code is ${e.code},message is ${e.message} `); + expect(e.code === ERROR_PARAMETER).assertTrue(); + } + console.info(TAG, 'end'); + }); + + /** + * @tc.name FileUriTest001 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor UnifiedRecord(UDType type, ValueType value) and sub image + */ + it('FileUriTest001', 0, async function (done) { + const TAG = 'FileUriTest001'; + console.info(TAG, 'start'); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.jpeg', + details : fileUriDetails + }; + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let arrayBuffer = new ArrayBuffer(4 * 200 * 200); + let opt = { editable: true, pixelFormat: 3, size: { height: 200, width: 200 }, alphaType: 3 }; + let pixelMapDetails = { + 'pixelMapKey1': 123, + 'pixelMapKey2': 'pixelMapValue', + 'pixelMapKey3': u8Array, + }; + let pixelMap = { + uniformDataType : 'openharmony.pixel-map', + pixelMap : image.createPixelMapSync(arrayBuffer, opt), + details : pixelMapDetails + }; + let record = new UDC.UnifiedRecord('general.file-uri', fileUri); + record.addEntry('openharmony.pixel-map', pixelMap); + let rawType = record.getType(); + expect(rawType).assertEqual('general.image'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + expect(rawRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + expect(rawDataTypes.includes('openharmony.pixel-map')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.image')).assertTrue(); + expect(unifiedData.hasType('general.jpeg')).assertTrue(); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + let records = data[0].getRecords(); + let rawType = records[0].getType(); + expect(rawType).assertEqual('general.image'); + let getRecordTypes = records[0].getTypes(); + expect(getRecordTypes.includes('general.file-uri')).assertTrue(); + expect(getRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getDataTypes = data[0].getTypes(); + expect(getDataTypes.includes('general.file-uri')).assertTrue(); + for (let i = 0; i < records.length; i++) { + if (records[i].getType() === 'general.image') { + let getImageUri = records[i].imageUri; + expect(getImageUri).assertEqual('www.xx.com'); + } + } + let getValue = records[0].getValue(); + expect(getValue.uniformDataType).assertEqual('general.file-uri'); + expect(getValue.oriUri).assertEqual('www.xx.com'); + expect(getValue.fileType).assertEqual('general.jpeg'); + expect(getValue.details.fileUriKey1).assertEqual(123); + expect(getValue.details.fileUriKey2).assertEqual('fileUriValue'); + + let getEntryFileUri = records[0].getEntry('general.file-uri'); + expect(getEntryFileUri.uniformDataType).assertEqual('general.file-uri'); + expect(getEntryFileUri.oriUri).assertEqual('www.xx.com'); + expect(getEntryFileUri.fileType).assertEqual('general.jpeg'); + expect(getEntryFileUri.details.fileUriKey1).assertEqual(123); + + let getEntryPixelMap = records[0].getEntry('openharmony.pixel-map'); + expect(getEntryPixelMap.uniformDataType).assertEqual('openharmony.pixel-map'); + getEntryPixelMap.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getEntryPixelMap.details.pixelMapKey1).assertEqual(123); + expect(getEntryPixelMap.details.pixelMapKey2).assertEqual('pixelMapValue'); + expect(data[0].hasType('general.file-uri')).assertTrue(); + expect(data[0].hasType('general.image')).assertTrue(); + expect(data[0].hasType('general.jpeg')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name FileUriTest002 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor UnifiedRecord(UDType type, ValueType value) + */ + it('FileUriTest002', 0, async function (done) { + const TAG = 'FileUriTest002'; + console.info(TAG, 'start'); + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + 'fileUriKey3': u8Array, + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.image', + details : fileUriDetails + }; + let record = new UDC.UnifiedRecord('general.file-uri', fileUri); + let rawType = record.getType(); + expect(rawType).assertEqual('general.image'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.image')).assertTrue(); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + expect(data.length).assertEqual(1); + let records = data[0].getRecords(); + expect(records.length).assertEqual(1); + let rawType = records[0].getType(); + expect(rawType).assertEqual('general.image'); + let rawRecordTypes = records[0].getTypes(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + + let rawDataTypes = data[0].getTypes(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + for (let i = 0; i < records.length; i++) { + if (records[i].getType() === 'general.image') { + let getImageUri = records[i].imageUri; + expect(getImageUri).assertEqual('www.xx.com'); + } + } + let getValue = records[0].getValue(); + expect(getValue.uniformDataType).assertEqual('general.file-uri'); + expect(getValue.oriUri).assertEqual('www.xx.com'); + expect(getValue.fileType).assertEqual('general.image'); + expect(getValue.details.fileUriKey1).assertEqual(123); + expect(getValue.details.fileUriKey2).assertEqual('fileUriValue'); + + let getEntry = records[0].getEntry('general.file-uri'); + expect(getEntry.uniformDataType).assertEqual('general.file-uri'); + expect(getEntry.oriUri).assertEqual('www.xx.com'); + expect(getEntry.fileType).assertEqual('general.image'); + expect(getEntry.details.fileUriKey1).assertEqual(123); + expect(data[0].hasType('general.file-uri')).assertTrue(); + expect(data[0].hasType('general.image')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name FileUriTest003 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor UDC + */ + it('FileUriTest003', 0, async function (done) { + const TAG = 'FileUriTest003'; + console.info(TAG, 'start'); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.jpeg', + details : fileUriDetails + }; + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let arrayBuffer = new ArrayBuffer(4 * 200 * 200); + let opt = { editable: true, pixelFormat: 3, size: { height: 200, width: 200 }, alphaType: 3 }; + let pixelMapDetails = { + 'pixelMapKey1': 123, + 'pixelMapKey2': 'pixelMapValue', + 'pixelMapKey3': u8Array, + }; + let pixelMap = { + uniformDataType : 'openharmony.pixel-map', + pixelMap : image.createPixelMapSync(arrayBuffer, opt), + details : pixelMapDetails + }; + let record = new UDC.File(); + record.details = { + name: 'test', + type: 'txt', + }; + record.uri = 'schema://com.samples.test/files/test.txt'; + record.addEntry('openharmony.pixel-map', pixelMap); + record.addEntry('general.file-uri', fileUri); + let rawType = record.getType(); + expect(rawType).assertEqual('general.file'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.file')).assertTrue(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + expect(rawRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.file')).assertTrue(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + expect(rawDataTypes.includes('openharmony.pixel-map')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.image')).assertTrue(); + expect(unifiedData.hasType('general.jpeg')).assertTrue(); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + let records = data[0].getRecords(); + let rawType = records[0].getType(); + expect(rawType).assertEqual('general.file'); + let getRecordTypes = records[0].getTypes(); + expect(getRecordTypes.includes('general.file')).assertTrue(); + expect(getRecordTypes.includes('general.file-uri')).assertTrue(); + expect(getRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getDataTypes = data[0].getTypes(); + expect(rawDataTypes.includes('general.file')).assertTrue(); + expect(getDataTypes.includes('general.file-uri')).assertTrue(); + expect(getDataTypes.includes('openharmony.pixel-map')).assertTrue(); + for (let i = 0; i < records.length; i++) { + if (records[i].getType() === 'general.file') { + let getUri = records[i].uri; + expect(getUri).assertEqual('schema://com.samples.test/files/test.txt'); + } + } + let getValue = records[0].getValue(); + expect(getValue).assertEqual(undefined); + + let getEntryFileUri = records[0].getEntry('general.file-uri'); + expect(getEntryFileUri.uniformDataType).assertEqual('general.file-uri'); + expect(getEntryFileUri.oriUri).assertEqual('www.xx.com'); + expect(getEntryFileUri.fileType).assertEqual('general.jpeg'); + expect(getEntryFileUri.details.fileUriKey1).assertEqual(123); + + let getEntryPixelMap = records[0].getEntry('openharmony.pixel-map'); + expect(getEntryPixelMap.uniformDataType).assertEqual('openharmony.pixel-map'); + getEntryPixelMap.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getEntryPixelMap.details.pixelMapKey1).assertEqual(123); + expect(getEntryPixelMap.details.pixelMapKey2).assertEqual('pixelMapValue'); + expect(data[0].hasType('general.file-uri')).assertTrue(); + expect(data[0].hasType('general.image')).assertTrue(); + expect(data[0].hasType('general.jpeg')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name FileUriTest004 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor no parameter + */ + it('FileUriTest004', 0, async function (done) { + const TAG = 'FileUriTest004'; + console.info(TAG, 'start'); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.vob', + details : fileUriDetails + }; + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let arrayBuffer = new ArrayBuffer(4 * 200 * 200); + let opt = { editable: true, pixelFormat: 3, size: { height: 200, width: 200 }, alphaType: 3 }; + let pixelMapDetails = { + 'pixelMapKey1': 123, + 'pixelMapKey2': 'pixelMapValue', + 'pixelMapKey3': u8Array, + }; + let pixelMap = { + uniformDataType : 'openharmony.pixel-map', + pixelMap : image.createPixelMapSync(arrayBuffer, opt), + details : pixelMapDetails + }; + let record = new UDC.UnifiedRecord(); + record.addEntry('general.file-uri', fileUri); + record.addEntry('openharmony.pixel-map', pixelMap); + let rawType = record.getType(); + expect(rawType).assertEqual('general.video'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + expect(rawRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + expect(rawDataTypes.includes('openharmony.pixel-map')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.video')).assertTrue(); + expect(unifiedData.hasType('general.vob')).assertTrue(); + let records = unifiedData.getRecords(); + expect(records.length).assertEqual(1); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + let records = data[0].getRecords(); + let rawType = records[0].getType(); + expect(rawType).assertEqual('general.video'); + let getRecordTypes = records[0].getTypes(); + expect(getRecordTypes.includes('general.file-uri')).assertTrue(); + expect(getRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getDataTypes = data[0].getTypes(); + expect(getDataTypes.includes('general.file-uri')).assertTrue(); + expect(getDataTypes.includes('openharmony.pixel-map')).assertTrue(); + for (let i = 0; i < records.length; i++) { + if (records[i].getType() === 'general.video') { + let getUri = records[i].videoUri; + expect(getUri).assertEqual('www.xx.com'); + } + } + let getValue = records[0].getValue(); + expect(getValue.uniformDataType).assertEqual('general.file-uri'); + expect(getValue.oriUri).assertEqual('www.xx.com'); + expect(getValue.fileType).assertEqual('general.vob'); + expect(getValue.details.fileUriKey1).assertEqual(123); + expect(getValue.details.fileUriKey2).assertEqual('fileUriValue'); + + let getEntryFileUri = records[0].getEntry('general.file-uri'); + expect(getEntryFileUri.uniformDataType).assertEqual('general.file-uri'); + expect(getEntryFileUri.oriUri).assertEqual('www.xx.com'); + expect(getEntryFileUri.fileType).assertEqual('general.vob'); + expect(getEntryFileUri.details.fileUriKey1).assertEqual(123); + + let getEntryPixelMap = records[0].getEntry('openharmony.pixel-map'); + expect(getEntryPixelMap.uniformDataType).assertEqual('openharmony.pixel-map'); + getEntryPixelMap.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getEntryPixelMap.details.pixelMapKey1).assertEqual(123); + expect(getEntryPixelMap.details.pixelMapKey2).assertEqual('pixelMapValue'); + expect(data[0].hasType('general.file-uri')).assertTrue(); + expect(data[0].hasType('general.video')).assertTrue(); + expect(data[0].hasType('general.vob')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name FileUriTest005 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor no parameter + */ + it('FileUriTest005', 0, async function (done) { + const TAG = 'FileUriTest005'; + console.info(TAG, 'start'); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.vob', + details : fileUriDetails + }; + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let arrayBuffer = new ArrayBuffer(4 * 200 * 200); + let opt = { editable: true, pixelFormat: 3, size: { height: 200, width: 200 }, alphaType: 3 }; + let pixelMapDetails = { + 'pixelMapKey1': 123, + 'pixelMapKey2': 'pixelMapValue', + 'pixelMapKey3': u8Array, + }; + let pixelMap = { + uniformDataType : 'openharmony.pixel-map', + pixelMap : image.createPixelMapSync(arrayBuffer, opt), + details : pixelMapDetails + }; + let record = new UDC.UnifiedRecord('general.vob', fileUri); + record.addEntry('openharmony.pixel-map', pixelMap); + let rawType = record.getType(); + expect(rawType).assertEqual('general.vob'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.vob')).assertTrue(); + expect(rawRecordTypes.includes('general.file-uri')).assertFalse(); + expect(rawRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.file-uri')).assertFalse(); + expect(rawDataTypes.includes('openharmony.pixel-map')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertFalse(); + expect(unifiedData.hasType('general.video')).assertTrue(); + expect(unifiedData.hasType('general.vob')).assertTrue(); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + let records = data[0].getRecords(); + let rawType = records[0].getType(); + expect(rawType).assertEqual('general.vob'); + let getRecordTypes = records[0].getTypes(); + expect(getRecordTypes.includes('general.vob')).assertTrue(); + expect(getRecordTypes.includes('general.file-uri')).assertFalse(); + expect(getRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getDataTypes = data[0].getTypes(); + expect(getDataTypes.includes('general.vob')).assertTrue(); + expect(getDataTypes.includes('general.file-uri')).assertFalse(); + expect(getDataTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getValue = records[0].getValue(); + expect(getValue.uniformDataType).assertEqual('general.file-uri'); + expect(getValue.oriUri).assertEqual('www.xx.com'); + expect(getValue.fileType).assertEqual('general.vob'); + expect(getValue.details.fileUriKey1).assertEqual(123); + expect(getValue.details.fileUriKey2).assertEqual('fileUriValue'); + + let getEntryFileUri = records[0].getEntry('general.file-uri'); + expect(getEntryFileUri).assertEqual(undefined); + + let getEntryVob = records[0].getEntry('general.vob'); + expect(getEntryVob.uniformDataType).assertEqual('general.file-uri'); + expect(getEntryVob.oriUri).assertEqual('www.xx.com'); + expect(getEntryVob.fileType).assertEqual('general.vob'); + expect(getEntryVob.details.fileUriKey1).assertEqual(123); + + let getEntryPixelMap = records[0].getEntry('openharmony.pixel-map'); + expect(getEntryPixelMap.uniformDataType).assertEqual('openharmony.pixel-map'); + getEntryPixelMap.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getEntryPixelMap.details.pixelMapKey1).assertEqual(123); + expect(getEntryPixelMap.details.pixelMapKey2).assertEqual('pixelMapValue'); + expect(data[0].hasType('general.file-uri')).assertFalse(); + expect(data[0].hasType('general.video')).assertTrue(); + expect(data[0].hasType('general.vob')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name FileUriTest006 + * @tc.desc + * @tc.type: FUNC + * @tc.require: test constructor no parameter + */ + it('FileUriTest006', 0, async function (done) { + const TAG = 'FileUriTest006'; + console.info(TAG, 'start'); + let fileUriDetails = { + 'fileUriKey1': 123, + 'fileUriKey2': 'fileUriValue', + }; + let fileUri = { + uniformDataType : 'general.file-uri', + oriUri : 'www.xx.com', + fileType : 'general.vob', + details : fileUriDetails + }; + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + let arrayBuffer = new ArrayBuffer(4 * 200 * 200); + let opt = { editable: true, pixelFormat: 3, size: { height: 200, width: 200 }, alphaType: 3 }; + let pixelMapDetails = { + 'pixelMapKey1': 123, + 'pixelMapKey2': 'pixelMapValue', + 'pixelMapKey3': u8Array, + }; + let pixelMap = { + uniformDataType : 'openharmony.pixel-map', + pixelMap : image.createPixelMapSync(arrayBuffer, opt), + details : pixelMapDetails + }; + let record = new UDC.UnifiedRecord('openharmony.pixel-map', pixelMap); + record.addEntry('general.file-uri', fileUri); + let rawType = record.getType(); + expect(rawType).assertEqual('openharmony.pixel-map'); + let rawRecordTypes = record.getTypes(); + expect(rawRecordTypes.includes('general.vob')).assertFalse(); + expect(rawRecordTypes.includes('general.file-uri')).assertTrue(); + expect(rawRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + let unifiedData = new UDC.UnifiedData(record); + let rawDataTypes = unifiedData.getTypes(); + expect(rawDataTypes.includes('general.vob')).assertFalse(); + expect(rawDataTypes.includes('general.file-uri')).assertTrue(); + expect(rawDataTypes.includes('openharmony.pixel-map')).assertTrue(); + expect(unifiedData.hasType('general.file-uri')).assertTrue(); + expect(unifiedData.hasType('general.video')).assertTrue(); + expect(unifiedData.hasType('general.vob')).assertTrue(); + expect(unifiedData.hasType('openharmony.pixel-map')).assertTrue(); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + let records = data[0].getRecords(); + let rawType = records[0].getType(); + expect(rawType).assertEqual('openharmony.pixel-map'); + let getRecordTypes = records[0].getTypes(); + expect(getRecordTypes.includes('general.vob')).assertFalse(); + expect(getRecordTypes.includes('general.file-uri')).assertTrue(); + expect(getRecordTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getDataTypes = data[0].getTypes(); + expect(getDataTypes.includes('general.vob')).assertFalse(); + expect(getDataTypes.includes('general.file-uri')).assertTrue(); + expect(getDataTypes.includes('openharmony.pixel-map')).assertTrue(); + + let getValue = records[0].getValue(); + expect(getValue.uniformDataType).assertEqual('openharmony.pixel-map'); + getValue.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getValue.details.pixelMapKey1).assertEqual(123); + expect(getValue.details.pixelMapKey2).assertEqual('pixelMapValue'); + + let getEntryFileUri = records[0].getEntry('general.file-uri'); + expect(getEntryFileUri.uniformDataType).assertEqual('general.file-uri'); + expect(getEntryFileUri.oriUri).assertEqual('www.xx.com'); + expect(getEntryFileUri.fileType).assertEqual('general.vob'); + expect(getEntryFileUri.details.fileUriKey1).assertEqual(123); + + let getEntryPixelMap = records[0].getEntry('openharmony.pixel-map'); + expect(getEntryPixelMap.uniformDataType).assertEqual('openharmony.pixel-map'); + getEntryPixelMap.pixelMap.getImageInfo().then((imageInfo)=>{ + expect(imageInfo.size.height).assertEqual(opt.size.height); + expect(imageInfo.pixelFormat).assertEqual(opt.pixelFormat); + }); + expect(getEntryPixelMap.details.pixelMapKey1).assertEqual(123); + expect(getEntryPixelMap.details.pixelMapKey2).assertEqual('pixelMapValue'); + expect(data[0].hasType('general.file-uri')).assertTrue(); + expect(data[0].hasType('general.video')).assertTrue(); + expect(data[0].hasType('general.vob')).assertTrue(); + expect(data[0].hasType('openharmony.pixel-map')).assertTrue(); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name MultiEntryTest007 + * @tc.desc Build a record with no params, then add entries. test if UnifiedData::getRecords() function works well. + * @tc.type: FUNC + * @tc.require: + */ + it('MultiEntryTest007', 0, async function (done) { + const TAG = 'MultiEntryTest007'; + console.info(TAG, 'start'); + + let plaintextValue = 'plaintextValue'; + let systemDefinedValue = 'systemDefinedValue'; + let hyperlinkValue = 'hyperlinkValue'; + let record = new UDC.UnifiedRecord(); + record.addEntry(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + record.addEntry(UTD.UniformDataType.HYPERLINK, hyperlinkValue); + record.addEntry('openharmony.app-item', systemDefinedValue); + + let recordFile = new UDC.File(); + recordFile.uri = 'schema://com.samples.test/files/test.txt'; + recordFile.addEntry(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + recordFile.addEntry(UTD.UniformDataType.HYPERLINK, hyperlinkValue); + + let recordAppDefined = new UDC.ApplicationDefinedRecord(); + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + recordAppDefined.applicationDefinedType = 'ApplicationDefinedType'; + recordAppDefined.rawData = u8Array; + + let recordAny = new UDC.UnifiedRecord('test.utd-id', 'test.utd-value'); + recordAny.addEntry(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + recordAny.addEntry(UTD.UniformDataType.HYPERLINK, hyperlinkValue); + + let unifiedData = new UDC.UnifiedData(record); + unifiedData.addRecord(recordFile); + unifiedData.addRecord(recordAppDefined); + unifiedData.addRecord(recordAny); + let records = unifiedData.getRecords(); + expect(records.length).assertEqual(4); + + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + console.info(TAG, `query start. The options: ${JSON.stringify(options)}`); + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + expect(data.length).assertEqual(1); + let records = data[0].getRecords(); + expect(records.length).assertEqual(4); + + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); + + /** + * @tc.name MultiEntryTest008 + * @tc.desc UDC to get first entry works well test + * @tc.type: FUNC + * @tc.require: + */ + it('MultiEntryTest008', 0, async function (done) { + const TAG = 'MultiEntryTest008'; + console.info(TAG, 'start'); + let plaintextValue = 'plaintextValue'; + let record = new UDC.UnifiedRecord(); + record.addEntry(UTD.UniformDataType.PLAIN_TEXT, plaintextValue); + let recordFile = new UDC.File(); + recordFile.uri = 'schema://com.samples.test/files/test.txt'; + let recordAppDefined = new UDC.ApplicationDefinedRecord(); + let u8Array = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + recordAppDefined.applicationDefinedType = 'ApplicationDefinedType'; + recordAppDefined.rawData = u8Array; + let recordAny = new UDC.UnifiedRecord('general.log', 'test.utd-value'); + let recordHyperlink = new UDC.UnifiedRecord('general.hyperlink', 'test.hyperlink'); + let unifiedData = new UDC.UnifiedData(record); + unifiedData.addRecord(recordFile); + unifiedData.addRecord(recordAppDefined); + unifiedData.addRecord(recordAny); + unifiedData.addRecord(recordHyperlink); + let records = unifiedData.getRecords(); + expect(records.length).assertEqual(5); + try { + UDC.insertData(optionsValid, unifiedData).then((data) => { + console.info(TAG, `insert success. The key: ${data}`); + let options = { key: data }; + console.info(TAG, `query start. The options: ${JSON.stringify(options)}`); + UDC.queryData(options).then((data) => { + console.info(TAG, 'query success.'); + expect(data.length).assertEqual(1); + let records = data[0].getRecords(); + let firstEntry1 = records[0].getEntry(UTD.UniformDataType.PLAIN_TEXT); + expect(firstEntry1.textContent).assertEqual('plaintextValue'); + let firstEntry2 = records[1].getEntry(UTD.UniformDataType.FILE); + expect(firstEntry2.oriUri).assertEqual('schema://com.samples.test/files/test.txt'); + let firstEntry3 = records[2].getEntry('ApplicationDefinedType'); + expect(firstEntry3.arrayBuffer.length).assertEqual(u8Array.length); + let firstEntry4 = records[3].getEntry('general.log'); + expect(firstEntry4).assertEqual('test.utd-value'); + let firstEntry5 = records[4].getEntry('general.hyperlink'); + expect(firstEntry5.url).assertEqual('test.hyperlink'); + UDC.deleteData(options).then((data) => { + console.info(TAG, 'delete success.'); + expect(data.length).assertEqual(1); + done(); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + }).catch(() => { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + }); + } catch (e) { + console.error(TAG, 'Unreachable code!'); + expect(null).assertFail(); + done(); + } + }); }); \ No newline at end of file diff --git a/udmf/framework/jskitsimpl/unittest/UdmfPromiseJsTest.js b/udmf/framework/jskitsimpl/unittest/UdmfPromiseJsTest.js index 3826e7b6..8b8ab2ca 100644 --- a/udmf/framework/jskitsimpl/unittest/UdmfPromiseJsTest.js +++ b/udmf/framework/jskitsimpl/unittest/UdmfPromiseJsTest.js @@ -286,8 +286,8 @@ describe('UdmfPromiseJSTest', function () { UDC.deleteData(optionsValid).then(() => { console.info(TAG, 'delete success.'); UDC.queryData(optionsValid).then(() => { - console.error(TAG, 'Unreachable code!'); - expect(null).assertFail(); + console.info(TAG, 'Unreachable code!'); + expect(err).assertUndefined(); done(); }).catch((err) => { console.info(TAG, 'query has no data.'); @@ -381,11 +381,11 @@ describe('UdmfPromiseJSTest', function () { expect(records[0].getType()).assertEqual(UTD.UniformDataType.PLAIN_TEXT); expect(records[0].textContent).assertEqual(TEXT_CONTEXT_01); UDC.queryData(options).then(() => { - console.error(TAG, 'Unreachable code!'); - expect(null).assertFail(); + console.info(TAG, 'Unreachable code!'); + expect(err).assertUndefined(); done(); }).catch(() => { - console.info(TAG, 'query has no data.'); + console.error(TAG, 'query has no data.'); done(); }); }).catch(() => { diff --git a/udmf/framework/ndkimpl/data/udmf.cpp b/udmf/framework/ndkimpl/data/udmf.cpp index c3f3c8c4..1e857a4f 100644 --- a/udmf/framework/ndkimpl/data/udmf.cpp +++ b/udmf/framework/ndkimpl/data/udmf.cpp @@ -35,6 +35,8 @@ #include "folder.h" #include "image.h" #include "video.h" +#include "utd.h" +#include "utd_client.h" using namespace OHOS::UDMF; @@ -47,6 +49,8 @@ static const std::map<std::string, UDType> FILE_TYPES = { { UDMF_META_IMAGE, UDType::IMAGE }, { UDMF_META_VIDEO, UDType::VIDEO } }; +static const std::set<std::string> FILE_SUB_TYPES = { + "general.image", "general.video", "general.audio", "general.folder" }; static void DestroyStringArray(char**& bufArray, unsigned int& count) { @@ -573,26 +577,32 @@ int OH_UdmfRecord_AddFileUri(OH_UdmfRecord* record, OH_UdsFileUri* fileUri) if (fileType == nullptr) { return UDMF_ERR; } - int32_t utdId = UtdUtils::GetUtdEnumFromUtdId(*fileType); - switch (utdId) { - case UDType::FILE: - AddUds<File>(record, fileUri, UDType::FILE); - break; - case UDType::AUDIO: - AddUds<Audio>(record, fileUri, UDType::AUDIO); - break; - case UDType::FOLDER: - AddUds<Folder>(record, fileUri, UDType::FOLDER); - break; - case UDType::IMAGE: - AddUds<Image>(record, fileUri, UDType::IMAGE); - break; - case UDType::VIDEO: - AddUds<Video>(record, fileUri, UDType::VIDEO); - break; - default: - AddUds<UnifiedRecord>(record, fileUri, UDType::FILE_URI); + std::map<UDType, std::function<void(OH_UdmfRecord*, OH_UdsFileUri*)>> addFileUriFuncs = { + {UDType::FILE, [](OH_UdmfRecord* record, OH_UdsFileUri* fileUri) + { AddUds<File>(record, fileUri, UDType::FILE); }}, + {UDType::IMAGE, [](OH_UdmfRecord* record, OH_UdsFileUri* fileUri) + { AddUds<Image>(record, fileUri, UDType::IMAGE); }}, + {UDType::VIDEO, [](OH_UdmfRecord* record, OH_UdsFileUri* fileUri) + { AddUds<Video>(record, fileUri, UDType::VIDEO); }}, + {UDType::AUDIO, [](OH_UdmfRecord* record, OH_UdsFileUri* fileUri) + { AddUds<Audio>(record, fileUri, UDType::AUDIO); }}, + {UDType::FOLDER, [](OH_UdmfRecord* record, OH_UdsFileUri* fileUri) + { AddUds<Folder>(record, fileUri, UDType::FOLDER); }}, + }; + int32_t utdId = UDType::FILE; + std::shared_ptr<TypeDescriptor> descriptor; + UtdClient::GetInstance().GetTypeDescriptor(*fileType, descriptor); + if (descriptor != nullptr) { + bool isFileType = false; + for (const auto &fileSub : FILE_SUB_TYPES) { + descriptor->BelongsTo(fileSub, isFileType); + if (isFileType) { + utdId = static_cast<UDType>(UtdUtils::GetUtdEnumFromUtdId(*fileType)); + break; + } + } } + addFileUriFuncs[static_cast<UDType>(utdId)](record, fileUri); return UDMF_E_OK; } diff --git a/udmf/framework/ndkimpl/unittest/BUILD.gn b/udmf/framework/ndkimpl/unittest/BUILD.gn index c393b4dd..ad8ece29 100644 --- a/udmf/framework/ndkimpl/unittest/BUILD.gn +++ b/udmf/framework/ndkimpl/unittest/BUILD.gn @@ -71,7 +71,23 @@ ohos_unittest("UdsTest") { ohos_unittest("UdmfTest") { module_out_path = module_output_path - sources = [ "udmf_test.cpp" ] + sources = [ + "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/audio.cpp", + "${udmf_framework_path}/innerkitsimpl/data/file.cpp", + "${udmf_framework_path}/innerkitsimpl/data/folder.cpp", + "${udmf_framework_path}/innerkitsimpl/data/html.cpp", + "${udmf_framework_path}/innerkitsimpl/data/image.cpp", + "${udmf_framework_path}/innerkitsimpl/data/link.cpp", + "${udmf_framework_path}/innerkitsimpl/data/plain_text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_appitem.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_form.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_pixelmap.cpp", + "${udmf_framework_path}/innerkitsimpl/data/system_defined_record.cpp", + "${udmf_framework_path}/innerkitsimpl/data/text.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", + "udmf_test.cpp", + ] configs = [ ":module_private_config" ] diff --git a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/BUILD.gn b/udmf/interfaces/components/BUILD.gn similarity index 47% rename from datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/BUILD.gn rename to udmf/interfaces/components/BUILD.gn index 9508a79a..53d18135 100644 --- a/datamgr_service/services/distributeddataservice/app/src/flowctrl_manager/BUILD.gn +++ b/udmf/interfaces/components/BUILD.gn @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Huawei Device Co., Ltd. +# Copyright (c) 2025 Huawei Device Co., Ltd. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,39 +11,41 @@ # See the License for the specific language governing permissions and # limitations under the License. +import("//build/config/components/ets_frontend/es2abc_config.gni") import("//build/ohos.gni") -import("//foundation/distributeddatamgr/datamgr_service/datamgr_service.gni") -ohos_source_set("distributeddata_flowctrl") { +es2abc_gen_abc("gen_udmfcomponents_abc") { + src_js = rebase_path("UdmfComponents.js") + dst_file = rebase_path(target_out_dir + "/udmfcomponents.abc") + in_puts = [ "UdmfComponents.js" ] + out_puts = [ target_out_dir + "/udmfcomponents.abc" ] + extra_args = [ "--module" ] +} + +gen_js_obj("udmfcomponents_abc") { + input = get_label_info(":gen_udmfcomponents_abc", "target_out_dir") + + "/udmfcomponents.abc" + output = target_out_dir + "/udmfcomponents_abc.o" + dep = ":gen_udmfcomponents_abc" +} + +ohos_shared_library("udmfcomponents") { branch_protector_ret = "pac_ret" sanitize = { + boundary_sanitize = true cfi = true cfi_cross_dso = true debug = false - boundary_sanitize = true + integer_overflow = true ubsan = true } - sources = [ "kvstore_flowctrl_manager.cpp" ] - - include_dirs = [ - "../../../adapter/include/account", - "../../src", - ] - - cflags_cc = [ - "-fvisibility=hidden", - "-Oz", - ] + sources = [ "udmfcomponents.cpp" ] - cflags = [ "-Oz" ] + deps = [ ":udmfcomponents_abc" ] - external_deps = [ - "c_utils:utils", - "json:nlohmann_json_static", - "kv_store:datamgr_common", - ] + external_deps = [ "napi:ace_napi" ] + relative_install_dir = "module/data" subsystem_name = "distributeddatamgr" - part_name = "datamgr_service" - defines = [ "OPENSSL_SUPPRESS_DEPRECATED" ] + part_name = "udmf" } diff --git a/udmf/interfaces/components/UdmfComponents.js b/udmf/interfaces/components/UdmfComponents.js new file mode 100644 index 00000000..7fddb8e2 --- /dev/null +++ b/udmf/interfaces/components/UdmfComponents.js @@ -0,0 +1,935 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +if (!("finalizeConstruction" in ViewPU.prototype)) { + Reflect.set(ViewPU.prototype, "finalizeConstruction", () => { }); +} + +let image = requireNapi('multimedia.image'); +let j = requireNapi('i18n'); +let display = requireNapi('display'); + +export var FormType; +(function (FormType) { + FormType[FormType["TYPE_BIG"] = 0] = "TYPE_BIG"; + FormType[FormType["TYPE_MID"] = 1] = "TYPE_MID"; + FormType[FormType["TYPE_SMALL"] = 2] = "TYPE_SMALL"; +})(FormType || (FormType = {})); +var TextType; +(function (TextType) { + TextType[TextType["TITLE"] = 0] = "TITLE"; + TextType[TextType["DESCRIPTION"] = 1] = "DESCRIPTION"; + TextType[TextType["APP_NAME"] = 2] = "APP_NAME"; +})(TextType || (TextType = {})); +const m = 'udmf.ContentFormCard'; +const o = + '82,73,70,70,60,3,0,0,87,69,66,80,86,80,56,32,48,3,0,0,144,67,0,157,1,42,36,2,76,1,62,145,72,161,76,37,164,163,34,3' + + '4,151,40,24,176,18,9,105,110,225,117,81,27,243,141,167,87,231,251,1,151,228,76,129,74,56,124,143,240,134,221,17,24' + + '5,145,49,195,251,155,103,15,145,254,16,219,162,62,178,38,56,127,115,108,225,242,63,194,27,116,71,214,68,199,15,238' + + ',109,156,62,71,248,67,110,136,250,200,152,225,253,205,179,135,200,255,8,109,209,31,89,19,28,63,185,182,112,249,31,' + + '225,13,186,35,235,34,99,135,247,54,206,31,35,252,33,183,68,125,100,76,112,254,230,217,195,228,75,0,41,63,219,242,2' + + '38,77,44,240,251,18,157,13,186,35,235,34,99,135,247,54,206,31,35,249,8,172,169,162,121,152,235,226,174,0,65,245,14' + + '5,49,195,251,155,103,15,145,254,16,219,50,4,52,148,102,170,225,73,64,87,161,183,68,125,100,76,112,254,230,217,195,' + + '228,71,209,214,155,210,69,175,155,95,117,236,130,111,176,161,115,26,13,253,205,179,135,200,255,8,109,209,31,89,19,' + + '28,63,185,182,112,248,134,3,147,196,80,183,60,143,240,134,221,17,245,145,49,195,251,155,103,9,153,121,194,183,243,' + + '118,43,147,107,248,164,83,185,180,54,232,143,172,137,142,31,220,219,56,124,136,157,203,110,159,181,177,87,164,132,' + + '51,246,217,120,189,13,186,35,235,34,99,134,241,245,180,72,132,116,112,254,7,167,195,150,227,244,98,234,67,237,155,' + + '35,135,102,236,204,223,23,161,183,68,125,100,75,176,70,248,207,116,46,59,232,218,137,15,41,225,38,20,162,105,88,3,' + + '59,221,52,249,17,46,76,68,130,195,148,187,103,15,145,253,241,76,10,132,82,146,126,208,179,241,65,64,84,151,15,193,' + + '27,58,174,246,254,217,195,225,201,8,103,237,178,241,122,27,116,71,210,161,106,19,234,133,230,77,60,101,201,227,55,' + + '59,2,148,71,237,122,200,152,222,202,193,86,94,164,111,28,63,185,180,88,205,133,69,41,39,237,156,62,237,252,33,183,' + + '68,126,68,34,111,88,1,159,60,108,76,112,252,104,245,218,227,1,255,172,137,142,31,220,219,56,124,143,239,99,182,153' + + ',157,89,206,237,156,41,135,174,215,24,15,76,90,90,193,245,145,49,195,251,155,103,15,145,18,140,226,36,22,28,165,21' + + '8,7,174,215,23,217,167,25,36,48,125,100,76,112,254,230,217,195,196,106,61,255,30,253,149,0,0,254,254,226,128,0,0,0' + + ',0,0,8,43,156,5,139,91,64,214,164,5,157,168,214,71,99,143,63,110,129,210,71,53,1,30,120,20,41,161,99,5,167,202,76,' + + '251,103,189,240,128,146,208,198,255,248,206,215,46,193,53,91,227,66,219,241,255,4,235,164,113,76,186,21,195,174,10' + + ',72,252,102,101,0,19,200,26,224,13,190,145,249,137,208,169,128,196,203,52,114,184,23,26,103,126,29,119,157,143,214' + + ',115,91,208,138,148,47,18,132,3,189,65,160,138,162,129,225,223,121,199,68,111,66,131,240,170,9,87,178,109,244,143,' + + '204,78,245,205,43,87,181,148,112,162,163,53,27,128,197,247,165,165,55,37,6,212,240,48,76,139,191,173,182,51,61,7,1' + + '38,70,81,93,158,178,96,58,63,135,99,61,33,123,114,106,17,205,205,245,73,209,248,208,230,67,84,83,67,62,174,199,125' + + ',7,42,68,205,119,254,54,95,35,146,246,87,229,105,194,49,134,23,113,205,13,105,146,10,231,32,0,26,210,69,47,127,104' + + ',73,141,205,245,214,23,231,110,132,188,27,13,88,8,43,145,225,60,68,0,42,15,95,85,238,25,204,75,166,163,127,0,0'; +const t = 1.2; +const u = 0.8; +const a1 = 0.6; +const b1 = 200; +const c1 = 200; +const d1 = 100; +const e1 = 137; +const f1 = 83; +const g1 = 70; +const h1 = 40; +const i1 = 50; +const j1 = 30; +const l1 = 2; +const m1 = '100%'; +const n1 = '#E6FFFFFF'; +const o1 = '#99182431'; +const q1 = '#CCCCCC'; +const s1 = '#55CCCCCC'; +const t1 = '#ff182431'; +const u1 = '#99182431'; +const v1 = 72; +const w1 = 59; +const q3 = 3.25; +const z1 = { + c2: 200, + d2: 120, + e2: 14, + f2: 16, + g2: 10, + h2: 14, + i2: 16, + j2: 4, + l2: 5, + m2: 10, + n2: 14, + o2: 12, + q2: 16, + s2: 6.5, + t2: 12, + u2: 10, + v2: 5 +}; +const a2 = { + c2: 36, + d2: 48, + w2: 14, + h2: 10, + i2: 12, + z2: 14, + a3: 16, + j2: 10, + e2: 14, + f2: 16, + g2: 14, + l2: 5, + m2: 10, + n2: 14, + o2: 12, + q2: 16, + s2: 6.5, + t2: 12, + u2: 10, + v2: 5 +}; +const b2 = { + c2: 24, + d2: 24, + w2: 8, + e2: 12, + f2: 14, + g2: 9, + h2: 10, + i2: 12, + z2: 12, + a3: 14, + j2: 4, + l2: 5, + m2: 10, + o2: 12, + q2: 12, + s2: 4, + n2: 12, + t2: 8, + u2: 8, + v2: 4 +}; + +export class ContentFormCard extends ViewPU { + constructor(parent, params, __localStorage, elmtId = -1, paramsLambda = undefined, extraInfo) { + super(parent, __localStorage, elmtId, extraInfo); + if (typeof paramsLambda === "function") { + this.paramsGenerator_ = paramsLambda; + } + this.b3 = new SynchedPropertySimpleOneWayPU(params.formType, this, 'formType'); + this.contentFormData = undefined; + this.formStyle = a2; + this.controller = new TextController(); + this.c3 = new ObservedPropertySimplePU(1, this, 'cardScale'); + this.d3 = new SynchedPropertySimpleOneWayPU(params.formWidth, this, 'formWidth'); + this.e3 = new SynchedPropertySimpleOneWayPU(params.formHeight, this, 'formHeight'); + this.f3 = new ObservedPropertySimplePU(0, this, 'cardWidth'); + this.g3 = new ObservedPropertySimplePU(0, this, 'cardHeight'); + this.h3 = new ObservedPropertyObjectPU(undefined, this, 'defaultThumbImage'); + this.i3 = new ObservedPropertyObjectPU(undefined, this, 'thumbImage'); + this.j3 = new ObservedPropertyObjectPU(undefined, this, 'appImage'); + this.l3 = new ObservedPropertySimplePU(1, this, 'lineCount'); + this.m3 = new ObservedPropertySimplePU(false, this, 'isMirrorLanguageType'); + this.handleOnClick = () => { + }; + this.setInitiallyProvidedValue(params); + this.declareWatch('formType', this.formTypeChange); + this.declareWatch('formWidth', this.formSizeChange); + this.declareWatch('formHeight', this.formSizeChange); + this.finalizeConstruction(); + } + + setInitiallyProvidedValue(params) { + if (params.formType === undefined) { + this.b3.set(FormType.TYPE_MID); + } + if (params.contentFormData !== undefined) { + this.contentFormData = params.contentFormData; + } + if (params.formStyle !== undefined) { + this.formStyle = params.formStyle; + } + if (params.controller !== undefined) { + this.controller = params.controller; + } + if (params.cardScale !== undefined) { + this.cardScale = params.cardScale; + } + if (params.formWidth === undefined) { + this.d3.set(0); + } + if (params.formHeight === undefined) { + this.e3.set(0); + } + if (params.cardWidth !== undefined) { + this.cardWidth = params.cardWidth; + } + if (params.cardHeight !== undefined) { + this.cardHeight = params.cardHeight; + } + if (params.defaultThumbImage !== undefined) { + this.defaultThumbImage = params.defaultThumbImage; + } + if (params.thumbImage !== undefined) { + this.thumbImage = params.thumbImage; + } + if (params.appImage !== undefined) { + this.appImage = params.appImage; + } + if (params.lineCount !== undefined) { + this.lineCount = params.lineCount; + } + if (params.isMirrorLanguageType !== undefined) { + this.isMirrorLanguageType = params.isMirrorLanguageType; + } + if (params.handleOnClick !== undefined) { + this.handleOnClick = params.handleOnClick; + } + } + + updateStateVars(params) { + this.b3.reset(params.formType); + this.d3.reset(params.formWidth); + this.e3.reset(params.formHeight); + } + + purgeVariableDependenciesOnElmtId(rmElmtId) { + this.b3.purgeDependencyOnElmtId(rmElmtId); + this.c3.purgeDependencyOnElmtId(rmElmtId); + this.d3.purgeDependencyOnElmtId(rmElmtId); + this.e3.purgeDependencyOnElmtId(rmElmtId); + this.f3.purgeDependencyOnElmtId(rmElmtId); + this.g3.purgeDependencyOnElmtId(rmElmtId); + this.h3.purgeDependencyOnElmtId(rmElmtId); + this.i3.purgeDependencyOnElmtId(rmElmtId); + this.j3.purgeDependencyOnElmtId(rmElmtId); + this.l3.purgeDependencyOnElmtId(rmElmtId); + this.m3.purgeDependencyOnElmtId(rmElmtId); + } + + aboutToBeDeleted() { + this.b3.aboutToBeDeleted(); + this.c3.aboutToBeDeleted(); + this.d3.aboutToBeDeleted(); + this.e3.aboutToBeDeleted(); + this.f3.aboutToBeDeleted(); + this.g3.aboutToBeDeleted(); + this.h3.aboutToBeDeleted(); + this.i3.aboutToBeDeleted(); + this.j3.aboutToBeDeleted(); + this.l3.aboutToBeDeleted(); + this.m3.aboutToBeDeleted(); + SubscriberManager.Get().delete(this.id__()); + this.aboutToBeDeletedInternal(); + } + + get formType() { + return this.b3.get(); + } + + set formType(newValue) { + this.b3.set(newValue); + } + + get cardScale() { + return this.c3.get(); + } + + set cardScale(newValue) { + this.c3.set(newValue); + } + + get formWidth() { + return this.d3.get(); + } + + set formWidth(newValue) { + this.d3.set(newValue); + } + + get formHeight() { + return this.e3.get(); + } + + set formHeight(newValue) { + this.e3.set(newValue); + } + + get cardWidth() { + return this.f3.get(); + } + + set cardWidth(newValue) { + this.f3.set(newValue); + } + + get cardHeight() { + return this.g3.get(); + } + + set cardHeight(newValue) { + this.g3.set(newValue); + } + + get defaultThumbImage() { + return this.h3.get(); + } + + set defaultThumbImage(newValue) { + this.h3.set(newValue); + } + + get thumbImage() { + return this.i3.get(); + } + + set thumbImage(newValue) { + this.i3.set(newValue); + } + + get appImage() { + return this.j3.get(); + } + + set appImage(newValue) { + this.j3.set(newValue); + } + + get lineCount() { + return this.l3.get(); + } + + set lineCount(newValue) { + this.l3.set(newValue); + } + + get isMirrorLanguageType() { + return this.m3.get(); + } + + set isMirrorLanguageType(newValue) { + this.m3.set(newValue); + } + + aboutToAppear() { + this.initSystemLanguage(); + this.initCardStyle(); + this.createPixelMap(); + } + + aboutToDisappear() { + this.contentFormData = undefined; + this.thumbImage = undefined; + this.appImage = undefined; + } + + formTypeChange() { + switch (this.formType) { + case FormType.TYPE_BIG: + this.formWidth = b1; + break; + case FormType.TYPE_MID: + this.formWidth = c1; + break; + default: + this.formWidth = e1; + break; + } + this.initCardStyle(); + } + + formSizeChange() { + this.initCardStyle(); + } + + initCardScale(n2, o2, q2) { + let r2 = this.formType === FormType.TYPE_SMALL ? a1 : u; + if (n2 > t) { + this.cardScale = t; + } else if (n2 < r2) { + this.cardScale = r2; + } else { + this.cardScale = n2; + } + this.cardWidth = o2 * this.cardScale; + this.cardHeight = + (this.contentFormData?.title === '' && this.formHeight > 0) ? this.formHeight : q2 * this.cardScale; + console.info(`${m}, widthScale:${this.cardScale}, cardScale: ${this.cardScale}, ` + + `cardWidth: ${this.cardWidth}, cardHeight: ${this.cardHeight}`); + } + + initCardStyle() { + let m2 = 1; + switch (this.formType) { + case FormType.TYPE_BIG: + this.formStyle = z1; + m2 = this.formWidth ? this.formWidth / b1 : 1; + this.initCardScale(m2, b1, b1); + break; + case FormType.TYPE_MID: + this.formStyle = a2; + m2 = this.formWidth ? this.formWidth / c1 : 1; + this.initCardScale(m2, c1, d1); + break; + default: + this.formStyle = b2; + m2 = this.formWidth ? this.formWidth / e1 : 1; + this.initCardScale(m2, e1, f1); + break; + } + } + + ThumbImage(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.size({ width: '100%' }); + Column.layoutWeight(this.formHeight > 0 ? 1 : 0); + Column.backgroundColor(this.thumbImage ? n1 : q1); + }, Column); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.formHeight > 0) { + this.ifElseBranchUpdateFunction(0, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.thumbImage ? this.thumbImage : this.defaultThumbImage); + Image.objectFit(ImageFit.Contain); + Image.width('100%'); + Image.layoutWeight(1); + Image.draggable(false); + }, Image); + }); + } else { + this.ifElseBranchUpdateFunction(1, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.thumbImage ? this.thumbImage : this.defaultThumbImage); + Image.objectFit(ImageFit.Contain); + Image.width('100%'); + Image.aspectRatio(this.getAspectRatio()); + Image.draggable(false); + }, Image); + }); + } + }, If); + If.pop(); + Column.pop(); + } + + CardDivider(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Divider.create(); + Divider.height(1); + Divider.opacity(0.5); + Divider.padding({ + left: this.formStyle.t2 * this.cardScale, + right: this.formStyle.t2 * this.cardScale + }); + }, Divider); + } + + AppView(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Row.create({ space: this.formStyle.s2 * this.cardScale }); + Row.padding({ + left: this.formStyle.t2 * this.cardScale, + right: this.formStyle.t2 * this.cardScale, + top: this.formStyle.v2 * this.cardScale, + bottom: this.formStyle.u2 * this.cardScale, + }); + }, Row); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.appImage); + Image.width(this.formStyle.o2 * this.cardScale); + Image.height(this.formStyle.o2 * this.cardScale); + Image.objectFit(ImageFit.Fill); + Image.alt({ + "id": -1, + "type": 20000, + params: ['sys.media.ohos_app_icon'], + "bundleName": "__harDefaultBundleName__", + "moduleName": "__harDefaultModuleName__" + }); + Image.borderRadius({ + "id": -1, + "type": 10002, + params: ['sys.float.corner_radius_level1'], + "bundleName": "__harDefaultBundleName__", + "moduleName": "__harDefaultModuleName__" + }); + Image.draggable(false); + }, Image); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Text.create(this.contentFormData?.appName ? this.contentFormData?.appName : ' '); + Text.fontSize(this.formStyle.m2 * this.cardScale); + Text.fontColor(o1); + Text.maxLines(1); + Text.lineHeight(this.formStyle.n2 * this.cardScale); + Text.textOverflow({ overflow: TextOverflow.Ellipsis }); + Text.constraintSize({ minWidth: this.getTextSize(TextType.APP_NAME, this.contentFormData?.appName) }); + Text.backgroundColor(this.getTextBackground(this.contentFormData?.appName)); + Text.fontWeight(FontWeight.Regular); + Text.borderRadius(this.contentFormData?.title === '' ? 0 : l1); + Text.direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr); + Text.maxFontScale(1); + Text.layoutWeight(1); + }, Text); + Text.pop(); + Row.pop(); + } + + TitleText(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Text.create(this.contentFormData?.title); + Text.fontSize(this.formStyle.e2 * this.cardScale); + Text.fontColor(t1); + Text.fontWeight(FontWeight.Bold); + Text.maxLines(1); + Text.textOverflow({ overflow: TextOverflow.Ellipsis }); + Text.height(this.formStyle.f2 * this.cardScale); + Text.margin({ top: this.formStyle.g2 * this.cardScale }); + Text.constraintSize({ minWidth: this.getTextSize(TextType.TITLE, this.contentFormData?.title) }); + Text.backgroundColor(this.getTextBackground(this.contentFormData?.title)); + Text.borderRadius(this.contentFormData?.title === '' ? 0 : l1); + Text.direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr); + Text.maxFontScale(1); + }, Text); + Text.pop(); + } + + Card4x4(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.size({ width: '100%', height: this.cardHeight }); + }, Column); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.thumbImage ? this.thumbImage : this.defaultThumbImage); + Image.objectFit(ImageFit.Cover); + Image.width('100%'); + Image.height(this.formStyle.d2 * this.cardScale); + Image.backgroundColor(this.thumbImage ? n1 : q1); + Image.draggable(false); + }, Image); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.alignItems(HorizontalAlign.Start); + Column.width('100%'); + Column.padding({ + left: this.formStyle.t2 * this.cardScale, + right: this.formStyle.t2 * this.cardScale + }); + Column.margin({ bottom: this.formStyle.l2 * this.cardScale }); + Column.justifyContent(FlexAlign.Center); + }, Column); + this.TitleText.bind(this)(); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Text.create(this.contentFormData?.description); + Text.fontSize(this.formStyle.h2 * this.cardScale); + Text.fontColor(u1); + Text.fontWeight(FontWeight.Regular); + Text.maxLines(1); + Text.textOverflow({ overflow: TextOverflow.Ellipsis }); + Text.constraintSize({ + minWidth: this.getTextSize(TextType.DESCRIPTION, this.contentFormData?.description) + }); + Text.height(this.formStyle.i2 * this.cardScale); + Text.margin({ top: this.formStyle.j2 * this.cardScale }); + Text.backgroundColor(this.getTextBackground(this.contentFormData?.description)); + Text.fontWeight(FontWeight.Regular); + Text.borderRadius(this.contentFormData?.description ? 0 : l1); + Text.direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr); + Text.maxFontScale(1); + }, Text); + Text.pop(); + Column.pop(); + this.CardDivider.bind(this)(); + this.AppView.bind(this)(); + Column.pop(); + } + + DescriptionView(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Text.create(this.contentFormData?.description ? this.contentFormData?.description : ' ', + { controller: this.controller }); + Text.fontColor(u1); + Text.fontWeight(FontWeight.Regular); + Text.maxLines(2); + Text.fontWeight(FontWeight.Regular); + Text.textOverflow({ overflow: TextOverflow.Ellipsis }); + Text.lineHeight((this.lineCount === 1 ? + (this.formStyle.a3 ? this.formStyle.a3 : + this.formStyle.i2) : this.formStyle.i2) * this.cardScale); + Text.fontSize(this.getDescriptionFontSize()); + Text.constraintSize({ + minWidth: this.getTextSize(TextType.DESCRIPTION, this.contentFormData?.description) + }); + Text.backgroundColor(this.getTextBackground(this.contentFormData?.description)); + Text.borderRadius(this.contentFormData?.description ? 0 : l1); + Text.onAreaChange(() => { + let l2 = this.controller.getLayoutManager(); + this.lineCount = l2.getLineCount(); + }); + Text.direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr); + Text.maxFontScale(1); + }, Text); + Text.pop(); + } + + Card4x2(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.size({ width: '100%' }); + Column.constraintSize(this.getThumbViewConstraintSize()); + }, Column); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.contentFormData?.title === '') { + this.ifElseBranchUpdateFunction(0, () => { + this.ThumbImage.bind(this)(); + }); + } else { + this.ifElseBranchUpdateFunction(1, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Row.create(); + Row.width('100%'); + Row.padding({ + left: this.formStyle.t2 * this.cardScale, + right: this.formStyle.t2 * this.cardScale + }); + Row.layoutWeight(1); + Row.margin({ bottom: this.formStyle.l2 * this.cardScale }); + Row.alignItems(VerticalAlign.Top); + }, Row); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create({ space: this.formStyle.j2 * this.cardScale }); + Column.layoutWeight(1); + Column.alignItems(HorizontalAlign.Start); + }, Column); + this.TitleText.bind(this)(); + this.DescriptionView.bind(this)(); + Column.pop(); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.thumbImage) { + this.ifElseBranchUpdateFunction(0, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.thumbImage); + Image.width(this.formStyle.c2 * this.cardScale); + Image.height(this.formStyle.d2 * this.cardScale); + Image.objectFit(this.thumbImage ? ImageFit.Cover : ImageFit.Contain); + Image.borderRadius(4); + Image.draggable(false); + Image.margin({ + right: this.isMirrorLanguageType ? (this.formStyle.w2 * this.cardScale) : 0, + left: this.isMirrorLanguageType ? 0 : (this.formStyle.w2 * this.cardScale), + top: this.formStyle.g2 * this.cardScale + }); + }, Image); + }); + } else { + this.ifElseBranchUpdateFunction(1, () => { + }); + } + }, If); + If.pop(); + Row.pop(); + }); + } + }, If); + If.pop(); + this.CardDivider.bind(this)(); + this.AppView.bind(this)(); + Column.pop(); + } + + Card2x1(parent = null) { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.size({ width: '100%' }); + Column.constraintSize(this.getThumbViewConstraintSize()); + }, Column); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.contentFormData?.title === '') { + this.ifElseBranchUpdateFunction(0, () => { + this.ThumbImage.bind(this)(); + }); + } else { + this.ifElseBranchUpdateFunction(1, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.width('100%'); + Column.padding({ + left: this.formStyle.t2 * this.cardScale, + right: this.formStyle.t2 * this.cardScale + }); + Column.layoutWeight(1); + Column.alignItems(HorizontalAlign.Start); + Column.margin({ bottom: this.formStyle.l2 * this.cardScale }); + }, Column); + this.TitleText.bind(this)(); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Row.create(); + Row.margin({ top: this.formStyle.j2 * this.cardScale }); + Row.layoutWeight(1); + }, Row); + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.layoutWeight(1); + Column.alignItems(HorizontalAlign.Start); + }, Column); + this.DescriptionView.bind(this)(); + Column.pop(); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.thumbImage) { + this.ifElseBranchUpdateFunction(0, () => { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Image.create(this.thumbImage); + Image.objectFit(ImageFit.Cover); + Image.borderRadius({ + "id": -1, + "type": 10002, + params: ['sys.float.corner_radius_level2'], + "bundleName": "__harDefaultBundleName__", + "moduleName": "__harDefaultModuleName__" + }); + Image.width(this.formStyle.c2 * this.cardScale); + Image.height(this.formStyle.d2 * this.cardScale); + Image.draggable(false); + Image.margin({ + left: this.isMirrorLanguageType ? 0 : (this.formStyle.w2 * this.cardScale), + right: this.isMirrorLanguageType ? (this.formStyle.w2 * this.cardScale) : 0 + }); + }, Image); + }); + } else { + this.ifElseBranchUpdateFunction(1, () => { + }); + } + }, If); + If.pop(); + Row.pop(); + Column.pop(); + }); + } + }, If); + If.pop(); + this.CardDivider.bind(this)(); + this.AppView.bind(this)(); + Column.pop(); + } + + initialRender() { + this.observeComponentCreation2((elmtId, isInitialRender) => { + Column.create(); + Column.borderRadius(this.formStyle.q2 * this.cardScale); + Column.clip(true); + Column.backgroundColor(n1); + Column.backgroundBlurStyle(BlurStyle.COMPONENT_ULTRA_THICK, { colorMode: ThemeColorMode.LIGHT, adaptiveColor: AdaptiveColor.DEFAULT, scale: 1.0 }); + Column.shadow(ShadowStyle.OUTER_DEFAULT_SM); + Column.width(this.cardWidth); + Column.onClick(() => { + if (!this.contentFormData?.linkUri) { + console.warn(`${m}, linkUri is null`); + return; + } + this.handleOnClick(); + try { + let context = getContext(this); + context.openLink(this.contentFormData?.linkUri, { appLinkingOnly: false, parameters: {} }); + } + catch (err) { + let error = err; + console.error(`${m}, Failed to openLink, code is ${error.code}, message is ${error.message}`); + } + }); + }, Column); + this.observeComponentCreation2((elmtId, isInitialRender) => { + If.create(); + if (this.initSystemLanguage() && this.formType === FormType.TYPE_BIG) { + this.ifElseBranchUpdateFunction(0, () => { + this.Card4x4.bind(this)(); + }); + } + else if (this.formType === FormType.TYPE_MID) { + this.ifElseBranchUpdateFunction(1, () => { + this.Card4x2.bind(this)(); + }); + } + else { + this.ifElseBranchUpdateFunction(2, () => { + this.Card2x1.bind(this)(); + }); + } + }, If); + If.pop(); + Column.pop(); + } + + initSystemLanguage() { + try { + this.isMirrorLanguageType = j.isRTL(j.System.getSystemLanguage()); + } catch (err) { + let error = err; + console.error(`${m}, Failed to init system language, code is ${error.code}, message is ${error.message}`); + } + return true; + } + + async getPixelMap(i2, callback) { + let j2 = undefined; + try { + j2 = image.createImageSource(i2.buffer); + let k2 = await j2?.createPixelMap(); + callback(k2); + j2.release(); + } catch (err) { + let error = err; + console.error(`${m}, Failed to create pixelMap, code is ${error.code}, message is ${error.message}`); + } + } + + transStringToUint8Array(g2) { + const arr = g2.split(','); + const h2 = new Uint8Array(arr.length); + arr.forEach((value, index) => { + h2[index] = parseInt(value); + }); + return h2; + } + + createPixelMap() { + let f2 = this.transStringToUint8Array(o); + this.getPixelMap(f2, (pixelMap) => { + this.defaultThumbImage = pixelMap; + }); + if (this.contentFormData && this.contentFormData?.thumbData) { + if (!(this.contentFormData?.thumbData instanceof Uint8Array)) { + console.error(`${m}, thumbData is not Uint8Array`); + return; + } + this.getPixelMap(this.contentFormData?.thumbData, (pixelMap) => { + this.thumbImage = pixelMap; + }); + } + if (this.contentFormData && this.contentFormData?.appIcon) { + if (!(this.contentFormData?.appIcon instanceof Uint8Array)) { + console.error(`${m}, appIcon is not Uint8Array`); + return; + } + this.getPixelMap(this.contentFormData?.appIcon, (pixelMap) => { + this.appImage = pixelMap; + }); + } + } + + getAspectRatio() { + let c2 = this.thumbImage?.getImageInfoSync().size; + let d2 = this.formType === FormType.TYPE_MID ? c1 : e1; + let e2 = this.formType === FormType.TYPE_MID ? v1 : w1; + if (c2 && this.thumbImage) { + if ((c2.width / c2.height) > (d2 / (e2 * u))) { + return d2 / (e2 * u); + } + if ((c2.width / c2.height) < (d2 / (e2 * t))) { + return d2 / (e2 * t); + } + return c2.width / c2.height; + } + return d2 / e2; + } + + getTextBackground(text) { + if (text && text.length > 0) { + return n1; + } + return s1; + } + + getTextSize(textType, text) { + if (textType === TextType.TITLE) { + if (text === '' || text === undefined || text === null) { + if (this.formType === FormType.TYPE_SMALL) { + return i1; + } + return g1; + } + return m1; + } + if (textType === TextType.APP_NAME) { + if (text === '' || text === undefined || text === null) { + if (this.formType === FormType.TYPE_SMALL) { + return j1; + } + return h1; + } + return m1; + } + return '100%'; + } + + getThumbViewConstraintSize() { + if (this.contentFormData?.title !== '') { + return { maxHeight: this.cardHeight, minHeight: this.cardHeight }; + } else { + if (this.formHeight > 0) { + return { + maxHeight: this.formHeight, + minHeight: this.formHeight + }; + } + return { + maxHeight: this.cardHeight * t, + minHeight: this.cardHeight * u + }; + } + } + + getDescriptionFontSize() { + return this.lineCount === 1 ? (this.formStyle.z2 ? this.formStyle.z2 : + this.formStyle.h2) : (this.formStyle.h2 * this.cardScale); + } + + rerender() { + this.updateDirtyElements(); + } +} + +export default { ContentFormCard, FormType }; \ No newline at end of file diff --git a/udmf/interfaces/components/source/UdmfComponents.ets b/udmf/interfaces/components/source/UdmfComponents.ets new file mode 100644 index 00000000..0377f35f --- /dev/null +++ b/udmf/interfaces/components/source/UdmfComponents.ets @@ -0,0 +1,660 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import common from '@ohos.app.ability.common'; +import I18n from '@ohos.i18n'; +import image from '@ohos.multimedia.image'; +import uniformDataStruct from '@ohos.data.uniformDataStruct'; +import { BusinessError } from '@kit.BasicServicesKit'; +import { display } from '@kit.ArkUI'; + +export enum FormType { + TYPE_BIG = 0, + TYPE_MID = 1, + TYPE_SMALL = 2 +} + +enum TextType { + TITLE = 0, + DESCRIPTION = 1, + APP_NAME = 2 +} + +const TAG: string = 'udmf.ContentFormCard'; +const defaultIcon: string = + '82,73,70,70,60,3,0,0,87,69,66,80,86,80,56,32,48,3,0,0,144,67,0,157,1,42,36,2,76,1,62,145,72,161,76,37,164,163,34,3' + + '4,151,40,24,176,18,9,105,110,225,117,81,27,243,141,167,87,231,251,1,151,228,76,129,74,56,124,143,240,134,221,17,24' + + '5,145,49,195,251,155,103,15,145,254,16,219,162,62,178,38,56,127,115,108,225,242,63,194,27,116,71,214,68,199,15,238' + + ',109,156,62,71,248,67,110,136,250,200,152,225,253,205,179,135,200,255,8,109,209,31,89,19,28,63,185,182,112,249,31,' + + '225,13,186,35,235,34,99,135,247,54,206,31,35,252,33,183,68,125,100,76,112,254,230,217,195,228,75,0,41,63,219,242,2' + + '38,77,44,240,251,18,157,13,186,35,235,34,99,135,247,54,206,31,35,249,8,172,169,162,121,152,235,226,174,0,65,245,14' + + '5,49,195,251,155,103,15,145,254,16,219,50,4,52,148,102,170,225,73,64,87,161,183,68,125,100,76,112,254,230,217,195,' + + '228,71,209,214,155,210,69,175,155,95,117,236,130,111,176,161,115,26,13,253,205,179,135,200,255,8,109,209,31,89,19,' + + '28,63,185,182,112,248,134,3,147,196,80,183,60,143,240,134,221,17,245,145,49,195,251,155,103,9,153,121,194,183,243,' + + '118,43,147,107,248,164,83,185,180,54,232,143,172,137,142,31,220,219,56,124,136,157,203,110,159,181,177,87,164,132,' + + '51,246,217,120,189,13,186,35,235,34,99,134,241,245,180,72,132,116,112,254,7,167,195,150,227,244,98,234,67,237,155,' + + '35,135,102,236,204,223,23,161,183,68,125,100,75,176,70,248,207,116,46,59,232,218,137,15,41,225,38,20,162,105,88,3,' + + '59,221,52,249,17,46,76,68,130,195,148,187,103,15,145,253,241,76,10,132,82,146,126,208,179,241,65,64,84,151,15,193,' + + '27,58,174,246,254,217,195,225,201,8,103,237,178,241,122,27,116,71,210,161,106,19,234,133,230,77,60,101,201,227,55,' + + '59,2,148,71,237,122,200,152,222,202,193,86,94,164,111,28,63,185,180,88,205,133,69,41,39,237,156,62,237,252,33,183,' + + '68,126,68,34,111,88,1,159,60,108,76,112,252,104,245,218,227,1,255,172,137,142,31,220,219,56,124,143,239,99,182,153' + + ',157,89,206,237,156,41,135,174,215,24,15,76,90,90,193,245,145,49,195,251,155,103,15,145,18,140,226,36,22,28,165,21' + + '8,7,174,215,23,217,167,25,36,48,125,100,76,112,254,230,217,195,196,106,61,255,30,253,149,0,0,254,254,226,128,0,0,0' + + ',0,0,8,43,156,5,139,91,64,214,164,5,157,168,214,71,99,143,63,110,129,210,71,53,1,30,120,20,41,161,99,5,167,202,76,' + + '251,103,189,240,128,146,208,198,255,248,206,215,46,193,53,91,227,66,219,241,255,4,235,164,113,76,186,21,195,174,10' + + ',72,252,102,101,0,19,200,26,224,13,190,145,249,137,208,169,128,196,203,52,114,184,23,26,103,126,29,119,157,143,214' + + ',115,91,208,138,148,47,18,132,3,189,65,160,138,162,129,225,223,121,199,68,111,66,131,240,170,9,87,178,109,244,143,' + + '204,78,245,205,43,87,181,148,112,162,163,53,27,128,197,247,165,165,55,37,6,212,240,48,76,139,191,173,182,51,61,7,1' + + '38,70,81,93,158,178,96,58,63,135,99,61,33,123,114,106,17,205,205,245,73,209,248,208,230,67,84,83,67,62,174,199,125' + + ',7,42,68,205,119,254,54,95,35,146,246,87,229,105,194,49,134,23,113,205,13,105,146,10,231,32,0,26,210,69,47,127,104' + + ',73,141,205,245,214,23,231,110,132,188,27,13,88,8,43,145,225,60,68,0,42,15,95,85,238,25,204,75,166,163,127,0,0'; +const MAX_CARD_SCALE: number = 1.2; +const MIN_CARD_SCALE: number = 0.8; +const SMALL_MIN_CARD_SCALE: number = 0.6; +const DEFAULT_BIG_CARD_SIZE: number = 200; +const DEFAULT_MID_CARD_WIDTH: number = 200; +const DEFAULT_MID_CARD_HEIGHT: number = 100; +const DEFAULT_SMALL_CARD_WIDTH: number = 137; +const DEFAULT_SMALL_CARD_HEIGHT: number = 83; +const DEFAULT_EMPTY_TITLE_WIDTH: number = 70; +const DEFAULT_EMPTY_APPNAME_WIDTH: number = 40; +const SMALL_EMPTY_TITLE_WIDTH: number = 50; +const SMALL_EMPTY_APPNAME_WIDTH: number = 30; +const DEFAULT_EMPTY_TEXT_RADIUS: number = 2; +const DEFAULT_EMPTY_DESCRIPTION_WIDTH: string = '100%'; +const CARD_BACKGROUND: string = '#E6FFFFFF'; +const APP_NAME_COLOR: string = '#99182431'; +const DEFAULT_THUMB_BACKGROUND: string = '#CCCCCC'; +const DEFAULT_FONT_BACKGROUND: string = '#55CCCCCC'; +const TITLE_FONT_COLOR: string = '#ff182431'; +const DESCRIPTION_FONT_COLOR: string = '#99182431'; +const DEFAULT_MID_IMAGE_HEIGHT: number = 72; // 4x2卡片，图片默认高度 +const DEFAULT_SMALL_IMAGE_HEIGHT: number = 59; // 2x1卡片，图片默认高度 +const DEFAULT_DENSITY: number = 3.25; + +interface CardStyle { + thumbWidth: number, + thumbHeight: number, + thumbMarginLeft?: number, + titleFontSize: number, + titleFontLineHeight: number, + titleFontMarginTop: number, + descriptionFontSize: number, + descriptionLineHeight: number, + maxDescriptionFontSize?: number, + maxDescriptionLineHeight?: number, + descriptionMarginTop: number, + dividerMarginTop: number, + appNameFontSize: number, + appIconSize: number, + cardRadius: number, + appNameMarginLeft: number, + appNameLineHeight: number, + cardPadding: number, + cardPaddingBottom: number, + cardPaddingTop: number +} + +const BIG_CARD_STYLE: CardStyle = { + thumbWidth: 200, + thumbHeight: 120, + titleFontSize: 14, + titleFontLineHeight: 16, + titleFontMarginTop: 10, + descriptionFontSize: 14, + descriptionLineHeight: 16, + descriptionMarginTop: 4, + dividerMarginTop: 5, + appNameFontSize: 10, + appNameLineHeight: 14, + appIconSize: 12, + cardRadius: 16, + appNameMarginLeft: 6.5, + cardPadding: 12, + cardPaddingBottom: 10, + cardPaddingTop: 5 +} + +const MID_CARD_STYLE: CardStyle = { + thumbWidth: 36, + thumbHeight: 48, + thumbMarginLeft: 14, + descriptionFontSize: 10, + descriptionLineHeight: 12, + maxDescriptionFontSize: 14, + maxDescriptionLineHeight: 16, + descriptionMarginTop: 10, + titleFontSize: 14, + titleFontLineHeight: 16, + titleFontMarginTop: 14, + dividerMarginTop: 5, + appNameFontSize: 10, + appNameLineHeight: 14, + appIconSize: 12, + cardRadius: 16, + appNameMarginLeft: 6.5, + cardPadding: 12, + cardPaddingBottom: 10, + cardPaddingTop: 5 +} + +const SMALL_CARD_STYLE: CardStyle = { + thumbWidth: 24, + thumbHeight: 24, + thumbMarginLeft: 8, + titleFontSize: 12, + titleFontLineHeight: 14, + titleFontMarginTop: 9, + descriptionFontSize: 10, + descriptionLineHeight: 12, + maxDescriptionFontSize: 12, + maxDescriptionLineHeight: 14, + descriptionMarginTop: 4, + dividerMarginTop: 5, + appNameFontSize: 10, + appIconSize: 12, + cardRadius: 12, + appNameMarginLeft: 4, + appNameLineHeight: 12, + cardPadding: 8, + cardPaddingBottom: 8, + cardPaddingTop: 4 +} + +@Preview +@Component +export struct ContentFormCard { + @Prop @Watch('formTypeChange') formType: FormType = FormType.TYPE_MID; + private contentFormData: uniformDataStruct.ContentForm | undefined = undefined; + private formStyle: CardStyle = MID_CARD_STYLE; + private controller: TextController = new TextController(); + @State cardScale: number = 1; + @Prop @Watch('formSizeChange') formWidth: number = 0; + @Prop @Watch('formSizeChange') formHeight: number = 0; + @State cardWidth: number = 0; + @State cardHeight: number = 0; + @State defaultThumbImage: image.PixelMap | undefined = undefined; + @State thumbImage: image.PixelMap | undefined = undefined; + @State appImage: image.PixelMap | undefined = undefined; + @State lineCount: number = 1; + @State isMirrorLanguageType: boolean = false; + private handleOnClick: () => void = () => { + }; + + aboutToAppear(): void { + this.initSystemLanguage(); + this.initCardStyle(); + this.createPixelMap(); + } + + aboutToDisappear(): void { + this.contentFormData = undefined; + this.thumbImage = undefined; + this.appImage = undefined; + } + + formTypeChange(): void { + switch (this.formType) { + case FormType.TYPE_BIG: + this.formWidth = DEFAULT_BIG_CARD_SIZE; + break; + case FormType.TYPE_MID: + this.formWidth = DEFAULT_MID_CARD_WIDTH; + break; + default: + this.formWidth = DEFAULT_SMALL_CARD_WIDTH; + break; + } + this.initCardStyle(); + } + + formSizeChange(): void { + this.initCardStyle(); + } + + initCardScale(widthScale: number, defaultWidth: number, defaultHeight: number): void { + let minScale = this.formType === FormType.TYPE_SMALL ? SMALL_MIN_CARD_SCALE : MIN_CARD_SCALE; + if (widthScale > MAX_CARD_SCALE) { + this.cardScale = MAX_CARD_SCALE; + } else if (widthScale < minScale) { + this.cardScale = minScale; + } else { + this.cardScale = widthScale; + } + this.cardWidth = defaultWidth * this.cardScale; + this.cardHeight = + (this.contentFormData?.title === '' && this.formHeight > 0) ? this.formHeight : defaultHeight * this.cardScale; + console.info(`${TAG}, widthScale:${this.cardScale}, cardScale: ${this.cardScale}, ` + + `cardWidth: ${this.cardWidth}, cardHeight: ${this.cardHeight}`); + } + + initCardStyle(): void { + let widthScale = 1; + switch (this.formType) { + case FormType.TYPE_BIG: + this.formStyle = BIG_CARD_STYLE; + widthScale = this.formWidth ? this.formWidth / DEFAULT_BIG_CARD_SIZE : 1; + this.initCardScale(widthScale, DEFAULT_BIG_CARD_SIZE, DEFAULT_BIG_CARD_SIZE); + break; + case FormType.TYPE_MID: + this.formStyle = MID_CARD_STYLE; + widthScale = this.formWidth ? this.formWidth / DEFAULT_MID_CARD_WIDTH : 1; + this.initCardScale(widthScale, DEFAULT_MID_CARD_WIDTH, DEFAULT_MID_CARD_HEIGHT); + break; + default: + this.formStyle = SMALL_CARD_STYLE; + widthScale = this.formWidth ? this.formWidth / DEFAULT_SMALL_CARD_WIDTH : 1; + this.initCardScale(widthScale, DEFAULT_SMALL_CARD_WIDTH, DEFAULT_SMALL_CARD_HEIGHT); + break; + } + } + + @Styles + thumbStyle() { + .width('100%') + .padding({ + left: this.formStyle.cardPadding * this.cardScale, + right: this.formStyle.cardPadding * this.cardScale + }) + .layoutWeight(1) + } + + @Builder + ThumbImage() { + Column() { + if (this.formHeight > 0) { + Image(this.thumbImage ? this.thumbImage : this.defaultThumbImage) + .objectFit(ImageFit.Contain) + .width('100%') + .layoutWeight(1) + .draggable(false) + } else { + Image(this.thumbImage ? this.thumbImage : this.defaultThumbImage) + .objectFit(ImageFit.Contain) + .width('100%') + .aspectRatio(this.getAspectRatio()) + .draggable(false) + } + } + .size({ width: '100%' }) + .layoutWeight(this.formHeight > 0 ? 1 : 0) + .backgroundColor(this.thumbImage ? CARD_BACKGROUND : DEFAULT_THUMB_BACKGROUND) + } + + @Builder + CardDivider() { + Divider() + .height(1)//.color(CARD_DIVIDER_COLOR) + .opacity(0.5) + .padding({ + left: this.formStyle.cardPadding * this.cardScale, + right: this.formStyle.cardPadding * this.cardScale + }) + } + + @Builder + AppView() { + Row({ space: this.formStyle.appNameMarginLeft * this.cardScale }) { + Image(this.appImage) + .width(this.formStyle.appIconSize * this.cardScale) + .height(this.formStyle.appIconSize * this.cardScale) + .objectFit(ImageFit.Fill) + .alt($r('sys.media.ohos_app_icon')) + .borderRadius($r('sys.float.corner_radius_level1')) + .draggable(false) + Text(this.contentFormData?.appName ? this.contentFormData?.appName : ' ') + .fontSize(this.formStyle.appNameFontSize * this.cardScale) + .fontColor(APP_NAME_COLOR) + .maxLines(1) + .lineHeight(this.formStyle.appNameLineHeight * this.cardScale) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .constraintSize({ minWidth: this.getTextSize(TextType.APP_NAME, this.contentFormData?.appName) }) + .backgroundColor(this.getTextBackground(this.contentFormData?.appName)) + .fontWeight(FontWeight.Regular) + .borderRadius(this.contentFormData?.title === '' ? 0 : DEFAULT_EMPTY_TEXT_RADIUS) + .direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr) + .maxFontScale(1) + .layoutWeight(1) + + } + .padding({ + left: this.formStyle.cardPadding * this.cardScale, + right: this.formStyle.cardPadding * this.cardScale, + top: this.formStyle.cardPaddingTop * this.cardScale, + bottom: this.formStyle.cardPaddingBottom * this.cardScale, + }) + } + + @Builder + TitleText() { + Text(this.contentFormData?.title) + .fontSize(this.formStyle.titleFontSize * this.cardScale) + .fontColor(TITLE_FONT_COLOR) + .fontWeight(FontWeight.Bold) + .maxLines(1) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .height(this.formStyle.titleFontLineHeight * this.cardScale) + .margin({ top: this.formStyle.titleFontMarginTop * this.cardScale }) + .constraintSize({ minWidth: this.getTextSize(TextType.TITLE, this.contentFormData?.title) }) + .backgroundColor(this.getTextBackground(this.contentFormData?.title)) + .borderRadius(this.contentFormData?.title === '' ? 0 : DEFAULT_EMPTY_TEXT_RADIUS) + .direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr) + .maxFontScale(1) + } + + @Builder + Card4x4() { + Column() { + Image(this.thumbImage ? this.thumbImage : this.defaultThumbImage) + .objectFit(ImageFit.Cover) + .width('100%') + .height(this.formStyle.thumbHeight * this.cardScale) + .backgroundColor(this.thumbImage ? CARD_BACKGROUND : DEFAULT_THUMB_BACKGROUND) + .draggable(false) + Column() { + this.TitleText() + Text(this.contentFormData?.description) + .fontSize(this.formStyle.descriptionFontSize * this.cardScale) + .fontColor(DESCRIPTION_FONT_COLOR) + .fontWeight(FontWeight.Regular) + .maxLines(1) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .constraintSize({ minWidth: this.getTextSize(TextType.DESCRIPTION, this.contentFormData?.description) }) + .height(this.formStyle.descriptionLineHeight * this.cardScale) + .margin({ top: this.formStyle.descriptionMarginTop * this.cardScale }) + .backgroundColor(this.getTextBackground(this.contentFormData?.description)) + .fontWeight(FontWeight.Regular) + .borderRadius(this.contentFormData?.description ? 0 : DEFAULT_EMPTY_TEXT_RADIUS) + .direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr) + .maxFontScale(1) + } + .alignItems(HorizontalAlign.Start) + .width('100%') + .padding({ + left: this.formStyle.cardPadding * this.cardScale, + right: this.formStyle.cardPadding * this.cardScale + }) + .margin({ bottom: this.formStyle.dividerMarginTop * this.cardScale }) + .justifyContent(FlexAlign.Center) + + this.CardDivider() + this.AppView() + } + .size({ width: '100%', height: this.cardHeight }) + } + + @Builder + DescriptionView() { + Text(this.contentFormData?.description ? this.contentFormData?.description : ' ', { controller: this.controller }) + .fontColor(DESCRIPTION_FONT_COLOR) + .fontWeight(FontWeight.Regular) + .maxLines(2) + .fontWeight(FontWeight.Regular) + .textOverflow({ overflow: TextOverflow.Ellipsis }) + .lineHeight((this.lineCount === 1 ? + (this.formStyle.maxDescriptionLineHeight ? this.formStyle.maxDescriptionLineHeight : + this.formStyle.descriptionLineHeight) : this.formStyle.descriptionLineHeight) * this.cardScale) + .fontSize(this.getDescriptionFontSize()) + .constraintSize({ minWidth: this.getTextSize(TextType.DESCRIPTION, this.contentFormData?.description) }) + .backgroundColor(this.getTextBackground(this.contentFormData?.description)) + .borderRadius(this.contentFormData?.description ? 0 : DEFAULT_EMPTY_TEXT_RADIUS) + .onAreaChange(() => { + let layoutManager: LayoutManager = this.controller.getLayoutManager(); + this.lineCount = layoutManager.getLineCount(); + }) + .direction(this.isMirrorLanguageType ? Direction.Rtl : Direction.Ltr) + .maxFontScale(1) + } + + @Builder + Card4x2() { + Column() { + if (this.contentFormData?.title === '') { // show the card only thumbData + this.ThumbImage() + } else { // show the card has thumbData and title,description + Row() { + Column({ space: this.formStyle.descriptionMarginTop * this.cardScale }) { + this.TitleText() + this.DescriptionView() + } + .layoutWeight(1) + .alignItems(HorizontalAlign.Start) + + if (this.thumbImage) { + Image(this.thumbImage) + .width(this.formStyle.thumbWidth * this.cardScale) + .height(this.formStyle.thumbHeight * this.cardScale) + .objectFit(this.thumbImage ? ImageFit.Cover : ImageFit.Contain) + .borderRadius(4) + .draggable(false) + .margin({ + right: this.isMirrorLanguageType ? (this.formStyle.thumbMarginLeft as number * this.cardScale) : 0, + left: this.isMirrorLanguageType ? 0 : (this.formStyle.thumbMarginLeft as number * this.cardScale), + top: this.formStyle.titleFontMarginTop * this.cardScale + }) + } + } + .thumbStyle() + .margin({ bottom: this.formStyle.dividerMarginTop * this.cardScale }) + .alignItems(VerticalAlign.Top) + } + + this.CardDivider(); + this.AppView(); + } + .size({ width: '100%' }) + .constraintSize(this.getThumbViewConstraintSize()) + } + + @Builder + Card2x1() { + Column() { + if (this.contentFormData?.title === '') { // show the card only thumbData + this.ThumbImage() + } else { + Column() { + this.TitleText() + Row() { + Column() { + this.DescriptionView() + } + .layoutWeight(1) + .alignItems(HorizontalAlign.Start) + + if (this.thumbImage) { + Image(this.thumbImage) + .objectFit(ImageFit.Cover) + .borderRadius($r('sys.float.corner_radius_level2')) + .width(this.formStyle.thumbWidth * this.cardScale) + .height(this.formStyle.thumbHeight * this.cardScale) + .draggable(false) + .margin({ + left: this.isMirrorLanguageType ? 0 : (this.formStyle.thumbMarginLeft as number * this.cardScale), + right: this.isMirrorLanguageType ? (this.formStyle.thumbMarginLeft as number * this.cardScale) : 0 + }) + } + } + .margin({ top: this.formStyle.descriptionMarginTop * this.cardScale }) + .layoutWeight(1) + } + .thumbStyle() + .alignItems(HorizontalAlign.Start) + .margin({ bottom: this.formStyle.dividerMarginTop * this.cardScale }) + } + this.CardDivider() + this.AppView() + } + .size({ width: '100%' }) + .constraintSize(this.getThumbViewConstraintSize()) + } + + build() { + Column() { + if (this.initSystemLanguage() && this.formType === FormType.TYPE_BIG) { + this.Card4x4(); + } else if (this.formType === FormType.TYPE_MID) { + this.Card4x2(); + } else { + this.Card2x1(); + } + } + .borderRadius(this.formStyle.cardRadius * this.cardScale) + .clip(true) + .backgroundColor(CARD_BACKGROUND) + .backgroundBlurStyle(BlurStyle.COMPONENT_ULTRA_THICK, + { colorMode: ThemeColorMode.LIGHT, adaptiveColor: AdaptiveColor.DEFAULT, scale: 1.0 }) + .shadow(ShadowStyle.OUTER_DEFAULT_SM) + .width(this.cardWidth) + .onClick(() => { + if (!this.contentFormData?.linkUri) { + console.warn(`${TAG}, linkUri is null`); + return; + } + this.handleOnClick(); + try { + let context = getContext(this) as common.UIAbilityContext; + context.openLink(this.contentFormData?.linkUri, { appLinkingOnly: false, parameters: {} }); + } catch (err) { + let error = err as BusinessError; + console.error(`${TAG}, Failed to openLink, code is ${error.code}, message is ${error.message}`); + } + }) + } + + initSystemLanguage(): boolean { + try { + this.isMirrorLanguageType = I18n.isRTL(I18n.System.getSystemLanguage()); + } catch (err) { + let error = err as BusinessError; + console.error(`${TAG}, Failed to init system language, code is ${error.code}, message is ${error.message}`); + } + return true; + } + + async getPixelMap(uint: Uint8Array, callback: Function): Promise<void> { + let imageResource: image.ImageSource | undefined = undefined; + try { + imageResource = image.createImageSource(uint.buffer); + let pixelMapData = await imageResource?.createPixelMap(); + callback(pixelMapData); + imageResource.release(); + } catch (err) { + let error = err as BusinessError; + console.error(`${TAG}, Failed to create pixelMap, code is ${error.code}, message is ${error.message}`); + } + } + + transStringToUint8Array(srcData: string): Uint8Array { + const arr: string[] = srcData.split(','); + const uint8Array = new Uint8Array(arr.length); + arr.forEach((value, index) => { + uint8Array[index] = parseInt(value); + }) + return uint8Array; + } + + createPixelMap(): void { + let defaultThumbData = this.transStringToUint8Array(defaultIcon); + this.getPixelMap(defaultThumbData, (pixelMap: image.PixelMap) => { + this.defaultThumbImage = pixelMap; + }) + + if (this.contentFormData && this.contentFormData?.thumbData) { + if (!(this.contentFormData?.thumbData instanceof Uint8Array)) { + console.error(`${TAG}, thumbData is not Uint8Array`); + return; + } + this.getPixelMap(this.contentFormData?.thumbData, (pixelMap: image.PixelMap) => { + this.thumbImage = pixelMap; + }) + } + if (this.contentFormData && this.contentFormData?.appIcon) { + if (!(this.contentFormData?.appIcon instanceof Uint8Array)) { + console.error(`${TAG}, appIcon is not Uint8Array`); + return; + } + this.getPixelMap(this.contentFormData?.appIcon, (pixelMap: image.PixelMap) => { + this.appImage = pixelMap; + }) + } + } + + getAspectRatio(): number { + let iamgeSize = this.thumbImage?.getImageInfoSync().size; + let defaultCardWidth = this.formType === FormType.TYPE_MID ? DEFAULT_MID_CARD_WIDTH : DEFAULT_SMALL_CARD_WIDTH; + let defaultImageHeight = + this.formType === FormType.TYPE_MID ? DEFAULT_MID_IMAGE_HEIGHT : DEFAULT_SMALL_IMAGE_HEIGHT; + if (iamgeSize && this.thumbImage) { + if ((iamgeSize.width / iamgeSize.height) > (defaultCardWidth / (defaultImageHeight * MIN_CARD_SCALE))) { + return defaultCardWidth / (defaultImageHeight * MIN_CARD_SCALE); + } + if ((iamgeSize.width / iamgeSize.height) < (defaultCardWidth / (defaultImageHeight * MAX_CARD_SCALE))) { + return defaultCardWidth / (defaultImageHeight * MAX_CARD_SCALE); + } + return iamgeSize.width / iamgeSize.height; + } + return defaultCardWidth / defaultImageHeight; + } + + getTextBackground(text: string | undefined): string { + if (text && text.length > 0) { + return CARD_BACKGROUND; + } + return DEFAULT_FONT_BACKGROUND; + } + + getTextSize(textType: TextType, text: string | undefined): number | string { + if (textType === TextType.TITLE) { + if (text === '' || text === undefined || text === null) { + if (this.formType === FormType.TYPE_SMALL) { + return SMALL_EMPTY_TITLE_WIDTH; + } + return DEFAULT_EMPTY_TITLE_WIDTH; + } + return DEFAULT_EMPTY_DESCRIPTION_WIDTH; + } + if (textType === TextType.APP_NAME) { + if (text === '' || text === undefined || text === null) { + if (this.formType === FormType.TYPE_SMALL) { + return SMALL_EMPTY_APPNAME_WIDTH; + } + return DEFAULT_EMPTY_APPNAME_WIDTH; + } + return DEFAULT_EMPTY_DESCRIPTION_WIDTH; + } + return '100%' + } + + getThumbViewConstraintSize(): ConstraintSizeOptions { + if (this.contentFormData?.title !== '') { + return { maxHeight: this.cardHeight, minHeight: this.cardHeight }; + } else { + if (this.formHeight > 0) { + return { + maxHeight: this.formHeight, + minHeight: this.formHeight + }; + } + return { + maxHeight: this.cardHeight * MAX_CARD_SCALE, + minHeight: this.cardHeight * MIN_CARD_SCALE + }; + } + } + + getDescriptionFontSize(): number { + return this.lineCount === 1 ? (this.formStyle.maxDescriptionFontSize ? this.formStyle.maxDescriptionFontSize : + this.formStyle.descriptionFontSize) : (this.formStyle.descriptionFontSize * this.cardScale); + } +} \ No newline at end of file diff --git a/udmf/interfaces/components/udmfcomponents.cpp b/udmf/interfaces/components/udmfcomponents.cpp new file mode 100644 index 00000000..d4897a01 --- /dev/null +++ b/udmf/interfaces/components/udmfcomponents.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "napi/native_api.h" +#include "napi/native_node_api.h" +#include "native_engine/native_engine.h" + +extern const char _binary_udmfcomponents_abc_start[]; +extern const char _binary_udmfcomponents_abc_end[]; + +extern "C" __attribute__((visibility("default"))) void NAPI_data_udmfComponents_GetABCCode(const char **buf, int *buflen) +{ + if (buf != nullptr) { + *buf = _binary_udmfcomponents_abc_start; + } + + if (buf != nullptr) { + *buflen = _binary_udmfcomponents_abc_end - _binary_udmfcomponents_abc_start; + } +} + +extern "C" __attribute__((constructor)) void UdmfComponentsRegisterModule(void) +{ + static napi_module udmfComponentsModule = { + .nm_version = 1, + .nm_flags = 0, + .nm_filename = nullptr, + .nm_modname = "data.udmfComponents", + .nm_priv = ((void *)0), + .reserved = { 0 }, + }; + napi_module_register(&udmfComponentsModule); +} \ No newline at end of file diff --git a/udmf/interfaces/innerkits/BUILD.gn b/udmf/interfaces/innerkits/BUILD.gn index 9eaa7da8..1e69192a 100644 --- a/udmf/interfaces/innerkits/BUILD.gn +++ b/udmf/interfaces/innerkits/BUILD.gn @@ -61,7 +61,6 @@ ohos_shared_library("udmf_client") { "${udmf_framework_path}/innerkitsimpl/common/progress_queue.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_key.cpp", "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", - "${udmf_framework_path}/innerkitsimpl/convert/ndk_data_conversion.cpp", "${udmf_framework_path}/innerkitsimpl/convert/udmf_conversion.cpp", "${udmf_framework_path}/innerkitsimpl/data/application_defined_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/audio.cpp", @@ -78,6 +77,7 @@ ohos_shared_library("udmf_client") { "${udmf_framework_path}/innerkitsimpl/data/text.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_data.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_data_helper.cpp", + "${udmf_framework_path}/innerkitsimpl/data/unified_html_record_process.cpp", "${udmf_framework_path}/innerkitsimpl/data/unified_record.cpp", "${udmf_framework_path}/innerkitsimpl/data/video.cpp", "${udmf_framework_path}/innerkitsimpl/service/progress_callback.cpp", @@ -87,8 +87,11 @@ ohos_shared_library("udmf_client") { public_configs = [ ":udmf_client_config" ] + deps = [ "../innerkits:utd_client" ] + external_deps = [ "ability_base:zuri", + "ability_runtime:dataobs_manager", "access_token:libaccesstoken_sdk", "app_file_service:fileuri_native", "bundle_framework:appexecfwk_core", diff --git a/udmf/interfaces/innerkits/client/getter_system.h b/udmf/interfaces/innerkits/client/getter_system.h index 41b674a0..6da57ecc 100644 --- a/udmf/interfaces/innerkits/client/getter_system.h +++ b/udmf/interfaces/innerkits/client/getter_system.h @@ -22,7 +22,7 @@ namespace OHOS { namespace UDMF { -class API_EXPORT GetterSystem { +class GetterSystem { public: class API_EXPORT GeneralGetter { public: @@ -31,9 +31,9 @@ public: }; using Creator = std::function<std::shared_ptr<GeneralGetter>()>; using Getter = std::shared_ptr<GeneralGetter>; - static GetterSystem &GetInstance(); - void RegisterCreator(const std::string& name, Creator creator); - Getter GetGetter(const std::string& name); + static API_EXPORT GetterSystem &GetInstance(); + void API_EXPORT RegisterCreator(const std::string& name, Creator creator); + Getter API_EXPORT GetGetter(const std::string& name); private: GetterSystem() = default; GetterSystem(const GetterSystem &) = delete; diff --git a/udmf/interfaces/innerkits/client/udmf_async_client.h b/udmf/interfaces/innerkits/client/udmf_async_client.h index 66137a66..b997be09 100644 --- a/udmf/interfaces/innerkits/client/udmf_async_client.h +++ b/udmf/interfaces/innerkits/client/udmf_async_client.h @@ -50,8 +50,10 @@ private: Status ProcessUnifiedData(std::unique_ptr<AsyncHelper> &asyncHelper); bool IsParamValid(const GetDataParams &params); +#ifndef IOS_PLATFORM ExecutorPool executor_; - std::unordered_map<std::string, std::unique_ptr<AsyncHelper>> asyncHelperMap_; +#endif + std::map<std::string, std::unique_ptr<AsyncHelper>> asyncHelperMap_; std::mutex mutex_; }; } // namespace diff --git a/udmf/interfaces/innerkits/client/udmf_client.h b/udmf/interfaces/innerkits/client/udmf_client.h index 90f0213b..b1b20bd2 100644 --- a/udmf/interfaces/innerkits/client/udmf_client.h +++ b/udmf/interfaces/innerkits/client/udmf_client.h @@ -30,22 +30,22 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT UdmfClient { +class UdmfClient { public: - static UdmfClient &GetInstance(); + static UdmfClient API_EXPORT &GetInstance(); - Status SetData(CustomOption &option, UnifiedData &unifiedData, std::string &key); - Status GetData(const QueryOption &query, UnifiedData &unifiedData); - Status GetBatchData(const QueryOption &query, std::vector<UnifiedData> &unifiedDataSet); - Status UpdateData(const QueryOption &query, UnifiedData &unifiedData); - Status DeleteData(const QueryOption &query, std::vector<UnifiedData> &unifiedDataSet); - Status GetSummary(const QueryOption &query, Summary& summary); - Status AddPrivilege(const QueryOption &query, Privilege &privilege); - Status Sync(const QueryOption &query, const std::vector<std::string> &devices); - Status IsRemoteData(const QueryOption &query, bool &result); - Status SetAppShareOption(const std::string &intention, enum ShareOptions shareOption); - Status RemoveAppShareOption(const std::string &intention); - Status GetAppShareOption(const std::string &intention, enum ShareOptions &shareOption); + Status API_EXPORT SetData(CustomOption &option, UnifiedData &unifiedData, std::string &key); + Status API_EXPORT GetData(const QueryOption &query, UnifiedData &unifiedData); + Status API_EXPORT GetBatchData(const QueryOption &query, std::vector<UnifiedData> &unifiedDataSet); + Status API_EXPORT UpdateData(const QueryOption &query, UnifiedData &unifiedData); + Status API_EXPORT DeleteData(const QueryOption &query, std::vector<UnifiedData> &unifiedDataSet); + Status API_EXPORT GetSummary(const QueryOption &query, Summary& summary); + Status API_EXPORT AddPrivilege(const QueryOption &query, Privilege &privilege); + Status API_EXPORT Sync(const QueryOption &query, const std::vector<std::string> &devices); + Status API_EXPORT IsRemoteData(const QueryOption &query, bool &result); + Status API_EXPORT SetAppShareOption(const std::string &intention, enum ShareOptions shareOption); + Status API_EXPORT RemoveAppShareOption(const std::string &intention); + Status API_EXPORT GetAppShareOption(const std::string &intention, enum ShareOptions &shareOption); Status GetDataFromCache(const QueryOption &query, UnifiedData &unifiedData); private: diff --git a/udmf/interfaces/innerkits/client/utd_client.h b/udmf/interfaces/innerkits/client/utd_client.h index 9f267527..71a7515c 100644 --- a/udmf/interfaces/innerkits/client/utd_client.h +++ b/udmf/interfaces/innerkits/client/utd_client.h @@ -31,21 +31,21 @@ namespace OHOS { namespace UDMF { class TypeDescriptor; class UtdChangeSubscriber; -class API_EXPORT UtdClient { +class UtdClient { public: - static UtdClient &GetInstance(); - Status GetTypeDescriptor(const std::string &typeId, std::shared_ptr<TypeDescriptor> &descriptor); - Status GetUniformDataTypeByFilenameExtension(const std::string &fileExtension, std::string &typeId, + static UtdClient API_EXPORT &GetInstance(); + Status API_EXPORT GetTypeDescriptor(const std::string &typeId, std::shared_ptr<TypeDescriptor> &descriptor); + Status API_EXPORT GetUniformDataTypeByFilenameExtension(const std::string &fileExtension, std::string &typeId, std::string belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypesByFilenameExtension(const std::string &fileExtension, + Status API_EXPORT GetUniformDataTypesByFilenameExtension(const std::string &fileExtension, std::vector<std::string> &typeIds, const std::string &belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypeByMIMEType(const std::string &mimeType, std::string &typeId, + Status API_EXPORT GetUniformDataTypeByMIMEType(const std::string &mimeType, std::string &typeId, std::string belongsTo = DEFAULT_TYPE_ID); - Status GetUniformDataTypesByMIMEType(const std::string &mimeType, std::vector<std::string> &typeIds, + Status API_EXPORT GetUniformDataTypesByMIMEType(const std::string &mimeType, std::vector<std::string> &typeIds, const std::string &belongsTo = DEFAULT_TYPE_ID); - Status IsUtd(std::string typeId, bool &result); - void InstallCustomUtds(const std::string &bundleName, const std::string &jsonStr, int32_t user); - void UninstallCustomUtds(const std::string &bundleName, int32_t user); + Status API_EXPORT IsUtd(std::string typeId, bool &result); + void API_EXPORT InstallCustomUtds(const std::string &bundleName, const std::string &jsonStr, int32_t user); + void API_EXPORT UninstallCustomUtds(const std::string &bundleName, int32_t user); private: UtdClient(); diff --git a/udmf/interfaces/innerkits/common/async_task_params.h b/udmf/interfaces/innerkits/common/async_task_params.h index a83645c9..a5ac1c30 100644 --- a/udmf/interfaces/innerkits/common/async_task_params.h +++ b/udmf/interfaces/innerkits/common/async_task_params.h @@ -20,18 +20,20 @@ #include <mutex> #include <string> +#ifndef IOS_PLATFORM #include "executor_pool.h" +#endif #include "progress_queue.h" -#include "unified_data.h" #include "unified_types.h" +#include "unified_data.h" namespace OHOS::UDMF { -enum class FileConflictOptions : uint32_t { +enum class FileConflictOptions: uint32_t { OVERWRITE = 0, SKIP = 1 }; -enum class ProgressIndicator : uint32_t { +enum class ProgressIndicator: uint32_t { NONE = 0, DEFAULT = 1 }; @@ -51,9 +53,11 @@ struct AsyncHelper { uint32_t sycnRetryTime = 0; int32_t lastProgress = 0; ProgressIndicator progressIndicator; +#ifndef IOS_PLATFORM ExecutorPool::TaskId invokeHapTask = ExecutorPool::INVALID_TASK_ID; ExecutorPool::TaskId getDataTask = ExecutorPool::INVALID_TASK_ID; ExecutorPool::TaskId progressTask = ExecutorPool::INVALID_TASK_ID; +#endif ProgressListener progressListener; FileConflictOptions fileConflictOptions; std::string businessUdKey; @@ -64,7 +68,7 @@ struct AsyncHelper { ProgressQueue progressQueue; }; -enum ListenerStatus : int32_t { +enum ListenerStatus: int32_t { FINISHED = 0, PROCESSING, CANCEL, diff --git a/udmf/interfaces/innerkits/common/progress_queue.h b/udmf/interfaces/innerkits/common/progress_queue.h index cca54133..506630f0 100644 --- a/udmf/interfaces/innerkits/common/progress_queue.h +++ b/udmf/interfaces/innerkits/common/progress_queue.h @@ -16,9 +16,7 @@ #ifndef PROGRESS_QUEUE_H #define PROGRESS_QUEUE_H -#include <mutex> #include <queue> - #include "unified_types.h" namespace OHOS::UDMF { @@ -30,14 +28,14 @@ public: void Cancel(); void SetClearable(const bool clearableFlag); bool Clear(); + bool IsClear() const; ProgressQueue() = default; ~ProgressQueue() = default; - ProgressQueue(const ProgressQueue &other) = delete; - ProgressQueue &operator=(const ProgressQueue &other) = delete; + ProgressQueue(const ProgressQueue& other) = delete; + ProgressQueue& operator=(const ProgressQueue& other) = delete; ProgressQueue(ProgressQueue &&other) = delete; - ProgressQueue &operator=(ProgressQueue &&other) = delete; - + ProgressQueue& operator=(ProgressQueue &&other) = delete; private: bool cancelFlag_ = false; bool clearableFlag_ = true; diff --git a/udmf/interfaces/innerkits/common/unified_key.h b/udmf/interfaces/innerkits/common/unified_key.h index 3cc724aa..dcff9fb9 100644 --- a/udmf/interfaces/innerkits/common/unified_key.h +++ b/udmf/interfaces/innerkits/common/unified_key.h @@ -31,6 +31,7 @@ struct API_EXPORT UnifiedKey { std::string bundleName; std::string groupId; std::string GetUnifiedKey(); + std::string GetPropertyKey() const; bool IsValid(); void PreliminaryWork(); bool CheckCharacter(std::string data, std::bitset<MAX_BIT_SIZE> rule); diff --git a/udmf/interfaces/innerkits/common/unified_meta.h b/udmf/interfaces/innerkits/common/unified_meta.h index 2da9724f..6c7e692e 100644 --- a/udmf/interfaces/innerkits/common/unified_meta.h +++ b/udmf/interfaces/innerkits/common/unified_meta.h @@ -63,6 +63,7 @@ constexpr const char* APP_ICON = "appIcon"; constexpr const char* APP_ICON_LENGTH = "appIconLen"; constexpr const char* LINK_URL = "linkUrl"; constexpr const char* APPLICATION_DEFINED_RECORD_MARK = "applicationDefinedRecordMark"; +constexpr const char* GENERAL_FILE_URI = "general.file-uri"; enum UDType : int32_t { ENTITY = 0, @@ -597,25 +598,25 @@ namespace ShareOptionsUtil { std::string API_EXPORT GetEnumStr(int32_t shareOption); } // namespace ShareOptionsUtil -class API_EXPORT UnifiedDataUtils { +class UnifiedDataUtils { public: - static bool IsValidType(int32_t value); - static bool IsValidIntention(int32_t value); + static bool API_EXPORT IsValidType(int32_t value); + static bool API_EXPORT IsValidIntention(int32_t value); static size_t GetVariantSize(UDVariant &variant); static size_t GetDetailsSize(UDDetails &details); - static bool IsPersist(const Intention &intention); - static bool IsPersist(const std::string &intention); - static Intention GetIntentionByString(const std::string &intention); - static bool IsValidOptions(const std::string &key, std::string &intention); + static bool API_EXPORT IsPersist(const Intention &intention); + static bool API_EXPORT IsPersist(const std::string &intention); + static Intention API_EXPORT GetIntentionByString(const std::string &intention); + static bool API_EXPORT IsValidOptions(const std::string &key, std::string &intention); }; struct Object; using ValueType = std::variant<std::monostate, int32_t, int64_t, double, bool, std::string, std::vector<uint8_t>, std::shared_ptr<OHOS::AAFwk::Want>, std::shared_ptr<OHOS::Media::PixelMap>, std::shared_ptr<Object>, nullptr_t>; -struct API_EXPORT Object { +struct Object { template<typename T> - bool GetValue(const std::string &key, T &value) + bool API_EXPORT GetValue(const std::string &key, T &value) { auto it = value_.find(key); if (it != value_.end() && std::holds_alternative<T>(it->second)) { @@ -635,6 +636,7 @@ namespace ObjectUtils { int64_t GetValueSize(const ValueType &value, bool isCalValueType); int64_t GetObjectValueSize(const std::shared_ptr<Object> object, bool isCalValueType); int64_t GetAllObjectSize(const std::shared_ptr<Object> object); + void API_EXPORT ProcessFileUriType(UDType &utdType, ValueType &value); template<typename T, typename... Types> bool ConvertVariant(T &&input, std::variant<Types...> &output) diff --git a/udmf/interfaces/innerkits/common/unified_types.h b/udmf/interfaces/innerkits/common/unified_types.h index baf632a5..18ddc287 100644 --- a/udmf/interfaces/innerkits/common/unified_types.h +++ b/udmf/interfaces/innerkits/common/unified_types.h @@ -63,6 +63,7 @@ struct Runtime { std::string deviceId; std::uint32_t recordTotalNum {}; uint32_t tokenId; + std::string sdkVersion; }; /* @@ -108,6 +109,12 @@ struct ProgressInfo { std::string srcDevName; }; +struct UriInfo { + std::string oriUri; + std::string authUri; + std::string dfsUri; + uint32_t position; +}; } // namespace UDMF } // namespace OHOS #endif // UDMF_UNIFIED_TYPES_H \ No newline at end of file diff --git a/udmf/interfaces/innerkits/convert/ndk_data_conversion.h b/udmf/interfaces/innerkits/convert/ndk_data_conversion.h index 511b1c91..9be03603 100644 --- a/udmf/interfaces/innerkits/convert/ndk_data_conversion.h +++ b/udmf/interfaces/innerkits/convert/ndk_data_conversion.h @@ -22,10 +22,10 @@ #include "error_code.h" namespace OHOS::UDMF { - class API_EXPORT NdkDataConversion { + class NdkDataConversion { public: - static Status GetNativeUnifiedData(OH_UdmfData* ndkData, std::shared_ptr<UnifiedData>& data); - static Status GetNdkUnifiedData(std::shared_ptr<UnifiedData> data, OH_UdmfData* ndkData); + static Status API_EXPORT GetNativeUnifiedData(OH_UdmfData* ndkData, std::shared_ptr<UnifiedData>& data); + static Status API_EXPORT GetNdkUnifiedData(std::shared_ptr<UnifiedData> data, OH_UdmfData* ndkData); }; } diff --git a/udmf/interfaces/innerkits/data/application_defined_record.h b/udmf/interfaces/innerkits/data/application_defined_record.h index 845a745a..6d0bf030 100644 --- a/udmf/interfaces/innerkits/data/application_defined_record.h +++ b/udmf/interfaces/innerkits/data/application_defined_record.h @@ -20,20 +20,20 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT ApplicationDefinedRecord : public UnifiedRecord { +class ApplicationDefinedRecord : public UnifiedRecord { public: - ApplicationDefinedRecord(); - explicit ApplicationDefinedRecord(std::string type); - explicit ApplicationDefinedRecord(std::string type, std::vector<uint8_t> &data); - ApplicationDefinedRecord(UDType type, ValueType value); + API_EXPORT ApplicationDefinedRecord(); + explicit API_EXPORT ApplicationDefinedRecord(std::string type); + explicit API_EXPORT ApplicationDefinedRecord(std::string type, std::vector<uint8_t> &data); + API_EXPORT ApplicationDefinedRecord(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::string GetApplicationDefinedType() const; - void SetApplicationDefinedType(const std::string &type); + std::string API_EXPORT GetApplicationDefinedType() const; + void API_EXPORT SetApplicationDefinedType(const std::string &type); - std::vector<uint8_t> GetRawData() const; - void SetRawData(const std::vector<uint8_t> &rawData); + std::vector<uint8_t> API_EXPORT GetRawData() const; + void API_EXPORT SetRawData(const std::vector<uint8_t> &rawData); void InitObject() override; protected: diff --git a/udmf/interfaces/innerkits/data/audio.h b/udmf/interfaces/innerkits/data/audio.h index c6eb27b2..5dff5eaa 100644 --- a/udmf/interfaces/innerkits/data/audio.h +++ b/udmf/interfaces/innerkits/data/audio.h @@ -20,11 +20,11 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT Audio : public File { +class Audio : public File { public: - Audio(); - explicit Audio(const std::string &uri); - Audio(UDType type, ValueType value); + API_EXPORT Audio(); + explicit API_EXPORT Audio(const std::string &uri); + API_EXPORT Audio(UDType type, ValueType value); }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/file.h b/udmf/interfaces/innerkits/data/file.h index 55156bf0..f533990b 100644 --- a/udmf/interfaces/innerkits/data/file.h +++ b/udmf/interfaces/innerkits/data/file.h @@ -20,27 +20,29 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT File : public UnifiedRecord { +class File : public UnifiedRecord { public: - File(); - explicit File(const std::string &uri); - File(UDType type, ValueType value); - int64_t GetSize() override; + API_EXPORT File(); + explicit API_EXPORT File(const std::string &uri); + API_EXPORT File(UDType type, ValueType value); + void API_EXPORT SetType(const UDType &type); + int64_t API_EXPORT GetSize() override; - std::string GetUri() const; - void SetUri(const std::string &uri); + std::string API_EXPORT GetUri() const; + void API_EXPORT SetUri(const std::string &uri); - std::string GetRemoteUri() const; - void SetRemoteUri(const std::string &uri); + std::string API_EXPORT GetRemoteUri() const; + void API_EXPORT SetRemoteUri(const std::string &uri); - void SetDetails(UDDetails &variantMap); - UDDetails GetDetails() const; + void API_EXPORT SetDetails(UDDetails &variantMap); + UDDetails API_EXPORT GetDetails() const; void InitObject() override; protected: std::string oriUri_; std::string remoteUri_; UDDetails details_; + std::string fileType_; }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/flexible_type.h b/udmf/interfaces/innerkits/data/flexible_type.h index 50db5905..b5b9303f 100644 --- a/udmf/interfaces/innerkits/data/flexible_type.h +++ b/udmf/interfaces/innerkits/data/flexible_type.h @@ -19,7 +19,6 @@ #include <vector> #include "error_code.h" #include "utd_common.h" -#include "visibility.h" namespace OHOS { namespace UDMF { constexpr const char* FLEXIBLE_TYPE_FLAG = "flex.z"; @@ -29,7 +28,7 @@ enum FlexibleTypeAttr : int32_t { FILE_EXTENTSION, INVALID_BUT }; -class API_EXPORT FlexibleType { +class FlexibleType { public: static std::string EscapeStr(const std::string &chs); static bool ParseFlexibleUtd(const std::string &typeId, TypeDescriptorCfg &flexibleTypeDescriptorCfg); diff --git a/udmf/interfaces/innerkits/data/folder.h b/udmf/interfaces/innerkits/data/folder.h index ab927e8a..d8df8604 100644 --- a/udmf/interfaces/innerkits/data/folder.h +++ b/udmf/interfaces/innerkits/data/folder.h @@ -20,11 +20,11 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT Folder : public File { +class Folder : public File { public: - Folder(); - explicit Folder(const std::string &uri); - Folder(UDType type, ValueType value); + API_EXPORT Folder(); + explicit API_EXPORT Folder(const std::string &uri); + API_EXPORT Folder(UDType type, ValueType value); }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/html.h b/udmf/interfaces/innerkits/data/html.h index d03e0572..f3bd188c 100644 --- a/udmf/interfaces/innerkits/data/html.h +++ b/udmf/interfaces/innerkits/data/html.h @@ -21,18 +21,18 @@ namespace OHOS { namespace UDMF { -class API_EXPORT Html : public Text { +class Html : public Text { public: - Html(); - explicit Html(const std::string &htmlContent, const std::string &plainContent); - Html(UDType type, ValueType value); + API_EXPORT Html(); + explicit API_EXPORT Html(const std::string &htmlContent, const std::string &plainContent); + API_EXPORT Html(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::string GetHtmlContent() const; - void SetHtmlContent(const std::string &htmlContent); - std::string GetPlainContent() const; - void SetPlainContent(const std::string &htmlContent); + std::string API_EXPORT GetHtmlContent() const; + void API_EXPORT SetHtmlContent(const std::string &htmlContent); + std::string API_EXPORT GetPlainContent() const; + void API_EXPORT SetPlainContent(const std::string &htmlContent); void InitObject() override; static bool CheckValue(const ValueType &value); diff --git a/udmf/interfaces/innerkits/data/image.h b/udmf/interfaces/innerkits/data/image.h index 1b8431e7..d76525d4 100644 --- a/udmf/interfaces/innerkits/data/image.h +++ b/udmf/interfaces/innerkits/data/image.h @@ -21,11 +21,11 @@ namespace OHOS { namespace UDMF { -class API_EXPORT Image : public File { +class Image : public File { public: - Image(); - explicit Image(const std::string &uri); - Image(UDType type, ValueType value); + API_EXPORT Image(); + explicit API_EXPORT Image(const std::string &uri); + API_EXPORT Image(UDType type, ValueType value); }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/link.h b/udmf/interfaces/innerkits/data/link.h index 123ac7c1..79aaaef7 100644 --- a/udmf/interfaces/innerkits/data/link.h +++ b/udmf/interfaces/innerkits/data/link.h @@ -21,19 +21,19 @@ namespace OHOS { namespace UDMF { -class API_EXPORT Link : public Text { +class Link : public Text { public: - Link(); - explicit Link(const std::string &url); - explicit Link(const std::string &url, const std::string &description); - Link(UDType type, ValueType value); + API_EXPORT Link(); + explicit API_EXPORT Link(const std::string &url); + explicit API_EXPORT Link(const std::string &url, const std::string &description); + API_EXPORT Link(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::string GetUrl() const; - void SetUrl(const std::string &url); - std::string GetDescription() const; - void SetDescription(const std::string &description); + std::string API_EXPORT GetUrl() const; + void API_EXPORT SetUrl(const std::string &url); + std::string API_EXPORT GetDescription() const; + void API_EXPORT SetDescription(const std::string &description); void InitObject() override; static bool CheckValue(const ValueType &value); diff --git a/udmf/interfaces/innerkits/data/plain_text.h b/udmf/interfaces/innerkits/data/plain_text.h index be4ce822..cbfbd4df 100644 --- a/udmf/interfaces/innerkits/data/plain_text.h +++ b/udmf/interfaces/innerkits/data/plain_text.h @@ -20,18 +20,18 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT PlainText : public Text { +class PlainText : public Text { public: - PlainText(); - explicit PlainText(const std::string &content, const std::string &abstract); - PlainText(UDType type, ValueType value); + API_EXPORT PlainText(); + explicit API_EXPORT PlainText(const std::string &content, const std::string &abstract); + API_EXPORT PlainText(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::string GetContent() const; - void SetContent(const std::string &content); - std::string GetAbstract() const; - void SetAbstract(const std::string &abstract); + std::string API_EXPORT GetContent() const; + void API_EXPORT SetContent(const std::string &content); + std::string API_EXPORT GetAbstract() const; + void API_EXPORT SetAbstract(const std::string &abstract); void InitObject() override; static bool CheckValue(const ValueType &value); diff --git a/udmf/interfaces/innerkits/data/system_defined_appitem.h b/udmf/interfaces/innerkits/data/system_defined_appitem.h index 65fd073c..61b3dc35 100644 --- a/udmf/interfaces/innerkits/data/system_defined_appitem.h +++ b/udmf/interfaces/innerkits/data/system_defined_appitem.h @@ -21,27 +21,27 @@ namespace OHOS { namespace UDMF { -class API_EXPORT SystemDefinedAppItem : public SystemDefinedRecord { +class SystemDefinedAppItem : public SystemDefinedRecord { public: - SystemDefinedAppItem(); - SystemDefinedAppItem(UDType type, ValueType value); + API_EXPORT SystemDefinedAppItem(); + API_EXPORT SystemDefinedAppItem(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::string GetAppId() const; - void SetAppId(const std::string &appId); - std::string GetAppName() const; - void SetAppName(const std::string &appName); - std::string GetAppIconId() const; - void SetAppIconId(const std::string &appIconId); - std::string GetAppLabelId() const; - void SetAppLabelId(const std::string &appLabelId); - std::string GetBundleName() const; - void SetBundleName(const std::string &bundleName); - std::string GetAbilityName() const; - void SetAbilityName(const std::string &abilityName); - void SetItems(UDDetails &details); - UDDetails GetItems(); + std::string API_EXPORT GetAppId() const; + void API_EXPORT SetAppId(const std::string &appId); + std::string API_EXPORT GetAppName() const; + void API_EXPORT SetAppName(const std::string &appName); + std::string API_EXPORT GetAppIconId() const; + void API_EXPORT SetAppIconId(const std::string &appIconId); + std::string API_EXPORT GetAppLabelId() const; + void API_EXPORT SetAppLabelId(const std::string &appLabelId); + std::string API_EXPORT GetBundleName() const; + void API_EXPORT SetBundleName(const std::string &bundleName); + std::string API_EXPORT GetAbilityName() const; + void API_EXPORT SetAbilityName(const std::string &abilityName); + void API_EXPORT SetItems(UDDetails &details); + UDDetails API_EXPORT GetItems(); void InitObject() override; static bool CheckValue(const ValueType &value); diff --git a/udmf/interfaces/innerkits/data/system_defined_form.h b/udmf/interfaces/innerkits/data/system_defined_form.h index 7ce952ce..a8981540 100644 --- a/udmf/interfaces/innerkits/data/system_defined_form.h +++ b/udmf/interfaces/innerkits/data/system_defined_form.h @@ -20,26 +20,26 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT SystemDefinedForm : public SystemDefinedRecord { +class SystemDefinedForm : public SystemDefinedRecord { public: - SystemDefinedForm(); - SystemDefinedForm(UDType type, ValueType value); + API_EXPORT SystemDefinedForm(); + API_EXPORT SystemDefinedForm(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - int32_t GetFormId() const; - void SetFormId(const int32_t &formId); - std::string GetFormName() const; - void SetFormName(const std::string &formName); - std::string GetBundleName() const; - void SetBundleName(const std::string &bundleName); - std::string GetAbilityName() const; - void SetAbilityName(const std::string &abilityName); - std::string GetModule() const; - void SetModule(const std::string &module); + int32_t API_EXPORT GetFormId() const; + void API_EXPORT SetFormId(const int32_t &formId); + std::string API_EXPORT GetFormName() const; + void API_EXPORT SetFormName(const std::string &formName); + std::string API_EXPORT GetBundleName() const; + void API_EXPORT SetBundleName(const std::string &bundleName); + std::string API_EXPORT GetAbilityName() const; + void API_EXPORT SetAbilityName(const std::string &abilityName); + std::string API_EXPORT GetModule() const; + void API_EXPORT SetModule(const std::string &module); - void SetItems(UDDetails &details); - UDDetails GetItems(); + void API_EXPORT SetItems(UDDetails &details); + UDDetails API_EXPORT GetItems(); void InitObject() override; @@ -49,7 +49,6 @@ private: std::string bundleName_; std::string abilityName_; std::string module_; - }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/system_defined_pixelmap.h b/udmf/interfaces/innerkits/data/system_defined_pixelmap.h index 04f59351..b37dc05d 100644 --- a/udmf/interfaces/innerkits/data/system_defined_pixelmap.h +++ b/udmf/interfaces/innerkits/data/system_defined_pixelmap.h @@ -20,16 +20,16 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT SystemDefinedPixelMap : public SystemDefinedRecord { +class SystemDefinedPixelMap : public SystemDefinedRecord { public: - SystemDefinedPixelMap(); - explicit SystemDefinedPixelMap(std::vector<uint8_t> &data); - SystemDefinedPixelMap(UDType type, ValueType value); + API_EXPORT SystemDefinedPixelMap(); + explicit API_EXPORT SystemDefinedPixelMap(std::vector<uint8_t> &data); + API_EXPORT SystemDefinedPixelMap(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - std::vector<uint8_t> GetRawData() const; - void SetRawData(const std::vector<uint8_t> &rawData); + std::vector<uint8_t> API_EXPORT GetRawData() const; + void API_EXPORT SetRawData(const std::vector<uint8_t> &rawData); void InitObject() override; private: diff --git a/udmf/interfaces/innerkits/data/system_defined_record.h b/udmf/interfaces/innerkits/data/system_defined_record.h index 010352a0..14a6b881 100644 --- a/udmf/interfaces/innerkits/data/system_defined_record.h +++ b/udmf/interfaces/innerkits/data/system_defined_record.h @@ -20,17 +20,17 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT SystemDefinedRecord : public UnifiedRecord { +class SystemDefinedRecord : public UnifiedRecord { public: - explicit SystemDefinedRecord(); - SystemDefinedRecord(UDType type, ValueType value); + explicit API_EXPORT SystemDefinedRecord(); + API_EXPORT SystemDefinedRecord(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - void AddProperty(const std::string &property, UDVariant &value); - UDVariant GetPropertyByName(const std::string &property) const; - void SetDetails(UDDetails &details); - UDDetails GetDetails() const; + void API_EXPORT AddProperty(const std::string &property, UDVariant &value); + UDVariant API_EXPORT GetPropertyByName(const std::string &property) const; + void API_EXPORT SetDetails(UDDetails &details); + UDDetails API_EXPORT GetDetails() const; void InitObject() override; diff --git a/udmf/interfaces/innerkits/data/text.h b/udmf/interfaces/innerkits/data/text.h index fcb10608..ec3a678e 100644 --- a/udmf/interfaces/innerkits/data/text.h +++ b/udmf/interfaces/innerkits/data/text.h @@ -21,16 +21,16 @@ namespace OHOS { namespace UDMF { constexpr int MAX_TEXT_LEN = 20 * 1024 * 1024; -class API_EXPORT Text : public UnifiedRecord { +class Text : public UnifiedRecord { public: - Text(); - explicit Text(UDDetails &variantMap); - Text(UDType type, ValueType value); + API_EXPORT Text(); + explicit API_EXPORT Text(UDDetails &variantMap); + API_EXPORT Text(UDType type, ValueType value); - int64_t GetSize() override; + int64_t API_EXPORT GetSize() override; - void SetDetails(UDDetails &variantMap); - UDDetails GetDetails() const; + void API_EXPORT SetDetails(UDDetails &variantMap); + UDDetails API_EXPORT GetDetails() const; void InitObject() override; diff --git a/udmf/interfaces/innerkits/data/type_descriptor.h b/udmf/interfaces/innerkits/data/type_descriptor.h index b9332ad7..27652dcc 100644 --- a/udmf/interfaces/innerkits/data/type_descriptor.h +++ b/udmf/interfaces/innerkits/data/type_descriptor.h @@ -18,33 +18,33 @@ #include <string> #include <set> -#include <memory> #include <vector> #include <map> +#include <memory> #include "error_code.h" #include "utd_common.h" #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT TypeDescriptor { +class TypeDescriptor { public: - TypeDescriptor(const std::string &typeId, const std::vector<std::string> &belongingToTypes, + API_EXPORT TypeDescriptor(const std::string &typeId, const std::vector<std::string> &belongingToTypes, const std::vector<std::string> &filenameExtensions, const std::vector<std::string> &mimeTypes, const std::string &description, const std::string &referenceURL, const std::string &iconFile); - TypeDescriptor(const TypeDescriptorCfg& typeDescriptorCfg); - ~TypeDescriptor(); - Status BelongsTo(const std::string &typeId, bool &checkResult); - Status IsLowerLevelType(const std::string &typeId, bool &checkResult); - Status IsHigherLevelType(const std::string &typeId, bool &checkResult); + API_EXPORT TypeDescriptor(const TypeDescriptorCfg& typeDescriptorCfg); + API_EXPORT ~TypeDescriptor(); + Status API_EXPORT BelongsTo(const std::string &typeId, bool &checkResult); + Status API_EXPORT IsLowerLevelType(const std::string &typeId, bool &checkResult); + Status API_EXPORT IsHigherLevelType(const std::string &typeId, bool &checkResult); - bool Equals(std::shared_ptr<TypeDescriptor> descriptor); - const std::string& GetTypeId() const; - std::vector<std::string> GetBelongingToTypes(); - std::string GetIconFile(); - std::string GetDescription(); - std::string GetReferenceURL(); - std::vector<std::string> GetFilenameExtensions(); - std::vector<std::string> GetMimeTypes(); + bool API_EXPORT Equals(std::shared_ptr<TypeDescriptor> descriptor); + const std::string& API_EXPORT GetTypeId() const; + std::vector<std::string> API_EXPORT GetBelongingToTypes(); + std::string API_EXPORT GetIconFile(); + std::string API_EXPORT GetDescription(); + std::string API_EXPORT GetReferenceURL(); + std::vector<std::string> API_EXPORT GetFilenameExtensions(); + std::vector<std::string> API_EXPORT GetMimeTypes(); private: bool CmpFlexibleTypeLevel(const std::string higherLevelTypeId, bool isFlexibleType); diff --git a/udmf/interfaces/innerkits/data/unified_data.h b/udmf/interfaces/innerkits/data/unified_data.h index 2ac36f2a..f0f36917 100644 --- a/udmf/interfaces/innerkits/data/unified_data.h +++ b/udmf/interfaces/innerkits/data/unified_data.h @@ -21,40 +21,47 @@ #include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT UnifiedData { +class UnifiedData { public: - UnifiedData(); - explicit UnifiedData(std::shared_ptr<UnifiedDataProperties> properties); + API_EXPORT UnifiedData(); + explicit API_EXPORT UnifiedData(std::shared_ptr<UnifiedDataProperties> properties); - int64_t GetSize(); + int64_t API_EXPORT GetSize(); std::string GetGroupId() const; - std::shared_ptr<Runtime> GetRuntime() const; - void SetRuntime(Runtime &runtime); + std::shared_ptr<Runtime> API_EXPORT GetRuntime() const; + void API_EXPORT SetRuntime(Runtime &runtime); - void AddRecord(const std::shared_ptr<UnifiedRecord> &record); - void AddRecords(const std::vector<std::shared_ptr<UnifiedRecord>> &records); - std::shared_ptr<UnifiedRecord> GetRecordAt(std::size_t index) const; - void SetRecords(std::vector<std::shared_ptr<UnifiedRecord>> records); - std::vector<std::shared_ptr<UnifiedRecord>> GetRecords() const; + void API_EXPORT AddRecord(const std::shared_ptr<UnifiedRecord> &record); + void API_EXPORT AddRecords(const std::vector<std::shared_ptr<UnifiedRecord>> &records); + std::shared_ptr<UnifiedRecord> API_EXPORT GetRecordAt(std::size_t index) const; + void API_EXPORT SetRecords(std::vector<std::shared_ptr<UnifiedRecord>> records); + std::vector<std::shared_ptr<UnifiedRecord>> API_EXPORT GetRecords() const; - std::vector<std::string> GetTypesLabels() const; - bool HasType(const std::string &type) const; - std::vector<std::string> GetEntriesTypes() const; - bool HasTypeInEntries(const std::string &type) const; + std::vector<std::string> API_EXPORT GetTypesLabels() const; + bool API_EXPORT HasType(const std::string &type) const; + bool API_EXPORT HasHigherFileType(const std::string &type) const; + std::vector<std::string> API_EXPORT GetEntriesTypes() const; + bool API_EXPORT HasTypeInEntries(const std::string &type) const; - bool IsEmpty() const; - bool IsValid(); - bool IsComplete(); + bool API_EXPORT IsEmpty() const; + bool API_EXPORT IsValid(); + bool API_EXPORT IsComplete(); bool HasFileType() const; + bool HasUriInfo() const; + void ClearUriInfo() const; - void SetProperties(std::shared_ptr<UnifiedDataProperties> properties); - std::shared_ptr<UnifiedDataProperties> GetProperties() const; + void API_EXPORT SetProperties(std::shared_ptr<UnifiedDataProperties> properties); + std::shared_ptr<UnifiedDataProperties> API_EXPORT GetProperties() const; - void SetDataId(uint32_t dataId); + void API_EXPORT SetDataId(uint32_t dataId); uint32_t GetDataId() const; void SetChannelName(const std::string &name); std::vector<std::string> GetFileUris() const; + bool API_EXPORT IsNeedTransferToEntries() const; + void API_EXPORT TransferToEntries(UnifiedData &data); + std::string API_EXPORT GetSdkVersion() const; + void API_EXPORT SetSdkVersion(const std::string &version); static constexpr int64_t MAX_DATA_SIZE = 200 * 1024 * 1024; @@ -67,6 +74,7 @@ private: std::shared_ptr<Runtime> runtime_; std::vector<std::shared_ptr<UnifiedRecord>> records_; std::shared_ptr<UnifiedDataProperties> properties_; + std::string sdkVersion_; }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/innerkits/data/unified_data_helper.h b/udmf/interfaces/innerkits/data/unified_data_helper.h index 0db8ae68..1b90cb4c 100644 --- a/udmf/interfaces/innerkits/data/unified_data_helper.h +++ b/udmf/interfaces/innerkits/data/unified_data_helper.h @@ -35,6 +35,7 @@ private: static bool SaveUDataToFile(const std::string &dataFile, UnifiedData &data); static bool LoadUDataFromFile(const std::string &dataFile, UnifiedData &data); static void CalRecordSummary(std::map<std::string, ValueType> &entries, Summary &summary); + static void ProcessTypeId(const ValueType &value, std::string &typeId); static std::string GetRootPath(); private: diff --git a/udmf/interfaces/innerkits/data/unified_data_properties.h b/udmf/interfaces/innerkits/data/unified_data_properties.h index 0def095c..565faf12 100644 --- a/udmf/interfaces/innerkits/data/unified_data_properties.h +++ b/udmf/interfaces/innerkits/data/unified_data_properties.h @@ -17,10 +17,9 @@ #include "want_params.h" #include "unified_meta.h" -#include "visibility.h" namespace OHOS { namespace UDMF { -class API_EXPORT UnifiedDataProperties { +class UnifiedDataProperties { public: std::string tag; AAFwk::WantParams extras; diff --git a/udmf/interfaces/innerkits/data/unified_html_record_process.h b/udmf/interfaces/innerkits/data/unified_html_record_process.h new file mode 100644 index 00000000..040c82b3 --- /dev/null +++ b/udmf/interfaces/innerkits/data/unified_html_record_process.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UDMF_UNIFIED_HTML_RECORD_PROCESS_H +#define UDMF_UNIFIED_HTML_RECORD_PROCESS_H + +#include "unified_data.h" +#include "visibility.h" +namespace OHOS { +namespace UDMF { +class UnifiedHtmlRecordProcess { +public: + static void API_EXPORT GetUriFromHtmlRecord(UnifiedData &unifiedData); + static void API_EXPORT RebuildHtmlRecord(UnifiedData &unifiedData); +private: + static std::vector<UriInfo> SplitHtmlStr(const std::string &htmlContent); + static std::vector<std::pair<std::string, uint32_t>> SplitHtmlWithImgLabel( + const std::string &html) noexcept; + static std::vector<UriInfo> SplitHtmlWithImgSrcLabel( + const std::vector<std::pair<std::string, uint32_t>> &matchs) noexcept; + static bool IsLocalURI(const std::string &uri) noexcept; + static void RebuildEntry(const std::vector<UriInfo> &uris, const ValueType &value); + static std::string RebuildHtmlContent(const std::string &str, const std::vector<UriInfo> &uris); + static std::vector<UriInfo> GetValueStr(const ValueType &value); + static void ProcessEntry(const std::shared_ptr<UnifiedRecord> &record); +}; +} // namespace UDMF +} // namespace OHOS +#endif // UDMF_UNIFIED_HTML_RECORD_PROCESS_H \ No newline at end of file diff --git a/udmf/interfaces/innerkits/data/unified_record.h b/udmf/interfaces/innerkits/data/unified_record.h index af23aef9..cbb13473 100644 --- a/udmf/interfaces/innerkits/data/unified_record.h +++ b/udmf/interfaces/innerkits/data/unified_record.h @@ -21,66 +21,76 @@ #include <vector> #include <mutex> #include "entry_getter.h" +#include "unified_meta.h" #include "visibility.h" #include "unified_types.h" namespace OHOS { namespace UDMF { -class API_EXPORT UnifiedRecord { +class UnifiedRecord { public: - UnifiedRecord(); - explicit UnifiedRecord(UDType type); - UnifiedRecord(UDType type, ValueType value); + API_EXPORT UnifiedRecord(); + explicit API_EXPORT UnifiedRecord(UDType type); + API_EXPORT UnifiedRecord(UDType type, ValueType value); virtual ~UnifiedRecord() = default; - UnifiedRecord(const UnifiedRecord& other) = delete; - UnifiedRecord& operator=(const UnifiedRecord& other) = delete; - UnifiedRecord(UnifiedRecord &&other) = delete; - UnifiedRecord& operator=(UnifiedRecord &&other) = delete; + API_EXPORT UnifiedRecord(const UnifiedRecord& other) = delete; + API_EXPORT UnifiedRecord& operator=(const UnifiedRecord& other) = delete; + API_EXPORT UnifiedRecord(UnifiedRecord &&other) = delete; + API_EXPORT UnifiedRecord& operator=(UnifiedRecord &&other) = delete; - UDType GetType() const; - std::vector<std::string> GetTypes() const; - void SetType(const UDType &type); - virtual int64_t GetSize(); + UDType API_EXPORT GetType() const; + std::vector<std::string> API_EXPORT GetTypes() const; + void API_EXPORT SetType(const UDType &type); + virtual int64_t API_EXPORT GetSize(); - std::string GetUid() const; - void SetUid(const std::string &id); - ValueType GetValue(); + std::string API_EXPORT GetUid() const; + void API_EXPORT SetUid(const std::string &id); + ValueType API_EXPORT GetValue(); void SetValue(const ValueType &value); - ValueType GetOriginValue() const; + ValueType API_EXPORT GetOriginValue() const; - void SetUtdId(const std::string &utdId); - std::set<std::string> GetUtdIds() const; - std::string GetUtdId() const; + void API_EXPORT SetUtdId(const std::string &utdId); + std::set<std::string> API_EXPORT GetUtdIds() const; + std::string API_EXPORT GetUtdId() const; - bool HasType(const std::string &utdId) const; - void AddEntry(const std::string &utdId, ValueType &&value); - ValueType GetEntry(const std::string &utdId); - std::shared_ptr<std::map<std::string, ValueType>> GetEntries(); - std::shared_ptr<std::map<std::string, ValueType>> GetInnerEntries() const; + bool API_EXPORT HasType(const std::string &utdId) const; + void API_EXPORT AddEntry(const std::string &utdId, ValueType &&value); + ValueType API_EXPORT GetEntry(const std::string &utdId); + std::shared_ptr<std::map<std::string, ValueType>> API_EXPORT GetEntries(); + std::shared_ptr<std::map<std::string, ValueType>> API_EXPORT GetInnerEntries() const; void SetInnerEntries(std::shared_ptr<std::map<std::string, ValueType>> entries); int64_t GetInnerEntriesSize() const; - void SetEntryGetter(const std::vector<std::string> &utdIds, const std::shared_ptr<EntryGetter> &entryGetter); - std::shared_ptr<EntryGetter> GetEntryGetter(); - void SetDataId(uint32_t dataId); - uint32_t GetDataId() const; - void SetRecordId(uint32_t recordId); - uint32_t GetRecordId() const; - void SetChannelName(const std::string &channelName); + void API_EXPORT SetEntryGetter(const std::vector<std::string> &utdIds, + const std::shared_ptr<EntryGetter> &entryGetter); + std::shared_ptr<EntryGetter> API_EXPORT GetEntryGetter(); + void API_EXPORT SetDataId(uint32_t dataId); + uint32_t API_EXPORT GetDataId() const; + void API_EXPORT SetRecordId(uint32_t recordId); + uint32_t API_EXPORT GetRecordId() const; + void API_EXPORT SetChannelName(const std::string &channelName); virtual void InitObject(); bool HasObject(); bool HasFileType(std::string &fileUri) const; void SetFileUri(const std::string &fileUri); + + std::vector<UriInfo> GetUris() const; + void SetUris(std::vector<UriInfo> uris); + void API_EXPORT ClearUris(); + void API_EXPORT ComputeUris(const std::function<bool(UriInfo &)> &action); protected: UDType dataType_; + std::string utdId_; ValueType value_; bool hasObject_ = false; private: + void AddFileUriType(std::set<std::string> &utdIds, const std::shared_ptr<Object> &fileUri) const; + std::string uid_; // unique identifier - std::string utdId_; std::shared_ptr<std::map<std::string, ValueType>> entries_ = std::make_shared<std::map<std::string, ValueType>>(); + std::vector<UriInfo> uris_; uint32_t dataId_ = 0; uint32_t recordId_ = 0; std::string channelName_; diff --git a/udmf/interfaces/jskits/BUILD.gn b/udmf/interfaces/jskits/BUILD.gn index c9c64606..12a5c2e7 100644 --- a/udmf/interfaces/jskits/BUILD.gn +++ b/udmf/interfaces/jskits/BUILD.gn @@ -80,6 +80,7 @@ ohos_shared_library("unifieddatachannel_napi") { deps = [ "${third_party_path}/bounds_checking_function:libsec_shared", "../innerkits:udmf_client", + "../innerkits:utd_client", "../jskits:udmf_js_common", ] @@ -187,6 +188,7 @@ ohos_shared_library("udmf_data_napi") { deps = [ "${third_party_path}/bounds_checking_function:libsec_shared", "../innerkits:udmf_client", + "../innerkits:utd_client", "../jskits:udmf_js_common", ] @@ -260,7 +262,9 @@ ohos_shared_library("intelligence_napi") { "${udmf_framework_path}/jskitsimpl/intelligence/aip_napi_utils.cpp", "${udmf_framework_path}/jskitsimpl/intelligence/i_aip_core_manager_impl.cpp", "${udmf_framework_path}/jskitsimpl/intelligence/image_embedding_napi.cpp", + "${udmf_framework_path}/jskitsimpl/intelligence/js_ability.cpp", "${udmf_framework_path}/jskitsimpl/intelligence/native_module_intelligence.cpp", + "${udmf_framework_path}/jskitsimpl/intelligence/retrieval_napi.cpp", "${udmf_framework_path}/jskitsimpl/intelligence/text_embedding_napi.cpp", ] @@ -269,6 +273,8 @@ ohos_shared_library("intelligence_napi") { deps = [] external_deps = [ + "ability_runtime:abilitykit_native", + "ability_runtime:napi_base_context", "c_utils:utils", "hilog:libhilog", "napi:ace_napi", diff --git a/udmf/interfaces/jskits/common/napi_data_utils.h b/udmf/interfaces/jskits/common/napi_data_utils.h index 3dc9e8b6..eefad858 100644 --- a/udmf/interfaces/jskits/common/napi_data_utils.h +++ b/udmf/interfaces/jskits/common/napi_data_utils.h @@ -131,6 +131,9 @@ private: TUPLE_SIZE }; static constexpr int32_t STR_MAX_SIZE = 256; + + static napi_status ProcessNapiObject(napi_env env, napi_value in, std::string &attributeName, + napi_value attributeValueNapi, std::shared_ptr<Object> object); }; #define LOG_ERROR_RETURN(condition, message, retVal) \ diff --git a/udmf/interfaces/jskits/data/unified_record_napi.h b/udmf/interfaces/jskits/data/unified_record_napi.h index 76d25bc1..f1f59cdb 100644 --- a/udmf/interfaces/jskits/data/unified_record_napi.h +++ b/udmf/interfaces/jskits/data/unified_record_napi.h @@ -47,6 +47,7 @@ private: static UnifiedRecordNapi *GetUnifiedRecord( napi_env env, napi_callback_info info, std::shared_ptr<ContextBase> ctxt); static void GetNativeValue(napi_env env, std::string type, napi_value valueNapi, ValueType &value); + static void ProcessNapiObject(napi_env env, std::string type, napi_value valueNapi, ValueType &value); }; } // namespace UDMF } // namespace OHOS diff --git a/udmf/interfaces/jskits/intelligence/aip_napi_error.h b/udmf/interfaces/jskits/intelligence/aip_napi_error.h index 73ba38cd..1dc8bb07 100644 --- a/udmf/interfaces/jskits/intelligence/aip_napi_error.h +++ b/udmf/interfaces/jskits/intelligence/aip_napi_error.h @@ -22,15 +22,58 @@ namespace OHOS { namespace DataIntelligence { -constexpr int32_t PARAM_EXCEPTION{ 401 }; -constexpr int32_t DEVICE_EXCEPTION{ 801 }; -constexpr int32_t INNER_ERROR{ 31300000 }; -const std::map<int32_t, std::string> ERROR_MESSAGES = { - { PARAM_EXCEPTION, "Params check failed." }, - { DEVICE_EXCEPTION, "The device does not support this API." }, - { INNER_ERROR, "Inner error." }, +enum NativeErrCode : int32_t { + NATIVE_DATABASE_INVALID_FILE_PATH = 27394059, + NATIVE_DATABASE_ALREADY_CLOSED = 27394078, + NATIVE_DATABASE_IS_BUSY = 27394082, + NATIVE_DATABASE_WAL_FILE_EXCEEDS_LIMIT = 27394095, + NATIVE_DATABASE_GENERIC_ERROD = 27394103, + NATIVE_DATABASE_CORRUPTED = 27394104, + NATIVE_DATABASE_PERMISSION_DENIED = 27394107, + NATIVE_DATABASE_FILE_IS_LOCKED = 27394109, + NATIVE_DATABASE_OUT_OF_MEMORY = 27394110, + NATIVE_DATABASE_IO_ERROR = 27394112, + NATIVE_E_RECALL_FAILURE = 28114948, + NATIVE_E_RERANK_FAILURE = 28114949, + NATIVE_E_INVALID_NUMBER = 28114950, + NATIVE_E_INVALID_PRIMARY_KEY = 28114951, + NATIVE_E_UNSUPPORTED_COMPOSITE_PRIMARY_KEY = 28114952, + NATIVE_E_EMPTY_STRING_FIELD = 28114953, + NATIVE_E_INVALID_FILTER_INPUT = 28114954, + NATIVE_E_INVALID_RECALL_FIELD = 28114955, + NATIVE_E_HIGH_VECTOR_SIMILARITY_THRESHOLD = 28114956, + NATIVE_E_RERANK_METHOD_MISMATCH = 28114957, + NATIVE_E_EMPTY_PARAMETER = 28114958, }; +enum TsErrCode : int32_t { + PARAM_EXCEPTION = 401, + DEVICE_EXCEPTION = 801, + INNER_ERROR = 31300000, + DATABASE_CORRUPTED = 31300001, + DATABASE_ALREADY_CLOSED = 31300002, + DATABASE_IS_BUSY = 31300003, + DATABASE_OUT_OF_MEMORY = 31300004, + DATABASE_GENERIC_ERROD = 31300100, + DATABASE_PERMISSION_DENIED = 31300101, + DATABASE_FILE_IS_LOCKED = 31300102, + DATABASE_IO_ERROR = 31300103, + DATABASE_WAL_FILE_EXCEEDS_LIMIT = 31300104, + DATABASE_INVALID_FILE_PATH = 31300105, + RETRIEVAL_E_RECALL_FAILURE = 31301000, + RETRIEVAL_E_RERANK_FAILURE = 31301001, + RETRIEVAL_E_INVALID_NUMBER = 31301002, + RETRIEVAL_E_INVALID_PRIMARY_KEY = 31301003, + RETRIEVAL_E_UNSUPPORTED_COMPOSITE_PRIMARY_KEY = 31301004, + RETRIEVAL_E_EMPTY_STRING_FIELD = 31301005, + RETRIEVAL_E_INVALID_FILTER_INPUT = 31301006, + RETRIEVAL_E_INVALID_RECALL_FIELD = 31301007, + RETRIEVAL_E_HIGH_VECTOR_SIMILARITY_THRESHOLD = 31301008, + RETRIEVAL_E_RERANK_METHOD_MISMATCH = 31301009, + RETRIEVAL_E_EMPTY_PARAMETER = 31301010, +}; + +int32_t ConvertErrCodeNative2Ts(int32_t nativeErrCode); napi_value CreateIntelligenceError(const napi_env &env, int32_t errorCode, const std::string &errorMsg); std::optional<std::string> GetIntelligenceErrMsg(int32_t errorCode); void ThrowIntelligenceErr(const napi_env &env, int32_t errorCode, const std::string &printMsg); diff --git a/udmf/interfaces/jskits/intelligence/aip_napi_utils.h b/udmf/interfaces/jskits/intelligence/aip_napi_utils.h index 753ea418..03565a4e 100644 --- a/udmf/interfaces/jskits/intelligence/aip_napi_utils.h +++ b/udmf/interfaces/jskits/intelligence/aip_napi_utils.h @@ -16,6 +16,8 @@ #ifndef AIP_NAPI_UTILS_H #define AIP_NAPI_UTILS_H +#include <type_traits> + #include "i_aip_core_manager.h" #include "napi/native_api.h" #include "napi/native_node_api.h" @@ -27,7 +29,8 @@ public: AipNapiUtils() = default; ~AipNapiUtils() = default; - static bool LoadAlgoLibrary(const std::string &algoPath, AipCoreManagerHandle &aipMgrHandler); + static int32_t FindOperaotrEnum(std::string operatorStr); + static bool LoadAlgoLibrary(const std::string &libraryName, AipCoreManagerHandle &aipMgrHandler, bool isSingleton); static bool UnLoadAlgoLibrary(AipCoreManagerHandle &aipMgrHandler); static IAipCoreManager *GetAlgoObj(AipCoreManagerHandle &aipMgrHandler); static bool ValidateArgsType(napi_env env, napi_value *args, size_t argc, @@ -42,7 +45,221 @@ public: static void CreateInt32Data(napi_env env, napi_value aipServiceValue, napi_value result, const std::string name, int32_t value); static void CreateDoubleData(napi_env env, double value, napi_value *result); + static void SetInt32Property(napi_env env, napi_value targetObj, int32_t value, const char *propName); + static void SetStringProperty(napi_env env, napi_value targetObj, std::string value, const char *propName); + static void SetPropertyName(napi_env env, napi_value targetObj, const char *propName, napi_value propValue); + static bool CheckModelConfig(napi_env env, napi_value value); + + static napi_status Convert2Value(napi_env env, napi_value in, bool &out); + static napi_status Convert2Value(napi_env env, napi_value in, std::vector<uint8_t> &out); + static napi_status Convert2Value(napi_env env, napi_value in, int32_t &out); + static napi_status Convert2Value(napi_env env, napi_value in, uint32_t &out); + static napi_status Convert2Value(napi_env env, napi_value in, int64_t &out); + static napi_status Convert2Value(napi_env env, napi_value in, float &out); + static napi_status Convert2Value(napi_env env, napi_value in, double &out); + static napi_status Convert2Value(napi_env env, napi_value in, std::string &out); + static napi_status Convert2Value(napi_env env, napi_value in, std::vector<float> &out); + + template <typename T> + struct is_shared_ptr : std::false_type {}; + template <typename U> + struct is_shared_ptr<std::shared_ptr<U>> : std::true_type {}; + + template<typename T> + static typename std::enable_if<!is_shared_ptr<T>::value, napi_status>::type + Convert2Value(napi_env env, napi_value in, T &out); + + template<typename T> + static typename std::enable_if<is_shared_ptr<T>::value, napi_status>::type + Convert2Value(napi_env env, napi_value in, T &out) + { + return Convert2Value(env, in, *out); + } + + static bool IsNull(napi_env env, napi_value value); + + static std::pair<napi_status, napi_value> GetInnerValue(napi_env env, napi_value in, const std::string &prop, + bool optional); + + template<typename T> + static inline napi_status GetNamedProperty( + napi_env env, napi_value in, const std::string &prop, T &value, bool optional = false) + { + auto [status, jsValue] = GetInnerValue(env, in, prop, optional); + if (jsValue == nullptr) { + return status; + } + return Convert2Value(env, jsValue, value); + }; + + template<typename T> + static napi_status Convert2Value(napi_env env, napi_value in, std::vector<T> &out); + + template<typename T> + static napi_status Convert2Value(napi_env env, napi_value in, std::map<std::string, T> &out); + + template<typename T> + static napi_status Convert2ValuePtr(napi_env env, napi_value in, std::shared_ptr<T> &out); + + static napi_status Convert2Value(napi_env env, napi_value in, RetrievalConfigStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, FilterInfoStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, RecallConditionStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, InvertedIndexStrategyStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, Bm25StrategyStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, ExactMatchingStrategyStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, OutOfOrderStrategyStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, InvertedIndexRecallConditionStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, ChannelRerankParamsStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, VectorQueryStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, RerankParamsStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, RetrievalConditionStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, VectorChannelRerankParamsStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, InvertedIndexRerankParamsStruct &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, RdbConfig &out); + static napi_status Convert2Value(napi_env env, const napi_value &in, CryptoParam &cryptoParam); + static napi_status Convert2Value(napi_env env, const napi_value &in, ContextParam &context); + static napi_status Convert2Value(napi_env env, const napi_value &in, napi_value &out); + + static napi_status Convert2JSValue(napi_env env, const std::vector<uint8_t> &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const std::vector<float> &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const std::string &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const int32_t &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const int64_t &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const uint64_t &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const double &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const bool &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const DataIntelligence::RetrievalResponseStruct &in, + napi_value &out); + static napi_status Convert2JSValue(napi_env env, const DataIntelligence::ItemInfoStruct &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const std::map<std::string, FieldType> &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, + const std::map<int32_t, std::map<std::string, RecallScoreStruct>> &in, napi_value &out); + static napi_status Convert2JSValue(napi_env env, const std::map<std::string, double> &in, napi_value &out); + template <typename T> + static napi_status Convert2JSValue(napi_env env, const std::vector<T> &value, napi_value &out); + + template <typename T> + static napi_status GetJSValue(napi_env env, const T &value, napi_value &out); + + template <typename T, typename First, typename... Types> + static napi_status GetJSValue(napi_env env, const T &value, napi_value &out); + + template <typename... Types> + static napi_status Convert2JSValue(napi_env env, const std::variant<Types...> &value, napi_value &out) + { + return GetJSValue<decltype(value), Types...>(env, value, out); + } }; + +template<typename T> +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value jsValue, std::vector<T> &value) +{ + bool isArray = false; + napi_is_array(env, jsValue, &isArray); + if (!isArray) { + return napi_invalid_arg; + } + + uint32_t arrLen = 0; + napi_get_array_length(env, jsValue, &arrLen); + if (arrLen == 0) { + return napi_ok; + } + + for (size_t i = 0; i < arrLen; ++i) { + napi_value element; + napi_get_element(env, jsValue, i, &element); + T item; + auto status = Convert2Value(env, element, item); + if (status != napi_ok) { + return napi_invalid_arg; + } + value.push_back(std::move(item)); + } + return napi_ok; +} + +template<typename T> +napi_status AipNapiUtils::Convert2Value(napi_env env, napi_value jsValue, std::map<std::string, T> &value) +{ + napi_value jsMapList = nullptr; + uint32_t jsCount = 0; + napi_status status = napi_get_property_names(env, jsValue, &jsMapList); + if (status != napi_ok) { + return napi_invalid_arg; + } + status = napi_get_array_length(env, jsMapList, &jsCount); + if (status != napi_ok || jsCount <= 0) { + return napi_invalid_arg; + } + napi_value jsKey = nullptr; + napi_value jsVal = nullptr; + for (uint32_t index = 0; index < jsCount; index++) { + status = napi_get_element(env, jsMapList, index, &jsKey); + if (status != napi_ok) { + return napi_invalid_arg; + } + std::string key; + int ret = Convert2Value(env, jsKey, key); + if (ret != napi_ok) { + return napi_invalid_arg; + } + status = napi_get_property(env, jsValue, jsKey, &jsVal); + if (status != napi_ok || jsVal == nullptr) { + return napi_invalid_arg; + } + T val; + ret = Convert2Value(env, jsVal, val); + if (ret != napi_ok) { + return napi_invalid_arg; + } + value.insert(std::pair<std::string, T>(key, val)); + } + return napi_ok; +} + + +template <typename T> +napi_status AipNapiUtils::Convert2JSValue(napi_env env, const std::vector<T> &value, + napi_value &out) +{ + napi_status status = napi_create_array_with_length(env, value.size(), &out); + if (status != napi_ok) { + return status; + } + for (size_t i = 0; i < value.size(); ++i) { + napi_value jsValue; + napi_create_object(env, &jsValue); + Convert2JSValue(env, value[i], jsValue); + napi_set_element(env, out, i, jsValue); + } + return napi_ok; +} + +template <typename T> +napi_status AipNapiUtils::GetJSValue(napi_env env, const T &value, napi_value &out) +{ + return napi_invalid_arg; +} + +template <typename T, typename First, typename... Types> +napi_status AipNapiUtils::GetJSValue(napi_env env, const T &value, napi_value &out) +{ + auto *val = std::get_if<First>(&value); + if (val != nullptr) { + Convert2JSValue(env, *val, out); + return napi_ok; + } + return GetJSValue<T, Types...>(env, value, out); +} + +#define LOG_ERROR_RETURN(condition, message, retVal) \ + do { \ + if (!(condition)) { \ + AIP_HILOGE(message); \ + return retVal; \ + } \ + } while (0) } // namespace DataIntelligence } // namespace OHOS #endif // AIP_NAPI_UTILS_H diff --git a/udmf/interfaces/jskits/intelligence/i_aip_core_manager.h b/udmf/interfaces/jskits/intelligence/i_aip_core_manager.h index d91e1e50..5770829e 100644 --- a/udmf/interfaces/jskits/intelligence/i_aip_core_manager.h +++ b/udmf/interfaces/jskits/intelligence/i_aip_core_manager.h @@ -15,8 +15,10 @@ #ifndef I_AIP_CORE_MANAGER_H #define I_AIP_CORE_MANAGER_H +#include <stdint.h> #include <string> #include <vector> +#include <map> namespace OHOS { namespace DataIntelligence { @@ -26,11 +28,234 @@ struct ModelConfigData { std::string cachePathValue; }; +struct CryptoParam { + mutable int32_t iterNum = 0; + int32_t encryptAlgo = 0; + int32_t hmacAlgo = 1; + int32_t kdfAlgo = 1; + uint32_t cryptoPageSize = 1024; + mutable std::vector<uint8_t> encryptKey_{}; +}; + +struct RdbConfig { + bool isEncrypt = false; + bool isSearchable = false; + bool isAutoClean = true; + bool vector = false; + bool allowRebuild = false; + bool isReadOnly = false; + int32_t securityLevel = 5; + int32_t tokenizer = 0; + int32_t haMode = 0; + std::string dataGroupId; + std::string name; + std::string customDir; + std::string rootDir; + std::string path; + std::vector<std::string> pluginLibs = {}; + CryptoParam cryptoParam; +}; + +struct ContextParam { + std::string bundleName; + std::string moduleName; + std::string baseDir; + int32_t area; + bool isSystemApp = false; + bool isStageMode = true; +}; + +using Blob = std::vector<uint8_t>; +using Bigint = std::vector<uint64_t>; +using FloatVector = std::vector<float>; +using FieldType = std::variant<int64_t, int32_t, double, std::string, bool, Blob, Bigint, FloatVector>; +struct ChannelConfigStruct { + int32_t channelType; + RdbConfig dbConfig; + ContextParam context; +}; + +enum TsChannelType { + VECTOR_DATABASE = 0, + INVERTED_INDEX_DATABASE, +}; + +enum StrategyType { + BM25 = 0, + EXACT_MATCH, + OUT_OF_ORDER_MATCH +}; + +struct RetrievalConfigStruct { + std::vector<ChannelConfigStruct> channelConfigs; +}; + +struct InvertedIndexStrategyStruct { + std::map<std::string, float> fieldWeight; + virtual StrategyType GetStrategy() = 0; + virtual ~InvertedIndexStrategyStruct() = default; +}; + +struct Bm25StrategyStruct : public InvertedIndexStrategyStruct { + float bm25Weight; + StrategyType GetStrategy() override + { + return StrategyType::BM25; + } + ~Bm25StrategyStruct() override = default; +}; + +struct ExactMatchingStrategyStruct : public InvertedIndexStrategyStruct { + float exactMatchingWeight; + StrategyType GetStrategy() override + { + return StrategyType::EXACT_MATCH; + } + ~ExactMatchingStrategyStruct() override = default; +}; + +struct OutOfOrderStrategyStruct : public InvertedIndexStrategyStruct { + float outOfOrderWeight; + std::map<std::string, int> fieldSlops; + StrategyType GetStrategy() override + { + return StrategyType::OUT_OF_ORDER_MATCH; + } + ~OutOfOrderStrategyStruct() override = default; +}; + +enum TsOperator { + OP_EQ = 0, + OP_NE, + OP_LT, + OP_LE, + OP_GE, + OP_GT, + OP_IN, + OP_NOT_IN, + OP_BETWEEN, + OP_LIKE, + OP_NOT_LIKE, + BUTT +}; + +struct FilterInfoStruct { + std::string field; + std::vector<std::string> fields; + TsOperator op; + std::string value; + std::pair<std::string, std::string> range; +}; + +struct RecallConditionStruct { + std::string fromClause; + std::vector<std::string> primaryKey; + std::vector<std::string> responseColumns; + std::vector<FilterInfoStruct> filters; + std::string recallName; + int32_t deepSize; + virtual TsChannelType GetChannelType() = 0; + virtual ~RecallConditionStruct() = default; +}; + +struct InvertedIndexRecallConditionStruct: public RecallConditionStruct { + std::string ftsTableName; + std::vector<std::shared_ptr<InvertedIndexStrategyStruct>> invertedIndexStrategies; + TsChannelType GetChannelType() override + { + return TsChannelType::INVERTED_INDEX_DATABASE; + } + ~InvertedIndexRecallConditionStruct() override = default; +}; + +struct VectorQueryStruct { + std::string vectorColumn; + std::vector<float> vectorValue; + double similarityThreshold = 1; +}; + +struct VectorRecallConditionStruct : public RecallConditionStruct { + VectorQueryStruct vectorQuery; + TsChannelType GetChannelType() override + { + return TsChannelType::VECTOR_DATABASE; + } +}; + +struct ChannelRerankParamsStruct { + std::map<std::string, double> recallWeights; + virtual TsChannelType GetChannelType() = 0; + virtual ~ChannelRerankParamsStruct() = default; +}; + +struct InvertedIndexRerankParamsStruct : public ChannelRerankParamsStruct { + TsChannelType GetChannelType() override + { + return TsChannelType::INVERTED_INDEX_DATABASE; + } + ~InvertedIndexRerankParamsStruct() override = default; +}; + +struct VectorChannelRerankParamsStruct : public ChannelRerankParamsStruct { + std::vector<double> rankLevelThresholds; + std::map<std::string, std::string> numberFilterCheckFields; + TsChannelType GetChannelType() override + { + return TsChannelType::VECTOR_DATABASE; + } + ~VectorChannelRerankParamsStruct() override = default; +}; + +enum TsRerankType { + RRF = 0, + FUSION_SCORE +}; + +struct RerankParamsStruct { + int32_t rankType{TsRerankType::RRF}; + std::map<int32_t, std::shared_ptr<ChannelRerankParamsStruct>> channelParams; + bool useScoreSoftmax{false}; +}; + +struct RetrievalConditionStruct { + std::vector<std::shared_ptr<RecallConditionStruct>> recallConditions; + RerankParamsStruct rerankParameter; + int topN = 500; +}; + +struct RecallScoreStruct { + double recallScore; + bool isOriginal; +}; + +struct ItemInfoStruct { + std::string primaryKey; + double score; + std::map<std::string, FieldType> fields; + std::map<int32_t, std::map<std::string, RecallScoreStruct>> recallScores; + std::map<std::string, double> features; + int32_t similarityLevel; +}; + +struct RetrievalResponseStruct { + std::vector<ItemInfoStruct> retrievalResults; +}; + +enum TsSimilarityLevel { + NONE = 0, + LOW, + MEDIUM, + HIGH +}; + class IAipCoreManager { public: IAipCoreManager() = default; virtual ~IAipCoreManager() = default; + virtual int32_t InitRetriever(const RetrievalConfigStruct& retrievalConfig) = 0; + virtual int32_t Retrieve(const std::string query, const RetrievalConditionStruct &condition, + RetrievalResponseStruct &retrievalResponse) = 0; virtual int32_t InitTextModel(const ModelConfigData &config) = 0; virtual int32_t InitImageModel(const ModelConfigData &config) = 0; virtual int32_t LoadTextModel() = 0; diff --git a/udmf/interfaces/jskits/intelligence/i_aip_core_manager_impl.h b/udmf/interfaces/jskits/intelligence/i_aip_core_manager_impl.h index 10955b1a..00ef61ea 100644 --- a/udmf/interfaces/jskits/intelligence/i_aip_core_manager_impl.h +++ b/udmf/interfaces/jskits/intelligence/i_aip_core_manager_impl.h @@ -27,6 +27,9 @@ class IAipCoreManagerImpl : public IAipCoreManager { public: IAipCoreManagerImpl() = default; ~IAipCoreManagerImpl() = default; + int32_t InitRetriever(const RetrievalConfigStruct& retrievalConfig) override; + int32_t Retrieve(const std::string query, const RetrievalConditionStruct &condition, + RetrievalResponseStruct &retrievalResponse) override; int32_t InitTextModel(const ModelConfigData &config) override; int32_t InitImageModel(const ModelConfigData &config) override; int32_t LoadTextModel() override; diff --git a/udmf/interfaces/jskits/intelligence/js_ability.h b/udmf/interfaces/jskits/intelligence/js_ability.h new file mode 100644 index 00000000..0e1602d1 --- /dev/null +++ b/udmf/interfaces/jskits/intelligence/js_ability.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DISTRIBUTEDDATAMGR_APPDATAMGR_JSABILITY_H +#define DISTRIBUTEDDATAMGR_APPDATAMGR_JSABILITY_H +#include <iostream> +#include <string> + +#include "ability.h" +#include "napi/native_api.h" +#include "napi/native_common.h" +#include "napi/native_node_api.h" +#include "napi_base_context.h" + +namespace OHOS { +namespace AppDataMgrJsKit { +class Context { +public: + explicit Context(std::shared_ptr<AbilityRuntime::Context> stageContext); + explicit Context(std::shared_ptr<AbilityRuntime::AbilityContext> abilityContext); + + std::string GetDatabaseDir(); + int GetSystemDatabaseDir(const std::string &dataGroupId, std::string &databaseDir); + std::string GetBundleName(); + std::string GetModuleName(); + std::string GetUri(); + std::string GetReadPermission(); + std::string GetWritePermission(); + int32_t GetArea() const; + bool IsSystemAppCalled(); + bool IsHasProxyDataConfig() const; + bool IsStageMode() const; + +private: + int32_t area_ = 0; + std::string databaseDir_; + std::string bundleName_; + std::string moduleName_; + std::string uri_; + std::string readPermission_; + std::string writePermission_; + bool hasProxyDataConfig_ = false; + bool isSystemAppCalled_ = false; + bool isStageMode_ = false; + std::shared_ptr<AbilityRuntime::Context> stageContext_; +}; + +class JSAbility final { +public: + static bool CheckContext(napi_env env, napi_callback_info info); + static std::shared_ptr<Context> GetContext(napi_env env, napi_value object); + static std::shared_ptr<Context> GetStageModeContext(napi_env env, napi_value value); + static std::shared_ptr<Context> GetCurrentAbility(napi_env env, napi_value value); + static int32_t GetHapVersion(napi_env env, napi_value value); + + static constexpr int32_t INVALID_HAP_VERSION = -1; +}; +} // namespace AppDataMgrJsKit +} // namespace OHOS + +#endif // DISTRIBUTEDDATAMGR_APPDATAMGR_JSABILITY_H diff --git a/udmf/interfaces/jskits/intelligence/native_module_intelligence.h b/udmf/interfaces/jskits/intelligence/native_module_intelligence.h index 747a6306..92826f90 100644 --- a/udmf/interfaces/jskits/intelligence/native_module_intelligence.h +++ b/udmf/interfaces/jskits/intelligence/native_module_intelligence.h @@ -19,5 +19,6 @@ #include "image_embedding_napi.h" #include "napi/native_node_api.h" #include "text_embedding_napi.h" +#include "retrieval_napi.h" #endif // NATIVE_MODULE_INTELLIGENCE_H \ No newline at end of file diff --git a/udmf/interfaces/jskits/intelligence/retrieval_napi.h b/udmf/interfaces/jskits/intelligence/retrieval_napi.h new file mode 100644 index 00000000..3074714c --- /dev/null +++ b/udmf/interfaces/jskits/intelligence/retrieval_napi.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RETRIEVAL_NAPI_H +#define RETRIEVAL_NAPI_H + +#include <string> +#include <map> +#include <uv.h> +#include <refbase.h> + +#include "i_aip_core_manager.h" +#include "napi/native_api.h" +#include "napi/native_node_api.h" + +namespace OHOS { +namespace DataIntelligence { +class RetrievalNapi; +struct AsyncGetRetriever { + napi_async_work asyncWork; + napi_deferred deferred; + RetrievalConfigStruct config; + IAipCoreManager* retrievalAipCoreManagerPtr; + int32_t ret; +}; +struct AsyncRetrieve { + napi_async_work asyncWork; + napi_deferred deferred; + std::string query; + RetrievalConditionStruct retrievalCondition; + IAipCoreManager* retrievalAipCoreManagerPtr; + RetrievalResponseStruct retrievalResponse; + napi_value res; + int32_t ret; +}; +class RetrievalNapi { +public: + RetrievalNapi(); + ~RetrievalNapi(); + + static napi_value Init(napi_env env, napi_value exports); + static napi_value GetRetriever(napi_env env, napi_callback_info info); + static napi_value Retrieve(napi_env env, napi_callback_info info); + static napi_value RetrieverConstructor(napi_env env, napi_callback_info info); + static napi_value WrapAipCoreManager(napi_env env, IAipCoreManager* retrievalAipCoreManager); + +private: + static bool LoadAsyncExecution(napi_env env, napi_deferred deferred); + static bool CreateAsyncGetRetrieverExecution(napi_env env, AsyncGetRetriever *asyncGetRetriever); + static bool CreateAsyncRetrieveExecution(napi_env env, AsyncRetrieve *asyncRetrieve); + static void GetRetrieverExecutionCB(napi_env env, void *data); + static void GetRetrieverCompleteCB(napi_env env, napi_status status, void *data); + static void RetrieveExecutionCB(napi_env env, void *data); + static void RetrieveCompleteCB(napi_env env, napi_status status, void *data); + napi_env env_; + static AipCoreManagerHandle retrievalAipCoreMgrHandle_; +}; +} // namespace DataIntelligence +} // namespace OHOS +#endif // RETRIEVAL_NAPI_H diff --git a/udmf/interfaces/ndk/BUILD.gn b/udmf/interfaces/ndk/BUILD.gn index 130c3eca..edf268c8 100644 --- a/udmf/interfaces/ndk/BUILD.gn +++ b/udmf/interfaces/ndk/BUILD.gn @@ -32,6 +32,7 @@ ohos_shared_library("libudmf") { sources = [ "${udmf_framework_path}/innerkitsimpl/common/unified_meta.cpp", "${udmf_framework_path}/innerkitsimpl/convert/data_params_conversion.cpp", + "${udmf_framework_path}/innerkitsimpl/convert/ndk_data_conversion.cpp", "${udmf_framework_path}/ndkimpl/data/data_provider_impl.cpp", "${udmf_framework_path}/ndkimpl/data/udmf.cpp", "${udmf_framework_path}/ndkimpl/data/uds.cpp", diff --git a/utils_native/CMakeLists.txt b/utils_native/CMakeLists.txt index 230865d8..18e577b1 100644 --- a/utils_native/CMakeLists.txt +++ b/utils_native/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.10.2) project(secure) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1y -fno-rtti -fvisibility=default -D_GNU_SOURCE") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdata-sections -ffunction-sections -D_GLIBC_MOCK") aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/base/src secureSrc) -- Gitee