diff --git a/IndexSDK/TestAscendIReduction.cpp b/IndexSDK/TestAscendIReduction.cpp index 8639cc600dffc714cba4cd225f32fbb380395ff7..06a704a39e963e1d654d3be4435a9111d3097c29 100644 --- a/IndexSDK/TestAscendIReduction.cpp +++ b/IndexSDK/TestAscendIReduction.cpp @@ -25,29 +25,30 @@ // 请在第36行导入对应的NN降维模型 namespace { - int g_dimin = 256; - int g_dimout = 64; - std::string g_nnom; - std::string g_metricTypeName = "INNER_PRODUCT"; - faiss::MetricType MetricType = g_metricTypeName == "L2" ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; +int g_dimin = 256; +int g_dimout = 64; +std::string g_nnom; +std::string g_metricTypeName = "INNER_PRODUCT"; +faiss::MetricType MetricType = g_metricTypeName == "L2" ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - void ReadModel() - { - std::string modelPath = "./"; // 导入对应的NN降维模型 - std::ifstream istrm(modelPath.c_str(), std::ios::binary); - std::stringstream buffer; - buffer << istrm.rdbuf(); - g_nnom = buffer.str(); - istrm.close(); - } +void ReadModel() +{ + std::string modelPath = "./"; // 导入对应的NN降维模型 + std::ifstream istrm(modelPath.c_str(), std::ios::binary); + std::stringstream buffer; + buffer << istrm.rdbuf(); + g_nnom = buffer.str(); + istrm.close(); +} - void TestSampleNNInfer() - { - std::vector deviceList = { 0 }; - int ntotal = 100000; - int maxSize = ntotal * g_dimin; - std::vector data(maxSize); - std::vector outputData(ntotal * g_dimout); +void TestSampleNNInfer() +{ + std::vector deviceList = { 0 }; + int ntotal = 100000; + int maxSize = ntotal * g_dimin; + std::vector data(maxSize); + std::vector outputData(ntotal * g_dimout); + try{ for (int i = 0; i < maxSize; i++) { data[i] = drand48(); } @@ -60,42 +61,50 @@ namespace { reinterpret_cast(outputData.data())); std::cout << "TestSampleNNInfer end " << std::endl; + } catch (std::exception &e) { + printf("%s\n", e.what()); } +} - void TestSampleNNReduce() - { - std::vector deviceList = { 0 }; - int ntotal = 100000; - int maxSize = ntotal * g_dimin; - std::vector data(maxSize); - std::vector outputData(ntotal * g_dimout); +void TestSampleNNReduce() +{ + std::vector deviceList = { 0 }; + int ntotal = 100000; + int maxSize = ntotal * g_dimin; + std::vector data(maxSize); + std::vector outputData(ntotal * g_dimout); + try { for (int i = 0; i < maxSize; i++) { data[i] = drand48(); } - + std::cout << "TestSampleNNReduce start " << std::endl; ReadModel(); - + faiss::ascend::ReductionConfig reductionConfig(deviceList, g_nnom.data(), g_nnom.size()); std::string method = "NN"; faiss::ascend::IReduction* reduction = CreateReduction(method, reductionConfig); reduction->train(ntotal, data.data()); reduction->reduce(ntotal, data.data(), outputData.data()); - + std::cout << "TestSampleNNReduce end " << std::endl; + } catch (std::exception &e) { + printf("%s\n", e.what()); } +} - void TestSamplePcarReduce() - { - std::vector deviceList = { 0 }; - int ntotal = 100000; - int maxSize = ntotal * g_dimin; - std::vector data(maxSize); - std::vector outputData(ntotal * g_dimout); +void TestSamplePcarReduce() +{ + std::vector deviceList = { 0 }; + int ntotal = 100000; + int maxSize = ntotal * g_dimin; + std::vector data(maxSize); + std::vector outputData(ntotal * g_dimout); + try { for (int i = 0; i < maxSize; i++) { data[i] = drand48(); } - + std::cout << "TestSamplePcarReduce start " << std::endl; // Pcar IReduction faiss::ascend::ReductionConfig reductionConfig(g_dimin, g_dimout, 0, false); @@ -103,10 +112,13 @@ namespace { faiss::ascend::IReduction* reduction = CreateReduction(method, reductionConfig); reduction->train(ntotal, data.data()); reduction->reduce(ntotal, data.data(), outputData.data()); - + std::cout << "TestSamplePcarReduce end " << std::endl; + } catch (std::exception &e) { + printf("%s\n", e.what()); } } +} int main(int argc, char **argv) { diff --git a/IndexSDK/TestAscendIndexAggressTs.cpp b/IndexSDK/TestAscendIndexAggressTs.cpp deleted file mode 100644 index 89cee55a3907b39c524cf6fcada5f9ec8bb72b9d..0000000000000000000000000000000000000000 --- a/IndexSDK/TestAscendIndexAggressTs.cpp +++ /dev/null @@ -1,450 +0,0 @@ -/* - * Copyright(C) 2023. Huawei Technologies Co.,Ltd. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - // 需要生成aicpu算子+flat算子(-d 512)+ mask算子 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "faiss/ascend/AscendIndexFlat.h" -#include "faiss/ascend/AscendIndexTS.h" -#include "faiss/impl/AuxIndexStructures.h" - -namespace { - using idx_t = int64_t; - using FeatureAttr = faiss::ascend::FeatureAttr; - using AttrFilter = faiss::ascend::AttrFilter; - const int DEVICE_ID = 0; - - inline double GetMillisecs() - { - struct timeval tv { - 0, 0 - }; - gettimeofday(&tv, nullptr); - return tv.tv_sec * 1e3 + tv.tv_usec * 1e-3; - } - - void FeatureGenerator(std::vector &features) - { - size_t n = features.size(); - for (size_t i = 0; i < n; ++i) { - features[i] = drand48(); - } - } - - void FeatureAttrGenerator(std::vector &attrs) - { - size_t n = attrs.size(); - int power = 4; - for (size_t i = 0; i < n; ++i) { - attrs[i].time = int32_t(i % power); - attrs[i].tokenId = int32_t(i % power); - } - } -} - -namespace faiss { -namespace ascend { -template -class AscendAggressBase { -public: - struct AggressPromisParam { - std::promise retval; - AggressParam param; - }; - - AscendAggressBase(uint32_t dim, int32_t core, std::initializer_list supportBatch) - : m_runningFlag(true), m_dim(dim), m_supportBatch(supportBatch) - { - m_workThread = std::move(std::thread(&AscendAggressBase::AggressWorkProcess, this)); - if (core != -1) { - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(core, &cpuset); - pthread_setaffinity_np(m_workThread.native_handle(), sizeof(cpu_set_t), &cpuset); - } - } - - virtual ~AscendAggressBase() - { - m_runningFlag.store(false, std::memory_order_acq_rel); - m_workThread.join(); - } - - void SetCondWaitQueueLen(int queueLen) - { - m_waitQueueLen.store(queueLen, std::memory_order_release); - } - - int32_t SearchWithAggres(AggressParam SearchParam) - { - if (!m_runningFlag.load(std::memory_order_acquire)) { - return -1; - } - - AggressPromisParam promisParam; - promisParam.param = SearchParam; - auto retval = promisParam.retval.get_future(); - { - std::lock_guard lk(m_mutex); - m_queue.push_back(&promisParam); - m_cond.notify_one(); - } // 解锁 - return retval.get(); - } - -protected: - // pure virtual function - virtual int32_t AggressSearch(AggressParam *) = 0; - virtual AggressParam *CreateBatchQueryData( - std::vector &batchQuery) = 0; - virtual void DestoryBatchQueryData(AggressParam *batchQueryData) = 0; - virtual void SpliceBatchQueryData( - AggressParam *batchQueryData, - std::vector &batchQuery) = 0; - - uint32_t BatchPadding(std::vector &batchQuery) - { - uint32_t paddingCnt = 0; - AggressPromisParam *paddingValue = batchQuery[0]; - for (auto i : m_supportBatch) { - if (i >= batchQuery.size()) { - paddingCnt = i - batchQuery.size(); - break; - } - } - - for (auto i = 0u; i < paddingCnt; i++) { - batchQuery.push_back(paddingValue); - } - return paddingCnt; - } - - static void RemovePadding(std::vector &batchQuery, uint32_t paddingCnt) - { - while (paddingCnt-- != 0) { - batchQuery.pop_back(); - } - } - - static void ReturnValue(std::vector *batchQuery, int32_t ret) - { - std::for_each(batchQuery->begin(), batchQuery->end(), - [ret](AggressPromisParam *query) { - query->retval.set_value(ret); - }); - } - - static void AgressSearchPorcess(AscendAggressBase *index, std::vector *batchQuery) - { - std::unique_ptr> batchQueryScope(batchQuery); - auto paddingCnt = index->BatchPadding(*batchQuery); - - // step3: 构造组合之后的查询数据 - auto deletor = [index](AggressParam *batchQueryData) { - index->DestoryBatchQueryData(batchQueryData); - }; - auto batchQueryData = std::unique_ptr( - index->CreateBatchQueryData(*batchQuery), deletor); - - // step4: 调用原生search接口进行查询 - auto ret = index->AggressSearch(batchQueryData.get()); - RemovePadding(*batchQuery, paddingCnt); - - if (ret != 0) { - ReturnValue(batchQuery, ret); - return; - } - - // step5: 分割查询之后的数据，并设置 - index->SpliceBatchQueryData(batchQueryData.get(), *batchQuery); - - // step6: 返回成功 - ReturnValue(batchQuery, ret); - } - - bool GetBatchQuery(std::vector *batchQuery) - { - std::unique_lock lk(m_mutex); - // step1: 等待条件满足 两个条件超时，或者队列超过阈值退出 - int waitTime = 15; - m_cond.wait_for(lk, std::chrono::milliseconds(waitTime), - [this]() { - return m_queue.size() >= - static_cast(m_waitQueueLen.load(std::memory_order_acquire)); - }); - - if (m_queue.empty()) { - return false; - } // 解锁 - - auto first = m_queue[0]; - auto filter = [first](AggressPromisParam *queryParam) { - return first->param.CanGroup(queryParam->param); - }; - - std::copy_if(m_queue.begin(), m_queue.end(), std::back_inserter(*batchQuery), filter); - m_queue.erase(std::remove_if(m_queue.begin(), m_queue.end(), filter), m_queue.end()); - return true; - } // 解锁 - - void AggressWorkProcess() - { - while (m_runningFlag.load(std::memory_order_acquire)) { - auto *batchQuery = new std::vector; - if (!GetBatchQuery(batchQuery)) { - delete batchQuery; - continue; - } - - auto task = std::thread(&AgressSearchPorcess, this, batchQuery); - task.detach(); // 该线程交由promise进行关闭,这里detach - } - } - - std::thread m_workThread; - std::vector m_queue; - std::atomic m_runningFlag; - std::condition_variable m_cond; - std::mutex m_mutex; - - uint32_t m_dim{0}; - std::atomic m_waitQueueLen{1}; - std::vector m_supportBatch; -}; - -struct AggressTsParam { - uint32_t count{0}; - float *features{nullptr}; - AttrFilter *attrFilter{nullptr}; - bool shareAttrFilter{false}; - uint32_t topk{0}; - int64_t *labels{nullptr}; - float *distances{nullptr}; - uint32_t *validNums{nullptr}; - bool enableTimeFilter{false}; - - // 用户需要实现CanGroup函数，组batch时将调用该函数进行判断是否能够组合成batch - bool CanGroup(AggressTsParam &other) { return topk == other.topk; } -}; - -using AgressIndexBase = AscendAggressBase; - -// 继承聚合基类和时空索引基类 需要实现 构造聚合 分割聚合 销毁聚合三个函数 -class AggressTsIndex : public AgressIndexBase, public AscendIndexTS { -public: - explicit AggressTsIndex(uint32_t dim = 0, int32_t core = -1, - std::initializer_list supportBatch = {}) - : AgressIndexBase(dim, core, supportBatch), AscendIndexTS() {} - - ~AggressTsIndex() override = default; - -private: - // 该函数由线程任务调用 - int32_t AggressSearch(AggressTsParam *batchQueryData) override - { - // 调用AscendIndexTS search接口进行查询 - return Search(batchQueryData->count, batchQueryData->features, - batchQueryData->attrFilter, batchQueryData->shareAttrFilter, - batchQueryData->topk, batchQueryData->labels, - batchQueryData->distances, batchQueryData->validNums, - batchQueryData->enableTimeFilter); - } - - // 构造聚合 - AggressTsParam *CreateBatchQueryData(std::vector &batchQuery) override - { - AggressTsParam *batchQueryData = new AggressTsParam; - batchQueryData->count = batchQuery.size(); - batchQueryData->features = new float[batchQueryData->count * m_dim]; - batchQueryData->attrFilter = new AttrFilter[batchQueryData->count]; - batchQueryData->shareAttrFilter = batchQuery[0]->param.shareAttrFilter; - batchQueryData->topk = batchQuery[0]->param.topk; - batchQueryData->labels = - new int64_t[batchQueryData->count * batchQueryData->topk]; - batchQueryData->distances = - new float[batchQueryData->count * batchQueryData->topk]; - batchQueryData->validNums = new uint32_t[batchQueryData->count]; - batchQueryData->enableTimeFilter = batchQuery[0]->param.enableTimeFilter; - - for (auto i = 0u; i < batchQueryData->count; i++) { - auto ret = memcpy_s(batchQueryData->features + i * m_dim, - m_dim * sizeof(float), batchQuery[i]->param.features, m_dim * sizeof(float)); - if (ret != 0) { - std::cerr << "ERROR: fail to memcpy features" << std::endl; - } - ret = memcpy_s(batchQueryData->attrFilter + i, sizeof(AttrFilter), - batchQuery[i]->param.attrFilter, sizeof(AttrFilter)); - if (ret != 0) { - std::cerr << "ERROR: fail to memcpy attrFilter" << std::endl; - } - } - return batchQueryData; - } - - // 分割聚合 - void SpliceBatchQueryData(AggressTsParam *batchQueryData, - std::vector &batchQuery) override - { - for (auto i = 0u; i < batchQuery.size(); i++) { - auto ret = memcpy_s(batchQuery[i]->param.labels, - batchQueryData->topk * sizeof(int64_t), - batchQueryData->labels + i * batchQueryData->topk, - batchQueryData->topk * sizeof(int64_t)); - if (ret != 0) { - std::cerr << "ERROR: fail to memcpy label" << std::endl; - return; - } - ret = memcpy_s(batchQuery[i]->param.distances, - batchQueryData->topk * sizeof(float), - batchQueryData->distances + i * batchQueryData->topk, - batchQueryData->topk * sizeof(float)); - if (ret != 0) { - std::cerr << "ERROR: fail to memcpy distances" << std::endl; - return; - } - *(batchQuery[i]->param.validNums) = batchQueryData->validNums[i]; - } - } - - // 销毁聚合 - void DestoryBatchQueryData(AggressTsParam *batchQueryData) override -{ - if (batchQueryData->features != nullptr) { - delete[] (batchQueryData->features); - } - if (batchQueryData->attrFilter != nullptr) { - delete[] (batchQueryData->attrFilter); - } - if (batchQueryData->labels != nullptr) { - delete[] (batchQueryData->labels); - } - if (batchQueryData->distances != nullptr) { - delete[] (batchQueryData->distances); - } - if (batchQueryData->validNums != nullptr) { - delete[] (batchQueryData->validNums); - } - delete batchQueryData; -} -}; -} -} - -void InitAndAdd(uint32_t dim, idx_t ntotal, uint32_t tokenNum, - faiss::ascend::AggressTsIndex *tsIndex, std::vector& features) -{ - auto res = tsIndex->Init(DEVICE_ID, dim, tokenNum, faiss::ascend::AlgorithmType::FLAT_IP_FP16); - EXPECT_EQ(res, 0); - printf("[---add ---------]\n"); - FeatureGenerator(features); - - std::vector labels; - for (int i = 0; i < ntotal; ++i) { - labels.push_back(i); - } - - std::vector attrs(ntotal); - FeatureAttrGenerator(attrs); - auto ts = GetMillisecs(); - res = tsIndex->AddFeature(ntotal, features.data(), attrs.data(), labels.data()); - auto te = GetMillisecs(); - printf("add %ld cost %f ms\n", ntotal, te - ts); - EXPECT_EQ(res, 0); - int64_t validNum = 0; - tsIndex->GetFeatureNum(&validNum); - EXPECT_EQ(validNum, ntotal); -} - -TEST(TestAscendIndexAggressTS, search) -{ - uint32_t dim = 512; - uint32_t tokenNum = 2500; - idx_t ntotal = 100000; - int queryNums = 100; - std::vector topks = {100}; - std::vector features(ntotal * dim); - faiss::ascend::AggressTsIndex *tsIndex = new faiss::ascend::AggressTsIndex(dim); - InitAndAdd(dim, ntotal, tokenNum, tsIndex, features); - - for (auto k : topks) - { - std::vector distances(queryNums * k, -1); - std::vector labelRes(queryNums * k, 10); - std::vector faissDistances(queryNums * k, -1); - std::vector faissLabelRes(queryNums * k, 10); - std::vector validNum(queryNums, 0); - uint32_t size = queryNums * dim; - std::vector querys(size); - querys.assign(features.begin(), features.begin() + size); - - uint32_t setlen = (uint32_t)((tokenNum + 7) / 8); - std::vector bitSet(setlen, 0); - - // 00001111 - bitSet[0] = 0x1 << 0 | 0x1 << 1 | 0x1 << 2 | 0x1 << 3; - AttrFilter filter{}; - filter.timesStart = 0; - filter.timesEnd = 3; - filter.tokenBitSet = bitSet.data(); - filter.tokenBitSetLen = setlen; - - std::vector queryFilters(queryNums, filter); - std::vector> searchGroup; - - for (auto query = 0; query < queryNums; query++) - { - faiss::ascend::AggressTsParam params; - params.count = 1; - params.features = querys.data() + query * dim; - params.attrFilter = queryFilters.data() + query; - params.topk = k; - params.labels = labelRes.data() + query * k; - params.distances = distances.data() + query * k; - params.validNums = validNum.data() + query; - params.enableTimeFilter = false; - - // 异步调用 组batch - searchGroup.emplace_back( - std::async(std::launch::async, &faiss::ascend::AggressTsIndex::SearchWithAggres, tsIndex, params)); - } - - std::for_each(searchGroup.begin(), searchGroup.end(), [](std::future &task) { - auto ret = task.get(); - EXPECT_EQ(ret, 0); - }); - } - - delete tsIndex; -} - -int main(int argc, char **argv) -{ - testing::InitGoogleTest(&argc, argv); - - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/IndexSDK/TestAscendIndexBinaryFlat.cpp b/IndexSDK/TestAscendIndexBinaryFlat.cpp index 6a197b659d75f8e3d3037d84a7b1ff68489d77d1..97cf3ac198a0d5baf9537d0185e6b18f9a77137a 100644 --- a/IndexSDK/TestAscendIndexBinaryFlat.cpp +++ b/IndexSDK/TestAscendIndexBinaryFlat.cpp @@ -61,28 +61,31 @@ TEST(TestAscendIndexBinaryFlat, QPS) printf("generate data\n"); std::vector base(ntotal * index.code_size, 0); FeatureGenerator(base); - - printf("add data\n"); - index.add(ntotal, base.data()); - int warmUpTimes = 10 ; - std::vector distw(127 * 10, 0); - std::vector labelw(127 * 10, 0); - for (int i = 0; i < warmUpTimes; i++) { - index.search(127, base.data(), 10, distw.data(), labelw.data()); - } - - for (size_t n = 0; n < searchNum.size(); n++) { - int k = 128; - int loopTimes = 10; - std::vector dist(searchNum[n] * k, 0); - std::vector label(searchNum[n] * k, 0); - double ts = GetMillisecs(); - for (int l = 0; l < loopTimes; l++) { - index.search(searchNum[n], base.data(), k, dist.data(), label.data()); + try { + printf("add data\n"); + index.add(ntotal, base.data()); + int warmUpTimes = 10 ; + std::vector distw(127 * 10, 0); + std::vector labelw(127 * 10, 0); + for (int i = 0; i < warmUpTimes; i++) { + index.search(127, base.data(), 10, distw.data(), labelw.data()); + } + + for (size_t n = 0; n < searchNum.size(); n++) { + int k = 128; + int loopTimes = 10; + std::vector dist(searchNum[n] * k, 0); + std::vector label(searchNum[n] * k, 0); + double ts = GetMillisecs(); + for (int l = 0; l < loopTimes; l++) { + index.search(searchNum[n], base.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], + MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); } - double te = GetMillisecs(); - printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], - MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + } catch (std::exception &e) { + printf("%s\n", e.what()); } } } // namespace diff --git a/IndexSDK/TestAscendIndexCluster.cpp b/IndexSDK/TestAscendIndexCluster.cpp index 72b76b59918aab6b6daa81a4a28f6096eec12ee2..8ae60ca1d0f57ac2526ad11ffbae7666a8278792 100644 --- a/IndexSDK/TestAscendIndexCluster.cpp +++ b/IndexSDK/TestAscendIndexCluster.cpp @@ -75,6 +75,7 @@ int main(int argc, char **argv) ret = index.AddFeatures(ntotal, addVec.data(), ids.data()); if (ret != 0) { printf("[ERROR] AddFeatures fail ret = %d \r\n", ret); + index.Finalize(); return 1; } @@ -83,26 +84,31 @@ int main(int argc, char **argv) uint32_t codeStartIdx = 0; uint32_t codeNum = 1000; float threshold = 0.75; - std::vector queryIdArr(nq); - std::iota(queryIdArr.begin(), queryIdArr.end(), start); - - bool aboveFilter = true; - std::vector> resDist(nq); - std::vector> resIdx(nq); - - ret = index.ComputeDistanceByThreshold(queryIdArr, codeStartIdx, codeNum, threshold, aboveFilter, resDist, resIdx); - if (ret != 0) { - printf("[ERROR] ComputeDistanceByThreshold fail ret = %d \r\n", ret); - return 1; - } - - for (uint32_t i = 0; i < nq; i++) { - uint32_t len = resDist[i].size(); - printf("queryFeature(%d/%d), %u feature dist greater than the threshold:\r\n", i, nq, len); - for (uint32_t j = 0; j < len; j++) { - printf(" id: %u, dist: %.4lf\r\n", resIdx[i][j], resDist[i][j]); + try { + std::vector queryIdArr(nq); + std::iota(queryIdArr.begin(), queryIdArr.end(), start); + + bool aboveFilter = true; + std::vector> resDist(nq); + std::vector> resIdx(nq); + + ret = index.ComputeDistanceByThreshold(queryIdArr, codeStartIdx, codeNum, threshold, aboveFilter, resDist, resIdx); + if (ret != 0) { + printf("[ERROR] ComputeDistanceByThreshold fail ret = %d \r\n", ret); + return 1; } + + for (uint32_t i = 0; i < nq; i++) { + uint32_t len = resDist[i].size(); + printf("queryFeature(%d/%d), %u feature dist greater than the threshold:\r\n", i, nq, len); + for (uint32_t j = 0; j < len; j++) { + printf(" id: %u, dist: %.4lf\r\n", resIdx[i][j], resDist[i][j]); + } + } + + index.Finalize(); + } catch (std::exception &e) { + index.Finalize(); + printf("%s\n", e.what()); } - - index.Finalize(); } diff --git a/IndexSDK/TestAscendIndexFlat.cpp b/IndexSDK/TestAscendIndexFlat.cpp index 18f36c0ea5222ae1250e2ed9a96d17588f4455d8..e590e3d6fd4cd1931d64b106a0f68321e276498e 100644 --- a/IndexSDK/TestAscendIndexFlat.cpp +++ b/IndexSDK/TestAscendIndexFlat.cpp @@ -137,49 +137,52 @@ TEST(TestAscendIndexFlat, QPS) int dim = 512; size_t ntotal = 7000000; size_t maxSize = ntotal * dim; - - std::vector data(maxSize); - for (size_t i = 0; i < maxSize; i++) { - data[i] = 1.0 * FastRand() / FAST_RAND_MAX; - } - - faiss::ascend::AscendIndexFlatConfig conf({ 0 }, 1024 * 1024 * 1500); - faiss::ascend::AscendIndexFlat index(dim, faiss::METRIC_L2, conf); - index.verbose = true; - - // 标准化 - Norm(data.data(), ntotal, dim); - - index.add(ntotal, data.data()); - { - size_t getTotal = 0; - for (size_t i = 0; i < conf.deviceList.size(); i++) { - size_t tmpTotal = index.getBaseSize(conf.deviceList[i]); - getTotal += tmpTotal; + try { + std::vector data(maxSize); + for (size_t i = 0; i < maxSize; i++) { + data[i] = 1.0 * FastRand() / FAST_RAND_MAX; } - EXPECT_EQ(getTotal, ntotal); - } - int warmUpTimes = 10 ; - std::vector distw(127 * 10, 0); - std::vector labelw(127 * 10, 0); - for (int i = 0; i < warmUpTimes; i++) { - index.search(127, data.data(), 10, distw.data(), labelw.data()); + faiss::ascend::AscendIndexFlatConfig conf({ 0 }, 1024 * 1024 * 1500); + faiss::ascend::AscendIndexFlat index(dim, faiss::METRIC_L2, conf); + index.verbose = true; + + // 标准化 + Norm(data.data(), ntotal, dim); + + index.add(ntotal, data.data()); + { + size_t getTotal = 0; + for (size_t i = 0; i < conf.deviceList.size(); i++) { + size_t tmpTotal = index.getBaseSize(conf.deviceList[i]); + getTotal += tmpTotal; + } + EXPECT_EQ(getTotal, ntotal); } - - std::vector searchNum = { 8, 16, 32, 64, 128, 256}; - for (size_t n = 0; n < searchNum.size(); n++) { - int k = 10; - int loopTimes = 100; - std::vector dist(searchNum[n] * k, 0); - std::vector label(searchNum[n] * k, 0); - double ts = GetMillisecs(); - for (int i = 0; i < loopTimes; i++) { - index.search(searchNum[n], data.data(), k, dist.data(), label.data()); + + int warmUpTimes = 10 ; + std::vector distw(127 * 10, 0); + std::vector labelw(127 * 10, 0); + for (int i = 0; i < warmUpTimes; i++) { + index.search(127, data.data(), 10, distw.data(), labelw.data()); + } + + std::vector searchNum = { 8, 16, 32, 64, 128, 256}; + for (size_t n = 0; n < searchNum.size(); n++) { + int k = 10; + int loopTimes = 100; + std::vector dist(searchNum[n] * k, 0); + std::vector label(searchNum[n] * k, 0); + double ts = GetMillisecs(); + for (int i = 0; i < loopTimes; i++) { + index.search(searchNum[n], data.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], + MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); } - double te = GetMillisecs(); - printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], - MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + } catch (std::exception &e) { + printf("%s\n", e.what()); } } @@ -192,32 +195,36 @@ TEST(TestAscendIndexFlat, Acc) { int queryNum = 8; printf("generate data\n"); std::vector data(maxSize); - for (size_t i = 0; i < maxSize; i++) { - data[i] = 1.0 * FastRand() / FAST_RAND_MAX; - } - faiss::ascend::AscendIndexFlatConfig conf({ 0 }, 1024 * 1024 * 1500); - faiss::ascend::AscendIndexFlat index(dim, faiss::METRIC_L2, conf); - index.verbose = true; + try { + for (size_t i = 0; i < maxSize; i++) { + data[i] = 1.0 * FastRand() / FAST_RAND_MAX; + } + faiss::ascend::AscendIndexFlatConfig conf({ 0 }, 1024 * 1024 * 1500); + faiss::ascend::AscendIndexFlat index(dim, faiss::METRIC_L2, conf); + index.verbose = true; + + // 标准化 + Norm(data.data(), ntotal, dim); - // 标准化 - Norm(data.data(), ntotal, dim); - - index.add(ntotal, data.data()); - printf("start search by npu\n"); - std::vector dist(queryNum * topk, 0); - std::vector label(queryNum * topk, 0); - index.search(queryNum, data.data(), topk, dist.data(), label.data()); - - printf("start add by cpu\n"); - faiss::IndexFlat faissIndex(dim, type); - faissIndex.add(ntotal, data.data()); - std::vector cpuDist(queryNum * topk, 0); - std::vector cpuLabel(queryNum * topk, 0); - printf("start search by cpu\n"); - faissIndex.search(queryNum, data.data(), topk, cpuDist.data(), cpuLabel.data()); - recallMap top = calRecall(label, cpuLabel.data(), queryNum); - printf("Recall %d: @1 = %.2f, @10 = %.2f, @100 = %.2f \n", topk, - top[RECMAP_KEY_1], top[RECMAP_KEY_10], top[RECMAP_KEY_100]); + index.add(ntotal, data.data()); + printf("start search by npu\n"); + std::vector dist(queryNum * topk, 0); + std::vector label(queryNum * topk, 0); + index.search(queryNum, data.data(), topk, dist.data(), label.data()); + + printf("start add by cpu\n"); + faiss::IndexFlat faissIndex(dim, type); + faissIndex.add(ntotal, data.data()); + std::vector cpuDist(queryNum * topk, 0); + std::vector cpuLabel(queryNum * topk, 0); + printf("start search by cpu\n"); + faissIndex.search(queryNum, data.data(), topk, cpuDist.data(), cpuLabel.data()); + recallMap top = calRecall(label, cpuLabel.data(), queryNum); + printf("Recall %d: @1 = %.2f, @10 = %.2f, @100 = %.2f \n", topk, + top[RECMAP_KEY_1], top[RECMAP_KEY_10], top[RECMAP_KEY_100]); + } catch (std::exception &e) { + printf("%s\n", e.what()); + } } diff --git a/IndexSDK/TestAscendIndexGreat.cpp b/IndexSDK/TestAscendIndexGreat.cpp index 9e86496ea1101b25df463ca6be463eca4dd2db64..ccbb83f3a6ff2133fef3e231712263bff8bfb4e7 100644 --- a/IndexSDK/TestAscendIndexGreat.cpp +++ b/IndexSDK/TestAscendIndexGreat.cpp @@ -75,50 +75,70 @@ TEST(TestAscendIndexGreat, Test_KMode_QPS) int convPQM = 128; int evaluationType = 0; int expandingFactor = 300; - AscendIndexGreatInitParams kParams(dim, degree, convPQM, evaluationType, expandingFactor); - auto index = std::make_shared(kParams); - - // 生成base底库数据 - std::vector data(ntotal); - Generate(ntotal * dim, data); - // 标准化 - Norm(data, dim); + try { + AscendIndexGreatInitParams kParams(dim, degree, convPQM, evaluationType, expandingFactor); + auto index = std::make_shared(kParams); + + // 生成base底库数据 + std::vector data(ntotal); + Generate(ntotal * dim, data); + // 标准化 + Norm(data, dim); + + // add底库 + index->Add(data); + size_t total = 0; + index->GetNTotal(total); + EXPECT_EQ(total, ntotal); + + // search检索 + int topk = 100; + int warmUpTimes = 10; + size_t nq = 9000; + std::vector distsWarm(nq * topk); + std::vector labelsWarm(nq * topk); + + // warm up + for (int i = 0; i < warmUpTimes; ++i) { + AscendIndexSearchParams searchParamsWarm {100, data, topk, distsWarm, labelsWarm}; + index->Search(searchParamsWarm); + } + + // search + std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; + int loopTimes = 100; + for (auto n : searchNum) { + std::vector queryData(data.begin(), data.begin() + n * dim); + std::vector dists(n * topk, 0); + std::vector labels(n * topk, 0); + double ts = GetMillisecs(); + for (int i = 0; i < loopTimes; ++i) { + AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; + index->Search(searchParams); + } + double te = GetMillisecs(); + printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", + ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); + } + } catch (std::exception &e) { + printf("%s\n", e.what()); + } +} - // add底库 - index->Add(data); - size_t total = 0; - index->GetNTotal(total); - EXPECT_EQ(total, ntotal); +void search_warm(int topk, std::vector &data, std::shared_ptr &index) +{ + // search检索 - int topk = 100; int warmUpTimes = 10; size_t nq = 9000; std::vector distsWarm(nq * topk); std::vector labelsWarm(nq * topk); - // warm up for (int i = 0; i < warmUpTimes; ++i) { AscendIndexSearchParams searchParamsWarm {100, data, topk, distsWarm, labelsWarm}; index->Search(searchParamsWarm); } - - // search - std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; - int loopTimes = 100; - for (auto n : searchNum) { - std::vector queryData(data.begin(), data.begin() + n * dim); - std::vector dists(n * topk, 0); - std::vector labels(n * topk, 0); - double ts = GetMillisecs(); - for (int i = 0; i < loopTimes; ++i) { - AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; - index->Search(searchParams); - } - double te = GetMillisecs(); - printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", - ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); - } } /** @@ -137,59 +157,52 @@ TEST(TestAscendIndexGreat, Test_AKMode_QPS) int convPQM = 128; int evaluationType = 0; int expandingFactor = 300; - + int topk = 100; int nlist = 1024; int subSpaceDim = 128; std::vector devices = {0}; - AscendIndexGreatInitParams kParams(dim, degree, convPQM, evaluationType, expandingFactor); - AscendIndexVstarInitParams aParams(dim, subSpaceDim, nlist, devices); - auto index = std::make_shared(aParams, kParams); - - // 添加码本 需要提前生成好码本路径 - std::string codebook = "/home/work/codebook_1024_1024_128/codebook_l1_l2.bin"; - auto ret = index->AddCodeBooks(codebook); - EXPECT_EQ(ret, 0); - - // 生成base底库数据 - std::vector data(ntotal); - Generate(ntotal * dim, data); - // 标准化 - Norm(data, dim); - - // add底库 - index->Add(data); - size_t total = 0; - index->GetNTotal(total); - EXPECT_EQ(total, ntotal); + try { + AscendIndexGreatInitParams kParams(dim, degree, convPQM, evaluationType, expandingFactor); + AscendIndexVstarInitParams aParams(dim, subSpaceDim, nlist, devices); + auto index = std::make_shared(aParams, kParams); - // search检索 - int topk = 100; - int warmUpTimes = 10; - size_t nq = 9000; - std::vector distsWarm(nq * topk); - std::vector labelsWarm(nq * topk); - - // warm up - for (int i = 0; i < warmUpTimes; ++i) { - AscendIndexSearchParams searchParamsWarm {100, data, topk, distsWarm, labelsWarm}; - index->Search(searchParamsWarm); - } - - // search - std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; - int loopTimes = 100; - for (auto n : searchNum) { - std::vector queryData(data.begin(), data.begin() + n * dim); - std::vector dists(n * topk, 0); - std::vector labels(n * topk, 0); - double ts = GetMillisecs(); - for (int i = 0; i < loopTimes; ++i) { - AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; - index->Search(searchParams); + // 添加码本 需要提前生成好码本路径 + std::string codebook = "/home/work/codebook_1024_1024_128/codebook_l1_l2.bin"; + auto ret = index->AddCodeBooks(codebook); + EXPECT_EQ(ret, 0); + + // 生成base底库数据 + std::vector data(ntotal); + Generate(ntotal * dim, data); + // 标准化 + Norm(data, dim); + + // add底库 + index->Add(data); + size_t total = 0; + index->GetNTotal(total); + EXPECT_EQ(total, ntotal); + search_warm(topk, data, index); + + + // search + std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; + int loopTimes = 100; + for (auto n : searchNum) { + std::vector queryData(data.begin(), data.begin() + n * dim); + std::vector dists(n * topk, 0); + std::vector labels(n * topk, 0); + double ts = GetMillisecs(); + for (int i = 0; i < loopTimes; ++i) { + AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; + index->Search(searchParams); + } + double te = GetMillisecs(); + printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", + ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); } - double te = GetMillisecs(); - printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", - ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); + } catch (std::exception &e) { + printf("%s\n", e.what()); } } } // namespace diff --git a/IndexSDK/TestAscendIndexIVFSP.cpp b/IndexSDK/TestAscendIndexIVFSP.cpp index 9dfb3162b8a74d49c1020c61f175cabb3db97279..b338e66346e7be6b4fb9d609c1776a604a7087d6 100644 --- a/IndexSDK/TestAscendIndexIVFSP.cpp +++ b/IndexSDK/TestAscendIndexIVFSP.cpp @@ -408,41 +408,68 @@ void RecallAndRecallFilter() // query data std::vector qData(QUERY_NUM * dim); std::vector gts(QUERY_NUM * TOPK, 0); - InitData(data, qData, gts, dim, ntotal); + try { + InitData(data, qData, gts, dim, ntotal); + + faiss::ascend::AscendIndexIVFSPConfig conf({0}); + conf.handleBatch = handleBatch; // 和OM算子保持一致 + conf.nprobe = handleBatch; // 64 32 128 16的倍数，且0 < nprobe <= nlist + conf.searchListSize = searchListSize; // 大于等于512 且为2的幂次。 + conf.filterable = true; + + faiss::ascend::AscendIndexIVFSP index(dim, nonzeroNum, nlist, + codeBookPath.c_str(), + faiss::ScalarQuantizer::QuantizerType::QT_8bit, + faiss::MetricType::METRIC_L2, conf); + index.setVerbose(true); + index.add(ntotal, data.data()); + std::cout << "index.ntotal: " << index.ntotal << std::endl; + + std::vector nprobeList = {handleBatch, handleBatch * 2, handleBatch / 2}; + for (int tmpNprobe : nprobeList) { + printf("-------------set nprobe: %d-------------------\n", tmpNprobe); + index.setNumProbes(tmpNprobe); + SearchData(index, batches, qData, dim, gts); + SearchFilter(index, batches, qData, dim, gts); + } + } catch (std::exception &e) { + printf("%s\n", e.what()); + } +} - faiss::ascend::AscendIndexIVFSPConfig conf({0}); - conf.handleBatch = handleBatch; // 和OM算子保持一致 - conf.nprobe = handleBatch; // 64 32 128 16的倍数，且0 < nprobe <= nlist - conf.searchListSize = searchListSize; // 大于等于512 且为2的幂次。 - conf.filterable = true; +void CreateMultiIndex(std::vector &indexes, + int dim, faiss::ascend::AscendIndexIVFSPConfig &conf) +{ + int nonzeroNum = 64; + int nlist = 256; + std::string basePath = " "; + // codeBook 码本 + std::string codeBookPath = basePath + "codebook.bin"; + for (int i = 0; i < INDEX_NUM; ++i) { + faiss::ascend::AscendIndexIVFSP* index; + if (i == 0) { + index = new faiss::ascend::AscendIndexIVFSP(dim, nonzeroNum, nlist, + codeBookPath.c_str(), faiss::ScalarQuantizer::QuantizerType::QT_8bit, + faiss::MetricType::METRIC_L2, conf); + } else { + index = new faiss::ascend::AscendIndexIVFSP(dim, nonzeroNum, nlist, + *(faiss::ascend::AscendIndexIVFSP*)indexes[0], faiss::ScalarQuantizer::QuantizerType::QT_8bit, + faiss::MetricType::METRIC_L2, conf); + } - faiss::ascend::AscendIndexIVFSP index(dim, nonzeroNum, nlist, - codeBookPath.c_str(), - faiss::ScalarQuantizer::QuantizerType::QT_8bit, - faiss::MetricType::METRIC_L2, conf); - index.setVerbose(true); - index.add(ntotal, data.data()); - std::cout << "index.ntotal: " << index.ntotal << std::endl; - - std::vector nprobeList = {handleBatch, handleBatch * 2, handleBatch / 2}; - for (int tmpNprobe : nprobeList) { - printf("-------------set nprobe: %d-------------------\n", tmpNprobe); - index.setNumProbes(tmpNprobe); - SearchData(index, batches, qData, dim, gts); - SearchFilter(index, batches, qData, dim, gts); + index->setVerbose(true); + indexes.emplace_back(index); + printf("create index:%d\n", i); } } void MultiSearchAndMultiSearchFilter() { // 数据集（特征数据、查询数据、groundtruth数据）、码本， 所在的目录。请根据实际情况填写。 - std::string basePath = " "; - // codeBook 码本 - std::string codeBookPath = basePath + "codebook.bin"; + // 参数值 dim、nlist、nonzeroNum、searchListSize，应该和使用的codeBook 码本保持一致，即和训练码本时指定的参数保持一致。 int dim = 256; - int nonzeroNum = 64; - int nlist = 256; + int handleBatch = 64; std::vector batches = {1, 2, 4, 8, 16, 32, 64}; int searchListSize = 32768; @@ -465,40 +492,32 @@ void MultiSearchAndMultiSearchFilter() conf.filterable = true; std::vector indexes; - for (int i = 0; i < INDEX_NUM; ++i) { - faiss::ascend::AscendIndexIVFSP* index; - if (i == 0) { - index = new faiss::ascend::AscendIndexIVFSP(dim, nonzeroNum, nlist, - codeBookPath.c_str(), faiss::ScalarQuantizer::QuantizerType::QT_8bit, - faiss::MetricType::METRIC_L2, conf); - } else { - index = new faiss::ascend::AscendIndexIVFSP(dim, nonzeroNum, nlist, - *(faiss::ascend::AscendIndexIVFSP*)indexes[0], faiss::ScalarQuantizer::QuantizerType::QT_8bit, - faiss::MetricType::METRIC_L2, conf); + try { + CreateMultiIndex(indexes, dim, conf); + + LoadAndSaveData(indexes, ntotal, data); + + std::vector nprobeList = {handleBatch, handleBatch * 2, handleBatch / 2}; + for (int tmpNprobe : nprobeList) { + printf("-------------set nprobe: %d-------------------\n", tmpNprobe); + for (int i = 0; i < INDEX_NUM; ++i) { + faiss::ascend::AscendIndexIVFSP* index = dynamic_cast(indexes[i]); + index->setNumProbes(tmpNprobe); + } + + MultiSearch(indexes, batches, qData, dim, gts); + MultiSearchWithSameFilter(indexes, batches, qData, dim, gts); + MultiSearchWithDifFilter(indexes, batches, qData, dim, gts); } - - index->setVerbose(true); - indexes.emplace_back(index); - printf("create index:%d\n", i); - } - - LoadAndSaveData(indexes, ntotal, data); - - std::vector nprobeList = {handleBatch, handleBatch * 2, handleBatch / 2}; - for (int tmpNprobe : nprobeList) { - printf("-------------set nprobe: %d-------------------\n", tmpNprobe); + for (int i = 0; i < INDEX_NUM; ++i) { - faiss::ascend::AscendIndexIVFSP* index = dynamic_cast(indexes[i]); - index->setNumProbes(tmpNprobe); + delete indexes[i]; } - - MultiSearch(indexes, batches, qData, dim, gts); - MultiSearchWithSameFilter(indexes, batches, qData, dim, gts); - MultiSearchWithDifFilter(indexes, batches, qData, dim, gts); - } - - for (int i = 0; i < INDEX_NUM; ++i) { - delete indexes[i]; + } catch (std::exception &e) { + for (int i = 0; i < INDEX_NUM; ++i) { + delete indexes[i]; + } + printf("%s\n", e.what()); } } diff --git a/IndexSDK/TestAscendIndexIVFSQTwithCpuFlat.cpp b/IndexSDK/TestAscendIndexIVFSQTwithCpuFlat.cpp index ace202096de29c5e8f66beb9cd36d7bbee8e9b2d..4068f04b80e7942ba1eccd7bab334a8e5afcc8c8 100644 --- a/IndexSDK/TestAscendIndexIVFSQTwithCpuFlat.cpp +++ b/IndexSDK/TestAscendIndexIVFSQTwithCpuFlat.cpp @@ -165,7 +165,7 @@ void SearchProccess(faiss::ascend::AscendIndexIVFSQT &index, size_t ntotal, std: { int fuzzyK = 3; float threshold = 1.6; - int trainSize = 500000; + int trainSize = learn.size() / DIM_IN; int topk = 100; int searchNum = 10240; index.verbose = true; @@ -207,51 +207,51 @@ void TestIVFSQT(int niter, int ncentroids) size_t learnNum = ntotal / 10; int gtNum = 100; int centroid = 256; - std::vector devices = { 0 }; - int64_t resourceSize = static_cast(1024 * 1024 * 1024); - faiss::ascend::AscendIndexIVFSQTConfig conf(devices, resourceSize); - conf.cp.niter = niter; - conf.useKmeansPP = true; - conf.cp.max_points_per_centroid = centroid; - faiss::ascend::AscendIndexIVFSQT index(DIM_IN, DIM_OUT, ncentroids, faiss::ScalarQuantizer::QuantizerType::QT_8bit, - faiss::METRIC_INNER_PRODUCT, conf); - printf("start generate data\n"); - std::vector baseInt8(ntotal * DIM_IN); - for (size_t i = 0; i < ntotal * DIM_IN; i++) { - baseInt8[i] = RandomInt8(); - } - std::vector learnInt8(learnNum * DIM_IN); - for (size_t i = 0; i < learnNum * DIM_IN; i++) { - learnInt8[i] = RandomInt8(); - } - std::vector queryInt8(queryNum * DIM_IN); - std::vector gt(queryNum * gtNum, 0); - for (size_t q = 0; q < queryNum; q++) { - size_t idx; - GeneratorRandomIndex(ntotal, idx); - for (size_t d = 0; d < DIM_IN; d++) { - queryInt8[q * DIM_IN + d] = baseInt8[idx * DIM_IN + d]; + try { + // resource size 1024 * 1024 * 1024 = 1GB + faiss::ascend::AscendIndexIVFSQTConfig conf({ 0 }, static_cast(1024 * 1024 * 1024)); + conf.cp.niter = niter; + conf.useKmeansPP = true; + conf.cp.max_points_per_centroid = centroid; + faiss::ascend::AscendIndexIVFSQT index(DIM_IN, DIM_OUT, ncentroids, + faiss::ScalarQuantizer::QuantizerType::QT_8bit, faiss::METRIC_INNER_PRODUCT, conf); + std::vector baseInt8(ntotal * DIM_IN); + for (size_t i = 0; i < ntotal * DIM_IN; i++) { + baseInt8[i] = RandomInt8(); } - gt[q * gtNum] = static_cast(idx); - } - - // int8 to float，除以128.0是为了将其映射到-1.0到1.0的区间内。 - float intToFloat = 128.0; - std::vector base(ntotal * DIM_IN); - for (size_t i = 0; i < ntotal * DIM_IN; i++) { - base[i] = static_cast(baseInt8[i]) / intToFloat; - } - std::vector query(queryNum * DIM_IN); - for (size_t i = 0; i < queryNum * DIM_IN; i++) { - query[i] = static_cast(queryInt8[i]) / intToFloat; - } - std::vector learn(learnNum * DIM_IN); - for (size_t i = 0; i < learnNum * DIM_IN; i++) { - learn[i] = static_cast(learnInt8[i]) / intToFloat; + std::vector learnInt8(learnNum * DIM_IN); + for (size_t i = 0; i < learnNum * DIM_IN; i++) { + learnInt8[i] = RandomInt8(); + } + std::vector queryInt8(queryNum * DIM_IN); + std::vector gt(queryNum * gtNum, 0); + for (size_t q = 0; q < queryNum; q++) { + size_t idx; + GeneratorRandomIndex(ntotal, idx); + for (size_t d = 0; d < DIM_IN; d++) { + queryInt8[q * DIM_IN + d] = baseInt8[idx * DIM_IN + d]; + } + gt[q * gtNum] = static_cast(idx); + } + // int8 to float，除以128.0是为了将其映射到-1.0到1.0的区间内。 + float intToFloat = 128.0; + std::vector base(ntotal * DIM_IN); + for (size_t i = 0; i < ntotal * DIM_IN; i++) { + base[i] = static_cast(baseInt8[i]) / intToFloat; + } + std::vector query(queryNum * DIM_IN); + for (size_t i = 0; i < queryNum * DIM_IN; i++) { + query[i] = static_cast(queryInt8[i]) / intToFloat; + } + std::vector learn(learnNum * DIM_IN); + for (size_t i = 0; i < learnNum * DIM_IN; i++) { + learn[i] = static_cast(learnInt8[i]) / intToFloat; + } + dataFloat dataBaseFloat(base, learn, query, gt); + SearchProccess(index, ntotal, base, learn, dataBaseFloat); + } catch (std::exception &e) { + printf("%s\n", e.what()); } - dataFloat dataBaseFloat(base, learn, query, gt); - printf("generate data ok\n"); - SearchProccess(index, ntotal, base, learn, dataBaseFloat); } } // namespace diff --git a/IndexSDK/TestAscendIndexInt8Flat.cpp b/IndexSDK/TestAscendIndexInt8Flat.cpp index 25fa8f87b5365bcbcae4db932137b168e163b559..40cced620cb0375af71e6062f82364e232d1553c 100644 --- a/IndexSDK/TestAscendIndexInt8Flat.cpp +++ b/IndexSDK/TestAscendIndexInt8Flat.cpp @@ -119,36 +119,39 @@ TEST(TestAscendIndexInt8Flat, QPS) int dim = 512; size_t ntotal = 7000000; std::vector searchNum = { 8, 16, 32, 64, 128, 256 }; - - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); - faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_L2, conf); - index.verbose = true; - - printf("generate data\n"); - std::vector base(ntotal * dim); - GenerateCodes(base.data(), ntotal, dim); - - printf("add data\n"); - index.add(ntotal, base.data()); - int warmUpTimes = 10 ; - std::vector distw(127 * 10, 0); - std::vector labelw(127 * 10, 0); - for (int i = 0; i < warmUpTimes; i++) { - index.search(127, base.data(), 10, distw.data(), labelw.data()); - } + try { + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); + faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_L2, conf); + index.verbose = true; + + printf("generate data\n"); + std::vector base(ntotal * dim); + GenerateCodes(base.data(), ntotal, dim); - for (size_t n = 0; n < searchNum.size(); n++) { - int k = 128; - int loopTimes = 10; - std::vector dist(searchNum[n] * k, 0); - std::vector label(searchNum[n] * k, 0); - double ts = GetMillisecs(); - for (int l = 0; l < loopTimes; l++) { - index.search(searchNum[n], base.data(), k, dist.data(), label.data()); + printf("add data\n"); + index.add(ntotal, base.data()); + int warmUpTimes = 10 ; + std::vector distw(127 * 10, 0); + std::vector labelw(127 * 10, 0); + for (int i = 0; i < warmUpTimes; i++) { + index.search(127, base.data(), 10, distw.data(), labelw.data()); } - double te = GetMillisecs(); - printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], - MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + + for (size_t n = 0; n < searchNum.size(); n++) { + int k = 128; + int loopTimes = 10; + std::vector dist(searchNum[n] * k, 0); + std::vector label(searchNum[n] * k, 0); + double ts = GetMillisecs(); + for (int l = 0; l < loopTimes; l++) { + index.search(searchNum[n], base.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], + MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + } + } catch (std::exception &e) { + printf("%s\n", e.what()); } } @@ -157,49 +160,52 @@ TEST(TestAscendIndexInt8Flat, Acc) size_t dim = 512; size_t ntotal = 1000000; int searchNum = 8; - - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); - faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_L2, conf); - index.verbose = true; - - printf("generate data\n"); - std::vector base(ntotal * dim); - GenerateCodes(base.data(), ntotal, dim); - - printf("add data\n"); - index.add(ntotal, base.data()); - printf("add finish\n"); - size_t k = 100; - std::vector dist(searchNum * k, 0); - std::vector label(searchNum * k, 0); - printf("search start\n"); - index.search(searchNum, base.data(), k, dist.data(), label.data()); - printf("search finish\n"); - printf("search compute distance by cpu\n"); - std::vector gtLabel; - for (int q = 0; q < searchNum; q++) { - std::vector cpuDist(ntotal, 0); - for (size_t i = 0; i < ntotal; i++) { - int sum = 0; - for (size_t d = 0; d < dim; d++) { - sum += (base[q * dim +d] - base[i * dim + d]) * (base[q * dim +d] - base[i * dim + d]); + try { + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); + faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_L2, conf); + index.verbose = true; + + printf("generate data\n"); + std::vector base(ntotal * dim); + GenerateCodes(base.data(), ntotal, dim); + + printf("add data\n"); + index.add(ntotal, base.data()); + printf("add finish\n"); + size_t k = 100; + std::vector dist(searchNum * k, 0); + std::vector label(searchNum * k, 0); + printf("search start\n"); + index.search(searchNum, base.data(), k, dist.data(), label.data()); + printf("search finish\n"); + printf("search compute distance by cpu\n"); + std::vector gtLabel; + for (int q = 0; q < searchNum; q++) { + std::vector cpuDist(ntotal, 0); + for (size_t i = 0; i < ntotal; i++) { + int sum = 0; + for (size_t d = 0; d < dim; d++) { + sum += (base[q * dim +d] - base[i * dim + d]) * (base[q * dim +d] - base[i * dim + d]); + } + cpuDist[i] = sqrt(sum); } - cpuDist[i] = sqrt(sum); - } - std::vector> cpuRes; - for (size_t i = 0; i < ntotal; ++i) { - cpuRes.push_back({i, cpuDist[i]}); - } - std::sort(cpuRes.begin(), cpuRes.end(), - [](std::pair a, std::pair b) -> bool {return a.second < b.second;}); - for (size_t d = 0; d < k; ++d) { - gtLabel.push_back(cpuRes[d].first); - } + std::vector> cpuRes; + for (size_t i = 0; i < ntotal; ++i) { + cpuRes.push_back({i, cpuDist[i]}); + } + std::sort(cpuRes.begin(), cpuRes.end(), + [](std::pair a, std::pair b) -> bool {return a.second < b.second;}); + for (size_t d = 0; d < k; ++d) { + gtLabel.push_back(cpuRes[d].first); + } + } + recallMap top = calRecall(label, gtLabel.data(), searchNum); + + printf("Recall %zu: @1 = %.2f, @10 = %.2f, @100 = %.2f \n", k, + top[RECMAP_KEY_1], top[RECMAP_KEY_10], top[RECMAP_KEY_100]); + } catch (std::exception &e) { + printf("%s\n", e.what()); } - recallMap top = calRecall(label, gtLabel.data(), searchNum); - - printf("Recall %zu: @1 = %.2f, @10 = %.2f, @100 = %.2f \n", k, - top[RECMAP_KEY_1], top[RECMAP_KEY_10], top[RECMAP_KEY_100]); } } // namespace diff --git a/IndexSDK/TestAscendIndexInt8FlatWithReduction.cpp b/IndexSDK/TestAscendIndexInt8FlatWithReduction.cpp index 4ff15f7a68ae3b1ec4e6714bcfbaa9fe5ea1836a..745fd239fa5ee78a574f44a087ce2cd9a4a89a87 100644 --- a/IndexSDK/TestAscendIndexInt8FlatWithReduction.cpp +++ b/IndexSDK/TestAscendIndexInt8FlatWithReduction.cpp @@ -100,39 +100,43 @@ TEST(TestAscendIndexInt8Flat, QPS) int dimIn = 256; int dimOut = 64; size_t ntotal = 7000000; - std::vector baseInt8(ntotal * dimOut); - std::vector queryInt8(SEARCH_NUM * dimOut); - - ReduceData(ntotal, dimIn, dimOut, baseInt8, queryInt8); - - // create index - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); - faiss::ascend::AscendIndexInt8Flat index(dimOut, faiss::METRIC_INNER_PRODUCT, conf); - index.verbose = true; - - printf("add data\n"); - index.add(ntotal, baseInt8.data()); - - int warmUpTimes = 10 ; - std::vector distw(127 * 10, 0); - std::vector labelw(127 * 10, 0); - for (int i = 0; i < warmUpTimes; i++) { - index.search(127, baseInt8.data(), 10, distw.data(), labelw.data()); - } + try { + std::vector baseInt8(ntotal * dimOut); + std::vector queryInt8(SEARCH_NUM * dimOut); - int k = 100; - int loopTimes = 10; - std::vector dist(SEARCH_NUM * k, 0); - std::vector label(SEARCH_NUM * k, 0); - - double ts = GetMillisecs(); - for (int l = 0; l < loopTimes; l++) { - // query data reduction and sq - index.search(SEARCH_NUM, queryInt8.data(), k, dist.data(), label.data()); + ReduceData(ntotal, dimIn, dimOut, baseInt8, queryInt8); + + // create index + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); + faiss::ascend::AscendIndexInt8Flat index(dimOut, faiss::METRIC_INNER_PRODUCT, conf); + index.verbose = true; + + printf("add data\n"); + index.add(ntotal, baseInt8.data()); + + int warmUpTimes = 10 ; + std::vector distw(127 * 10, 0); + std::vector labelw(127 * 10, 0); + for (int i = 0; i < warmUpTimes; i++) { + index.search(127, baseInt8.data(), 10, distw.data(), labelw.data()); + } + + int k = 100; + int loopTimes = 10; + std::vector dist(SEARCH_NUM * k, 0); + std::vector label(SEARCH_NUM * k, 0); + + double ts = GetMillisecs(); + for (int l = 0; l < loopTimes; l++) { + // query data reduction and sq + index.search(SEARCH_NUM, queryInt8.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + printf("base:%zu, dim:%d, search num:%d, topk:%d, QPS:%.4f\n", ntotal, dimOut, SEARCH_NUM, + k, MILLI_SECOND * SEARCH_NUM * loopTimes / (te - ts)); + } catch (std::exception &e) { + printf("%s\n", e.what()); } - double te = GetMillisecs(); - printf("base:%zu, dim:%d, search num:%d, topk:%d, QPS:%.4f\n", ntotal, dimOut, SEARCH_NUM, - k, MILLI_SECOND * SEARCH_NUM * loopTimes / (te - ts)); } } // namespace diff --git a/IndexSDK/TestAscendIndexInt8FlatWithSQ.cpp b/IndexSDK/TestAscendIndexInt8FlatWithSQ.cpp index 378dc8ebda837607c113ecc93c41410fe236f7b1..f7d78fdd6a5371e546061fa5f97cce1e2bfc3545 100644 --- a/IndexSDK/TestAscendIndexInt8FlatWithSQ.cpp +++ b/IndexSDK/TestAscendIndexInt8FlatWithSQ.cpp @@ -107,47 +107,50 @@ TEST(TestAscendIndexInt8Flat, QPS) int dim = 512; size_t ntotal = 7000000; std::vector searchNum = { 8, 16, 32, 64, 128, 256 }; - - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); - faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_INNER_PRODUCT, conf); - index.verbose = true; - - printf("start generate data\n"); - std::vector base(ntotal * dim); - std::vector baseFp(ntotal * dim); - for (size_t i = 0; i < ntotal * dim; i++) { - baseFp[i] = drand48(); - } - printf("generate data finished\n"); - faiss::ScalarQuantizer sq = faiss::ScalarQuantizer(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit); - sq.train(ntotal, baseFp.data()); - sqEncode(sq, baseFp.data(), base.data(), ntotal); - printf("add data\n"); - index.add(ntotal, base.data()); - int warmUpTimes = 10 ; - std::vector distw(127 * 10, 0); - std::vector labelw(127 * 10, 0); - for (int i = 0; i < warmUpTimes; i++) { - index.search(127, base.data(), 10, distw.data(), labelw.data()); - } - - for (size_t n = 0; n < searchNum.size(); n++) { - int k = 100; - int loopTimes = 10; - - std::vector dist(searchNum[n] * k, 0); - std::vector label(searchNum[n] * k, 0); - double ts = GetMillisecs(); - // start to quantize query data - std::vector query(searchNum[n] * dim); - sqEncode(sq, baseFp.data(), query.data(), searchNum[n]); - - for (int l = 0; l < loopTimes; l++) { - index.search(searchNum[n], query.data(), k, dist.data(), label.data()); + try { + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); + faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_INNER_PRODUCT, conf); + index.verbose = true; + + printf("start generate data\n"); + std::vector base(ntotal * dim); + std::vector baseFp(ntotal * dim); + for (size_t i = 0; i < ntotal * dim; i++) { + baseFp[i] = drand48(); + } + printf("generate data finished\n"); + faiss::ScalarQuantizer sq = faiss::ScalarQuantizer(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit); + sq.train(ntotal, baseFp.data()); + sqEncode(sq, baseFp.data(), base.data(), ntotal); + printf("add data\n"); + index.add(ntotal, base.data()); + int warmUpTimes = 10 ; + std::vector distw(127 * 10, 0); + std::vector labelw(127 * 10, 0); + for (int i = 0; i < warmUpTimes; i++) { + index.search(127, base.data(), 10, distw.data(), labelw.data()); } - double te = GetMillisecs(); - printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], - MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + + for (size_t n = 0; n < searchNum.size(); n++) { + int k = 100; + int loopTimes = 10; + + std::vector dist(searchNum[n] * k, 0); + std::vector label(searchNum[n] * k, 0); + double ts = GetMillisecs(); + // start to quantize query data + std::vector query(searchNum[n] * dim); + sqEncode(sq, baseFp.data(), query.data(), searchNum[n]); + + for (int l = 0; l < loopTimes; l++) { + index.search(searchNum[n], query.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + printf("case[%zu]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", n, ntotal, dim, searchNum[n], + MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); + } + } catch (std::exception &e) { + printf("%s\n", e.what()); } } @@ -156,38 +159,41 @@ TEST(TestAscendIndexInt8Flat, Recall) int dim = 512; size_t ntotal = 7000000; int searchNum = 8; - - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); - faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_INNER_PRODUCT, conf); - index.verbose = true; - - printf("start generate data\n"); - std::vector base(ntotal * dim); - std::vector baseFp(ntotal * dim); - for (size_t i = 0; i < ntotal * dim; i++) { - baseFp[i] = drand48(); - } - printf("generate data finished\n"); - faiss::ScalarQuantizer sq = faiss::ScalarQuantizer(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit); - sq.train(ntotal, baseFp.data()); - sqEncode(sq, baseFp.data(), base.data(), ntotal); - printf("add data\n"); - index.add(ntotal, base. - data()); - - int k = 100; - std::vector gt(searchNum * k, 0); - for (int i = 0; i < searchNum; i++) { - gt[i * k] = i; + try { + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }, 1024 * 1024 * 1024); + faiss::ascend::AscendIndexInt8Flat index(dim, faiss::METRIC_INNER_PRODUCT, conf); + index.verbose = true; + + printf("start generate data\n"); + std::vector base(ntotal * dim); + std::vector baseFp(ntotal * dim); + for (size_t i = 0; i < ntotal * dim; i++) { + baseFp[i] = drand48(); + } + printf("generate data finished\n"); + faiss::ScalarQuantizer sq = faiss::ScalarQuantizer(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit); + sq.train(ntotal, baseFp.data()); + sqEncode(sq, baseFp.data(), base.data(), ntotal); + printf("add data\n"); + index.add(ntotal, base. + data()); + + int k = 100; + std::vector gt(searchNum * k, 0); + for (int i = 0; i < searchNum; i++) { + gt[i * k] = i; + } + std::vector dist(searchNum * k, 0); + std::vector label(searchNum * k, 0); + // start to quantize query data + std::vector query(searchNum * dim); + sqEncode(sq, baseFp.data(), query.data(), searchNum); + index.search(searchNum, query.data(), k, dist.data(), label.data()); + recallMap Top = calRecall(label, gt.data(), searchNum); + printf("TOPK %d: t1 = %.2f, t10 = %.2f, t100 = %.2f\n", k, Top[1], Top[10], Top[100]); + } catch (std::exception &e) { + printf("%s\n", e.what()); } - std::vector dist(searchNum * k, 0); - std::vector label(searchNum * k, 0); - // start to quantize query data - std::vector query(searchNum * dim); - sqEncode(sq, baseFp.data(), query.data(), searchNum); - index.search(searchNum, query.data(), k, dist.data(), label.data()); - recallMap Top = calRecall(label, gt.data(), searchNum); - printf("TOPK %d: t1 = %.2f, t10 = %.2f, t100 = %.2f\n", k, Top[1], Top[10], Top[100]); } } // namespace diff --git a/IndexSDK/TestAscendIndexSQ.cpp b/IndexSDK/TestAscendIndexSQ.cpp index 404dc190ae0941ac969ece6ffd5274ce0af7e882..7459d92aa6a8f4e560fae9638e38110e046eed5b 100644 --- a/IndexSDK/TestAscendIndexSQ.cpp +++ b/IndexSDK/TestAscendIndexSQ.cpp @@ -83,61 +83,66 @@ void Norm(float *data, size_t n, int dim) } } +size_t search_warm(faiss::ascend::AscendIndexSQ &index, faiss::ascend::AscendIndexSQConfig &conf) +{ + size_t getTotal = 0; + for (size_t k = 0; k < conf.deviceList.size(); k++) { + size_t tmpTotal = index.getBaseSize(conf.deviceList[k]); + getTotal += tmpTotal; + } + return getTotal; +} TEST(TestAscendIndexSQ, QPS) { std::vector dim = { 256, 512 }; std::vector ntotal = { 7000000 }; std::vector searchNum = { 1, 2, 4, 8, 16, 32, 48, 64, 96 }; - - size_t maxSize = ntotal.back() * dim.back(); - std::vector data(maxSize); - for (size_t i = 0; i < maxSize; i++) { - data[i] = 1.0 * FastRand() / FAST_RAND_MAX; - } + try { + size_t maxSize = ntotal.back() * dim.back(); + std::vector data(maxSize); + for (size_t i = 0; i < maxSize; i++) { + data[i] = 1.0 * FastRand() / FAST_RAND_MAX; + } + + Norm(data.data(), ntotal.back(), dim.back()); - Norm(data.data(), ntotal.back(), dim.back()); - - for (size_t i = 0; i < dim.size(); i++) { - faiss::ascend::AscendIndexSQConfig conf({ 0 }, 1024 * 1024 * 1500); - faiss::ascend::AscendIndexSQ index(dim[i], faiss::ScalarQuantizer::QuantizerType::QT_8bit, faiss::METRIC_L2, - conf); - - for (size_t j = 0; j < ntotal.size(); j++) { - index.reset(); - for (auto deviceId : conf.deviceList) { - int len = index.getBaseSize(deviceId); - ASSERT_EQ(len, 0); - } - - index.train(ntotal[j], data.data()); - index.add(ntotal[j], data.data()); - { - size_t getTotal = 0; - for (size_t k = 0; k < conf.deviceList.size(); k++) { - size_t tmpTotal = index.getBaseSize(conf.deviceList[k]); - getTotal += tmpTotal; + for (size_t i = 0; i < dim.size(); i++) { + faiss::ascend::AscendIndexSQConfig conf({ 0 }, 1024 * 1024 * 1500); + faiss::ascend::AscendIndexSQ index(dim[i], faiss::ScalarQuantizer::QuantizerType::QT_8bit, faiss::METRIC_L2, + conf); + + for (size_t j = 0; j < ntotal.size(); j++) { + index.reset(); + for (auto deviceId : conf.deviceList) { + int len = index.getBaseSize(deviceId); + ASSERT_EQ(len, 0); } - EXPECT_EQ(getTotal, ntotal[j]); - } - - { - for (size_t n = 0; n < searchNum.size(); n++) { - int k = 100; - int loopTimes = 100; - std::vector dist(searchNum[n] * k, 0); - std::vector label(searchNum[n] * k, 0); - double ts = GetMillisecs(); - for (int l = 0; l < loopTimes; l++) { - index.search(searchNum[n], data.data(), k, dist.data(), label.data()); + + index.train(ntotal[j], data.data()); + index.add(ntotal[j], data.data()); + EXPECT_EQ(search_warm(index, conf), ntotal[j]); + + { + for (size_t n = 0; n < searchNum.size(); n++) { + int k = 100; + int loopTimes = 100; + std::vector dist(searchNum[n] * k, 0); + std::vector label(searchNum[n] * k, 0); + double ts = GetMillisecs(); + for (int l = 0; l < loopTimes; l++) { + index.search(searchNum[n], data.data(), k, dist.data(), label.data()); + } + double te = GetMillisecs(); + int cases = i * ntotal.size() * searchNum.size() + j * searchNum.size() + n; + printf("case[%d]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", cases, ntotal[j], dim[i], + searchNum[n], MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); } - double te = GetMillisecs(); - int cases = i * ntotal.size() * searchNum.size() + j * searchNum.size() + n; - printf("case[%d]: base:%zu, dim:%d, search num:%d, QPS:%.4f\n", cases, ntotal[j], dim[i], - searchNum[n], MILLI_SECOND * searchNum[n] * loopTimes / (te - ts)); } } } + } catch (std::exception &e) { + printf("%s\n", e.what()); } } } // namespace diff --git a/IndexSDK/TestAscendIndexVStar.cpp b/IndexSDK/TestAscendIndexVStar.cpp index d6e5d3daaa2abb1f4488937f45c615d0212971ae..fa6a05572bd932405d93cc157e27ae9adec18640 100644 --- a/IndexSDK/TestAscendIndexVStar.cpp +++ b/IndexSDK/TestAscendIndexVStar.cpp @@ -83,54 +83,59 @@ TEST(TestAscendIndexVstar, Test_Search_Func) int subSpaceDim = 128; std::vector devices = {0}; bool verbose = false; - AscendIndexVstarInitParams aParams(dim, subSpaceDim, nlist, devices, verbose); - auto index = std::make_shared(aParams); + try { - // 添加码本 需要提前生成好码本路径 - std::string codebook = "/home/work/codebook_256_1024_128/codebook_l1_l2.bin"; - auto ret = index->AddCodeBooksByPath(codebook); - EXPECT_EQ(ret, 0); - - // 生成base底库数据 - std::vector data(ntotal); - Generate(ntotal * dim, data); - // 标准化 - Norm(data, dim); - - // add底库 - index->Add(data); - size_t total = 0; - index->GetNTotal(total); - EXPECT_EQ(total, ntotal); + AscendIndexVstarInitParams aParams(dim, subSpaceDim, nlist, devices, verbose); + auto index = std::make_shared(aParams); - // search检索 - int topk = 100; - int warmUpTimes = 10; - size_t nq = 9000; - std::vector distsWarm(nq * topk); - std::vector labelsWarm(nq * topk); - - // warm up - for (int i = 0; i < warmUpTimes; ++i) { - AscendIndexSearchParams searchParamsWarm {100, data, topk, distsWarm, labelsWarm}; - index->Search(searchParamsWarm); - } - - // search - std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; - int loopTimes = 100; - for (auto n : searchNum) { - std::vector queryData(data.begin(), data.begin() + n * dim); - std::vector dists(n * topk, 0); - std::vector labels(n * topk, 0); - double ts = GetMillisecs(); - for (int i = 0; i < loopTimes; ++i) { - AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; - index->Search(searchParams); + // 添加码本 需要提前生成好码本路径 + std::string codebook = "/home/work/codebook_256_1024_128/codebook_l1_l2.bin"; + auto ret = index->AddCodeBooksByPath(codebook); + EXPECT_EQ(ret, 0); + + // 生成base底库数据 + std::vector data(ntotal); + Generate(ntotal * dim, data); + // 标准化 + Norm(data, dim); + + // add底库 + index->Add(data); + size_t total = 0; + index->GetNTotal(total); + EXPECT_EQ(total, ntotal); + + // search检索 + int topk = 100; + int warmUpTimes = 10; + size_t nq = 9000; + std::vector distsWarm(nq * topk); + std::vector labelsWarm(nq * topk); + + // warm up + for (int i = 0; i < warmUpTimes; ++i) { + AscendIndexSearchParams searchParamsWarm {100, data, topk, distsWarm, labelsWarm}; + index->Search(searchParamsWarm); + } + + // search + std::vector searchNum = {1, 8, 16, 32, 64, 128, 256}; + int loopTimes = 100; + for (auto n : searchNum) { + std::vector queryData(data.begin(), data.begin() + n * dim); + std::vector dists(n * topk, 0); + std::vector labels(n * topk, 0); + double ts = GetMillisecs(); + for (int i = 0; i < loopTimes; ++i) { + AscendIndexSearchParams searchParams {n, queryData, topk, dists, labels}; + index->Search(searchParams); + } + double te = GetMillisecs(); + printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", + ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); } - double te = GetMillisecs(); - printf("base:%zu, dim:%d, search num:%zu, QPS:%.4f\n", - ntotal, dim, n, MILLI_SECOND * n * loopTimes / (te - ts)); + } catch (std::exception &e) { + printf("%s\n", e.what()); } } } // namespace diff --git a/IndexSDK/TestAscendMultiSearch.cpp b/IndexSDK/TestAscendMultiSearch.cpp index 8f47c0cc9d8b17d59ac0a29836a46131ca67eee7..aff3e92527c14e69a1cc6bbe674c3167b6dfd610 100644 --- a/IndexSDK/TestAscendMultiSearch.cpp +++ b/IndexSDK/TestAscendMultiSearch.cpp @@ -83,26 +83,18 @@ struct IDFilter { timeRange[0] = 0; timeRange[1] = -1; } - + uint8_t cameraIdMask[16] = { 0xFF }; // cid uint32_t timeRange[2] = { 0 }; // 时间戳 }; -void AscendIndexSQMultiSearchFilter() +void DoSearchFilter(int ntotal, int dim, std::vector &data) { - int dim = 64; int searchNum = 2; int indexNum = 10; + std::vector indexes(indexNum); faiss::ascend::AscendIndexSQConfig conf({ 0 }); conf.filterable = true; - - int ntotal = 128; - std::vector data(dim * ntotal); - for (int i = 0; i < dim * ntotal; i++) { - data[i] = drand48(); - } - Norm(data.data(), ntotal, dim); - std::vector ids(ntotal, 0); int seed = std::chrono::system_clock::now().time_since_epoch().count(); std::default_random_engine e1(seed); @@ -115,7 +107,6 @@ void AscendIndexSQMultiSearchFilter() } std::cout << "AscendIndex SQ MultiSearch with different filter start" << std::endl; - std::vector indexes(indexNum); for (int i = 0; i < indexNum; i++) { indexes[i] = new faiss::ascend::AscendIndexSQ(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit, METRIC_TYPE, conf); @@ -147,25 +138,33 @@ void AscendIndexSQMultiSearchFilter() for (int i = 0; i < indexNum; i++) { delete indexes[i]; } - std::cout << "AscendIndex SQ MultiSearch with different filter end" << std::endl; } -void IndexSQMultiSearchFilter() +void AscendIndexSQMultiSearchFilter() { int dim = 64; + try { + int ntotal = 128; + std::vector data(dim * ntotal); + for (int i = 0; i < dim * ntotal; i++) { + data[i] = drand48(); + } + Norm(data.data(), ntotal, dim); + DoSearchFilter(ntotal, dim, data); + std::cout << "AscendIndex SQ MultiSearch with different filter end" << std::endl; + } catch (std::exception &e) { + printf("%s\n", e.what()); + } +} + +void DoIndexSearchFilter(int ntotal, int dim, std::vector &data) +{ int searchNum = 2; int indexNum = 10; + std::vector indexes(indexNum); + std::vector ids(ntotal, 0); faiss::ascend::AscendIndexSQConfig conf({ 0 }); conf.filterable = true; - - int ntotal = 128; - std::vector data(dim * ntotal); - for (int i = 0; i < dim * ntotal; i++) { - data[i] = drand48(); - } - Norm(data.data(), ntotal, dim); - - std::vector ids(ntotal, 0); int seed = std::chrono::system_clock::now().time_since_epoch().count(); std::default_random_engine e1(seed); std::uniform_int_distribution id(0, std::numeric_limits::max()); @@ -177,7 +176,6 @@ void IndexSQMultiSearchFilter() } std::cout << "Index SQ MultiSearch with different filter start" << std::endl; - std::vector indexes(indexNum); for (int i = 0; i < indexNum; i++) { indexes[i] = new faiss::ascend::AscendIndexSQ(dim, faiss::ScalarQuantizer::QuantizerType::QT_8bit, METRIC_TYPE, conf); @@ -205,11 +203,26 @@ void IndexSQMultiSearchFilter() std::vector dist(indexNum * k * searchNum, 0); std::vector label(indexNum * k * searchNum, 0); faiss::ascend::SearchWithFilter(indexes, searchNum, data.data(), k, dist.data(), label.data(), multiFilters, false); - for (int i = 0; i < indexNum; i++) { delete indexes[i]; } - std::cout << "Index SQ MultiSearch with different filter end" << std::endl; +} + +void IndexSQMultiSearchFilter() +{ + int dim = 64; + try { + int ntotal = 128; + std::vector data(dim * ntotal); + for (int i = 0; i < dim * ntotal; i++) { + data[i] = drand48(); + } + Norm(data.data(), ntotal, dim); + DoIndexSearchFilter(ntotal, dim, data); + std::cout << "Index SQ MultiSearch with different filter end" << std::endl; + } catch (std::exception &e) { + printf("%s\n", e.what()); + } } void AscendIndexSQMultiSearch() @@ -221,32 +234,39 @@ void AscendIndexSQMultiSearch() std::vector searchNum = { 1, 2, 4, 8 }; size_t maxSize = ntotal * dim; std::vector data(maxSize); - for (size_t i = 0; i < dim * ntotal; i++) { - data[i] = drand48(); - } - Norm(data.data(), ntotal, dim); - - std::cout << "AscendIndex SQ MultiSearch start" << std::endl; - faiss::ascend::AscendIndexSQConfig conf({ 0 }); std::vector indexes(indexNum); - for (int i = 0; i < indexNum; i++) { - indexes[i] = new faiss::ascend::AscendIndexSQ(dim, - faiss::ScalarQuantizer::QuantizerType::QT_8bit, METRIC_TYPE, conf); - - indexes[i]->train(ntotal, data.data()); - indexes[i]->add(ntotal, data.data()); - } - - for (size_t i = 0; i < searchNum.size(); i++) { - std::vector dist(indexNum * searchNum[i] * k, 0); - std::vector label(indexNum * searchNum[i] * k, 0); - - Search(indexes, searchNum[i], data.data(), k, dist.data(), label.data(), false); - } - for (int i = 0; i < indexNum; i++) { - delete indexes[i]; + try{ + for (size_t i = 0; i < dim * ntotal; i++) { + data[i] = drand48(); + } + Norm(data.data(), ntotal, dim); + + std::cout << "AscendIndex SQ MultiSearch start" << std::endl; + faiss::ascend::AscendIndexSQConfig conf({ 0 }); + for (int i = 0; i < indexNum; i++) { + indexes[i] = new faiss::ascend::AscendIndexSQ(dim, + faiss::ScalarQuantizer::QuantizerType::QT_8bit, METRIC_TYPE, conf); + + indexes[i]->train(ntotal, data.data()); + indexes[i]->add(ntotal, data.data()); + } + + for (size_t i = 0; i < searchNum.size(); i++) { + std::vector dist(indexNum * searchNum[i] * k, 0); + std::vector label(indexNum * searchNum[i] * k, 0); + + Search(indexes, searchNum[i], data.data(), k, dist.data(), label.data(), false); + } + for (int i = 0; i < indexNum; i++) { + delete indexes[i]; + } + std::cout << "AscendIndex SQ MultiSearch end" << std::endl; + } catch (std::exception &e) { + for (int i = 0; i < indexNum; i++) { + delete indexes[i]; + } + printf("%s\n", e.what()); } - std::cout << "AscendIndex SQ MultiSearch end" << std::endl; } void AscendIndexInt8MultiSearch() @@ -258,32 +278,40 @@ void AscendIndexInt8MultiSearch() std::vector searchNum = { 1, 2, 4, 8 }; std::vector> data(indexNum, std::vector(dim * ntotal)); - for (int i = 0; i < indexNum; i++) { - GenerateCodes(data[i].data(), ntotal, dim); - } - - std::cout << "AscendIndex Int8 MultiSearch start" << std::endl; - faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }); std::vector indexes(indexNum); - - for (int i = 0; i < indexNum; i++) { - indexes[i] = new faiss::ascend::AscendIndexInt8Flat(dim, METRIC_TYPE, conf); - - indexes[i]->add(ntotal, data[i].data()); - } + try { - for (size_t i = 0; i < searchNum.size(); i++) { - std::vector dist(indexNum * searchNum[i] * k, 0); - std::vector label(indexNum * searchNum[i] * k, 0); - std::vector query(dim * searchNum[i]); - GenerateCodes(query.data(), searchNum[i], dim); - - Search(indexes, searchNum[i], query.data(), k, dist.data(), label.data(), false); - } - for (int i = 0; i < indexNum; i++) { - delete indexes[i]; + for (int i = 0; i < indexNum; i++) { + GenerateCodes(data[i].data(), ntotal, dim); + } + + std::cout << "AscendIndex Int8 MultiSearch start" << std::endl; + faiss::ascend::AscendIndexInt8FlatConfig conf({ 0 }); + + for (int i = 0; i < indexNum; i++) { + indexes[i] = new faiss::ascend::AscendIndexInt8Flat(dim, METRIC_TYPE, conf); + + indexes[i]->add(ntotal, data[i].data()); + } + + for (size_t i = 0; i < searchNum.size(); i++) { + std::vector dist(indexNum * searchNum[i] * k, 0); + std::vector label(indexNum * searchNum[i] * k, 0); + std::vector query(dim * searchNum[i]); + GenerateCodes(query.data(), searchNum[i], dim); + + Search(indexes, searchNum[i], query.data(), k, dist.data(), label.data(), false); + } + for (int i = 0; i < indexNum; i++) { + delete indexes[i]; + } + std::cout << "AscendIndex Int8 MultiSearch end" << std::endl; + } catch (std::exception &e) { + for (int i = 0; i < indexNum; i++) { + delete indexes[i]; + } + printf("%s\n", e.what()); } - std::cout << "AscendIndex Int8 MultiSearch end" << std::endl; } } diff --git a/IndexSDK/dependencies.cmake b/IndexSDK/dependencies.cmake index 3ec5ccf32e51188e5b8ab64e0779e93e8388404a..27137c8f2afa6ca35fc8a7b8465a4b51f2f93bad 100644 --- a/IndexSDK/dependencies.cmake +++ b/IndexSDK/dependencies.cmake @@ -1,5 +1,5 @@ SET(MXINDEX_HOME /home/work/FeatureRetrieval/mxIndex/ CACHE STRING "") -SET(FAISS_HOME /usr/local/faiss/faiss1.7.4 CACHE STRING "") +SET(FAISS_HOME /usr/local/faiss/faiss1.10.0 CACHE STRING "") SET(GTEST_HOME /usr/local/gtest CACHE STRING "") SET(DRIVER_HOME /usr/local/Ascend/driver/ CACHE STRING "") SET(OPENBLAS /opt/OpenBLAS/ CACHE STRING "")